{
  "schema_version": 5,
  "results_version": "0bdaa447",
  "updated": "2026-11-03T18:59:28Z",
  "sure_threshold": 0.99,
  "dataset": {
    "name": "questions-v2.sqlite",
    "sha256": "797ad6c868bc1c9485bc54203f58839ad995b0443b7149f6a9e36cf0d07df4d7",
    "graded_sha256": "797ad6c868bc1c9485bc54203f58839ad995b0443b7149f6a9e36cf0d07df4d7"
  },
  "excluded_since_run": 0,
  "questions": 2198,
  "suspect": 12,
  "excluded": {
    "excluded": 12
  },
  "tyt_mix": {
    "reading": 33,
    "grammar": 7
  },
  "confidence_bins": [
    [
      0.0,
      0.25
    ],
    [
      0.25,
      0.5
    ],
    [
      0.5,
      0.75
    ],
    [
      0.75,
      0.9
    ],
    [
      0.9,
      1.0
    ]
  ],
  "coverages": [
    0.2,
    0.5,
    0.8
  ],
  "units": [
    {
      "number": 1,
      "name": "SÖZCÜKTE VE SÖZ ÖBEKLERİNDE ANLAM",
      "english": "Word and phrase meanings",
      "questions": 131
    },
    {
      "number": 2,
      "name": "CÜMLE ANLAMI",
      "english": "Sentence meanings",
      "questions": 154
    },
    {
      "number": 3,
      "name": "ANLATIM TEKNİKLERİ VE DÜŞÜNCEYİ GELİŞTİRME YOLLARI",
      "english": "How a writer explains and develops ideas",
      "questions": 70
    },
    {
      "number": 4,
      "name": "PARAGRAFTA KONU VE ANA DÜŞÜNCE",
      "english": "Topic and main point of a paragraph",
      "questions": 98
    },
    {
      "number": 5,
      "name": "PARAGRAFIN YAPISI",
      "english": "How a paragraph fits together",
      "questions": 106
    },
    {
      "number": 6,
      "name": "PARAGRAFTA YARDIMCI DÜŞÜNCELER",
      "english": "Details that support a paragraph’s main point",
      "questions": 97
    },
    {
      "number": 7,
      "name": "SÖZCÜK TÜRLERİ",
      "english": "Word types, such as nouns and verbs",
      "questions": 140
    },
    {
      "number": 8,
      "name": "TAMLAMALAR",
      "english": "Groups of words built around a noun",
      "questions": 65
    },
    {
      "number": 9,
      "name": "FİİLLER (EYLEMLER)",
      "english": "Verbs and their forms",
      "questions": 91
    },
    {
      "number": 10,
      "name": "FİİLİMSİLER",
      "english": "Verb forms used like nouns or modifiers",
      "questions": 72
    },
    {
      "number": 11,
      "name": "EKLER",
      "english": "Word endings and what they do",
      "questions": 79
    },
    {
      "number": 12,
      "name": "SÖZCÜK YAPISI",
      "english": "How words are built",
      "questions": 69
    },
    {
      "number": 13,
      "name": "CÜMLENİN ÖGELERİ",
      "english": "The roles of words and phrases in a sentence",
      "questions": 145
    },
    {
      "number": 14,
      "name": "FİİL ÇATISI",
      "english": "How verbs show who acts and who receives the action",
      "questions": 74
    },
    {
      "number": 15,
      "name": "CÜMLE ÇEŞİTLERİ",
      "english": "Types of sentences",
      "questions": 85
    },
    {
      "number": 16,
      "name": "KARMA DİL BİLGİSİ",
      "english": "Questions combining several grammar topics",
      "questions": 220
    },
    {
      "number": 17,
      "name": "SES BİLGİSİ",
      "english": "Sound patterns and changes",
      "questions": 77
    },
    {
      "number": 18,
      "name": "YAZIM KURALLARI",
      "english": "Spelling",
      "questions": 133
    },
    {
      "number": 19,
      "name": "NOKTALAMA İŞARETLERİ",
      "english": "Punctuation",
      "questions": 117
    },
    {
      "number": 20,
      "name": "ANLATIM BOZUKLUKLARI",
      "english": "Problems in sentence wording",
      "questions": 175
    }
  ],
  "prices": [
    {
      "provider": "baselabs",
      "model": "baselabs/elephant-1",
      "basis": "Baseten's list price for the same model",
      "source": "https://www.baseten.co/library/elephant-1/",
      "date": "2026-11-01",
      "change": null,
      "applies_to": "maker",
      "extra_input_per_request": 0,
      "rates": {
        "input_tokens": 0.035,
        "cached_input_tokens": 0.035,
        "cache_write_input_tokens": 0.035,
        "cache_write_1h_input_tokens": 0.035,
        "output_tokens": 0.0
      },
      "text": "$0.035 input, output free per 1M tokens"
    },
    {
      "provider": "baselabs",
      "model": "baselabs/mammoth-1",
      "basis": "Baseten's list price for the same model",
      "source": "https://www.baseten.co/library/mammoth-1/",
      "date": "2026-11-01",
      "change": null,
      "applies_to": "maker",
      "extra_input_per_request": 0,
      "rates": {
        "input_tokens": 0.1,
        "cached_input_tokens": 0.1,
        "cache_write_input_tokens": 0.1,
        "cache_write_1h_input_tokens": 0.1,
        "output_tokens": 0.0
      },
      "text": "$0.10 input, output free per 1M tokens"
    },
    {
      "provider": "celeris",
      "model": "celeris-1-decision",
      "basis": "Celeris's decision model price",
      "source": "https://docs.celeris.ai/pricing",
      "date": "2026-10-10",
      "change": null,
      "applies_to": "billed",
      "extra_input_per_request": 0,
      "rates": {
        "input_tokens": 0.04,
        "cached_input_tokens": 0.04,
        "cache_write_input_tokens": 0.04,
        "cache_write_1h_input_tokens": 0.04,
        "output_tokens": 0.0
      },
      "text": "$0.04 input, output free per 1M tokens"
    },
    {
      "provider": "claude",
      "model": "claude-fable-5-1",
      "basis": "Anthropic's API list price",
      "source": "https://platform.claude.com/docs/en/about-claude/pricing",
      "date": "2026-10-28",
      "change": null,
      "applies_to": "billed",
      "extra_input_per_request": 0,
      "rates": {
        "input_tokens": 10.0,
        "cached_input_tokens": 0.25,
        "cache_write_input_tokens": 12.5,
        "cache_write_1h_input_tokens": 20.0,
        "output_tokens": 50.0
      },
      "text": "$10 input, $0.25 cached input, $12.50 5-minute cache writes, $20 1-hour cache writes, $50 output per 1M tokens"
    },
    {
      "provider": "claude",
      "model": "claude-opus-5-5",
      "basis": "Anthropic's API list price",
      "source": "https://platform.claude.com/docs/en/about-claude/pricing",
      "date": "2026-10-08",
      "change": null,
      "applies_to": "billed",
      "extra_input_per_request": 0,
      "rates": {
        "input_tokens": 4.0,
        "cached_input_tokens": 0.2,
        "cache_write_input_tokens": 5.0,
        "cache_write_1h_input_tokens": 8.0,
        "output_tokens": 20.0
      },
      "text": "$4 input, $0.20 cached input, $5 5-minute cache writes, $8 1-hour cache writes, $20 output per 1M tokens"
    },
    {
      "provider": "claude",
      "model": "claude-sonnet-5-5",
      "basis": "Anthropic's API list price",
      "source": "https://platform.claude.com/docs/en/about-claude/pricing",
      "date": "2026-10-08",
      "change": null,
      "applies_to": "billed",
      "extra_input_per_request": 0,
      "rates": {
        "input_tokens": 2.0,
        "cached_input_tokens": 0.1,
        "cache_write_input_tokens": 2.5,
        "cache_write_1h_input_tokens": 4.0,
        "output_tokens": 10.0
      },
      "text": "$2 input, $0.10 cached input, $2.50 5-minute cache writes, $4 1-hour cache writes, $10 output per 1M tokens"
    },
    {
      "provider": "clef",
      "model": "clef",
      "basis": "Cloudflare's list price for the same model",
      "source": "https://developers.cloudflare.com/workers-ai/models/clef/",
      "date": "2026-10-18",
      "change": null,
      "applies_to": "maker",
      "extra_input_per_request": 0,
      "rates": {
        "input_tokens": 0.24,
        "cached_input_tokens": 0.24,
        "cache_write_input_tokens": 0.24,
        "cache_write_1h_input_tokens": 0.24,
        "output_tokens": 0.0
      },
      "text": "$0.24 input, output free per 1M tokens"
    },
    {
      "provider": "clef",
      "model": "clef-flash",
      "basis": "Cloudflare's list price for the same model",
      "source": "https://developers.cloudflare.com/workers-ai/models/clef-flash/",
      "date": "2026-10-18",
      "change": null,
      "applies_to": "maker",
      "extra_input_per_request": 0,
      "rates": {
        "input_tokens": 0.038,
        "cached_input_tokens": 0.038,
        "cache_write_input_tokens": 0.038,
        "cache_write_1h_input_tokens": 0.038,
        "output_tokens": 0.0
      },
      "text": "$0.038 input, output free per 1M tokens"
    },
    {
      "provider": "drex-max",
      "model": "drex-max-v1",
      "basis": "Nace.AI's list price for the same model",
      "source": "https://nace.ai/drex",
      "date": "2026-10-30",
      "change": null,
      "applies_to": "maker",
      "extra_input_per_request": 0,
      "rates": {
        "input_tokens": 0.06,
        "cached_input_tokens": 0.06,
        "cache_write_input_tokens": 0.06,
        "cache_write_1h_input_tokens": 0.06,
        "output_tokens": 0.0
      },
      "text": "$0.06 input, output free per 1M tokens"
    },
    {
      "provider": "fastino",
      "model": "fastino/GLiDE",
      "basis": "Fastino's GLiDE price",
      "source": "https://docs.fastino.ai/pricing",
      "date": "2026-10-08",
      "change": "$0.04 per 1M input tokens since, as listed on 22 October 2026",
      "applies_to": "billed",
      "extra_input_per_request": 0,
      "rates": {
        "input_tokens": 0.15,
        "cached_input_tokens": 0.15,
        "cache_write_input_tokens": 0.15,
        "cache_write_1h_input_tokens": 0.15,
        "output_tokens": 0.0
      },
      "text": "$0.15 input, output free per 1M tokens"
    },
    {
      "provider": "gemini",
      "model": "gemini-3.8-flash",
      "basis": "Google's Gemini API price, introductory",
      "source": "https://ai.google.dev/gemini-api/docs/pricing",
      "date": "2026-10-08",
      "change": "doubles on 1 January 2027, to $1.50 input and $7.50 output per 1M tokens",
      "applies_to": "billed",
      "extra_input_per_request": 0,
      "rates": {
        "input_tokens": 0.75,
        "cached_input_tokens": 0.075,
        "cache_write_input_tokens": 0.75,
        "cache_write_1h_input_tokens": 0.75,
        "output_tokens": 3.75
      },
      "text": "$0.75 input, $0.075 cached input, $3.75 output per 1M tokens"
    },
    {
      "provider": "gemini",
      "model": "gemini-4-argon",
      "basis": "Google's Gemini API regular price (from 1 December 2026)",
      "source": "https://ai.google.dev/gemini-api/docs/pricing",
      "date": "2026-10-28",
      "change": "introductory until 30 November 2026: $2.00 input, $10.00 output and $0.10 cached input per 1M tokens",
      "applies_to": "billed",
      "extra_input_per_request": 0,
      "rates": {
        "input_tokens": 4.0,
        "cached_input_tokens": 0.2,
        "cache_write_input_tokens": 4.0,
        "cache_write_1h_input_tokens": 4.0,
        "output_tokens": 20.0
      },
      "text": "$4 input, $0.20 cached input, $20 output per 1M tokens"
    },
    {
      "provider": "glide",
      "model": "glide-v1.2",
      "basis": "Fastino's list price for the same model",
      "source": "https://docs.fastino.ai/pricing",
      "date": "2026-10-22",
      "change": null,
      "applies_to": "maker",
      "extra_input_per_request": 0,
      "rates": {
        "input_tokens": 0.04,
        "cached_input_tokens": 0.04,
        "cache_write_input_tokens": 0.04,
        "cache_write_1h_input_tokens": 0.04,
        "output_tokens": 0.0
      },
      "text": "$0.04 input, output free per 1M tokens"
    },
    {
      "provider": "jev",
      "model": "jev-1.13.0",
      "basis": "TypeSafe's System One price",
      "source": "https://typesafe.ai/blog/introducing-system-one-models-and-jev",
      "date": "2026-10-08",
      "change": null,
      "applies_to": "billed",
      "extra_input_per_request": 0,
      "rates": {
        "input_tokens": 0.042,
        "cached_input_tokens": 0.042,
        "cache_write_input_tokens": 0.042,
        "cache_write_1h_input_tokens": 0.042,
        "output_tokens": 0.0
      },
      "text": "$0.042 input, output free per 1M tokens"
    },
    {
      "provider": "jev",
      "model": "jev-1.14.0",
      "basis": "TypeSafe's System One price",
      "source": "https://docs.typesafe.ai/models",
      "date": "2026-10-16",
      "change": null,
      "applies_to": "billed",
      "extra_input_per_request": 0,
      "rates": {
        "input_tokens": 0.042,
        "cached_input_tokens": 0.042,
        "cache_write_input_tokens": 0.042,
        "cache_write_1h_input_tokens": 0.042,
        "output_tokens": 0.0
      },
      "text": "$0.042 input, output free per 1M tokens"
    },
    {
      "provider": "liquid",
      "model": "d1-ultra",
      "basis": "Liquid AI's d1-ultra price",
      "source": "https://www.liquid.ai/blog/d1-ultra",
      "date": "2026-10-16",
      "change": null,
      "applies_to": "billed",
      "extra_input_per_request": 0,
      "rates": {
        "input_tokens": 0.042,
        "cached_input_tokens": 0.042,
        "cache_write_input_tokens": 0.042,
        "cache_write_1h_input_tokens": 0.042,
        "output_tokens": 0.0
      },
      "text": "$0.042 input, output free per 1M tokens"
    },
    {
      "provider": "liquid",
      "model": "d1.1",
      "basis": "Liquid AI's price for d1.1, as its API bills it",
      "source": "https://www.liquid.ai/pricing",
      "date": "2026-10-24",
      "change": null,
      "applies_to": "billed",
      "extra_input_per_request": 0,
      "rates": {
        "input_tokens": 0.02,
        "cached_input_tokens": 0.02,
        "cache_write_input_tokens": 0.02,
        "cache_write_1h_input_tokens": 0.02,
        "output_tokens": 0.0
      },
      "text": "$0.02 input, output free per 1M tokens"
    },
    {
      "provider": "liquid",
      "model": "d1:free",
      "basis": "Liquid's list price for the same model",
      "source": "https://www.liquid.ai/pricing",
      "date": "2026-10-18",
      "change": null,
      "applies_to": "free tier",
      "extra_input_per_request": 0,
      "rates": {
        "input_tokens": 0.02,
        "cached_input_tokens": 0.02,
        "cache_write_input_tokens": 0.02,
        "cache_write_1h_input_tokens": 0.02,
        "output_tokens": 0.0
      },
      "text": "$0.02 input, output free per 1M tokens"
    },
    {
      "provider": "meragpt",
      "model": "decider-1",
      "basis": "meraGPT's Decider 1 price",
      "source": "https://meragpt.com/docs/models",
      "date": "2026-10-19",
      "change": null,
      "applies_to": "billed",
      "extra_input_per_request": 0,
      "rates": {
        "input_tokens": 0.04,
        "cached_input_tokens": 0.04,
        "cache_write_input_tokens": 0.04,
        "cache_write_1h_input_tokens": 0.04,
        "output_tokens": 0.0
      },
      "text": "$0.04 input, output free per 1M tokens"
    },
    {
      "provider": "nace",
      "model": "drex-max-v1.1",
      "basis": "Nace.AI's list price (launch price $0.03 per 1M input tokens, 50% off, is what Nace bills)",
      "source": "https://nace.ai/drex",
      "date": "2026-10-30",
      "change": null,
      "applies_to": "billed",
      "extra_input_per_request": 0,
      "rates": {
        "input_tokens": 0.06,
        "cached_input_tokens": 0.06,
        "cache_write_input_tokens": 0.06,
        "cache_write_1h_input_tokens": 0.06,
        "output_tokens": 0.0
      },
      "text": "$0.06 input, output free per 1M tokens"
    },
    {
      "provider": "nace",
      "model": "drex-v2.0",
      "basis": "Nace.AI's list price",
      "source": "https://console.nace.ai/docs/drex/reference/pricing",
      "date": "2026-10-30",
      "change": null,
      "applies_to": "billed",
      "extra_input_per_request": 0,
      "rates": {
        "input_tokens": 0.06,
        "cached_input_tokens": 0.06,
        "cache_write_input_tokens": 0.06,
        "cache_write_1h_input_tokens": 0.06,
        "output_tokens": 0.0
      },
      "text": "$0.06 input, output free per 1M tokens"
    },
    {
      "provider": "openai",
      "model": "gpt-6-astra",
      "basis": "OpenAI's API list price",
      "source": "https://developers.openai.com/api/docs/pricing",
      "date": "2026-10-08",
      "change": null,
      "applies_to": "billed",
      "extra_input_per_request": 0,
      "rates": {
        "input_tokens": 10.0,
        "cached_input_tokens": 1.0,
        "cache_write_input_tokens": 12.5,
        "cache_write_1h_input_tokens": 10.0,
        "output_tokens": 50.0
      },
      "text": "$10 input, $1 cached input, $12.50 cache writes, $50 output per 1M tokens"
    },
    {
      "provider": "openai",
      "model": "gpt-6-luna",
      "basis": "OpenAI's API list price",
      "source": "https://developers.openai.com/api/docs/pricing",
      "date": "2026-10-08",
      "change": null,
      "applies_to": "billed",
      "extra_input_per_request": 0,
      "rates": {
        "input_tokens": 0.1,
        "cached_input_tokens": 0.01,
        "cache_write_input_tokens": 0.125,
        "cache_write_1h_input_tokens": 0.1,
        "output_tokens": 0.5
      },
      "text": "$0.10 input, $0.01 cached input, $0.125 cache writes, $0.50 output per 1M tokens"
    },
    {
      "provider": "openai",
      "model": "gpt-6.1-sol",
      "basis": "OpenAI's API list price",
      "source": "https://developers.openai.com/api/docs/pricing",
      "date": "2026-10-08",
      "change": null,
      "applies_to": "billed",
      "extra_input_per_request": 0,
      "rates": {
        "input_tokens": 2.0,
        "cached_input_tokens": 0.1,
        "cache_write_input_tokens": 2.5,
        "cache_write_1h_input_tokens": 2.0,
        "output_tokens": 10.0
      },
      "text": "$2 input, $0.10 cached input, $2.50 cache writes, $10 output per 1M tokens"
    },
    {
      "provider": "openai",
      "model": "gpt-6.1-terra",
      "basis": "OpenAI's API list price",
      "source": "https://developers.openai.com/api/docs/pricing",
      "date": "2026-10-28",
      "change": null,
      "applies_to": "billed",
      "extra_input_per_request": 0,
      "rates": {
        "input_tokens": 1.0,
        "cached_input_tokens": 0.05,
        "cache_write_input_tokens": 1.25,
        "cache_write_1h_input_tokens": 1.0,
        "output_tokens": 5.0
      },
      "text": "$1 input, $0.05 cached input, $1.25 cache writes, $5 output per 1M tokens"
    },
    {
      "provider": "openai-decisions",
      "model": "gpt-6-luna",
      "basis": "OpenAI's Decisions API price",
      "source": "https://developers.openai.com/api/docs/guides/decisions#pricing-and-availability",
      "date": "2026-10-08",
      "change": null,
      "applies_to": "billed",
      "extra_input_per_request": 0,
      "rates": {
        "input_tokens": 0.1,
        "cached_input_tokens": 0.1,
        "cache_write_input_tokens": 0.1,
        "cache_write_1h_input_tokens": 0.1,
        "output_tokens": 0.0
      },
      "text": "$0.10 input, output free per 1M tokens"
    },
    {
      "provider": "openrouter",
      "model": "inception/mercury-decide",
      "basis": "OpenRouter's Mercury Decide price",
      "source": "https://openrouter.ai/inception/mercury-decide",
      "date": "2026-10-17",
      "change": null,
      "applies_to": "billed",
      "extra_input_per_request": 0,
      "rates": {
        "input_tokens": 0.02,
        "cached_input_tokens": 0.02,
        "cache_write_input_tokens": 0.02,
        "cache_write_1h_input_tokens": 0.02,
        "output_tokens": 0.0
      },
      "text": "$0.02 input, output free per 1M tokens"
    },
    {
      "provider": "openrouter",
      "model": "microsoft/microsoft-decision-1",
      "basis": "OpenRouter's Microsoft-Decision-1 price",
      "source": "https://openrouter.ai/microsoft/microsoft-decision-1",
      "date": "2026-10-17",
      "change": null,
      "applies_to": "billed",
      "extra_input_per_request": 0,
      "rates": {
        "input_tokens": 0.042,
        "cached_input_tokens": 0.042,
        "cache_write_input_tokens": 0.042,
        "cache_write_1h_input_tokens": 0.042,
        "output_tokens": 0.0
      },
      "text": "$0.042 input, output free per 1M tokens"
    },
    {
      "provider": "openrouter-chat",
      "model": "moonshotai/kimi-k3.1-pro",
      "basis": "OpenRouter's price for Moonshot AI's endpoint (mxfp4)",
      "source": "https://openrouter.ai/moonshotai/kimi-k3.1-pro",
      "date": "2026-10-28",
      "change": null,
      "applies_to": "billed",
      "extra_input_per_request": 0,
      "rates": {
        "input_tokens": 2.0,
        "cached_input_tokens": 0.2,
        "cache_write_input_tokens": 2.0,
        "cache_write_1h_input_tokens": 2.0,
        "output_tokens": 10.0
      },
      "text": "$2 input, $0.20 cached input, $10 output per 1M tokens"
    },
    {
      "provider": "openrouter-chat",
      "model": "z-ai/glm-5.3",
      "basis": "OpenRouter's price for Fireworks' spot endpoint, which served nearly every request",
      "source": "https://openrouter.ai/z-ai/glm-5.3",
      "date": "2026-10-28",
      "change": null,
      "applies_to": "billed",
      "extra_input_per_request": 0,
      "rates": {
        "input_tokens": 0.98,
        "cached_input_tokens": 0.182,
        "cache_write_input_tokens": 0.98,
        "cache_write_1h_input_tokens": 0.98,
        "output_tokens": 3.08
      },
      "text": "$0.98 input, $0.182 cached input, $3.08 output per 1M tokens"
    },
    {
      "provider": "perplexity",
      "model": "pplx-decider-v1-27b",
      "basis": "Perplexity's Decisions API price",
      "source": "https://docs.perplexity.ai/docs/getting-started/pricing",
      "date": "2026-10-04",
      "change": "$0.02 per 1M input tokens since, as listed on 8 October 2026",
      "applies_to": "billed",
      "extra_input_per_request": 0,
      "rates": {
        "input_tokens": 0.04,
        "cached_input_tokens": 0.04,
        "cache_write_input_tokens": 0.04,
        "cache_write_1h_input_tokens": 0.04,
        "output_tokens": 0.0
      },
      "text": "$0.04 input, output free per 1M tokens"
    },
    {
      "provider": "perplexity",
      "model": "pplx-decider-v1.1-27b",
      "basis": "Perplexity's Decisions API price",
      "source": "https://docs.perplexity.ai/docs/getting-started/pricing",
      "date": "2026-10-09",
      "change": null,
      "applies_to": "billed",
      "extra_input_per_request": 0,
      "rates": {
        "input_tokens": 0.02,
        "cached_input_tokens": 0.02,
        "cache_write_input_tokens": 0.02,
        "cache_write_1h_input_tokens": 0.02,
        "output_tokens": 0.0
      },
      "text": "$0.02 input, output free per 1M tokens"
    },
    {
      "provider": "pplx-decider",
      "model": "pplx-decider-flash-v1.2",
      "basis": "Perplexity's list price for the same model",
      "source": "https://docs.perplexity.ai/docs/getting-started/pricing",
      "date": "2026-10-18",
      "change": null,
      "applies_to": "maker",
      "extra_input_per_request": 3,
      "rates": {
        "input_tokens": 0.01,
        "cached_input_tokens": 0.01,
        "cache_write_input_tokens": 0.01,
        "cache_write_1h_input_tokens": 0.01,
        "output_tokens": 0.0
      },
      "text": "$0.01 input, output free per 1M tokens, plus 3 billed input tokens per request"
    },
    {
      "provider": "pplx-decider",
      "model": "pplx-decider-v1.2",
      "basis": "Perplexity's list price for the same model",
      "source": "https://docs.perplexity.ai/docs/getting-started/pricing",
      "date": "2026-10-18",
      "change": null,
      "applies_to": "maker",
      "extra_input_per_request": 3,
      "rates": {
        "input_tokens": 0.02,
        "cached_input_tokens": 0.02,
        "cache_write_input_tokens": 0.02,
        "cache_write_1h_input_tokens": 0.02,
        "output_tokens": 0.0
      },
      "text": "$0.02 input, output free per 1M tokens, plus 3 billed input tokens per request"
    },
    {
      "provider": "subq",
      "model": "subq-snap-1.1",
      "basis": "SubQ's list price",
      "source": "https://docs.subq.ai/using-the-api/models/",
      "date": "2026-11-01",
      "change": null,
      "applies_to": "billed",
      "extra_input_per_request": 0,
      "rates": {
        "input_tokens": 0.04,
        "cached_input_tokens": 0.04,
        "cache_write_input_tokens": 0.04,
        "cache_write_1h_input_tokens": 0.04,
        "output_tokens": 0.0
      },
      "text": "$0.04 input, output free per 1M tokens"
    },
    {
      "provider": "subq",
      "model": "subq-snap-preview",
      "basis": "SubQ's Decisions API price",
      "source": "https://docs.subq.ai/using-the-api/models/",
      "date": "2026-10-13",
      "change": null,
      "applies_to": "billed",
      "extra_input_per_request": 0,
      "rates": {
        "input_tokens": 0.05,
        "cached_input_tokens": 0.05,
        "cache_write_input_tokens": 0.05,
        "cache_write_1h_input_tokens": 0.05,
        "output_tokens": 0.0
      },
      "text": "$0.05 input, output free per 1M tokens"
    },
    {
      "provider": "vllm",
      "model": "qwen38-27b-bf16",
      "basis": "Alibaba's list price for the same model",
      "source": "https://www.alibabacloud.com/help/en/model-studio/qwen3-8-27b",
      "date": "2026-10-18",
      "change": null,
      "applies_to": "maker",
      "extra_input_per_request": 0,
      "rates": {
        "input_tokens": 0.5,
        "cached_input_tokens": 0.1,
        "cache_write_input_tokens": 0.5,
        "cache_write_1h_input_tokens": 0.5,
        "output_tokens": 3.0
      },
      "text": "$0.50 input, $0.10 cached input, $3 output per 1M tokens"
    },
    {
      "provider": "vllm-verbal",
      "model": "qwen38-27b-bf16",
      "basis": "Alibaba's list price for the same model",
      "source": "https://www.alibabacloud.com/help/en/model-studio/qwen3-8-27b",
      "date": "2026-10-18",
      "change": null,
      "applies_to": "maker",
      "extra_input_per_request": 0,
      "rates": {
        "input_tokens": 0.5,
        "cached_input_tokens": 0.1,
        "cache_write_input_tokens": 0.5,
        "cache_write_1h_input_tokens": 0.5,
        "output_tokens": 3.0
      },
      "text": "$0.50 input, $0.10 cached input, $3 output per 1M tokens"
    },
    {
      "provider": "vllm-vote",
      "model": "qwen38-27b-bf16",
      "basis": "Alibaba's list price for the same model",
      "source": "https://www.alibabacloud.com/help/en/model-studio/qwen3-8-27b",
      "date": "2026-10-18",
      "change": null,
      "applies_to": "maker",
      "extra_input_per_request": 0,
      "rates": {
        "input_tokens": 0.5,
        "cached_input_tokens": 0.1,
        "cache_write_input_tokens": 0.5,
        "cache_write_1h_input_tokens": 0.5,
        "output_tokens": 3.0
      },
      "text": "$0.50 input, $0.10 cached input, $3 output per 1M tokens"
    },
    {
      "provider": "vllm-yes-no",
      "model": "qwen38-27b-bf16",
      "basis": "Alibaba's list price for the same model",
      "source": "https://www.alibabacloud.com/help/en/model-studio/qwen3-8-27b",
      "date": "2026-10-18",
      "change": null,
      "applies_to": "maker",
      "extra_input_per_request": 0,
      "rates": {
        "input_tokens": 0.5,
        "cached_input_tokens": 0.1,
        "cache_write_input_tokens": 0.5,
        "cache_write_1h_input_tokens": 0.5,
        "output_tokens": 3.0
      },
      "text": "$0.50 input, $0.10 cached input, $3 output per 1M tokens"
    },
    {
      "provider": "xai",
      "model": "grok-4.7",
      "basis": "xAI's API list price",
      "source": "https://docs.x.ai/docs/models/grok-4.7",
      "date": "2026-10-28",
      "change": null,
      "applies_to": "billed",
      "extra_input_per_request": 0,
      "rates": {
        "input_tokens": 2.0,
        "cached_input_tokens": 0.5,
        "cache_write_input_tokens": 2.0,
        "cache_write_1h_input_tokens": 2.0,
        "output_tokens": 6.0
      },
      "text": "$2 input, $0.50 cached input, $6 output per 1M tokens"
    }
  ],
  "runs": [
    {
      "run": 140,
      "model": "gemini-3.8-flash",
      "name": "Gemini 3.8 Flash",
      "variant": null,
      "reasoning": "high",
      "provider": "gemini",
      "note": null,
      "parameters": null,
      "active_parameters": null,
      "access": "Gemini API",
      "questions_per_request": 16,
      "concurrency": 4,
      "correct": 2156,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 637
        },
        "grammar": {
          "questions": 1542,
          "correct": 1519
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 2151
        }
      },
      "tyt_mix": 0.9734949364936257,
      "questions_per_minute": 51.00400853213506,
      "request_seconds": 2585.6791220029118,
      "median_request_seconds": 53.66963235649746,
      "p90_request_seconds": 134.23365150109166,
      "billing": "api",
      "api_equivalent_usd": 5.31718875,
      "cost_basis": "api",
      "requests": 138,
      "hardware": null,
      "input_tokens": 483830,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 1321151,
      "confidence": {
        "source": "stated",
        "label": "Written estimate",
        "bins": [
          {
            "questions": 0,
            "correct": 0
          },
          {
            "questions": 0,
            "correct": 0
          },
          {
            "questions": 1,
            "correct": 0
          },
          {
            "questions": 7,
            "correct": 1
          },
          {
            "questions": 2190,
            "correct": 2155
          }
        ],
        "auroc": 0.8327259475218659,
        "auroc_interval": [
          0.7525756872017904,
          0.9011506960046598
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.9027100718830042,
            "auroc_interval": [
              0.8166444147752558,
              0.9584406663232011
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.7695423190313994,
            "auroc_interval": [
              0.6546877678112324,
              0.8733423342334233
            ]
          }
        },
        "aurc": 0.006165393746823294,
        "ece": 0.010841674249322482,
        "brier": 0.01698002729754322,
        "coverage_accuracy": {
          "0.20": 0.9943470887507067,
          "0.50": 0.9943470887507067,
          "0.80": 0.9943470887507067
        },
        "coverage_intervals": {
          "0.20": [
            0.9904440697020798,
            0.9972283813747229
          ],
          "0.50": [
            0.9904440697020798,
            0.9972283813747229
          ],
          "0.80": [
            0.9904440697020798,
            0.9972283813747229
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0057
          ],
          [
            0.04,
            0.0057
          ],
          [
            0.06,
            0.0057
          ],
          [
            0.08,
            0.0057
          ],
          [
            0.1,
            0.0057
          ],
          [
            0.12,
            0.0057
          ],
          [
            0.14,
            0.0057
          ],
          [
            0.16,
            0.0057
          ],
          [
            0.18,
            0.0057
          ],
          [
            0.2,
            0.0057
          ],
          [
            0.22,
            0.0057
          ],
          [
            0.24,
            0.0057
          ],
          [
            0.26,
            0.0057
          ],
          [
            0.28,
            0.0057
          ],
          [
            0.3,
            0.0057
          ],
          [
            0.32,
            0.0057
          ],
          [
            0.34,
            0.0057
          ],
          [
            0.36,
            0.0057
          ],
          [
            0.38,
            0.0057
          ],
          [
            0.4,
            0.0057
          ],
          [
            0.42,
            0.0057
          ],
          [
            0.44,
            0.0057
          ],
          [
            0.46,
            0.0057
          ],
          [
            0.48,
            0.0057
          ],
          [
            0.5,
            0.0057
          ],
          [
            0.52,
            0.0057
          ],
          [
            0.54,
            0.0057
          ],
          [
            0.56,
            0.0057
          ],
          [
            0.58,
            0.0057
          ],
          [
            0.6,
            0.0057
          ],
          [
            0.62,
            0.0057
          ],
          [
            0.64,
            0.0057
          ],
          [
            0.66,
            0.0057
          ],
          [
            0.68,
            0.0057
          ],
          [
            0.7,
            0.0057
          ],
          [
            0.72,
            0.0057
          ],
          [
            0.74,
            0.0057
          ],
          [
            0.76,
            0.0057
          ],
          [
            0.78,
            0.0057
          ],
          [
            0.8,
            0.0057
          ],
          [
            0.82,
            0.0055
          ],
          [
            0.84,
            0.0054
          ],
          [
            0.86,
            0.0053
          ],
          [
            0.88,
            0.0057
          ],
          [
            0.9,
            0.0071
          ],
          [
            0.92,
            0.0086
          ],
          [
            0.94,
            0.0099
          ],
          [
            0.96,
            0.0112
          ],
          [
            0.98,
            0.0125
          ],
          [
            1.0,
            0.0191
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 1,
            "mean_confidence": 0.6,
            "accuracy": 0.0
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 7,
            "mean_confidence": 0.8428571428571426,
            "accuracy": 0.14285714285714285
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 2190,
            "mean_confidence": 0.9923881278538862,
            "accuracy": 0.9840182648401826
          }
        ],
        "median_right": 1.0,
        "median_wrong": 0.95,
        "sure": {
          "questions": 1845,
          "correct": 1835,
          "interval": [
            0.9900512791575007,
            0.9970532948633348
          ]
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 1,
          "correct": 0,
          "interval": [
            0.0,
            0.7934567085261071
          ]
        },
        "ece_refit": 0.00892553033970141,
        "brier_refit": 0.017539275961408893,
        "brier_base": 0.018757831705756473,
        "refit_slopes": [
          0.2587321746273532,
          0.2832978624233587,
          0.27348574886360777,
          0.3519947450080838,
          0.26746730012401315
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        127,
        153,
        66,
        96,
        99,
        96,
        138,
        65,
        91,
        71,
        78,
        67,
        142,
        73,
        84,
        217,
        77,
        133,
        115,
        168
      ],
      "started": "2026-10-07T19:47:57Z",
      "updated": "2026-10-07T21:20:08Z",
      "format_of": null,
      "group": "a",
      "latency": {
        "source": "run",
        "runs": [
          140
        ],
        "concurrency": 4,
        "median_request_seconds": 53.66963235649746,
        "p90_request_seconds": 134.23365150109166
      },
      "single_question_latency": null
    },
    {
      "run": 277,
      "model": "gpt-6-astra",
      "name": "GPT-6 Astra",
      "variant": null,
      "reasoning": "high",
      "provider": "openai",
      "note": null,
      "parameters": null,
      "active_parameters": null,
      "access": "Codex CLI, ChatGPT subscription",
      "questions_per_request": 16,
      "concurrency": 1,
      "correct": 2153,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 637
        },
        "grammar": {
          "questions": 1542,
          "correct": 1516
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 2149
        }
      },
      "tyt_mix": 0.9731544695675556,
      "questions_per_minute": 33.629793554323186,
      "request_seconds": 3921.522735100065,
      "median_request_seconds": 21.670585694999318,
      "p90_request_seconds": 49.642695360598736,
      "billing": "subscription",
      "api_equivalent_usd": 13.33209,
      "cost_basis": "subscription",
      "requests": 138,
      "hardware": null,
      "input_tokens": 1163564,
      "cached_input_tokens": 265600,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 81737,
      "confidence": {
        "source": "stated",
        "label": "Written estimate",
        "bins": [
          {
            "questions": 3,
            "correct": 2
          },
          {
            "questions": 3,
            "correct": 1
          },
          {
            "questions": 11,
            "correct": 9
          },
          {
            "questions": 25,
            "correct": 16
          },
          {
            "questions": 2156,
            "correct": 2125
          }
        ],
        "auroc": 0.8900345770759147,
        "auroc_interval": [
          0.836810384793695,
          0.9369320815948076
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.9533999834751714,
            "auroc_interval": [
              0.9275118858954041,
              0.9742194042349563
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.8448853257560381,
            "auroc_interval": [
              0.7577947161481686,
              0.9170957095709571
            ]
          }
        },
        "aurc": 0.0032113620469389047,
        "ece": 0.0065000000000037935,
        "brier": 0.018129200636942674,
        "coverage_accuracy": {
          "0.20": 1.0,
          "0.50": 0.9970361065038816,
          "0.80": 0.996125224500211
        },
        "coverage_intervals": {
          "0.20": [
            1.0,
            1.0
          ],
          "0.50": [
            0.9947144150965808,
            0.9991413098393058
          ],
          "0.80": [
            0.9932000059388595,
            0.9988822343080126
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0
          ],
          [
            0.04,
            0.0
          ],
          [
            0.06,
            0.0
          ],
          [
            0.08,
            0.0
          ],
          [
            0.1,
            0.0
          ],
          [
            0.12,
            0.0
          ],
          [
            0.14,
            0.0
          ],
          [
            0.16,
            0.0
          ],
          [
            0.18,
            0.0
          ],
          [
            0.2,
            0.0
          ],
          [
            0.22,
            0.0
          ],
          [
            0.24,
            0.0003
          ],
          [
            0.26,
            0.0007
          ],
          [
            0.28,
            0.0011
          ],
          [
            0.3,
            0.0013
          ],
          [
            0.32,
            0.0016
          ],
          [
            0.34,
            0.0018
          ],
          [
            0.36,
            0.002
          ],
          [
            0.38,
            0.0022
          ],
          [
            0.4,
            0.0024
          ],
          [
            0.42,
            0.0025
          ],
          [
            0.44,
            0.0026
          ],
          [
            0.46,
            0.0028
          ],
          [
            0.48,
            0.0029
          ],
          [
            0.5,
            0.003
          ],
          [
            0.52,
            0.0031
          ],
          [
            0.54,
            0.0031
          ],
          [
            0.56,
            0.0032
          ],
          [
            0.58,
            0.0033
          ],
          [
            0.6,
            0.0034
          ],
          [
            0.62,
            0.0034
          ],
          [
            0.64,
            0.0035
          ],
          [
            0.66,
            0.0036
          ],
          [
            0.68,
            0.0036
          ],
          [
            0.7,
            0.0037
          ],
          [
            0.72,
            0.0037
          ],
          [
            0.74,
            0.0038
          ],
          [
            0.76,
            0.0038
          ],
          [
            0.78,
            0.0038
          ],
          [
            0.8,
            0.0039
          ],
          [
            0.82,
            0.0041
          ],
          [
            0.84,
            0.0049
          ],
          [
            0.86,
            0.0057
          ],
          [
            0.88,
            0.0064
          ],
          [
            0.9,
            0.0071
          ],
          [
            0.92,
            0.0079
          ],
          [
            0.94,
            0.0088
          ],
          [
            0.96,
            0.0117
          ],
          [
            0.98,
            0.0139
          ],
          [
            1.0,
            0.0205
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 4,
            "mean_confidence": 0.21250000000000002,
            "accuracy": 0.5
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 1,
            "mean_confidence": 0.35,
            "accuracy": 0.0
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 1,
            "mean_confidence": 0.45,
            "accuracy": 1.0
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 2,
            "mean_confidence": 0.515,
            "accuracy": 1.0
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 6,
            "mean_confidence": 0.645,
            "accuracy": 0.6666666666666666
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 8,
            "mean_confidence": 0.7474999999999999,
            "accuracy": 0.75
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 20,
            "mean_confidence": 0.8545000000000001,
            "accuracy": 0.65
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 2156,
            "mean_confidence": 0.9888807977736588,
            "accuracy": 0.985621521335807
          }
        ],
        "median_right": 0.99,
        "median_wrong": 0.96,
        "sure": {
          "questions": 1793,
          "correct": 1786,
          "interval": [
            0.9919629649507112,
            0.9981076122220346
          ]
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 14,
          "correct": 9,
          "interval": [
            0.3876400468214042,
            0.8365550926279727
          ]
        },
        "ece_refit": 0.005550159603769864,
        "brier_refit": 0.018528120305340724,
        "brier_base": 0.020076204043327585,
        "refit_slopes": [
          1.0198556796032585,
          1.1634421606027185,
          1.1612631612273636,
          1.0100661460861688,
          1.0538539000054599
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        128,
        152,
        65,
        95,
        101,
        96,
        139,
        64,
        91,
        71,
        78,
        66,
        142,
        73,
        84,
        214,
        76,
        132,
        114,
        172
      ],
      "started": "2026-10-28T00:51:50Z",
      "updated": "2026-10-28T01:57:12Z",
      "format_of": null,
      "group": "a",
      "latency": {
        "source": "run",
        "runs": [
          277
        ],
        "concurrency": 1,
        "median_request_seconds": 21.670585694999318,
        "p90_request_seconds": 49.642695360598736
      },
      "single_question_latency": null
    },
    {
      "run": 276,
      "model": "gpt-6.1-sol",
      "name": "GPT-6.1 Sol",
      "variant": null,
      "reasoning": "high",
      "provider": "openai",
      "note": "It stopped after 1,792 answers when the service reported the model at capacity, and was resumed for the other 406 questions; the usage and cost of the failed request are unknown.",
      "parameters": null,
      "active_parameters": null,
      "access": "Codex CLI, ChatGPT subscription",
      "questions_per_request": 16,
      "concurrency": 1,
      "correct": 2149,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 633
        },
        "grammar": {
          "questions": 1542,
          "correct": 1516
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 2145
        }
      },
      "tyt_mix": 0.9681239817626774,
      "questions_per_minute": 27.420445672643766,
      "request_seconds": 4809.549836440885,
      "median_request_seconds": 24.741423804487567,
      "p90_request_seconds": 70.32583790899079,
      "billing": "subscription",
      "api_equivalent_usd": 3.565574,
      "cost_basis": "subscription",
      "requests": 138,
      "hardware": null,
      "input_tokens": 1163522,
      "cached_input_tokens": 208000,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 163373,
      "confidence": {
        "source": "stated",
        "label": "Written estimate",
        "bins": [
          {
            "questions": 3,
            "correct": 1
          },
          {
            "questions": 3,
            "correct": 1
          },
          {
            "questions": 11,
            "correct": 8
          },
          {
            "questions": 20,
            "correct": 13
          },
          {
            "questions": 2161,
            "correct": 2126
          }
        ],
        "auroc": 0.8789660117187871,
        "auroc_interval": [
          0.8284532200110558,
          0.9278642619838633
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.947386496325297,
            "auroc_interval": [
              0.8948360124830713,
              0.9792124542124542
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.815366856099046,
            "auroc_interval": [
              0.7374633304572907,
              0.8982812059735137
            ]
          }
        },
        "aurc": 0.00394303052141354,
        "ece": 0.009818016378528189,
        "brier": 0.01932666060054595,
        "coverage_accuracy": {
          "0.20": 1.0,
          "0.50": 0.9959160612618406,
          "0.80": 0.9946044753621909
        },
        "coverage_intervals": {
          "0.20": [
            1.0,
            1.0
          ],
          "0.50": [
            0.9930038388914716,
            0.9983462471706068
          ],
          "0.80": [
            0.9908339861893458,
            0.9978225708723937
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0
          ],
          [
            0.04,
            0.0
          ],
          [
            0.06,
            0.0
          ],
          [
            0.08,
            0.0
          ],
          [
            0.1,
            0.0
          ],
          [
            0.12,
            0.0
          ],
          [
            0.14,
            0.0
          ],
          [
            0.16,
            0.0
          ],
          [
            0.18,
            0.0
          ],
          [
            0.2,
            0.0
          ],
          [
            0.22,
            0.0
          ],
          [
            0.24,
            0.0003
          ],
          [
            0.26,
            0.0009
          ],
          [
            0.28,
            0.0013
          ],
          [
            0.3,
            0.0018
          ],
          [
            0.32,
            0.0021
          ],
          [
            0.34,
            0.0024
          ],
          [
            0.36,
            0.0027
          ],
          [
            0.38,
            0.003
          ],
          [
            0.4,
            0.0032
          ],
          [
            0.42,
            0.0034
          ],
          [
            0.44,
            0.0036
          ],
          [
            0.46,
            0.0038
          ],
          [
            0.48,
            0.0039
          ],
          [
            0.5,
            0.0041
          ],
          [
            0.52,
            0.0042
          ],
          [
            0.54,
            0.0043
          ],
          [
            0.56,
            0.0045
          ],
          [
            0.58,
            0.0046
          ],
          [
            0.6,
            0.0047
          ],
          [
            0.62,
            0.0048
          ],
          [
            0.64,
            0.0048
          ],
          [
            0.66,
            0.0049
          ],
          [
            0.68,
            0.005
          ],
          [
            0.7,
            0.0051
          ],
          [
            0.72,
            0.0052
          ],
          [
            0.74,
            0.0052
          ],
          [
            0.76,
            0.0053
          ],
          [
            0.78,
            0.0053
          ],
          [
            0.8,
            0.0054
          ],
          [
            0.82,
            0.0054
          ],
          [
            0.84,
            0.0058
          ],
          [
            0.86,
            0.0064
          ],
          [
            0.88,
            0.007
          ],
          [
            0.9,
            0.0075
          ],
          [
            0.92,
            0.0081
          ],
          [
            0.94,
            0.0089
          ],
          [
            0.96,
            0.0125
          ],
          [
            0.98,
            0.0156
          ],
          [
            1.0,
            0.0223
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 4,
            "mean_confidence": 0.21750000000000003,
            "accuracy": 0.5
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 1,
            "mean_confidence": 0.35,
            "accuracy": 0.0
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 1,
            "mean_confidence": 0.4,
            "accuracy": 0.0
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 3,
            "mean_confidence": 0.5366666666666667,
            "accuracy": 1.0
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 5,
            "mean_confidence": 0.6559999999999999,
            "accuracy": 0.4
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 5,
            "mean_confidence": 0.736,
            "accuracy": 1.0
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 18,
            "mean_confidence": 0.8566666666666667,
            "accuracy": 0.6111111111111112
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 2161,
            "mean_confidence": 0.9890282276723762,
            "accuracy": 0.983803794539565
          }
        ],
        "median_right": 0.99,
        "median_wrong": 0.96,
        "sure": {
          "questions": 1826,
          "correct": 1816,
          "interval": [
            0.9899480689473389,
            0.997022606547196
          ]
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 14,
          "correct": 7,
          "interval": [
            0.2679886791947528,
            0.7320113208052472
          ]
        },
        "ece_refit": 0.005951373191439028,
        "brier_refit": 0.01987911731815504,
        "brier_base": 0.021825656934849978,
        "refit_slopes": [
          1.04695876215782,
          1.1994130183046028,
          1.2755722610656928,
          1.0534390576921764,
          1.0894183469643701
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        128,
        151,
        64,
        94,
        101,
        95,
        138,
        64,
        91,
        71,
        78,
        66,
        142,
        73,
        84,
        217,
        76,
        132,
        112,
        172
      ],
      "started": "2026-10-28T00:51:50Z",
      "updated": "2026-10-28T02:12:27Z",
      "format_of": null,
      "group": "ab",
      "latency": {
        "source": "run",
        "runs": [
          276
        ],
        "concurrency": 1,
        "median_request_seconds": 24.741423804487567,
        "p90_request_seconds": 70.32583790899079
      },
      "single_question_latency": null
    },
    {
      "run": 48,
      "model": "gpt-6-astra",
      "name": "GPT-6 Astra",
      "variant": null,
      "reasoning": "low",
      "provider": "openai",
      "note": null,
      "parameters": null,
      "active_parameters": null,
      "access": "Codex CLI, ChatGPT subscription",
      "questions_per_request": 16,
      "concurrency": 1,
      "correct": 2145,
      "invalid": 0,
      "answered_questions": 2210,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 636
        },
        "grammar": {
          "questions": 1542,
          "correct": 1509
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 2142
        }
      },
      "tyt_mix": 0.9711024247888392,
      "questions_per_minute": 61.21174024165934,
      "request_seconds": 2166.2511060215766,
      "median_request_seconds": null,
      "p90_request_seconds": null,
      "billing": "subscription",
      "api_equivalent_usd": 11.88588,
      "cost_basis": "subscription",
      "requests": 139,
      "hardware": null,
      "input_tokens": 1167698,
      "cached_input_tokens": 240000,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 47378,
      "confidence": {
        "source": "stated",
        "label": "Written estimate",
        "bins": [
          {
            "questions": 3,
            "correct": 1
          },
          {
            "questions": 3,
            "correct": 1
          },
          {
            "questions": 11,
            "correct": 5
          },
          {
            "questions": 35,
            "correct": 24
          },
          {
            "questions": 2146,
            "correct": 2114
          }
        ],
        "auroc": 0.9175968685402648,
        "auroc_interval": [
          0.880409324939965,
          0.9517436243463325
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.9613993710691824,
            "auroc_interval": [
              0.9323558933180385,
              0.9818226885896059
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.8922224230375324,
            "auroc_interval": [
              0.8351557743048508,
              0.941222700198544
            ]
          }
        },
        "aurc": 0.002778055628708045,
        "ece": 0.008130118289359167,
        "brier": 0.019536897179253866,
        "coverage_accuracy": {
          "0.20": 1.0,
          "0.50": 0.997991037184887,
          "0.80": 0.996552134171561
        },
        "coverage_intervals": {
          "0.20": [
            0.9999999999999999,
            1.0000000000000002
          ],
          "0.50": [
            0.9962446189539265,
            0.9995938578877263
          ],
          "0.80": [
            0.99381993891648,
            0.9988716192114185
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0
          ],
          [
            0.04,
            0.0
          ],
          [
            0.06,
            0.0
          ],
          [
            0.08,
            0.0
          ],
          [
            0.1,
            0.0
          ],
          [
            0.12,
            0.0
          ],
          [
            0.14,
            0.0
          ],
          [
            0.16,
            0.0
          ],
          [
            0.18,
            0.0
          ],
          [
            0.2,
            0.0
          ],
          [
            0.22,
            0.0
          ],
          [
            0.24,
            0.0
          ],
          [
            0.26,
            0.0
          ],
          [
            0.28,
            0.0001
          ],
          [
            0.3,
            0.0004
          ],
          [
            0.32,
            0.0006
          ],
          [
            0.34,
            0.0009
          ],
          [
            0.36,
            0.0011
          ],
          [
            0.38,
            0.0012
          ],
          [
            0.4,
            0.0014
          ],
          [
            0.42,
            0.0015
          ],
          [
            0.44,
            0.0017
          ],
          [
            0.46,
            0.0018
          ],
          [
            0.48,
            0.0019
          ],
          [
            0.5,
            0.002
          ],
          [
            0.52,
            0.0021
          ],
          [
            0.54,
            0.0022
          ],
          [
            0.56,
            0.0023
          ],
          [
            0.58,
            0.0023
          ],
          [
            0.6,
            0.0024
          ],
          [
            0.62,
            0.0025
          ],
          [
            0.64,
            0.0025
          ],
          [
            0.66,
            0.0026
          ],
          [
            0.68,
            0.0027
          ],
          [
            0.7,
            0.0027
          ],
          [
            0.72,
            0.0028
          ],
          [
            0.74,
            0.0028
          ],
          [
            0.76,
            0.0028
          ],
          [
            0.78,
            0.0029
          ],
          [
            0.8,
            0.0034
          ],
          [
            0.82,
            0.0041
          ],
          [
            0.84,
            0.0048
          ],
          [
            0.86,
            0.0054
          ],
          [
            0.88,
            0.006
          ],
          [
            0.9,
            0.0074
          ],
          [
            0.92,
            0.0089
          ],
          [
            0.94,
            0.0098
          ],
          [
            0.96,
            0.0121
          ],
          [
            0.98,
            0.0158
          ],
          [
            1.0,
            0.0241
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 3,
            "mean_confidence": 0.22,
            "accuracy": 0.3333333333333333
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 2,
            "mean_confidence": 0.32499999999999996,
            "accuracy": 0.0
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 1,
            "mean_confidence": 0.45,
            "accuracy": 1.0
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 4,
            "mean_confidence": 0.5449999999999999,
            "accuracy": 0.75
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 1,
            "mean_confidence": 0.69,
            "accuracy": 0.0
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 17,
            "mean_confidence": 0.7494117647058822,
            "accuracy": 0.47058823529411764
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 24,
            "mean_confidence": 0.8616666666666667,
            "accuracy": 0.75
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 2146,
            "mean_confidence": 0.988536812674749,
            "accuracy": 0.9850885368126747
          }
        ],
        "median_right": 0.99,
        "median_wrong": 0.94,
        "sure": {
          "questions": 1725,
          "correct": 1720,
          "interval": [
            0.9932323994929521,
            0.9987613146795898
          ]
        },
        "sure_before_audit": {
          "questions": 1728,
          "correct": 1720,
          "interval": [
            0.9908907883305993,
            0.9976522747301333
          ]
        },
        "doubtful": {
          "questions": 11,
          "correct": 5,
          "interval": [
            0.21270957543988045,
            0.7199122443100118
          ]
        },
        "ece_refit": 0.005341433891123305,
        "brier_refit": 0.019079494093511784,
        "brier_base": 0.023541520954383208,
        "refit_slopes": [
          1.1852737539068712,
          1.3815980561378234,
          1.2553958567084262,
          1.3136941925335306,
          1.2687701598707959
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": {
        "run": 37,
        "changed": {
          "questions": 372,
          "older_correct": 288,
          "newer_correct": 361,
          "flipped": 77,
          "changed_answer": 79,
          "newer_wrong": 11,
          "repeated_mistakes": 7,
          "p": 3.975760896174872e-20
        },
        "unchanged": {
          "questions": 1814,
          "older_correct": 1770,
          "newer_correct": 1772,
          "flipped": 24,
          "changed_answer": 25,
          "newer_wrong": 42,
          "repeated_mistakes": 30,
          "p": 0.8388197422027588
        },
        "added": {
          "questions": 12,
          "newer_correct": 12
        }
      },
      "units": [
        128,
        153,
        65,
        95,
        100,
        95,
        139,
        64,
        91,
        71,
        78,
        65,
        142,
        73,
        84,
        211,
        74,
        132,
        113,
        172
      ],
      "started": "2026-10-04T08:31:16Z",
      "updated": "2026-10-04T09:07:22Z",
      "format_of": null,
      "group": "abc",
      "latency": {
        "source": "sample",
        "runs": [
          69,
          78,
          82,
          87,
          91
        ],
        "repeats": [
          1,
          2,
          3,
          4,
          5
        ],
        "questions": 200,
        "concurrency": 1,
        "median_request_seconds": 13.043534992961213,
        "p90_request_seconds": 16.78220434060786
      },
      "single_question_latency": {
        "run": 142,
        "questions": 200,
        "median_request_seconds": 6.135667477443349,
        "p90_request_seconds": 8.567763022321742
      }
    },
    {
      "run": 191,
      "model": "gpt-6-astra",
      "name": "GPT-6 Astra",
      "variant": null,
      "reasoning": "low",
      "provider": "openai",
      "note": null,
      "parameters": null,
      "active_parameters": null,
      "access": "Codex CLI, ChatGPT subscription",
      "questions_per_request": 1,
      "concurrency": 1,
      "correct": 2141,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 638
        },
        "grammar": {
          "questions": 1542,
          "correct": 1503
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 2137
        }
      },
      "tyt_mix": 0.9729367348391383,
      "questions_per_minute": 8.554164640275534,
      "request_seconds": 15417.05187424965,
      "median_request_seconds": 6.41418632400746,
      "p90_request_seconds": 9.219829414092237,
      "billing": "subscription",
      "api_equivalent_usd": 58.769456,
      "cost_basis": "subscription",
      "requests": 2198,
      "hardware": null,
      "input_tokens": 10201437,
      "cached_input_tokens": 5527296,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 130015,
      "confidence": {
        "source": "stated",
        "label": "Written estimate",
        "bins": [
          {
            "questions": 0,
            "correct": 0
          },
          {
            "questions": 5,
            "correct": 2
          },
          {
            "questions": 7,
            "correct": 1
          },
          {
            "questions": 17,
            "correct": 11
          },
          {
            "questions": 2169,
            "correct": 2127
          }
        ],
        "auroc": 0.8606774994468891,
        "auroc_interval": [
          0.8142625666794109,
          0.9067377042500305
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.9334291187739464,
            "auroc_interval": [
              0.8743045538841954,
              0.9697787776527934
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.8236944913591621,
            "auroc_interval": [
              0.757736409823885,
              0.8819344415463075
            ]
          }
        },
        "aurc": 0.005155817197363828,
        "ece": 0.013358962693360663,
        "brier": 0.02226428889899909,
        "coverage_accuracy": {
          "0.20": 1.0,
          "0.50": 0.9946640604814703,
          "0.80": 0.9920354081712893
        },
        "coverage_intervals": {
          "0.20": [
            1.0,
            1.0
          ],
          "0.50": [
            0.9915748186209632,
            0.9972116176360506
          ],
          "0.80": [
            0.9875164995442114,
            0.9957829436531399
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0
          ],
          [
            0.04,
            0.0
          ],
          [
            0.06,
            0.0
          ],
          [
            0.08,
            0.0
          ],
          [
            0.1,
            0.0
          ],
          [
            0.12,
            0.0
          ],
          [
            0.14,
            0.0
          ],
          [
            0.16,
            0.0
          ],
          [
            0.18,
            0.0
          ],
          [
            0.2,
            0.0
          ],
          [
            0.22,
            0.0
          ],
          [
            0.24,
            0.0
          ],
          [
            0.26,
            0.0
          ],
          [
            0.28,
            0.0
          ],
          [
            0.3,
            0.0007
          ],
          [
            0.32,
            0.0014
          ],
          [
            0.34,
            0.002
          ],
          [
            0.36,
            0.0026
          ],
          [
            0.38,
            0.0031
          ],
          [
            0.4,
            0.0036
          ],
          [
            0.42,
            0.004
          ],
          [
            0.44,
            0.0044
          ],
          [
            0.46,
            0.0047
          ],
          [
            0.48,
            0.005
          ],
          [
            0.5,
            0.0053
          ],
          [
            0.52,
            0.0056
          ],
          [
            0.54,
            0.0059
          ],
          [
            0.56,
            0.0061
          ],
          [
            0.58,
            0.0063
          ],
          [
            0.6,
            0.0065
          ],
          [
            0.62,
            0.0067
          ],
          [
            0.64,
            0.0069
          ],
          [
            0.66,
            0.007
          ],
          [
            0.68,
            0.0072
          ],
          [
            0.7,
            0.0073
          ],
          [
            0.72,
            0.0075
          ],
          [
            0.74,
            0.0076
          ],
          [
            0.76,
            0.0077
          ],
          [
            0.78,
            0.0079
          ],
          [
            0.8,
            0.008
          ],
          [
            0.82,
            0.0081
          ],
          [
            0.84,
            0.0083
          ],
          [
            0.86,
            0.0092
          ],
          [
            0.88,
            0.01
          ],
          [
            0.9,
            0.0108
          ],
          [
            0.92,
            0.0118
          ],
          [
            0.94,
            0.0136
          ],
          [
            0.96,
            0.0153
          ],
          [
            0.98,
            0.0176
          ],
          [
            1.0,
            0.0259
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 3,
            "mean_confidence": 0.25,
            "accuracy": 0.6666666666666666
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 2,
            "mean_confidence": 0.4,
            "accuracy": 0.0
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 1,
            "mean_confidence": 0.55,
            "accuracy": 1.0
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 4,
            "mean_confidence": 0.6325,
            "accuracy": 0.0
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 4,
            "mean_confidence": 0.7475,
            "accuracy": 0.25
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 15,
            "mean_confidence": 0.8573333333333332,
            "accuracy": 0.6666666666666666
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 2169,
            "mean_confidence": 0.9896187183033687,
            "accuracy": 0.9806362378976486
          }
        ],
        "median_right": 0.99,
        "median_wrong": 0.97,
        "sure": {
          "questions": 1839,
          "correct": 1824,
          "interval": [
            0.9865853790384032,
            0.9950508071644697
          ]
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 10,
          "correct": 3,
          "interval": [
            0.10778928748621183,
            0.6032267800204347
          ]
        },
        "ece_refit": 0.0027419500066747735,
        "brier_refit": 0.021185065315210815,
        "brier_base": 0.025277112815578567,
        "refit_slopes": [
          1.2017140434904103,
          1.4121585480243923,
          1.266934444082122,
          1.2836577901145678,
          1.1232302023068093
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        128,
        152,
        66,
        96,
        101,
        95,
        137,
        63,
        91,
        70,
        78,
        66,
        141,
        73,
        84,
        211,
        75,
        133,
        109,
        172
      ],
      "started": "2026-10-13T16:20:09Z",
      "updated": "2026-10-13T20:37:11Z",
      "format_of": 48,
      "group": "abcd",
      "latency": {
        "source": "run",
        "runs": [
          191
        ],
        "concurrency": 1,
        "median_request_seconds": 6.41418632400746,
        "p90_request_seconds": 9.219829414092237
      },
      "single_question_latency": null
    },
    {
      "run": 188,
      "model": "gemini-3.8-flash",
      "name": "Gemini 3.8 Flash",
      "variant": null,
      "reasoning": "low",
      "provider": "gemini",
      "note": null,
      "parameters": null,
      "active_parameters": null,
      "access": "Gemini API",
      "questions_per_request": 1,
      "concurrency": 1,
      "correct": 2135,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 631
        },
        "grammar": {
          "questions": 1542,
          "correct": 1504
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 2131
        }
      },
      "tyt_mix": 0.9642468701559583,
      "questions_per_minute": 19.128837453656914,
      "request_seconds": 6894.302924550604,
      "median_request_seconds": 2.717947673998424,
      "p90_request_seconds": 4.674682790008956,
      "billing": "api",
      "api_equivalent_usd": 2.11764075,
      "cost_basis": "api",
      "requests": 2198,
      "hardware": null,
      "input_tokens": 680631,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 428578,
      "confidence": {
        "source": "stated",
        "label": "Written estimate",
        "bins": [
          {
            "questions": 0,
            "correct": 0
          },
          {
            "questions": 2,
            "correct": 1
          },
          {
            "questions": 5,
            "correct": 1
          },
          {
            "questions": 47,
            "correct": 29
          },
          {
            "questions": 2144,
            "correct": 2104
          }
        ],
        "auroc": 0.8421508494108025,
        "auroc_interval": [
          0.7898236432719191,
          0.8858766233766234
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.8913470681458003,
            "auroc_interval": [
              0.8433743990384616,
              0.940071803369235
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.8099804031354984,
            "auroc_interval": [
              0.7375650856764177,
              0.8733480989294943
            ]
          }
        },
        "aurc": 0.007247573429098589,
        "ece": 0.006160145586908525,
        "brier": 0.02447661510464058,
        "coverage_accuracy": {
          "0.20": 0.9959316517493898,
          "0.50": 0.9959316517493898,
          "0.80": 0.9877657204267254
        },
        "coverage_intervals": {
          "0.20": [
            0.9919807538091419,
            0.9991836734693877
          ],
          "0.50": [
            0.991980753809142,
            0.9991836734693877
          ],
          "0.80": [
            0.9829962986595384,
            0.9915938173411502
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0041
          ],
          [
            0.04,
            0.0041
          ],
          [
            0.06,
            0.0041
          ],
          [
            0.08,
            0.0041
          ],
          [
            0.1,
            0.0041
          ],
          [
            0.12,
            0.0041
          ],
          [
            0.14,
            0.0041
          ],
          [
            0.16,
            0.0041
          ],
          [
            0.18,
            0.0041
          ],
          [
            0.2,
            0.0041
          ],
          [
            0.22,
            0.0041
          ],
          [
            0.24,
            0.0041
          ],
          [
            0.26,
            0.0041
          ],
          [
            0.28,
            0.0041
          ],
          [
            0.3,
            0.0041
          ],
          [
            0.32,
            0.0041
          ],
          [
            0.34,
            0.0041
          ],
          [
            0.36,
            0.0041
          ],
          [
            0.38,
            0.0041
          ],
          [
            0.4,
            0.0041
          ],
          [
            0.42,
            0.0041
          ],
          [
            0.44,
            0.0041
          ],
          [
            0.46,
            0.0041
          ],
          [
            0.48,
            0.0041
          ],
          [
            0.5,
            0.0041
          ],
          [
            0.52,
            0.0041
          ],
          [
            0.54,
            0.0041
          ],
          [
            0.56,
            0.0041
          ],
          [
            0.58,
            0.0042
          ],
          [
            0.6,
            0.0043
          ],
          [
            0.62,
            0.0044
          ],
          [
            0.64,
            0.0043
          ],
          [
            0.66,
            0.0055
          ],
          [
            0.68,
            0.0066
          ],
          [
            0.7,
            0.0077
          ],
          [
            0.72,
            0.0087
          ],
          [
            0.74,
            0.0097
          ],
          [
            0.76,
            0.0106
          ],
          [
            0.78,
            0.0114
          ],
          [
            0.8,
            0.0122
          ],
          [
            0.82,
            0.013
          ],
          [
            0.84,
            0.0137
          ],
          [
            0.86,
            0.0144
          ],
          [
            0.88,
            0.0151
          ],
          [
            0.9,
            0.0158
          ],
          [
            0.92,
            0.0164
          ],
          [
            0.94,
            0.0169
          ],
          [
            0.96,
            0.0175
          ],
          [
            0.98,
            0.0203
          ],
          [
            1.0,
            0.0287
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 1,
            "mean_confidence": 0.25,
            "accuracy": 1.0
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 1,
            "mean_confidence": 0.3,
            "accuracy": 0.0
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 5,
            "mean_confidence": 0.51,
            "accuracy": 0.2
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 3,
            "mean_confidence": 0.75,
            "accuracy": 0.6666666666666666
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 44,
            "mean_confidence": 0.8477272727272733,
            "accuracy": 0.6136363636363636
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 2144,
            "mean_confidence": 0.9815251865671758,
            "accuracy": 0.9813432835820896
          }
        ],
        "median_right": 1.0,
        "median_wrong": 0.95,
        "sure": {
          "questions": 1365,
          "correct": 1359,
          "interval": [
            0.9904430302635897,
            0.9979839725393643
          ]
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 7,
          "correct": 2,
          "interval": [
            0.0822171657090155,
            0.6410709098517873
          ]
        },
        "ece_refit": 0.007891635691462938,
        "brier_refit": 0.02602231626647306,
        "brier_base": 0.02786726821140324,
        "refit_slopes": [
          0.3828690017304012,
          0.3968758968652139,
          0.32760206445387274,
          0.36504244426170723,
          0.3827809490935771
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        126,
        152,
        64,
        96,
        98,
        95,
        137,
        64,
        91,
        71,
        76,
        66,
        142,
        74,
        83,
        215,
        74,
        130,
        112,
        169
      ],
      "started": "2026-10-13T16:20:09Z",
      "updated": "2026-10-13T18:15:07Z",
      "format_of": 123,
      "group": "abcd",
      "latency": {
        "source": "run",
        "runs": [
          188
        ],
        "concurrency": 1,
        "median_request_seconds": 2.717947673998424,
        "p90_request_seconds": 4.674682790008956
      },
      "single_question_latency": null
    },
    {
      "run": 190,
      "model": "gpt-6.1-sol",
      "name": "GPT-6.1 Sol",
      "variant": null,
      "reasoning": "low",
      "provider": "openai",
      "note": null,
      "parameters": null,
      "active_parameters": null,
      "access": "Codex CLI, ChatGPT subscription",
      "questions_per_request": 1,
      "concurrency": 1,
      "correct": 2126,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 630
        },
        "grammar": {
          "questions": 1542,
          "correct": 1496
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 2122
        }
      },
      "tyt_mix": 0.9620813364018854,
      "questions_per_minute": 8.103710483239363,
      "request_seconds": 16274.026604573679,
      "median_request_seconds": 6.613393928506412,
      "p90_request_seconds": 9.831893868802581,
      "billing": "subscription",
      "api_equivalent_usd": 12.2396048,
      "cost_basis": "subscription",
      "requests": 2198,
      "hardware": null,
      "input_tokens": 10202072,
      "cached_input_tokens": 5136768,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 159532,
      "confidence": {
        "source": "stated",
        "label": "Written estimate",
        "bins": [
          {
            "questions": 1,
            "correct": 0
          },
          {
            "questions": 6,
            "correct": 3
          },
          {
            "questions": 5,
            "correct": 3
          },
          {
            "questions": 18,
            "correct": 10
          },
          {
            "questions": 2168,
            "correct": 2110
          }
        ],
        "auroc": 0.8591577819588168,
        "auroc_interval": [
          0.8134712672592096,
          0.9023442151046738
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.9397741147741148,
            "auroc_interval": [
              0.8916889185580774,
              0.9718239174075136
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.8144762845849802,
            "auroc_interval": [
              0.748975840745183,
              0.8764447616240284
            ]
          }
        },
        "aurc": 0.00698838107641348,
        "ece": 0.017714740673342787,
        "brier": 0.029051106915377614,
        "coverage_accuracy": {
          "0.20": 0.9984717103331952,
          "0.50": 0.9932438414119905,
          "0.80": 0.9914358903350872
        },
        "coverage_intervals": {
          "0.20": [
            0.9971641190667739,
            0.9994882126057889
          ],
          "0.50": [
            0.9895745445563241,
            0.9966362503432398
          ],
          "0.80": [
            0.9870523917995444,
            0.9954016943661914
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0
          ],
          [
            0.04,
            0.0
          ],
          [
            0.06,
            0.0
          ],
          [
            0.08,
            0.0
          ],
          [
            0.1,
            0.0
          ],
          [
            0.12,
            0.0
          ],
          [
            0.14,
            0.0
          ],
          [
            0.16,
            0.0
          ],
          [
            0.18,
            0.0006
          ],
          [
            0.2,
            0.0015
          ],
          [
            0.22,
            0.0023
          ],
          [
            0.24,
            0.003
          ],
          [
            0.26,
            0.0035
          ],
          [
            0.28,
            0.004
          ],
          [
            0.3,
            0.0044
          ],
          [
            0.32,
            0.0048
          ],
          [
            0.34,
            0.0051
          ],
          [
            0.36,
            0.0054
          ],
          [
            0.38,
            0.0057
          ],
          [
            0.4,
            0.0059
          ],
          [
            0.42,
            0.0061
          ],
          [
            0.44,
            0.0063
          ],
          [
            0.46,
            0.0065
          ],
          [
            0.48,
            0.0066
          ],
          [
            0.5,
            0.0068
          ],
          [
            0.52,
            0.0069
          ],
          [
            0.54,
            0.007
          ],
          [
            0.56,
            0.0071
          ],
          [
            0.58,
            0.0072
          ],
          [
            0.6,
            0.0073
          ],
          [
            0.62,
            0.0074
          ],
          [
            0.64,
            0.0075
          ],
          [
            0.66,
            0.0076
          ],
          [
            0.68,
            0.0077
          ],
          [
            0.7,
            0.0078
          ],
          [
            0.72,
            0.0078
          ],
          [
            0.74,
            0.0079
          ],
          [
            0.76,
            0.0079
          ],
          [
            0.78,
            0.008
          ],
          [
            0.8,
            0.0086
          ],
          [
            0.82,
            0.0098
          ],
          [
            0.84,
            0.0111
          ],
          [
            0.86,
            0.0122
          ],
          [
            0.88,
            0.0133
          ],
          [
            0.9,
            0.0146
          ],
          [
            0.92,
            0.0168
          ],
          [
            0.94,
            0.0189
          ],
          [
            0.96,
            0.02
          ],
          [
            0.98,
            0.024
          ],
          [
            1.0,
            0.0328
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 2,
            "mean_confidence": 0.225,
            "accuracy": 0.5
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 3,
            "mean_confidence": 0.34333333333333327,
            "accuracy": 0.6666666666666666
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 2,
            "mean_confidence": 0.44999999999999996,
            "accuracy": 0.0
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 2,
            "mean_confidence": 0.535,
            "accuracy": 0.5
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 2,
            "mean_confidence": 0.645,
            "accuracy": 0.5
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 4,
            "mean_confidence": 0.7549999999999999,
            "accuracy": 0.75
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 15,
            "mean_confidence": 0.8613333333333334,
            "accuracy": 0.5333333333333333
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 2168,
            "mean_confidence": 0.9876462177121805,
            "accuracy": 0.9732472324723247
          }
        ],
        "median_right": 0.99,
        "median_wrong": 0.97,
        "sure": {
          "questions": 1741,
          "correct": 1727,
          "interval": [
            0.9865470638705058,
            0.9952039443585508
          ]
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 11,
          "correct": 5,
          "interval": [
            0.21270957543988045,
            0.7199122443100118
          ]
        },
        "ece_refit": 0.007707009158749494,
        "brier_refit": 0.028098716554039928,
        "brier_base": 0.03170565713565758,
        "refit_slopes": [
          1.1343562570462733,
          1.4080455141912043,
          1.253162343239531,
          1.2824061850515291,
          1.1701851140540194
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        127,
        150,
        64,
        94,
        99,
        96,
        137,
        64,
        89,
        71,
        78,
        65,
        141,
        73,
        84,
        209,
        72,
        131,
        110,
        172
      ],
      "started": "2026-10-13T16:20:09Z",
      "updated": "2026-10-13T20:51:28Z",
      "format_of": 124,
      "group": "bcd",
      "latency": {
        "source": "run",
        "runs": [
          190
        ],
        "concurrency": 1,
        "median_request_seconds": 6.613393928506412,
        "p90_request_seconds": 9.831893868802581
      },
      "single_question_latency": null
    },
    {
      "run": 124,
      "model": "gpt-6.1-sol",
      "name": "GPT-6.1 Sol",
      "variant": null,
      "reasoning": "low",
      "provider": "openai",
      "note": null,
      "parameters": null,
      "active_parameters": null,
      "access": "Codex CLI, ChatGPT subscription",
      "questions_per_request": 16,
      "concurrency": 1,
      "correct": 2125,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 631
        },
        "grammar": {
          "questions": 1542,
          "correct": 1494
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 2121
        }
      },
      "tyt_mix": 0.9631119804023915,
      "questions_per_minute": 60.83084796503379,
      "request_seconds": 2167.9789845409687,
      "median_request_seconds": 13.26978919148678,
      "p90_request_seconds": 24.609638098091818,
      "billing": "subscription",
      "api_equivalent_usd": 2.48456,
      "cost_basis": "subscription",
      "requests": 138,
      "hardware": null,
      "input_tokens": 1159990,
      "cached_input_tokens": 198400,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 54154,
      "confidence": {
        "source": "stated",
        "label": "Written estimate",
        "bins": [
          {
            "questions": 3,
            "correct": 1
          },
          {
            "questions": 5,
            "correct": 3
          },
          {
            "questions": 11,
            "correct": 8
          },
          {
            "questions": 43,
            "correct": 28
          },
          {
            "questions": 2136,
            "correct": 2085
          }
        ],
        "auroc": 0.8585688960515713,
        "auroc_interval": [
          0.8103745494436608,
          0.9019869675278273
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.967987321711569,
            "auroc_interval": [
              0.9458984375,
              0.9837190131307778
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.8009956492637216,
            "auroc_interval": [
              0.7372203031552897,
              0.862030562030562
            ]
          }
        },
        "aurc": 0.007414674863018657,
        "ece": 0.01493630573248991,
        "brier": 0.028964012738853503,
        "coverage_accuracy": {
          "0.20": 0.99800796812749,
          "0.50": 0.9932068207724121,
          "0.80": 0.9893609575138238
        },
        "coverage_intervals": {
          "0.20": [
            0.993801652892562,
            1.0
          ],
          "0.50": [
            0.9894110092415956,
            0.9967465802762838
          ],
          "0.80": [
            0.9846940559440559,
            0.9937285642631529
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.002
          ],
          [
            0.04,
            0.002
          ],
          [
            0.06,
            0.002
          ],
          [
            0.08,
            0.002
          ],
          [
            0.1,
            0.002
          ],
          [
            0.12,
            0.002
          ],
          [
            0.14,
            0.002
          ],
          [
            0.16,
            0.002
          ],
          [
            0.18,
            0.002
          ],
          [
            0.2,
            0.002
          ],
          [
            0.22,
            0.002
          ],
          [
            0.24,
            0.0024
          ],
          [
            0.26,
            0.0031
          ],
          [
            0.28,
            0.0036
          ],
          [
            0.3,
            0.0041
          ],
          [
            0.32,
            0.0045
          ],
          [
            0.34,
            0.0049
          ],
          [
            0.36,
            0.0052
          ],
          [
            0.38,
            0.0055
          ],
          [
            0.4,
            0.0058
          ],
          [
            0.42,
            0.006
          ],
          [
            0.44,
            0.0062
          ],
          [
            0.46,
            0.0064
          ],
          [
            0.48,
            0.0066
          ],
          [
            0.5,
            0.0068
          ],
          [
            0.52,
            0.0069
          ],
          [
            0.54,
            0.0071
          ],
          [
            0.56,
            0.0072
          ],
          [
            0.58,
            0.0074
          ],
          [
            0.6,
            0.0075
          ],
          [
            0.62,
            0.0076
          ],
          [
            0.64,
            0.0077
          ],
          [
            0.66,
            0.0078
          ],
          [
            0.68,
            0.0079
          ],
          [
            0.7,
            0.0079
          ],
          [
            0.72,
            0.008
          ],
          [
            0.74,
            0.0084
          ],
          [
            0.76,
            0.0092
          ],
          [
            0.78,
            0.0099
          ],
          [
            0.8,
            0.0106
          ],
          [
            0.82,
            0.0113
          ],
          [
            0.84,
            0.012
          ],
          [
            0.86,
            0.0124
          ],
          [
            0.88,
            0.0127
          ],
          [
            0.9,
            0.013
          ],
          [
            0.92,
            0.0146
          ],
          [
            0.94,
            0.0167
          ],
          [
            0.96,
            0.0206
          ],
          [
            0.98,
            0.0257
          ],
          [
            1.0,
            0.0332
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 4,
            "mean_confidence": 0.23249999999999998,
            "accuracy": 0.5
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 2,
            "mean_confidence": 0.32499999999999996,
            "accuracy": 0.5
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 2,
            "mean_confidence": 0.435,
            "accuracy": 0.5
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 3,
            "mean_confidence": 0.5366666666666666,
            "accuracy": 0.6666666666666666
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 5,
            "mean_confidence": 0.632,
            "accuracy": 0.6
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 10,
            "mean_confidence": 0.7550000000000001,
            "accuracy": 0.7
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 36,
            "mean_confidence": 0.8597222222222223,
            "accuracy": 0.6666666666666666
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 2136,
            "mean_confidence": 0.986999063670418,
            "accuracy": 0.976123595505618
          }
        ],
        "median_right": 0.99,
        "median_wrong": 0.94,
        "sure": {
          "questions": 1610,
          "correct": 1597,
          "interval": [
            0.9862337881881241,
            0.995275177747566
          ]
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 16,
          "correct": 9,
          "interval": [
            0.33178195606776517,
            0.7690163666481448
          ]
        },
        "ece_refit": 0.009922443389027528,
        "brier_refit": 0.028303679357028138,
        "brier_base": 0.03211905909655759,
        "refit_slopes": [
          0.8690905395195768,
          0.9748854168363457,
          0.9948074644867254,
          1.0171167637580596,
          0.9714977166913915
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        128,
        151,
        64,
        94,
        99,
        95,
        138,
        63,
        91,
        71,
        78,
        65,
        142,
        73,
        82,
        208,
        72,
        130,
        110,
        171
      ],
      "started": "2026-10-05T20:36:36Z",
      "updated": "2026-10-05T21:12:45Z",
      "format_of": null,
      "group": "cd",
      "latency": {
        "source": "run",
        "runs": [
          124
        ],
        "concurrency": 1,
        "median_request_seconds": 13.26978919148678,
        "p90_request_seconds": 24.609638098091818
      },
      "single_question_latency": null
    },
    {
      "run": 557,
      "model": "gemini-4-argon",
      "name": "Gemini 4 Argon",
      "variant": null,
      "reasoning": "low",
      "provider": "gemini",
      "note": null,
      "parameters": null,
      "active_parameters": null,
      "access": "Gemini API",
      "questions_per_request": 16,
      "concurrency": 1,
      "correct": 2118,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 636
        },
        "grammar": {
          "questions": 1542,
          "correct": 1482
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 2114
        }
      },
      "tyt_mix": 0.968038222454209,
      "questions_per_minute": 152.28388138219916,
      "request_seconds": 866.0141756500816,
      "median_request_seconds": 5.459406626003329,
      "p90_request_seconds": 7.905744231294375,
      "billing": "api",
      "api_equivalent_usd": 3.925196,
      "cost_basis": "api",
      "requests": 138,
      "hardware": null,
      "input_tokens": 767554,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 42749,
      "confidence": {
        "source": "stated",
        "label": "Written estimate",
        "bins": [
          {
            "questions": 0,
            "correct": 0
          },
          {
            "questions": 0,
            "correct": 0
          },
          {
            "questions": 6,
            "correct": 1
          },
          {
            "questions": 15,
            "correct": 7
          },
          {
            "questions": 2177,
            "correct": 2110
          }
        ],
        "auroc": 0.8553883380547687,
        "auroc_interval": [
          0.8129560233660311,
          0.891328133767254
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.9621855345911949,
            "auroc_interval": [
              0.938521888185654,
              0.9831761006289308
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.842661943319838,
            "auroc_interval": [
              0.7950333436724566,
              0.8837870575675454
            ]
          }
        },
        "aurc": 0.007794885961728752,
        "ece": 0.02185623293903984,
        "brier": 0.03162174704276615,
        "coverage_accuracy": {
          "0.20": 0.9987063389391979,
          "0.50": 0.9929853740665678,
          "0.80": 0.9878983658075077
        },
        "coverage_intervals": {
          "0.20": [
            0.9960053262316911,
            1.0000000000000002
          ],
          "0.50": [
            0.9895906938162171,
            0.9959442749768792
          ],
          "0.80": [
            0.9827566964285714,
            0.9926357182983505
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0013
          ],
          [
            0.04,
            0.0013
          ],
          [
            0.06,
            0.0013
          ],
          [
            0.08,
            0.0013
          ],
          [
            0.1,
            0.0013
          ],
          [
            0.12,
            0.0013
          ],
          [
            0.14,
            0.0013
          ],
          [
            0.16,
            0.0013
          ],
          [
            0.18,
            0.0013
          ],
          [
            0.2,
            0.0013
          ],
          [
            0.22,
            0.0013
          ],
          [
            0.24,
            0.0013
          ],
          [
            0.26,
            0.0013
          ],
          [
            0.28,
            0.0013
          ],
          [
            0.3,
            0.0013
          ],
          [
            0.32,
            0.0013
          ],
          [
            0.34,
            0.0013
          ],
          [
            0.36,
            0.0017
          ],
          [
            0.38,
            0.0027
          ],
          [
            0.4,
            0.0036
          ],
          [
            0.42,
            0.0044
          ],
          [
            0.44,
            0.0052
          ],
          [
            0.46,
            0.0058
          ],
          [
            0.48,
            0.0064
          ],
          [
            0.5,
            0.007
          ],
          [
            0.52,
            0.0075
          ],
          [
            0.54,
            0.008
          ],
          [
            0.56,
            0.0085
          ],
          [
            0.58,
            0.0089
          ],
          [
            0.6,
            0.0093
          ],
          [
            0.62,
            0.0096
          ],
          [
            0.64,
            0.01
          ],
          [
            0.66,
            0.0103
          ],
          [
            0.68,
            0.0106
          ],
          [
            0.7,
            0.0109
          ],
          [
            0.72,
            0.0112
          ],
          [
            0.74,
            0.0114
          ],
          [
            0.76,
            0.0117
          ],
          [
            0.78,
            0.0119
          ],
          [
            0.8,
            0.0121
          ],
          [
            0.82,
            0.0123
          ],
          [
            0.84,
            0.0125
          ],
          [
            0.86,
            0.013
          ],
          [
            0.88,
            0.0134
          ],
          [
            0.9,
            0.0161
          ],
          [
            0.92,
            0.0191
          ],
          [
            0.94,
            0.0219
          ],
          [
            0.96,
            0.0246
          ],
          [
            0.98,
            0.0283
          ],
          [
            1.0,
            0.0364
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 3,
            "mean_confidence": 0.5,
            "accuracy": 0.0
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 2,
            "mean_confidence": 0.6,
            "accuracy": 0.0
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 2,
            "mean_confidence": 0.725,
            "accuracy": 0.5
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 14,
            "mean_confidence": 0.8428571428571426,
            "accuracy": 0.5
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 2177,
            "mean_confidence": 0.9876389526871886,
            "accuracy": 0.9692237023426734
          }
        ],
        "median_right": 0.99,
        "median_wrong": 0.95,
        "sure": {
          "questions": 1842,
          "correct": 1819,
          "interval": [
            0.9813325235735316,
            0.9916653761379063
          ]
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 5,
          "correct": 0,
          "interval": [
            0.0,
            0.43449149475208104
          ]
        },
        "ece_refit": 0.004693471691863756,
        "brier_refit": 0.02907319351580202,
        "brier_base": 0.03510111751548249,
        "refit_slopes": [
          1.0626894967536764,
          1.2740415731482782,
          1.1066905066038477,
          1.1308890396363034,
          1.1267453627995414
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        127,
        154,
        63,
        97,
        102,
        93,
        135,
        64,
        91,
        71,
        75,
        66,
        139,
        73,
        82,
        202,
        73,
        131,
        112,
        168
      ],
      "started": "2026-11-03T18:31:52Z",
      "updated": "2026-11-03T18:46:19Z",
      "format_of": null,
      "group": "cde",
      "latency": {
        "source": "run",
        "runs": [
          557
        ],
        "concurrency": 1,
        "median_request_seconds": 5.459406626003329,
        "p90_request_seconds": 7.905744231294375
      },
      "single_question_latency": null
    },
    {
      "run": 310,
      "model": "claude-fable-5-1",
      "name": "Claude Fable 5.1",
      "variant": null,
      "reasoning": "high",
      "provider": "claude",
      "note": null,
      "parameters": null,
      "active_parameters": null,
      "access": "Claude Code CLI, Claude subscription",
      "questions_per_request": 16,
      "concurrency": 1,
      "correct": 2113,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 627
        },
        "grammar": {
          "questions": 1542,
          "correct": 1486
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 2110
        }
      },
      "tyt_mix": 0.9571735807946601,
      "questions_per_minute": 27.111672851118513,
      "request_seconds": 4864.325441082445,
      "median_request_seconds": 30.3313984014967,
      "p90_request_seconds": 64.38088531556423,
      "billing": "subscription",
      "api_equivalent_usd": 37.4582275,
      "cost_basis": "subscription",
      "requests": 138,
      "hardware": null,
      "input_tokens": 1239453,
      "cached_input_tokens": 324710,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 914467,
      "output_tokens": 381699,
      "confidence": {
        "source": "stated",
        "label": "Written estimate",
        "bins": [
          {
            "questions": 0,
            "correct": 0
          },
          {
            "questions": 38,
            "correct": 22
          },
          {
            "questions": 254,
            "correct": 214
          },
          {
            "questions": 801,
            "correct": 777
          },
          {
            "questions": 1105,
            "correct": 1100
          }
        ],
        "auroc": 0.8582890231340998,
        "auroc_interval": [
          0.8195304750358205,
          0.8936852803171265
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.9335643183193092,
            "auroc_interval": [
              0.8779354341967772,
              0.9749639398310324
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.8176612670640262,
            "auroc_interval": [
              0.7617674418604651,
              0.8670806892637879
            ]
          }
        },
        "aurc": 0.007881676988024843,
        "ece": 0.11954504094631425,
        "brier": 0.04882238398544131,
        "coverage_accuracy": {
          "0.20": 0.9993156336268718,
          "0.50": 0.9954889653151555,
          "0.80": 0.9862767490519763
        },
        "coverage_intervals": {
          "0.20": [
            0.9982538722352977,
            0.9999345177068614
          ],
          "0.50": [
            0.9909960429142773,
            0.9990903124399613
          ],
          "0.80": [
            0.9807507987220447,
            0.9912249477857874
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0
          ],
          [
            0.04,
            0.0
          ],
          [
            0.06,
            0.0
          ],
          [
            0.08,
            0.0
          ],
          [
            0.1,
            0.0
          ],
          [
            0.12,
            0.0
          ],
          [
            0.14,
            0.0
          ],
          [
            0.16,
            0.0
          ],
          [
            0.18,
            0.0
          ],
          [
            0.2,
            0.0007
          ],
          [
            0.22,
            0.0013
          ],
          [
            0.24,
            0.0017
          ],
          [
            0.26,
            0.0022
          ],
          [
            0.28,
            0.0025
          ],
          [
            0.3,
            0.0028
          ],
          [
            0.32,
            0.0031
          ],
          [
            0.34,
            0.0033
          ],
          [
            0.36,
            0.0035
          ],
          [
            0.38,
            0.0037
          ],
          [
            0.4,
            0.0039
          ],
          [
            0.42,
            0.004
          ],
          [
            0.44,
            0.0042
          ],
          [
            0.46,
            0.0043
          ],
          [
            0.48,
            0.0044
          ],
          [
            0.5,
            0.0045
          ],
          [
            0.52,
            0.0052
          ],
          [
            0.54,
            0.006
          ],
          [
            0.56,
            0.0068
          ],
          [
            0.58,
            0.0075
          ],
          [
            0.6,
            0.0082
          ],
          [
            0.62,
            0.0088
          ],
          [
            0.64,
            0.0094
          ],
          [
            0.66,
            0.01
          ],
          [
            0.68,
            0.0105
          ],
          [
            0.7,
            0.011
          ],
          [
            0.72,
            0.0115
          ],
          [
            0.74,
            0.012
          ],
          [
            0.76,
            0.0126
          ],
          [
            0.78,
            0.0132
          ],
          [
            0.8,
            0.0137
          ],
          [
            0.82,
            0.0143
          ],
          [
            0.84,
            0.0147
          ],
          [
            0.86,
            0.0151
          ],
          [
            0.88,
            0.0163
          ],
          [
            0.9,
            0.0179
          ],
          [
            0.92,
            0.0195
          ],
          [
            0.94,
            0.0231
          ],
          [
            0.96,
            0.0266
          ],
          [
            0.98,
            0.0313
          ],
          [
            1.0,
            0.0387
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 1,
            "mean_confidence": 0.25,
            "accuracy": 0.0
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 10,
            "mean_confidence": 0.31999999999999995,
            "accuracy": 0.5
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 27,
            "mean_confidence": 0.4277777777777778,
            "accuracy": 0.6296296296296297
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 56,
            "mean_confidence": 0.5142857142857145,
            "accuracy": 0.75
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 85,
            "mean_confidence": 0.6105882352941182,
            "accuracy": 0.8117647058823529
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 208,
            "mean_confidence": 0.7228365384615386,
            "accuracy": 0.9375
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 706,
            "mean_confidence": 0.8362606232294699,
            "accuracy": 0.9702549575070821
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 1105,
            "mean_confidence": 0.9179095022624393,
            "accuracy": 0.995475113122172
          }
        ],
        "median_right": 0.9,
        "median_wrong": 0.6,
        "sure": {
          "questions": 0,
          "correct": 0,
          "interval": null
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 179,
          "correct": 133,
          "interval": [
            0.6743665161346157,
            0.8014551787094462
          ]
        },
        "ece_refit": 0.004927273798078389,
        "brier_refit": 0.03201638591406321,
        "brier_base": 0.037226249541556566,
        "refit_slopes": [
          1.584059085803524,
          1.7420762988934309,
          1.6450682695212202,
          1.8232936094388659,
          1.761611267326474
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        125,
        150,
        64,
        96,
        97,
        95,
        135,
        63,
        91,
        71,
        78,
        66,
        139,
        74,
        84,
        207,
        73,
        128,
        108,
        169
      ],
      "started": "2026-10-28T14:33:55Z",
      "updated": "2026-10-28T15:55:00Z",
      "format_of": null,
      "group": "de",
      "latency": {
        "source": "run",
        "runs": [
          310
        ],
        "concurrency": 1,
        "median_request_seconds": 30.3313984014967,
        "p90_request_seconds": 64.38088531556423
      },
      "single_question_latency": null
    },
    {
      "run": 278,
      "model": "claude-opus-5-5",
      "name": "Claude Opus 5.5",
      "variant": null,
      "reasoning": "high",
      "provider": "claude",
      "note": null,
      "parameters": null,
      "active_parameters": null,
      "access": "Claude Code CLI, Claude subscription",
      "questions_per_request": 16,
      "concurrency": 1,
      "correct": 2088,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 627
        },
        "grammar": {
          "questions": 1542,
          "correct": 1461
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 2086
        }
      },
      "tyt_mix": 0.9543363564107432,
      "questions_per_minute": 39.02604924802538,
      "request_seconds": 3379.2813400570594,
      "median_request_seconds": 22.122042668488575,
      "p90_request_seconds": 44.978391354210906,
      "billing": "subscription",
      "api_equivalent_usd": 14.306400199999999,
      "cost_basis": "subscription",
      "requests": 138,
      "hardware": null,
      "input_tokens": 1247691,
      "cached_input_tokens": 318161,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 929252,
      "output_tokens": 340382,
      "confidence": {
        "source": "stated",
        "label": "Written estimate",
        "bins": [
          {
            "questions": 1,
            "correct": 1
          },
          {
            "questions": 65,
            "correct": 37
          },
          {
            "questions": 382,
            "correct": 332
          },
          {
            "questions": 1026,
            "correct": 1000
          },
          {
            "questions": 724,
            "correct": 718
          }
        ],
        "auroc": 0.8372278822709858,
        "auroc_interval": [
          0.7948792016806723,
          0.8769483949993923
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.9171478853874498,
            "auroc_interval": [
              0.8642580645161291,
              0.9608425452345598
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.8061449539888965,
            "auroc_interval": [
              0.7512468227325612,
              0.8551464826953581
            ]
          }
        },
        "aurc": 0.013546016026022567,
        "ece": 0.14620109190173014,
        "brier": 0.06557611464968154,
        "coverage_accuracy": {
          "0.20": 0.9939624257346251,
          "0.50": 0.9886675678030848,
          "0.80": 0.981631945463618
        },
        "coverage_intervals": {
          "0.20": [
            0.9871925316821532,
            0.9987479417319438
          ],
          "0.50": [
            0.982283810389217,
            0.9940049699730793
          ],
          "0.80": [
            0.9750466797898528,
            0.9873401216062551
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.007
          ],
          [
            0.04,
            0.0076
          ],
          [
            0.06,
            0.0076
          ],
          [
            0.08,
            0.0057
          ],
          [
            0.1,
            0.0045
          ],
          [
            0.12,
            0.0038
          ],
          [
            0.14,
            0.0036
          ],
          [
            0.16,
            0.0046
          ],
          [
            0.18,
            0.0054
          ],
          [
            0.2,
            0.006
          ],
          [
            0.22,
            0.0066
          ],
          [
            0.24,
            0.007
          ],
          [
            0.26,
            0.0074
          ],
          [
            0.28,
            0.0077
          ],
          [
            0.3,
            0.0079
          ],
          [
            0.32,
            0.0082
          ],
          [
            0.34,
            0.0085
          ],
          [
            0.36,
            0.0089
          ],
          [
            0.38,
            0.0093
          ],
          [
            0.4,
            0.0097
          ],
          [
            0.42,
            0.0101
          ],
          [
            0.44,
            0.0104
          ],
          [
            0.46,
            0.0108
          ],
          [
            0.48,
            0.0111
          ],
          [
            0.5,
            0.0113
          ],
          [
            0.52,
            0.0116
          ],
          [
            0.54,
            0.0118
          ],
          [
            0.56,
            0.012
          ],
          [
            0.58,
            0.0122
          ],
          [
            0.6,
            0.0123
          ],
          [
            0.62,
            0.0128
          ],
          [
            0.64,
            0.0133
          ],
          [
            0.66,
            0.0138
          ],
          [
            0.68,
            0.0143
          ],
          [
            0.7,
            0.0147
          ],
          [
            0.72,
            0.0155
          ],
          [
            0.74,
            0.0163
          ],
          [
            0.76,
            0.017
          ],
          [
            0.78,
            0.0177
          ],
          [
            0.8,
            0.0184
          ],
          [
            0.82,
            0.0189
          ],
          [
            0.84,
            0.0194
          ],
          [
            0.86,
            0.0207
          ],
          [
            0.88,
            0.0235
          ],
          [
            0.9,
            0.0264
          ],
          [
            0.92,
            0.0291
          ],
          [
            0.94,
            0.0321
          ],
          [
            0.96,
            0.0362
          ],
          [
            0.98,
            0.0407
          ],
          [
            1.0,
            0.05
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 6,
            "mean_confidence": 0.24166666666666667,
            "accuracy": 0.3333333333333333
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 20,
            "mean_confidence": 0.3249999999999999,
            "accuracy": 0.4
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 40,
            "mean_confidence": 0.42874999999999996,
            "accuracy": 0.7
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 102,
            "mean_confidence": 0.525490196078431,
            "accuracy": 0.7843137254901961
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 155,
            "mean_confidence": 0.6116129032258059,
            "accuracy": 0.8516129032258064
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 325,
            "mean_confidence": 0.7308307692307681,
            "accuracy": 0.9569230769230769
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 826,
            "mean_confidence": 0.839648910411627,
            "accuracy": 0.9794188861985472
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 724,
            "mean_confidence": 0.9144751381215382,
            "accuracy": 0.9917127071823204
          }
        ],
        "median_right": 0.85,
        "median_wrong": 0.6,
        "sure": {
          "questions": 0,
          "correct": 0,
          "interval": null
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 323,
          "correct": 250,
          "interval": [
            0.7253155128701995,
            0.8162312180232977
          ]
        },
        "ece_refit": 0.006655589085001782,
        "brier_refit": 0.04032216631468203,
        "brier_base": 0.04757495173364269,
        "refit_slopes": [
          1.5351339133839872,
          1.667319893251849,
          1.4998586398700586,
          1.7430898442917737,
          1.8351614958072473
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        126,
        151,
        63,
        97,
        96,
        94,
        136,
        64,
        89,
        71,
        78,
        65,
        141,
        71,
        81,
        207,
        70,
        120,
        104,
        164
      ],
      "started": "2026-10-28T00:51:50Z",
      "updated": "2026-10-28T01:48:10Z",
      "format_of": null,
      "group": "ef",
      "latency": {
        "source": "run",
        "runs": [
          278
        ],
        "concurrency": 1,
        "median_request_seconds": 22.122042668488575,
        "p90_request_seconds": 44.978391354210906
      },
      "single_question_latency": null
    },
    {
      "run": 275,
      "model": "gpt-6.1-terra",
      "name": "GPT-6.1 Terra",
      "variant": null,
      "reasoning": "high",
      "provider": "openai",
      "note": null,
      "parameters": null,
      "active_parameters": null,
      "access": "Codex CLI, ChatGPT subscription",
      "questions_per_request": 16,
      "concurrency": 1,
      "correct": 2072,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 631
        },
        "grammar": {
          "questions": 1542,
          "correct": 1441
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 2070
        }
      },
      "tyt_mix": 0.9570970647084875,
      "questions_per_minute": 113.26624110162051,
      "request_seconds": 1164.3363346160622,
      "median_request_seconds": 7.102677927498007,
      "p90_request_seconds": 12.638287538199801,
      "billing": "subscription",
      "api_equivalent_usd": 1.1309,
      "cost_basis": "subscription",
      "requests": 138,
      "hardware": null,
      "input_tokens": 1163550,
      "cached_input_tokens": 380800,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 65822,
      "confidence": {
        "source": "stated",
        "label": "Written estimate",
        "bins": [
          {
            "questions": 5,
            "correct": 4
          },
          {
            "questions": 7,
            "correct": 1
          },
          {
            "questions": 17,
            "correct": 11
          },
          {
            "questions": 108,
            "correct": 76
          },
          {
            "questions": 2061,
            "correct": 1980
          }
        ],
        "auroc": 0.8120212048783477,
        "auroc_interval": [
          0.7686381317618326,
          0.850444642216519
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.9256418383518225,
            "auroc_interval": [
              0.8678193548387096,
              0.9679245283018868
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.7769700634185556,
            "auroc_interval": [
              0.7280996265716537,
              0.8212640224358975
            ]
          }
        },
        "aurc": 0.01851789581061053,
        "ece": 0.02824840764332354,
        "brier": 0.04924818016378526,
        "coverage_accuracy": {
          "0.20": 0.9875686513946643,
          "0.50": 0.98623267228425,
          "0.80": 0.9745616235660286
        },
        "coverage_intervals": {
          "0.20": [
            0.98044440982731,
            0.9930367112545
          ],
          "0.50": [
            0.9795653738190598,
            0.9917716787436663
          ],
          "0.80": [
            0.966284463492096,
            0.9813336818433779
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0074
          ],
          [
            0.04,
            0.0074
          ],
          [
            0.06,
            0.0074
          ],
          [
            0.08,
            0.0091
          ],
          [
            0.1,
            0.0102
          ],
          [
            0.12,
            0.0109
          ],
          [
            0.14,
            0.0115
          ],
          [
            0.16,
            0.0119
          ],
          [
            0.18,
            0.0122
          ],
          [
            0.2,
            0.0124
          ],
          [
            0.22,
            0.0126
          ],
          [
            0.24,
            0.0128
          ],
          [
            0.26,
            0.0129
          ],
          [
            0.28,
            0.0131
          ],
          [
            0.3,
            0.0132
          ],
          [
            0.32,
            0.0133
          ],
          [
            0.34,
            0.0133
          ],
          [
            0.36,
            0.0134
          ],
          [
            0.38,
            0.0135
          ],
          [
            0.4,
            0.0135
          ],
          [
            0.42,
            0.0136
          ],
          [
            0.44,
            0.0136
          ],
          [
            0.46,
            0.0137
          ],
          [
            0.48,
            0.0137
          ],
          [
            0.5,
            0.0138
          ],
          [
            0.52,
            0.0138
          ],
          [
            0.54,
            0.0138
          ],
          [
            0.56,
            0.0139
          ],
          [
            0.58,
            0.0139
          ],
          [
            0.6,
            0.0139
          ],
          [
            0.62,
            0.0139
          ],
          [
            0.64,
            0.0154
          ],
          [
            0.66,
            0.0168
          ],
          [
            0.68,
            0.0181
          ],
          [
            0.7,
            0.0194
          ],
          [
            0.72,
            0.0206
          ],
          [
            0.74,
            0.0217
          ],
          [
            0.76,
            0.0228
          ],
          [
            0.78,
            0.0242
          ],
          [
            0.8,
            0.0254
          ],
          [
            0.82,
            0.0272
          ],
          [
            0.84,
            0.0289
          ],
          [
            0.86,
            0.0309
          ],
          [
            0.88,
            0.0329
          ],
          [
            0.9,
            0.0349
          ],
          [
            0.92,
            0.037
          ],
          [
            0.94,
            0.0398
          ],
          [
            0.96,
            0.0446
          ],
          [
            0.98,
            0.0492
          ],
          [
            1.0,
            0.0573
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 6,
            "mean_confidence": 0.20833333333333334,
            "accuracy": 0.6666666666666666
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 3,
            "mean_confidence": 0.3333333333333333,
            "accuracy": 0.3333333333333333
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 3,
            "mean_confidence": 0.44333333333333336,
            "accuracy": 0.0
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 5,
            "mean_confidence": 0.5599999999999999,
            "accuracy": 0.6
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 6,
            "mean_confidence": 0.6433333333333333,
            "accuracy": 0.5
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 21,
            "mean_confidence": 0.76,
            "accuracy": 0.6190476190476191
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 93,
            "mean_confidence": 0.8577419354838708,
            "accuracy": 0.7311827956989247
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 2061,
            "mean_confidence": 0.9811838913149079,
            "accuracy": 0.9606986899563319
          }
        ],
        "median_right": 0.99,
        "median_wrong": 0.94,
        "sure": {
          "questions": 1363,
          "correct": 1344,
          "interval": [
            0.9783301784584166,
            0.9910579349850137
          ]
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 23,
          "correct": 11,
          "interval": [
            0.2923686953994572,
            0.6703757013578151
          ]
        },
        "ece_refit": 0.004834169479357083,
        "brier_refit": 0.048065563695641224,
        "brier_base": 0.05408489066889625,
        "refit_slopes": [
          0.8347234166126128,
          0.991013726659991,
          0.878616808198628,
          0.8205883847448608,
          0.824239445811547
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        125,
        152,
        64,
        95,
        101,
        94,
        136,
        62,
        89,
        71,
        75,
        63,
        138,
        70,
        83,
        194,
        67,
        126,
        100,
        167
      ],
      "started": "2026-10-28T00:51:50Z",
      "updated": "2026-10-28T01:11:15Z",
      "format_of": null,
      "group": "fg",
      "latency": {
        "source": "run",
        "runs": [
          275
        ],
        "concurrency": 1,
        "median_request_seconds": 7.102677927498007,
        "p90_request_seconds": 12.638287538199801
      },
      "single_question_latency": null
    },
    {
      "run": 192,
      "model": "claude-opus-5-5",
      "name": "Claude Opus 5.5",
      "variant": null,
      "reasoning": "low",
      "provider": "claude",
      "note": null,
      "parameters": null,
      "active_parameters": null,
      "access": "Claude Code CLI, Claude subscription",
      "questions_per_request": 1,
      "concurrency": 1,
      "correct": 2051,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 628
        },
        "grammar": {
          "questions": 1542,
          "correct": 1423
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 2046
        }
      },
      "tyt_mix": 0.9512813972984088,
      "questions_per_minute": 14.516400274394256,
      "request_seconds": 9084.896910195122,
      "median_request_seconds": 3.7071781139966333,
      "p90_request_seconds": 5.883561807710794,
      "billing": "subscription",
      "api_equivalent_usd": 25.5178246,
      "cost_basis": "subscription",
      "requests": 2198,
      "hardware": null,
      "input_tokens": 3481382,
      "cached_input_tokens": 1403883,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 2073103,
      "output_tokens": 431732,
      "confidence": {
        "source": "stated",
        "label": "Written estimate",
        "bins": [
          {
            "questions": 0,
            "correct": 0
          },
          {
            "questions": 65,
            "correct": 25
          },
          {
            "questions": 268,
            "correct": 210
          },
          {
            "questions": 1091,
            "correct": 1044
          },
          {
            "questions": 774,
            "correct": 772
          }
        ],
        "auroc": 0.8817301664693181,
        "auroc_interval": [
          0.8552272412359706,
          0.9070379417033057
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.9479356232939036,
            "auroc_interval": [
              0.925619073920648,
              0.9707911511061117
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.8571723840625498,
            "auroc_interval": [
              0.8221939648120882,
              0.8904754976137819
            ]
          }
        },
        "aurc": 0.01208182981847769,
        "ece": 0.10196087352138282,
        "brier": 0.059664467697907196,
        "coverage_accuracy": {
          "0.20": 0.9988456021164501,
          "0.50": 0.9929916945772753,
          "0.80": 0.9790193564397386
        },
        "coverage_intervals": {
          "0.20": [
            0.9959147152442579,
            1.0
          ],
          "0.50": [
            0.9872938714564418,
            0.9972652689152234
          ],
          "0.80": [
            0.9718237704918032,
            0.9855600621578466
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0
          ],
          [
            0.04,
            0.0
          ],
          [
            0.06,
            0.0
          ],
          [
            0.08,
            -0.0
          ],
          [
            0.1,
            0.0
          ],
          [
            0.12,
            0.0
          ],
          [
            0.14,
            0.0001
          ],
          [
            0.16,
            0.0006
          ],
          [
            0.18,
            0.0009
          ],
          [
            0.2,
            0.0012
          ],
          [
            0.22,
            0.0014
          ],
          [
            0.24,
            0.0016
          ],
          [
            0.26,
            0.0017
          ],
          [
            0.28,
            0.0018
          ],
          [
            0.3,
            0.0021
          ],
          [
            0.32,
            0.0023
          ],
          [
            0.34,
            0.0025
          ],
          [
            0.36,
            0.0029
          ],
          [
            0.38,
            0.0036
          ],
          [
            0.4,
            0.0043
          ],
          [
            0.42,
            0.0049
          ],
          [
            0.44,
            0.0054
          ],
          [
            0.46,
            0.0059
          ],
          [
            0.48,
            0.0064
          ],
          [
            0.5,
            0.007
          ],
          [
            0.52,
            0.0077
          ],
          [
            0.54,
            0.0084
          ],
          [
            0.56,
            0.009
          ],
          [
            0.58,
            0.0096
          ],
          [
            0.6,
            0.0102
          ],
          [
            0.62,
            0.0107
          ],
          [
            0.64,
            0.0112
          ],
          [
            0.66,
            0.0116
          ],
          [
            0.68,
            0.012
          ],
          [
            0.7,
            0.0131
          ],
          [
            0.72,
            0.0148
          ],
          [
            0.74,
            0.0165
          ],
          [
            0.76,
            0.0181
          ],
          [
            0.78,
            0.0196
          ],
          [
            0.8,
            0.021
          ],
          [
            0.82,
            0.0228
          ],
          [
            0.84,
            0.0253
          ],
          [
            0.86,
            0.0275
          ],
          [
            0.88,
            0.0297
          ],
          [
            0.9,
            0.033
          ],
          [
            0.92,
            0.0379
          ],
          [
            0.94,
            0.0425
          ],
          [
            0.96,
            0.0475
          ],
          [
            0.98,
            0.0559
          ],
          [
            1.0,
            0.0669
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 1,
            "mean_confidence": 0.25,
            "accuracy": 0.0
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 21,
            "mean_confidence": 0.3357142857142856,
            "accuracy": 0.2857142857142857
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 43,
            "mean_confidence": 0.425581395348837,
            "accuracy": 0.4418604651162791
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 51,
            "mean_confidence": 0.5333333333333335,
            "accuracy": 0.7058823529411765
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 125,
            "mean_confidence": 0.6015199999999999,
            "accuracy": 0.752
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 171,
            "mean_confidence": 0.7301169590643275,
            "accuracy": 0.8713450292397661
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 1012,
            "mean_confidence": 0.8447727272727299,
            "accuracy": 0.9634387351778656
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 774,
            "mean_confidence": 0.9324806201550361,
            "accuracy": 0.9974160206718347
          }
        ],
        "median_right": 0.88,
        "median_wrong": 0.6,
        "sure": {
          "questions": 1,
          "correct": 1,
          "interval": [
            0.20654329147389294,
            1.0
          ]
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 241,
          "correct": 155,
          "interval": [
            0.5808570824452763,
            0.7009577725466858
          ]
        },
        "ece_refit": 0.00543041273668052,
        "brier_refit": 0.04854622311883117,
        "brier_base": 0.0624604750405318,
        "refit_slopes": [
          1.7155185099051558,
          1.9135611912412036,
          1.7790892098860236,
          1.8211751768993845,
          1.9340675037843547
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        128,
        149,
        60,
        95,
        100,
        96,
        134,
        63,
        89,
        71,
        77,
        64,
        139,
        70,
        78,
        199,
        66,
        113,
        96,
        164
      ],
      "started": "2026-10-13T16:20:09Z",
      "updated": "2026-10-13T18:51:39Z",
      "format_of": 47,
      "group": "fgh",
      "latency": {
        "source": "run",
        "runs": [
          192
        ],
        "concurrency": 1,
        "median_request_seconds": 3.7071781139966333,
        "p90_request_seconds": 5.883561807710794
      },
      "single_question_latency": null
    },
    {
      "run": 123,
      "model": "gemini-3.8-flash",
      "name": "Gemini 3.8 Flash",
      "variant": null,
      "reasoning": "low",
      "provider": "gemini",
      "note": null,
      "parameters": null,
      "active_parameters": null,
      "access": "Gemini API",
      "questions_per_request": 16,
      "concurrency": 1,
      "correct": 2050,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 624
        },
        "grammar": {
          "questions": 1542,
          "correct": 1426
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 2047
        }
      },
      "tyt_mix": 0.9465913764196008,
      "questions_per_minute": 399.91697973147666,
      "request_seconds": 329.7684436618583,
      "median_request_seconds": 2.1872003605240025,
      "p90_request_seconds": 2.6911692536843477,
      "billing": "api",
      "api_equivalent_usd": 0.5903475,
      "cost_basis": "api",
      "requests": 138,
      "hardware": null,
      "input_tokens": 483830,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 60660,
      "confidence": {
        "source": "stated",
        "label": "Written estimate",
        "bins": [
          {
            "questions": 0,
            "correct": 0
          },
          {
            "questions": 1,
            "correct": 1
          },
          {
            "questions": 3,
            "correct": 0
          },
          {
            "questions": 39,
            "correct": 24
          },
          {
            "questions": 2155,
            "correct": 2025
          }
        ],
        "auroc": 0.765596572181938,
        "auroc_interval": [
          0.7246943765281174,
          0.802016129032258
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.8043369391025641,
            "auroc_interval": [
              0.7083712465878071,
              0.8861677419354839
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.7625985394399575,
            "auroc_interval": [
              0.7202386101499423,
              0.8013830532212886
            ]
          }
        },
        "aurc": 0.02684836217262224,
        "ece": 0.03474977252048743,
        "brier": 0.05971137397634213,
        "coverage_accuracy": {
          "0.20": 0.9843184559710495,
          "0.50": 0.9811757404261361,
          "0.80": 0.9598300086907391
        },
        "coverage_intervals": {
          "0.20": [
            0.9754672897196262,
            0.9924623115577889
          ],
          "0.50": [
            0.9726028068885212,
            0.9886076661323434
          ],
          "0.80": [
            0.9507848771381875,
            0.9681232290682815
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0157
          ],
          [
            0.04,
            0.0157
          ],
          [
            0.06,
            0.0157
          ],
          [
            0.08,
            0.0157
          ],
          [
            0.1,
            0.0157
          ],
          [
            0.12,
            0.0157
          ],
          [
            0.14,
            0.0157
          ],
          [
            0.16,
            0.0157
          ],
          [
            0.18,
            0.0157
          ],
          [
            0.2,
            0.0157
          ],
          [
            0.22,
            0.0157
          ],
          [
            0.24,
            0.0157
          ],
          [
            0.26,
            0.0157
          ],
          [
            0.28,
            0.0157
          ],
          [
            0.3,
            0.0157
          ],
          [
            0.32,
            0.0157
          ],
          [
            0.34,
            0.0157
          ],
          [
            0.36,
            0.0157
          ],
          [
            0.38,
            0.0157
          ],
          [
            0.4,
            0.0156
          ],
          [
            0.42,
            0.0155
          ],
          [
            0.44,
            0.0154
          ],
          [
            0.46,
            0.0164
          ],
          [
            0.48,
            0.0177
          ],
          [
            0.5,
            0.0188
          ],
          [
            0.52,
            0.0199
          ],
          [
            0.54,
            0.0217
          ],
          [
            0.56,
            0.0237
          ],
          [
            0.58,
            0.0256
          ],
          [
            0.6,
            0.0274
          ],
          [
            0.62,
            0.029
          ],
          [
            0.64,
            0.0306
          ],
          [
            0.66,
            0.032
          ],
          [
            0.68,
            0.0334
          ],
          [
            0.7,
            0.0347
          ],
          [
            0.72,
            0.0359
          ],
          [
            0.74,
            0.0371
          ],
          [
            0.76,
            0.0381
          ],
          [
            0.78,
            0.0392
          ],
          [
            0.8,
            0.0402
          ],
          [
            0.82,
            0.0411
          ],
          [
            0.84,
            0.042
          ],
          [
            0.86,
            0.0428
          ],
          [
            0.88,
            0.0437
          ],
          [
            0.9,
            0.046
          ],
          [
            0.92,
            0.0498
          ],
          [
            0.94,
            0.0534
          ],
          [
            0.96,
            0.0569
          ],
          [
            0.98,
            0.0603
          ],
          [
            1.0,
            0.0673
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 1,
            "mean_confidence": 0.4,
            "accuracy": 1.0
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 1,
            "mean_confidence": 0.5,
            "accuracy": 0.0
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 2,
            "mean_confidence": 0.7,
            "accuracy": 0.0
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 39,
            "mean_confidence": 0.8430769230769235,
            "accuracy": 0.6153846153846154
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 2155,
            "mean_confidence": 0.9698375870069751,
            "accuracy": 0.9396751740139211
          }
        ],
        "median_right": 0.98,
        "median_wrong": 0.95,
        "sure": {
          "questions": 976,
          "correct": 961,
          "interval": [
            0.9747976246987011,
            0.9906645479626759
          ]
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 2,
          "correct": 1,
          "interval": [
            0.09452865480086614,
            0.9054713451991339
          ]
        },
        "ece_refit": 0.008492513656288768,
        "brier_refit": 0.06012368254290253,
        "brier_base": 0.06286850312130109,
        "refit_slopes": [
          0.19838847758099945,
          0.2651897343583401,
          0.22354347734766247,
          0.22878117116277033,
          0.232743214994192
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        126,
        149,
        62,
        96,
        96,
        95,
        133,
        62,
        91,
        68,
        72,
        64,
        130,
        73,
        79,
        190,
        66,
        128,
        104,
        166
      ],
      "started": "2026-10-05T20:04:57Z",
      "updated": "2026-10-05T20:10:28Z",
      "format_of": null,
      "group": "fghi",
      "latency": {
        "source": "run",
        "runs": [
          123
        ],
        "concurrency": 1,
        "median_request_seconds": 2.1872003605240025,
        "p90_request_seconds": 2.6911692536843477
      },
      "single_question_latency": null
    },
    {
      "run": 311,
      "model": "grok-4.7",
      "name": "Grok 4.7",
      "variant": null,
      "reasoning": "high",
      "provider": "xai",
      "note": "xAI did not answer 31 attempts within the time limit, and the runner sent each such request again. The run stopped twice and was resumed from the last answered question: at 1,696 answers, after one request timed out six times (the limit then went from 15 to 30 minutes), and at 2,144, when a retry sent an expired sign-in token. The usage and cost of the timed-out attempts are unknown.",
      "parameters": null,
      "active_parameters": null,
      "access": "xAI Responses API, SuperGrok subscription",
      "questions_per_request": 16,
      "concurrency": 4,
      "correct": 2038,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 623
        },
        "grammar": {
          "questions": 1542,
          "correct": 1415
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 2036
        }
      },
      "tyt_mix": 0.9440853757394578,
      "questions_per_minute": 6.595969744562386,
      "request_seconds": 19994.027429965063,
      "median_request_seconds": 401.9361946709978,
      "p90_request_seconds": 1228.6228537980119,
      "billing": "subscription",
      "api_equivalent_usd": 31.414164,
      "cost_basis": "subscription",
      "requests": 138,
      "hardware": null,
      "input_tokens": 808614,
      "cached_input_tokens": 215424,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 5020012,
      "confidence": {
        "source": "stated",
        "label": "Written estimate",
        "bins": [
          {
            "questions": 1,
            "correct": 0
          },
          {
            "questions": 91,
            "correct": 59
          },
          {
            "questions": 469,
            "correct": 378
          },
          {
            "questions": 848,
            "correct": 819
          },
          {
            "questions": 789,
            "correct": 782
          }
        ],
        "auroc": 0.8488208415112856,
        "auroc_interval": [
          0.8201860006421724,
          0.8760292076597314
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.9193054136874361,
            "auroc_interval": [
              0.8838391426282052,
              0.951602988462266
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.8237055173756991,
            "auroc_interval": [
              0.7852437166793602,
              0.8579925771057195
            ]
          }
        },
        "aurc": 0.0165252439882545,
        "ece": 0.11871246587807233,
        "brier": 0.07360759781619654,
        "coverage_accuracy": {
          "0.20": 1.0,
          "0.50": 0.9898729484716746,
          "0.80": 0.9719157567485592
        },
        "coverage_intervals": {
          "0.20": [
            1.0,
            1.0
          ],
          "0.50": [
            0.9840947927816273,
            0.9954483386436049
          ],
          "0.80": [
            0.963710885984894,
            0.9796995903504779
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0
          ],
          [
            0.04,
            0.0
          ],
          [
            0.06,
            0.0
          ],
          [
            0.08,
            0.0
          ],
          [
            0.1,
            0.0
          ],
          [
            0.12,
            0.0
          ],
          [
            0.14,
            0.0
          ],
          [
            0.16,
            0.0
          ],
          [
            0.18,
            0.0
          ],
          [
            0.2,
            0.0
          ],
          [
            0.22,
            0.0
          ],
          [
            0.24,
            0.0001
          ],
          [
            0.26,
            0.0021
          ],
          [
            0.28,
            0.0039
          ],
          [
            0.3,
            0.0054
          ],
          [
            0.32,
            0.0067
          ],
          [
            0.34,
            0.0079
          ],
          [
            0.36,
            0.0088
          ],
          [
            0.38,
            0.0089
          ],
          [
            0.4,
            0.0092
          ],
          [
            0.42,
            0.0094
          ],
          [
            0.44,
            0.0093
          ],
          [
            0.46,
            0.0095
          ],
          [
            0.48,
            0.0099
          ],
          [
            0.5,
            0.0101
          ],
          [
            0.52,
            0.0101
          ],
          [
            0.54,
            0.0101
          ],
          [
            0.56,
            0.0127
          ],
          [
            0.58,
            0.0153
          ],
          [
            0.6,
            0.0161
          ],
          [
            0.62,
            0.016
          ],
          [
            0.64,
            0.0164
          ],
          [
            0.66,
            0.0172
          ],
          [
            0.68,
            0.0179
          ],
          [
            0.7,
            0.0186
          ],
          [
            0.72,
            0.0194
          ],
          [
            0.74,
            0.0216
          ],
          [
            0.76,
            0.0235
          ],
          [
            0.78,
            0.0253
          ],
          [
            0.8,
            0.0281
          ],
          [
            0.82,
            0.0312
          ],
          [
            0.84,
            0.0345
          ],
          [
            0.86,
            0.0384
          ],
          [
            0.88,
            0.0424
          ],
          [
            0.9,
            0.0455
          ],
          [
            0.92,
            0.0485
          ],
          [
            0.94,
            0.0528
          ],
          [
            0.96,
            0.0608
          ],
          [
            0.98,
            0.0668
          ],
          [
            1.0,
            0.0728
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 4,
            "mean_confidence": 0.245,
            "accuracy": 0.5
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 17,
            "mean_confidence": 0.3323529411764706,
            "accuracy": 0.6470588235294118
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 71,
            "mean_confidence": 0.4295774647887322,
            "accuracy": 0.647887323943662
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 96,
            "mean_confidence": 0.5405208333333328,
            "accuracy": 0.6770833333333334
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 183,
            "mean_confidence": 0.6324590163934424,
            "accuracy": 0.8032786885245902
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 301,
            "mean_confidence": 0.7357142857142849,
            "accuracy": 0.893687707641196
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 737,
            "mean_confidence": 0.8437313432835845,
            "accuracy": 0.9715061058344641
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 789,
            "mean_confidence": 0.9239923954372569,
            "accuracy": 0.991128010139417
          }
        ],
        "median_right": 0.86,
        "median_wrong": 0.62,
        "sure": {
          "questions": 0,
          "correct": 0,
          "interval": null
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 371,
          "correct": 271,
          "interval": [
            0.6831139704154376,
            0.7730787253792659
          ]
        },
        "ece_refit": 0.012685528505652943,
        "brier_refit": 0.05843769294503204,
        "brier_base": 0.06760579465800974,
        "refit_slopes": [
          1.5315311865466716,
          1.5541532037330157,
          1.4886049205749632,
          1.5891089869979191,
          1.4708680697183578
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        125,
        152,
        60,
        94,
        98,
        94,
        131,
        63,
        87,
        72,
        74,
        61,
        141,
        72,
        81,
        187,
        69,
        115,
        100,
        162
      ],
      "started": "2026-10-28T14:33:55Z",
      "updated": "2026-10-28T22:00:47Z",
      "format_of": null,
      "group": "ghij",
      "latency": {
        "source": "run",
        "runs": [
          311
        ],
        "concurrency": 4,
        "median_request_seconds": 401.9361946709978,
        "p90_request_seconds": 1228.6228537980119
      },
      "single_question_latency": null
    },
    {
      "run": 121,
      "model": "deepseek-v4.1-flash",
      "name": "DeepSeek-V4.1-Flash, first release",
      "variant": null,
      "reasoning": "on",
      "provider": "vllm",
      "note": null,
      "parameters": 552000000000,
      "active_parameters": [
        8000000000,
        16000000000
      ],
      "access": "SGLang, self-hosted, published mixed precision; first-release weights (revision fb2764a), not the 1017 update",
      "questions_per_request": 16,
      "concurrency": 8,
      "correct": 2023,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 613
        },
        "grammar": {
          "questions": 1542,
          "correct": 1410
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 2020
        }
      },
      "tyt_mix": 0.9309417113504793,
      "questions_per_minute": 19.260714724220154,
      "request_seconds": 6847.09793422994,
      "median_request_seconds": 365.14285134349484,
      "p90_request_seconds": 686.8833388963016,
      "billing": "local",
      "api_equivalent_usd": null,
      "cost_basis": null,
      "requests": 138,
      "hardware": "4 × RTX PRO 6000 Blackwell (96 GB); FP4 experts, FP8 and BF16 layers",
      "input_tokens": 669316,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 2212141,
      "confidence": {
        "source": "probability",
        "label": "Answer letter probability",
        "bins": [
          {
            "questions": 0,
            "correct": 0
          },
          {
            "questions": 0,
            "correct": 0
          },
          {
            "questions": 0,
            "correct": 0
          },
          {
            "questions": 0,
            "correct": 0
          },
          {
            "questions": 2198,
            "correct": 2023
          }
        ],
        "auroc": 0.561861450462538,
        "auroc_interval": [
          0.5197339736501216,
          0.6051804510346025
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.598012064190599,
            "auroc_interval": [
              0.5086850649350649,
              0.6810838709677419
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.5574736728992048,
            "auroc_interval": [
              0.5061183606466153,
              0.6138800545206724
            ]
          }
        },
        "aurc": 0.0644329020491846,
        "ece": 0.07960198328526213,
        "brier": 0.07960859691673716,
        "coverage_accuracy": {
          "0.20": 0.9317561419472248,
          "0.50": 0.9326660600545951,
          "0.80": 0.9306187443130118
        },
        "coverage_intervals": {
          "0.20": [
            0.9072305593451568,
            0.9567000911577028
          ],
          "0.50": [
            0.9173103134938664,
            0.9480874316939891
          ],
          "0.80": [
            0.9169699727024568,
            0.9425352753755121
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0227
          ],
          [
            0.04,
            0.0227
          ],
          [
            0.06,
            0.0455
          ],
          [
            0.08,
            0.0455
          ],
          [
            0.1,
            0.0546
          ],
          [
            0.12,
            0.0455
          ],
          [
            0.14,
            0.052
          ],
          [
            0.16,
            0.0597
          ],
          [
            0.18,
            0.0632
          ],
          [
            0.2,
            0.0682
          ],
          [
            0.22,
            0.0703
          ],
          [
            0.24,
            0.0645
          ],
          [
            0.26,
            0.0612
          ],
          [
            0.28,
            0.0617
          ],
          [
            0.3,
            0.0598
          ],
          [
            0.32,
            0.0626
          ],
          [
            0.34,
            0.0642
          ],
          [
            0.36,
            0.0632
          ],
          [
            0.38,
            0.0658
          ],
          [
            0.4,
            0.066
          ],
          [
            0.42,
            0.0672
          ],
          [
            0.44,
            0.0651
          ],
          [
            0.46,
            0.0663
          ],
          [
            0.48,
            0.0663
          ],
          [
            0.5,
            0.0673
          ],
          [
            0.52,
            0.0674
          ],
          [
            0.54,
            0.0666
          ],
          [
            0.56,
            0.0658
          ],
          [
            0.58,
            0.0705
          ],
          [
            0.6,
            0.0705
          ],
          [
            0.62,
            0.0695
          ],
          [
            0.64,
            0.069
          ],
          [
            0.66,
            0.0696
          ],
          [
            0.68,
            0.0703
          ],
          [
            0.7,
            0.0689
          ],
          [
            0.72,
            0.0714
          ],
          [
            0.74,
            0.0707
          ],
          [
            0.76,
            0.0706
          ],
          [
            0.78,
            0.07
          ],
          [
            0.8,
            0.0694
          ],
          [
            0.82,
            0.0727
          ],
          [
            0.84,
            0.0728
          ],
          [
            0.86,
            0.0758
          ],
          [
            0.88,
            0.0781
          ],
          [
            0.9,
            0.0789
          ],
          [
            0.92,
            0.0796
          ],
          [
            0.94,
            0.0799
          ],
          [
            0.96,
            0.0787
          ],
          [
            0.98,
            0.0789
          ],
          [
            1.0,
            0.0796
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 2198,
            "mean_confidence": 0.9999841488903577,
            "accuracy": 0.9203821656050956
          }
        ],
        "median_right": 0.9999997494873384,
        "median_wrong": 0.9999996215664603,
        "sure": {
          "questions": 2198,
          "correct": 2023,
          "interval": [
            0.9083178247126321,
            0.9309796070372266
          ]
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 0,
          "correct": 0,
          "interval": null
        },
        "ece_refit": 0.005784995062470635,
        "brier_refit": 0.07321441096810519,
        "brier_base": 0.07332355322110665,
        "refit_slopes": [
          0.1439718980670833,
          0.1496437067090975,
          0.08478049629756118,
          0.14508454447568944,
          0.14655911197914528
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        125,
        150,
        59,
        92,
        94,
        93,
        129,
        63,
        89,
        70,
        75,
        61,
        136,
        70,
        81,
        190,
        66,
        125,
        98,
        157
      ],
      "started": "2026-10-05T19:02:51Z",
      "updated": "2026-10-05T20:56:59Z",
      "format_of": null,
      "group": "hij",
      "latency": {
        "source": "run",
        "runs": [
          121
        ],
        "concurrency": 8,
        "median_request_seconds": 365.14285134349484,
        "p90_request_seconds": 686.8833388963016
      },
      "single_question_latency": null
    },
    {
      "run": 274,
      "model": "gpt-6.1-terra",
      "name": "GPT-6.1 Terra",
      "variant": null,
      "reasoning": "low",
      "provider": "openai",
      "note": null,
      "parameters": null,
      "active_parameters": null,
      "access": "Codex CLI, ChatGPT subscription",
      "questions_per_request": 16,
      "concurrency": 1,
      "correct": 1997,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 629
        },
        "grammar": {
          "questions": 1542,
          "correct": 1368
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 1993
        }
      },
      "tyt_mix": 0.9462971256050109,
      "questions_per_minute": 206.76412628602517,
      "request_seconds": 637.8282459770853,
      "median_request_seconds": 4.235071693503414,
      "p90_request_seconds": 5.358385625202208,
      "billing": "subscription",
      "api_equivalent_usd": 0.936888,
      "cost_basis": "subscription",
      "requests": 138,
      "hardware": null,
      "input_tokens": 1163508,
      "cached_input_tokens": 396800,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 30068,
      "confidence": {
        "source": "stated",
        "label": "Written estimate",
        "bins": [
          {
            "questions": 4,
            "correct": 2
          },
          {
            "questions": 2,
            "correct": 1
          },
          {
            "questions": 17,
            "correct": 7
          },
          {
            "questions": 125,
            "correct": 80
          },
          {
            "questions": 2050,
            "correct": 1907
          }
        ],
        "auroc": 0.8198230181092534,
        "auroc_interval": [
          0.7871864104340068,
          0.8488369024528626
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.8853265029735617,
            "auroc_interval": [
              0.8004475063298593,
              0.9532347351896224
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.8013418363917456,
            "auroc_interval": [
              0.7650453753173431,
              0.8343257003943968
            ]
          }
        },
        "aurc": 0.0282707955133511,
        "ece": 0.05959963603276847,
        "brier": 0.07876869881710646,
        "coverage_accuracy": {
          "0.20": 0.9828884634354599,
          "0.50": 0.97966662600699,
          "0.80": 0.9587429528249674
        },
        "coverage_intervals": {
          "0.20": [
            0.9755681818181818,
            0.9886019017604576
          ],
          "0.50": [
            0.9712235518968921,
            0.9862959681554955
          ],
          "0.80": [
            0.9484525014244045,
            0.9673928490222828
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0
          ],
          [
            0.04,
            0.0
          ],
          [
            0.06,
            0.0046
          ],
          [
            0.08,
            0.0091
          ],
          [
            0.1,
            0.0117
          ],
          [
            0.12,
            0.0135
          ],
          [
            0.14,
            0.0148
          ],
          [
            0.16,
            0.0158
          ],
          [
            0.18,
            0.0165
          ],
          [
            0.2,
            0.0171
          ],
          [
            0.22,
            0.0176
          ],
          [
            0.24,
            0.018
          ],
          [
            0.26,
            0.0184
          ],
          [
            0.28,
            0.0186
          ],
          [
            0.3,
            0.0189
          ],
          [
            0.32,
            0.0191
          ],
          [
            0.34,
            0.0193
          ],
          [
            0.36,
            0.0195
          ],
          [
            0.38,
            0.0197
          ],
          [
            0.4,
            0.0198
          ],
          [
            0.42,
            0.0199
          ],
          [
            0.44,
            0.02
          ],
          [
            0.46,
            0.0201
          ],
          [
            0.48,
            0.0202
          ],
          [
            0.5,
            0.0203
          ],
          [
            0.52,
            0.0204
          ],
          [
            0.54,
            0.0205
          ],
          [
            0.56,
            0.0206
          ],
          [
            0.58,
            0.0206
          ],
          [
            0.6,
            0.0213
          ],
          [
            0.62,
            0.0232
          ],
          [
            0.64,
            0.025
          ],
          [
            0.66,
            0.0267
          ],
          [
            0.68,
            0.0283
          ],
          [
            0.7,
            0.0298
          ],
          [
            0.72,
            0.0313
          ],
          [
            0.74,
            0.0325
          ],
          [
            0.76,
            0.0336
          ],
          [
            0.78,
            0.037
          ],
          [
            0.8,
            0.0413
          ],
          [
            0.82,
            0.0451
          ],
          [
            0.84,
            0.0487
          ],
          [
            0.86,
            0.052
          ],
          [
            0.88,
            0.0551
          ],
          [
            0.9,
            0.0615
          ],
          [
            0.92,
            0.0669
          ],
          [
            0.94,
            0.0724
          ],
          [
            0.96,
            0.0777
          ],
          [
            0.98,
            0.0845
          ],
          [
            1.0,
            0.0914
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 4,
            "mean_confidence": 0.2,
            "accuracy": 0.5
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 2,
            "mean_confidence": 0.32499999999999996,
            "accuracy": 0.5
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 2,
            "mean_confidence": 0.55,
            "accuracy": 0.5
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 6,
            "mean_confidence": 0.65,
            "accuracy": 0.3333333333333333
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 21,
            "mean_confidence": 0.7399999999999999,
            "accuracy": 0.6190476190476191
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 113,
            "mean_confidence": 0.8492920353982296,
            "accuracy": 0.6283185840707964
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 2050,
            "mean_confidence": 0.9789951219512318,
            "accuracy": 0.9302439024390244
          }
        ],
        "median_right": 0.99,
        "median_wrong": 0.94,
        "sure": {
          "questions": 1306,
          "correct": 1279,
          "interval": [
            0.970087348715641,
            0.9857534185954597
          ]
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 14,
          "correct": 6,
          "interval": [
            0.21380509930400526,
            0.6740973309713064
          ]
        },
        "ece_refit": 0.015224676552838498,
        "brier_refit": 0.07139019809114178,
        "brier_base": 0.08315358717727672,
        "refit_slopes": [
          0.9714424462024931,
          1.0665658864769951,
          1.008551076881422,
          0.9754639179750753,
          1.0263860435224172
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        127,
        150,
        64,
        94,
        98,
        96,
        131,
        58,
        87,
        69,
        73,
        54,
        130,
        67,
        80,
        180,
        55,
        120,
        98,
        166
      ],
      "started": "2026-10-28T00:51:38Z",
      "updated": "2026-10-28T01:02:17Z",
      "format_of": null,
      "group": "ijk",
      "latency": {
        "source": "run",
        "runs": [
          274
        ],
        "concurrency": 1,
        "median_request_seconds": 4.235071693503414,
        "p90_request_seconds": 5.358385625202208
      },
      "single_question_latency": null
    },
    {
      "run": 47,
      "model": "claude-opus-5-5",
      "name": "Claude Opus 5.5",
      "variant": null,
      "reasoning": "low",
      "provider": "claude",
      "note": null,
      "parameters": null,
      "active_parameters": null,
      "access": "Claude Code CLI, Claude subscription",
      "questions_per_request": 16,
      "concurrency": 1,
      "correct": 1988,
      "invalid": 0,
      "answered_questions": 2210,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 622
        },
        "grammar": {
          "questions": 1542,
          "correct": 1366
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 1987
        }
      },
      "tyt_mix": 0.9372667939957611,
      "questions_per_minute": 71.89577090051554,
      "request_seconds": 1844.3365769522497,
      "median_request_seconds": null,
      "p90_request_seconds": null,
      "billing": "subscription",
      "api_equivalent_usd": 10.896595,
      "cost_basis": "subscription",
      "requests": 139,
      "hardware": null,
      "input_tokens": 1248840,
      "cached_input_tokens": 328215,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 920343,
      "output_tokens": 173354,
      "confidence": {
        "source": "stated",
        "label": "Written estimate",
        "bins": [
          {
            "questions": 2,
            "correct": 1
          },
          {
            "questions": 205,
            "correct": 111
          },
          {
            "questions": 568,
            "correct": 490
          },
          {
            "questions": 977,
            "correct": 941
          },
          {
            "questions": 446,
            "correct": 445
          }
        ],
        "auroc": 0.8398665804349909,
        "auroc_interval": [
          0.810289418549744,
          0.8671906425320207
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.8845517306601097,
            "auroc_interval": [
              0.8410194174757282,
              0.9207959512307339
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.8226074803673632,
            "auroc_interval": [
              0.7854622636308507,
              0.855900286944046
            ]
          }
        },
        "aurc": 0.025639199934659004,
        "ece": 0.1653457688808004,
        "brier": 0.10085882620564149,
        "coverage_accuracy": {
          "0.20": 0.9977252047315741,
          "0.50": 0.980202293738956,
          "0.80": 0.9577330401374987
        },
        "coverage_intervals": {
          "0.20": [
            0.9909132212630622,
            1.0
          ],
          "0.50": [
            0.9710047846889953,
            0.9876867686768678
          ],
          "0.80": [
            0.9482654185022027,
            0.966347451692286
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0111
          ],
          [
            0.04,
            0.0112
          ],
          [
            0.06,
            0.0076
          ],
          [
            0.08,
            0.0057
          ],
          [
            0.1,
            0.0045
          ],
          [
            0.12,
            0.0038
          ],
          [
            0.14,
            0.0032
          ],
          [
            0.16,
            0.0028
          ],
          [
            0.18,
            0.0025
          ],
          [
            0.2,
            0.0023
          ],
          [
            0.22,
            0.0037
          ],
          [
            0.24,
            0.0058
          ],
          [
            0.26,
            0.0076
          ],
          [
            0.28,
            0.0092
          ],
          [
            0.3,
            0.0105
          ],
          [
            0.32,
            0.0117
          ],
          [
            0.34,
            0.0127
          ],
          [
            0.36,
            0.0137
          ],
          [
            0.38,
            0.0145
          ],
          [
            0.4,
            0.0152
          ],
          [
            0.42,
            0.0159
          ],
          [
            0.44,
            0.0169
          ],
          [
            0.46,
            0.018
          ],
          [
            0.48,
            0.0189
          ],
          [
            0.5,
            0.0198
          ],
          [
            0.52,
            0.0206
          ],
          [
            0.54,
            0.0214
          ],
          [
            0.56,
            0.0221
          ],
          [
            0.58,
            0.0231
          ],
          [
            0.6,
            0.024
          ],
          [
            0.62,
            0.0249
          ],
          [
            0.64,
            0.0257
          ],
          [
            0.66,
            0.0271
          ],
          [
            0.68,
            0.0289
          ],
          [
            0.7,
            0.0305
          ],
          [
            0.72,
            0.0323
          ],
          [
            0.74,
            0.035
          ],
          [
            0.76,
            0.0375
          ],
          [
            0.78,
            0.04
          ],
          [
            0.8,
            0.0423
          ],
          [
            0.82,
            0.0445
          ],
          [
            0.84,
            0.0465
          ],
          [
            0.86,
            0.0486
          ],
          [
            0.88,
            0.0527
          ],
          [
            0.9,
            0.0566
          ],
          [
            0.92,
            0.0619
          ],
          [
            0.94,
            0.0703
          ],
          [
            0.96,
            0.0783
          ],
          [
            0.98,
            0.0857
          ],
          [
            1.0,
            0.0955
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 7,
            "mean_confidence": 0.2357142857142857,
            "accuracy": 0.5714285714285714
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 64,
            "mean_confidence": 0.32187500000000013,
            "accuracy": 0.46875
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 136,
            "mean_confidence": 0.40735294117647,
            "accuracy": 0.5735294117647058
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 152,
            "mean_confidence": 0.5141447368421048,
            "accuracy": 0.8026315789473685
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 265,
            "mean_confidence": 0.6058490566037711,
            "accuracy": 0.8679245283018868
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 347,
            "mean_confidence": 0.728242074927952,
            "accuracy": 0.9337175792507204
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 781,
            "mean_confidence": 0.8316645326504548,
            "accuracy": 0.9667093469910372
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 446,
            "mean_confidence": 0.9102914798206211,
            "accuracy": 0.9977578475336323
          }
        ],
        "median_right": 0.8,
        "median_wrong": 0.5,
        "sure": {
          "questions": 0,
          "correct": 0,
          "interval": null
        },
        "sure_before_audit": {
          "questions": 0,
          "correct": 0,
          "interval": null
        },
        "doubtful": {
          "questions": 624,
          "correct": 464,
          "interval": [
            0.7079109015548006,
            0.7762876606303111
          ]
        },
        "ece_refit": 0.013180411538190976,
        "brier_refit": 0.0698607041220345,
        "brier_base": 0.08645000333687364,
        "refit_slopes": [
          1.5130388235866603,
          1.5846956139730646,
          1.5754465806958524,
          1.6182150157472301,
          1.6700146645246663
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": {
        "run": 36,
        "changed": {
          "questions": 372,
          "older_correct": 224,
          "newer_correct": 337,
          "flipped": 137,
          "changed_answer": 146,
          "newer_wrong": 35,
          "repeated_mistakes": 14,
          "p": 7.0458837577401e-25
        },
        "unchanged": {
          "questions": 1814,
          "older_correct": 1621,
          "newer_correct": 1639,
          "flipped": 174,
          "changed_answer": 189,
          "newer_wrong": 175,
          "repeated_mistakes": 82,
          "p": 0.19733841594358562
        },
        "added": {
          "questions": 12,
          "newer_correct": 12
        }
      },
      "units": [
        125,
        149,
        60,
        96,
        101,
        91,
        137,
        64,
        86,
        70,
        75,
        58,
        139,
        71,
        79,
        163,
        54,
        111,
        99,
        160
      ],
      "started": "2026-10-04T08:31:15Z",
      "updated": "2026-10-04T09:02:00Z",
      "format_of": null,
      "group": "jk",
      "latency": {
        "source": "sample",
        "runs": [
          67,
          77,
          81,
          86,
          90
        ],
        "repeats": [
          1,
          2,
          3,
          4,
          5
        ],
        "questions": 200,
        "concurrency": 1,
        "median_request_seconds": 13.402853598003276,
        "p90_request_seconds": 19.79555194299901
      },
      "single_question_latency": null
    },
    {
      "run": 193,
      "model": "claude-sonnet-5-5",
      "name": "Claude Sonnet 5.5",
      "variant": null,
      "reasoning": "high",
      "provider": "claude",
      "note": null,
      "parameters": null,
      "active_parameters": null,
      "access": "Claude Code CLI, Claude subscription",
      "questions_per_request": 1,
      "concurrency": 1,
      "correct": 1960,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 618
        },
        "grammar": {
          "questions": 1542,
          "correct": 1342
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 1956
        }
      },
      "tyt_mix": 0.9295125707823226,
      "questions_per_minute": 11.214821323849451,
      "request_seconds": 11759.438353203528,
      "median_request_seconds": 4.643365134485066,
      "p90_request_seconds": 8.381210892091621,
      "billing": "subscription",
      "api_equivalent_usd": 17.0292028,
      "cost_basis": "subscription",
      "requests": 2198,
      "hardware": null,
      "input_tokens": 3285774,
      "cached_input_tokens": 1200108,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 2081270,
      "output_tokens": 857532,
      "confidence": {
        "source": "stated",
        "label": "Written estimate",
        "bins": [
          {
            "questions": 3,
            "correct": 1
          },
          {
            "questions": 125,
            "correct": 62
          },
          {
            "questions": 428,
            "correct": 328
          },
          {
            "questions": 949,
            "correct": 882
          },
          {
            "questions": 693,
            "correct": 687
          }
        ],
        "auroc": 0.8380080603670039,
        "auroc_interval": [
          0.8142523942619476,
          0.8619197274509933
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.8927354794753875,
            "auroc_interval": [
              0.8520094562647754,
              0.9293561573837507
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.8238822652757078,
            "auroc_interval": [
              0.7959304127134025,
              0.8514827619973034
            ]
          }
        },
        "aurc": 0.028767491793356383,
        "ece": 0.09837579617834623,
        "brier": 0.08956101000909918,
        "coverage_accuracy": {
          "0.20": 0.9936705859339108,
          "0.50": 0.9833329634479239,
          "0.80": 0.9475729838499557
        },
        "coverage_intervals": {
          "0.20": [
            0.9856625084519343,
            1.0
          ],
          "0.50": [
            0.9760848601735777,
            0.9904995151136606
          ],
          "0.80": [
            0.9362896044012586,
            0.9592062628997536
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0
          ],
          [
            0.04,
            0.0022
          ],
          [
            0.06,
            0.0049
          ],
          [
            0.08,
            0.0061
          ],
          [
            0.1,
            0.0061
          ],
          [
            0.12,
            0.0062
          ],
          [
            0.14,
            0.0063
          ],
          [
            0.16,
            0.0063
          ],
          [
            0.18,
            0.0063
          ],
          [
            0.2,
            0.0063
          ],
          [
            0.22,
            0.0062
          ],
          [
            0.24,
            0.0068
          ],
          [
            0.26,
            0.0074
          ],
          [
            0.28,
            0.0079
          ],
          [
            0.3,
            0.0084
          ],
          [
            0.32,
            0.0089
          ],
          [
            0.34,
            0.01
          ],
          [
            0.36,
            0.011
          ],
          [
            0.38,
            0.0119
          ],
          [
            0.4,
            0.0127
          ],
          [
            0.42,
            0.0134
          ],
          [
            0.44,
            0.0141
          ],
          [
            0.46,
            0.0149
          ],
          [
            0.48,
            0.0158
          ],
          [
            0.5,
            0.0167
          ],
          [
            0.52,
            0.0174
          ],
          [
            0.54,
            0.0181
          ],
          [
            0.56,
            0.0188
          ],
          [
            0.58,
            0.0208
          ],
          [
            0.6,
            0.0243
          ],
          [
            0.62,
            0.0276
          ],
          [
            0.64,
            0.0307
          ],
          [
            0.66,
            0.0336
          ],
          [
            0.68,
            0.0364
          ],
          [
            0.7,
            0.0389
          ],
          [
            0.72,
            0.0414
          ],
          [
            0.74,
            0.0437
          ],
          [
            0.76,
            0.0464
          ],
          [
            0.78,
            0.0494
          ],
          [
            0.8,
            0.0524
          ],
          [
            0.82,
            0.0564
          ],
          [
            0.84,
            0.0602
          ],
          [
            0.86,
            0.0638
          ],
          [
            0.88,
            0.0673
          ],
          [
            0.9,
            0.0712
          ],
          [
            0.92,
            0.0768
          ],
          [
            0.94,
            0.083
          ],
          [
            0.96,
            0.0901
          ],
          [
            0.98,
            0.0987
          ],
          [
            1.0,
            0.1083
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 9,
            "mean_confidence": 0.23333333333333334,
            "accuracy": 0.3333333333333333
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 46,
            "mean_confidence": 0.32173913043478264,
            "accuracy": 0.43478260869565216
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 73,
            "mean_confidence": 0.41438356164383516,
            "accuracy": 0.547945205479452
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 104,
            "mean_confidence": 0.5187499999999997,
            "accuracy": 0.6538461538461539
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 230,
            "mean_confidence": 0.6005217391304324,
            "accuracy": 0.7869565217391304
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 94,
            "mean_confidence": 0.7131914893617018,
            "accuracy": 0.8404255319148937
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 949,
            "mean_confidence": 0.8378292939936764,
            "accuracy": 0.9293993677555321
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 693,
            "mean_confidence": 0.9269985569985522,
            "accuracy": 0.9913419913419913
          }
        ],
        "median_right": 0.85,
        "median_wrong": 0.6,
        "sure": {
          "questions": 0,
          "correct": 0,
          "interval": null
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 462,
          "correct": 312,
          "interval": [
            0.6313318615436868,
            0.7164258311999411
          ]
        },
        "ece_refit": 0.008422014389942258,
        "brier_refit": 0.07915890619252773,
        "brier_base": 0.09665777834761399,
        "refit_slopes": [
          1.388787627390687,
          1.3638613086287816,
          1.2677084470359488,
          1.3557656320677889,
          1.3583466868403913
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        124,
        148,
        58,
        94,
        101,
        93,
        131,
        61,
        86,
        67,
        75,
        61,
        130,
        65,
        80,
        184,
        57,
        102,
        89,
        154
      ],
      "started": "2026-10-13T16:20:09Z",
      "updated": "2026-10-13T19:36:13Z",
      "format_of": 55,
      "group": "k",
      "latency": {
        "source": "run",
        "runs": [
          193
        ],
        "concurrency": 1,
        "median_request_seconds": 4.643365134485066,
        "p90_request_seconds": 8.381210892091621
      },
      "single_question_latency": null
    },
    {
      "run": 55,
      "model": "claude-sonnet-5-5",
      "name": "Claude Sonnet 5.5",
      "variant": null,
      "reasoning": "high",
      "provider": "claude",
      "note": null,
      "parameters": null,
      "active_parameters": null,
      "access": "Claude Code CLI, Claude subscription",
      "questions_per_request": 16,
      "concurrency": 1,
      "correct": 1869,
      "invalid": 0,
      "answered_questions": 2210,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 559
        },
        "grammar": {
          "questions": 1542,
          "correct": 1310
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 1868
        }
      },
      "tyt_mix": 0.8516812284489577,
      "questions_per_minute": 37.778240432131966,
      "request_seconds": 3509.9570144939353,
      "median_request_seconds": null,
      "p90_request_seconds": null,
      "billing": "subscription",
      "api_equivalent_usd": 8.0760773,
      "cost_basis": "subscription",
      "requests": 139,
      "hardware": null,
      "input_tokens": 1215038,
      "cached_input_tokens": 301633,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 913127,
      "output_tokens": 439285,
      "confidence": {
        "source": "stated",
        "label": "Written estimate",
        "bins": [
          {
            "questions": 17,
            "correct": 5
          },
          {
            "questions": 279,
            "correct": 150
          },
          {
            "questions": 785,
            "correct": 655
          },
          {
            "questions": 862,
            "correct": 814
          },
          {
            "questions": 255,
            "correct": 245
          }
        ],
        "auroc": 0.781242021073311,
        "auroc_interval": [
          0.7520556418760119,
          0.8097837635952683
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.8130866975268798,
            "auroc_interval": [
              0.7643322830588748,
              0.8555715118215118
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.7709150434324822,
            "auroc_interval": [
              0.7369377140019342,
              0.804098426860182
            ]
          }
        },
        "aurc": 0.06268237187990797,
        "ece": 0.15961328480436646,
        "brier": 0.13529667879890808,
        "coverage_accuracy": {
          "0.20": 0.9599196516177774,
          "0.50": 0.9484268434189548,
          "0.80": 0.915435084526603
        },
        "coverage_intervals": {
          "0.20": [
            0.9421571125265393,
            0.9750326394695535
          ],
          "0.50": [
            0.9340591275269798,
            0.960281271664851
          ],
          "0.80": [
            0.901752995939388,
            0.9294257868039361
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0104
          ],
          [
            0.04,
            0.0392
          ],
          [
            0.06,
            0.0404
          ],
          [
            0.08,
            0.0398
          ],
          [
            0.1,
            0.0394
          ],
          [
            0.12,
            0.0391
          ],
          [
            0.14,
            0.0384
          ],
          [
            0.16,
            0.039
          ],
          [
            0.18,
            0.0396
          ],
          [
            0.2,
            0.0401
          ],
          [
            0.22,
            0.0405
          ],
          [
            0.24,
            0.0408
          ],
          [
            0.26,
            0.0411
          ],
          [
            0.28,
            0.0413
          ],
          [
            0.3,
            0.0415
          ],
          [
            0.32,
            0.0432
          ],
          [
            0.34,
            0.0445
          ],
          [
            0.36,
            0.0456
          ],
          [
            0.38,
            0.0466
          ],
          [
            0.4,
            0.0474
          ],
          [
            0.42,
            0.0483
          ],
          [
            0.44,
            0.049
          ],
          [
            0.46,
            0.0497
          ],
          [
            0.48,
            0.0507
          ],
          [
            0.5,
            0.0516
          ],
          [
            0.52,
            0.0536
          ],
          [
            0.54,
            0.0562
          ],
          [
            0.56,
            0.0586
          ],
          [
            0.58,
            0.0609
          ],
          [
            0.6,
            0.0622
          ],
          [
            0.62,
            0.0625
          ],
          [
            0.64,
            0.0647
          ],
          [
            0.66,
            0.0668
          ],
          [
            0.68,
            0.0687
          ],
          [
            0.7,
            0.0706
          ],
          [
            0.72,
            0.0723
          ],
          [
            0.74,
            0.0739
          ],
          [
            0.76,
            0.0764
          ],
          [
            0.78,
            0.0797
          ],
          [
            0.8,
            0.0846
          ],
          [
            0.82,
            0.0892
          ],
          [
            0.84,
            0.0936
          ],
          [
            0.86,
            0.0978
          ],
          [
            0.88,
            0.1015
          ],
          [
            0.9,
            0.108
          ],
          [
            0.92,
            0.1158
          ],
          [
            0.94,
            0.1232
          ],
          [
            0.96,
            0.1311
          ],
          [
            0.98,
            0.1397
          ],
          [
            1.0,
            0.1497
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 6,
            "mean_confidence": 0.145,
            "accuracy": 0.16666666666666666
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 24,
            "mean_confidence": 0.22708333333333341,
            "accuracy": 0.375
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 101,
            "mean_confidence": 0.3267326732673273,
            "accuracy": 0.46534653465346537
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 165,
            "mean_confidence": 0.414242424242424,
            "accuracy": 0.593939393939394
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 255,
            "mean_confidence": 0.5127450980392152,
            "accuracy": 0.7450980392156863
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 337,
            "mean_confidence": 0.6043026706231422,
            "accuracy": 0.8783382789317508
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 302,
            "mean_confidence": 0.7180463576158918,
            "accuracy": 0.8940397350993378
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 753,
            "mean_confidence": 0.8316201859229801,
            "accuracy": 0.9468791500664011
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 255,
            "mean_confidence": 0.9138823529411781,
            "accuracy": 0.9607843137254902
          }
        ],
        "median_right": 0.8,
        "median_wrong": 0.5,
        "sure": {
          "questions": 0,
          "correct": 0,
          "interval": null
        },
        "sure_before_audit": {
          "questions": 0,
          "correct": 0,
          "interval": null
        },
        "doubtful": {
          "questions": 888,
          "correct": 641,
          "interval": [
            0.6914669605526395,
            0.7503155263822602
          ]
        },
        "ece_refit": 0.019904266294240233,
        "brier_refit": 0.10628128315414606,
        "brier_base": 0.1273579137048501,
        "refit_slopes": [
          1.189079764947056,
          1.1949074496794316,
          1.2141432594459831,
          1.2559753431687235,
          1.39580395763525
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": {
        "run": 39,
        "changed": {
          "questions": 372,
          "older_correct": 213,
          "newer_correct": 323,
          "flipped": 130,
          "changed_answer": 148,
          "newer_wrong": 49,
          "repeated_mistakes": 21,
          "p": 4.263429420007923e-25
        },
        "unchanged": {
          "questions": 1814,
          "older_correct": 1526,
          "newer_correct": 1534,
          "flipped": 194,
          "changed_answer": 234,
          "newer_wrong": 280,
          "repeated_mistakes": 147,
          "p": 0.615380740884383
        },
        "added": {
          "questions": 12,
          "newer_correct": 12
        }
      },
      "units": [
        105,
        138,
        48,
        91,
        87,
        90,
        131,
        59,
        79,
        68,
        75,
        58,
        130,
        68,
        77,
        170,
        61,
        101,
        94,
        139
      ],
      "started": "2026-10-04T09:11:47Z",
      "updated": "2026-10-04T10:10:17Z",
      "format_of": null,
      "group": "l",
      "latency": {
        "source": "sample",
        "runs": [
          63,
          80,
          88,
          93,
          98
        ],
        "repeats": [
          1,
          2,
          3,
          4,
          5
        ],
        "questions": 200,
        "concurrency": 1,
        "median_request_seconds": 20.488734460028354,
        "p90_request_seconds": 37.048832100233994
      },
      "single_question_latency": null
    },
    {
      "run": 320,
      "model": "moonshotai/kimi-k3.1-pro",
      "name": "Kimi K3.1 Pro",
      "variant": null,
      "reasoning": "low",
      "provider": "openrouter-chat",
      "note": null,
      "parameters": null,
      "active_parameters": null,
      "access": "OpenRouter chat API, Moonshot AI's endpoint (mxfp4), fallbacks off",
      "questions_per_request": 16,
      "concurrency": 4,
      "correct": 1819,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 615
        },
        "grammar": {
          "questions": 1542,
          "correct": 1204
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 1817
        }
      },
      "tyt_mix": 0.9100782263294424,
      "questions_per_minute": 60.023205399428846,
      "request_seconds": 2197.1502375188866,
      "median_request_seconds": 44.8976706439862,
      "p90_request_seconds": 144.5165325260954,
      "billing": "api",
      "api_equivalent_usd": 4.5005352,
      "cost_basis": "api",
      "requests": 138,
      "hardware": null,
      "input_tokens": 788026,
      "cached_input_tokens": 11776,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 294568,
      "confidence": {
        "source": "stated",
        "label": "Written estimate",
        "bins": [
          {
            "questions": 13,
            "correct": 3
          },
          {
            "questions": 47,
            "correct": 11
          },
          {
            "questions": 58,
            "correct": 29
          },
          {
            "questions": 59,
            "correct": 26
          },
          {
            "questions": 2021,
            "correct": 1750
          }
        ],
        "auroc": 0.7502643599298522,
        "auroc_interval": [
          0.7219610995341642,
          0.7780382206790882
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.816597263533611,
            "auroc_interval": [
              0.7463436775039505,
              0.8862903225806451
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.7342893019324146,
            "auroc_interval": [
              0.7022192996078735,
              0.7629254618462118
            ]
          }
        },
        "aurc": 0.08729043705762139,
        "ece": 0.1256551410373169,
        "brier": 0.1406611010009099,
        "coverage_accuracy": {
          "0.20": 0.927274589093628,
          "0.50": 0.921514065244702,
          "0.80": 0.8936705852723255
        },
        "coverage_intervals": {
          "0.20": [
            0.9122535872311198,
            0.942303512201611
          ],
          "0.50": [
            0.9083494132522661,
            0.9353157962062093
          ],
          "0.80": [
            0.877701986399609,
            0.9079545454545455
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0446
          ],
          [
            0.04,
            0.0446
          ],
          [
            0.06,
            0.0503
          ],
          [
            0.08,
            0.0583
          ],
          [
            0.1,
            0.0631
          ],
          [
            0.12,
            0.0663
          ],
          [
            0.14,
            0.0686
          ],
          [
            0.16,
            0.0703
          ],
          [
            0.18,
            0.0717
          ],
          [
            0.2,
            0.0727
          ],
          [
            0.22,
            0.0736
          ],
          [
            0.24,
            0.0743
          ],
          [
            0.26,
            0.0749
          ],
          [
            0.28,
            0.0755
          ],
          [
            0.3,
            0.0759
          ],
          [
            0.32,
            0.0763
          ],
          [
            0.34,
            0.0767
          ],
          [
            0.36,
            0.077
          ],
          [
            0.38,
            0.0773
          ],
          [
            0.4,
            0.0775
          ],
          [
            0.42,
            0.0778
          ],
          [
            0.44,
            0.078
          ],
          [
            0.46,
            0.0782
          ],
          [
            0.48,
            0.0783
          ],
          [
            0.5,
            0.0785
          ],
          [
            0.52,
            0.0786
          ],
          [
            0.54,
            0.0788
          ],
          [
            0.56,
            0.0789
          ],
          [
            0.58,
            0.079
          ],
          [
            0.6,
            0.0791
          ],
          [
            0.62,
            0.0792
          ],
          [
            0.64,
            0.0793
          ],
          [
            0.66,
            0.0808
          ],
          [
            0.68,
            0.085
          ],
          [
            0.7,
            0.0889
          ],
          [
            0.72,
            0.0926
          ],
          [
            0.74,
            0.0961
          ],
          [
            0.76,
            0.0994
          ],
          [
            0.78,
            0.1025
          ],
          [
            0.8,
            0.1063
          ],
          [
            0.82,
            0.1122
          ],
          [
            0.84,
            0.1178
          ],
          [
            0.86,
            0.1206
          ],
          [
            0.88,
            0.1223
          ],
          [
            0.9,
            0.129
          ],
          [
            0.92,
            0.1343
          ],
          [
            0.94,
            0.1443
          ],
          [
            0.96,
            0.1503
          ],
          [
            0.98,
            0.1594
          ],
          [
            1.0,
            0.1724
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 3,
            "mean_confidence": 0.13333333333333333,
            "accuracy": 0.3333333333333333
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 14,
            "mean_confidence": 0.21428571428571433,
            "accuracy": 0.21428571428571427
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 20,
            "mean_confidence": 0.33949999999999986,
            "accuracy": 0.2
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 23,
            "mean_confidence": 0.44130434782608696,
            "accuracy": 0.2608695652173913
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 19,
            "mean_confidence": 0.5431578947368422,
            "accuracy": 0.47368421052631576
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 28,
            "mean_confidence": 0.6339285714285714,
            "accuracy": 0.4642857142857143
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 27,
            "mean_confidence": 0.7399999999999999,
            "accuracy": 0.5925925925925926
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 43,
            "mean_confidence": 0.8402325581395351,
            "accuracy": 0.3953488372093023
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 2021,
            "mean_confidence": 0.9843988124690859,
            "accuracy": 0.8659079663532905
          }
        ],
        "median_right": 0.99,
        "median_wrong": 0.97,
        "sure": {
          "questions": 1436,
          "correct": 1322,
          "interval": [
            0.9054814839745762,
            0.9334996928437627
          ]
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 107,
          "correct": 36,
          "interval": [
            0.2539713264752684,
            0.4302627517047651
          ]
        },
        "ece_refit": 0.026297100558171353,
        "brier_refit": 0.12106359461004967,
        "brier_base": 0.14281439299431933,
        "refit_slopes": [
          0.5210207027968878,
          0.7011732560128098,
          0.5764249713357896,
          0.5772012273008552,
          0.559057871723632
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        120,
        152,
        57,
        96,
        96,
        94,
        116,
        55,
        79,
        67,
        64,
        51,
        116,
        62,
        69,
        148,
        40,
        103,
        84,
        150
      ],
      "started": "2026-10-29T10:35:16Z",
      "updated": "2026-10-29T11:11:54Z",
      "format_of": null,
      "group": "l",
      "latency": {
        "source": "run",
        "runs": [
          320
        ],
        "concurrency": 4,
        "median_request_seconds": 44.8976706439862,
        "p90_request_seconds": 144.5165325260954
      },
      "single_question_latency": null
    },
    {
      "run": 160,
      "model": "glm-5.3-flash",
      "name": "GLM-5.3-Flash",
      "variant": null,
      "reasoning": "on",
      "provider": "vllm-stated",
      "note": null,
      "parameters": 320000000000,
      "active_parameters": [
        18000000000
      ],
      "access": "TensorFold, self-hosted, EXL3 4-bit experts",
      "questions_per_request": 16,
      "concurrency": 8,
      "correct": 1732,
      "invalid": 128,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 586
        },
        "grammar": {
          "questions": 1542,
          "correct": 1146
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 1729
        }
      },
      "tyt_mix": 0.8670248291733891,
      "questions_per_minute": 34.1746540558756,
      "request_seconds": 3859.0002925670015,
      "median_request_seconds": 102.61677862299985,
      "p90_request_seconds": 639.2771307083002,
      "billing": "local",
      "api_equivalent_usd": null,
      "cost_basis": null,
      "requests": 138,
      "hardware": "4 × RTX PRO 6000 Blackwell (96 GB); EXL3 4-bit experts, FP8 dense layers and KV cache",
      "input_tokens": 557949,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 2218925,
      "confidence": {
        "source": "stated",
        "label": "Written estimate",
        "bins": [
          {
            "questions": 0,
            "correct": 0
          },
          {
            "questions": 81,
            "correct": 33
          },
          {
            "questions": 514,
            "correct": 358
          },
          {
            "questions": 696,
            "correct": 598
          },
          {
            "questions": 779,
            "correct": 743
          }
        ],
        "auroc": 0.7604575208057176,
        "auroc_interval": [
          0.7340235045792788,
          0.7876946251532335
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.8415894685519258,
            "auroc_interval": [
              0.796294671259022,
              0.8840134079391891
            ]
          },
          "form": {
            "questions": 1414,
            "auroc": 0.7231252116381444,
            "auroc_interval": [
              0.690356575420969,
              0.7567973576946293
            ]
          }
        },
        "aurc": 0.0682724112587261,
        "ece": 0.04548309178743938,
        "brier": 0.12034507246376812,
        "coverage_accuracy": {
          "0.20": 0.9743235631810402,
          "0.50": 0.9327893893670132,
          "0.80": 0.8938844971453667
        },
        "coverage_intervals": {
          "0.20": [
            0.9612247962247963,
            0.9853064104651794
          ],
          "0.50": [
            0.9183787115199368,
            0.9464918149255498
          ],
          "0.80": [
            0.8800144997583375,
            0.9096147646679562
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0038
          ],
          [
            0.04,
            0.0102
          ],
          [
            0.06,
            0.0124
          ],
          [
            0.08,
            0.0134
          ],
          [
            0.1,
            0.0141
          ],
          [
            0.12,
            0.0145
          ],
          [
            0.14,
            0.0148
          ],
          [
            0.16,
            0.0151
          ],
          [
            0.18,
            0.0208
          ],
          [
            0.2,
            0.0257
          ],
          [
            0.22,
            0.0297
          ],
          [
            0.24,
            0.033
          ],
          [
            0.26,
            0.0358
          ],
          [
            0.28,
            0.0382
          ],
          [
            0.3,
            0.0403
          ],
          [
            0.32,
            0.0421
          ],
          [
            0.34,
            0.0437
          ],
          [
            0.36,
            0.0452
          ],
          [
            0.38,
            0.047
          ],
          [
            0.4,
            0.0512
          ],
          [
            0.42,
            0.055
          ],
          [
            0.44,
            0.0585
          ],
          [
            0.46,
            0.0617
          ],
          [
            0.48,
            0.0645
          ],
          [
            0.5,
            0.0672
          ],
          [
            0.52,
            0.0697
          ],
          [
            0.54,
            0.0719
          ],
          [
            0.56,
            0.0741
          ],
          [
            0.58,
            0.0771
          ],
          [
            0.6,
            0.0803
          ],
          [
            0.62,
            0.0832
          ],
          [
            0.64,
            0.086
          ],
          [
            0.66,
            0.0886
          ],
          [
            0.68,
            0.0895
          ],
          [
            0.7,
            0.0903
          ],
          [
            0.72,
            0.0923
          ],
          [
            0.74,
            0.0959
          ],
          [
            0.76,
            0.0994
          ],
          [
            0.78,
            0.1027
          ],
          [
            0.8,
            0.1061
          ],
          [
            0.82,
            0.1102
          ],
          [
            0.84,
            0.1146
          ],
          [
            0.86,
            0.1189
          ],
          [
            0.88,
            0.123
          ],
          [
            0.9,
            0.1282
          ],
          [
            0.92,
            0.1333
          ],
          [
            0.94,
            0.1391
          ],
          [
            0.96,
            0.1455
          ],
          [
            0.98,
            0.1532
          ],
          [
            1.0,
            0.1633
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 18,
            "mean_confidence": 0.33333333333333326,
            "accuracy": 0.3888888888888889
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 63,
            "mean_confidence": 0.4238095238095234,
            "accuracy": 0.4126984126984127
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 161,
            "mean_confidence": 0.5298136645962724,
            "accuracy": 0.6024844720496895
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 182,
            "mean_confidence": 0.6151098901098895,
            "accuracy": 0.7087912087912088
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 280,
            "mean_confidence": 0.7194642857142838,
            "accuracy": 0.8142857142857143
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 587,
            "mean_confidence": 0.8331345826235184,
            "accuracy": 0.8551959114139693
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 779,
            "mean_confidence": 0.922721437740688,
            "accuracy": 0.9537869062901155
          }
        ],
        "median_right": 0.85,
        "median_wrong": 0.7,
        "sure": {
          "questions": 9,
          "correct": 9,
          "interval": [
            0.7008472464490407,
            1.0
          ]
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 424,
          "correct": 259,
          "interval": [
            0.5636431702559835,
            0.6560643102695193
          ]
        },
        "ece_refit": 0.011736648161086367,
        "brier_refit": 0.1183204394139952,
        "brier_base": 0.13674129133601315,
        "refit_slopes": [
          1.0897543572757171,
          1.1003259483754384,
          1.1619146758269352,
          1.0754919436478023,
          1.040811182810583
        ],
        "refit_questions": 2070,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        116,
        142,
        55,
        94,
        87,
        92,
        117,
        55,
        79,
        65,
        70,
        53,
        122,
        66,
        72,
        138,
        0,
        86,
        82,
        141
      ],
      "started": "2026-10-11T10:53:32Z",
      "updated": "2026-10-11T11:57:51Z",
      "format_of": null,
      "group": "m",
      "latency": {
        "source": "run",
        "runs": [
          160
        ],
        "concurrency": 8,
        "median_request_seconds": 102.61677862299985,
        "p90_request_seconds": 639.2771307083002
      },
      "single_question_latency": null
    },
    {
      "run": 199,
      "model": "d1-ultra",
      "name": "Liquid d1-ultra",
      "variant": null,
      "reasoning": null,
      "provider": "liquid",
      "note": null,
      "parameters": null,
      "active_parameters": null,
      "access": "Liquid AI decisions API",
      "questions_per_request": 1,
      "concurrency": 1,
      "correct": 1694,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 604
        },
        "grammar": {
          "questions": 1542,
          "correct": 1090
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 1691
        }
      },
      "tyt_mix": 0.8833066416753661,
      "questions_per_minute": 241.99067818307225,
      "request_seconds": 544.9796702508902,
      "median_request_seconds": 0.22344711299228948,
      "p90_request_seconds": 0.350752345994988,
      "billing": "api",
      "api_equivalent_usd": 0.031515876,
      "cost_basis": "api",
      "requests": 2198,
      "hardware": null,
      "input_tokens": 750378,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 0,
      "confidence": {
        "source": "probability",
        "label": "Option score",
        "bins": [
          {
            "questions": 7,
            "correct": 1
          },
          {
            "questions": 479,
            "correct": 205
          },
          {
            "questions": 491,
            "correct": 337
          },
          {
            "questions": 419,
            "correct": 357
          },
          {
            "questions": 802,
            "correct": 794
          }
        ],
        "auroc": 0.8496291767395664,
        "auroc_interval": [
          0.8329334436477294,
          0.8660828542284866
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.9064887926642894,
            "auroc_interval": [
              0.8759800427655025,
              0.9328299776286354
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.8079016806040432,
            "auroc_interval": [
              0.7860320214215546,
              0.8293124836772003
            ]
          }
        },
        "aurc": 0.06666732724801985,
        "ece": 0.03629871483620005,
        "brier": 0.1291910911352421,
        "coverage_accuracy": {
          "0.20": 1.0,
          "0.50": 0.9572338489535942,
          "0.80": 0.8589626933575978
        },
        "coverage_intervals": {
          "0.20": [
            1.0,
            1.0
          ],
          "0.50": [
            0.9444949954504095,
            0.9699590350477925
          ],
          "0.80": [
            0.8420454545454545,
            0.8762477313974592
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0
          ],
          [
            0.04,
            0.0
          ],
          [
            0.06,
            0.0
          ],
          [
            0.08,
            0.0
          ],
          [
            0.1,
            0.0
          ],
          [
            0.12,
            0.0
          ],
          [
            0.14,
            0.0
          ],
          [
            0.16,
            0.0
          ],
          [
            0.18,
            0.0
          ],
          [
            0.2,
            0.0
          ],
          [
            0.22,
            0.0
          ],
          [
            0.24,
            0.0
          ],
          [
            0.26,
            0.0
          ],
          [
            0.28,
            0.0
          ],
          [
            0.3,
            0.0
          ],
          [
            0.32,
            0.0014
          ],
          [
            0.34,
            0.004
          ],
          [
            0.36,
            0.0088
          ],
          [
            0.38,
            0.012
          ],
          [
            0.4,
            0.0171
          ],
          [
            0.42,
            0.0227
          ],
          [
            0.44,
            0.0279
          ],
          [
            0.46,
            0.0298
          ],
          [
            0.48,
            0.0379
          ],
          [
            0.5,
            0.0428
          ],
          [
            0.52,
            0.0463
          ],
          [
            0.54,
            0.0522
          ],
          [
            0.56,
            0.0569
          ],
          [
            0.58,
            0.0628
          ],
          [
            0.6,
            0.0713
          ],
          [
            0.62,
            0.0748
          ],
          [
            0.64,
            0.0808
          ],
          [
            0.66,
            0.0862
          ],
          [
            0.68,
            0.0943
          ],
          [
            0.7,
            0.1011
          ],
          [
            0.72,
            0.1059
          ],
          [
            0.74,
            0.1147
          ],
          [
            0.76,
            0.1236
          ],
          [
            0.78,
            0.1315
          ],
          [
            0.8,
            0.141
          ],
          [
            0.82,
            0.1489
          ],
          [
            0.84,
            0.1572
          ],
          [
            0.86,
            0.1668
          ],
          [
            0.88,
            0.1784
          ],
          [
            0.9,
            0.1871
          ],
          [
            0.92,
            0.1944
          ],
          [
            0.94,
            0.2038
          ],
          [
            0.96,
            0.2119
          ],
          [
            0.98,
            0.2205
          ],
          [
            1.0,
            0.2293
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 56,
            "mean_confidence": 0.276077435294843,
            "accuracy": 0.39285714285714285
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 201,
            "mean_confidence": 0.3500646339544253,
            "accuracy": 0.40298507462686567
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 229,
            "mean_confidence": 0.44793650294249115,
            "accuracy": 0.4497816593886463
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 200,
            "mean_confidence": 0.5486927126394172,
            "accuracy": 0.615
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 177,
            "mean_confidence": 0.6503535792510212,
            "accuracy": 0.7288135593220338
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 241,
            "mean_confidence": 0.7516161170991935,
            "accuracy": 0.7800829875518672
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 292,
            "mean_confidence": 0.8530497756742561,
            "accuracy": 0.8698630136986302
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 802,
            "mean_confidence": 0.9610143304759956,
            "accuracy": 0.9900249376558603
          }
        ],
        "median_right": 0.8843267784091986,
        "median_wrong": 0.4753555300193727,
        "sure": {
          "questions": 90,
          "correct": 90,
          "interval": [
            0.9590629315783191,
            0.9999999999999999
          ]
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 863,
          "correct": 458,
          "interval": [
            0.4973477180015374,
            0.563793786974689
          ]
        },
        "ece_refit": 0.01387378156266704,
        "brier_refit": 0.12852338496102722,
        "brier_base": 0.17705161269398065,
        "refit_slopes": [
          1.136248257005341,
          1.1546622120804522,
          1.2214547936923263,
          1.1542284227949682,
          1.180004784637523
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        116,
        148,
        60,
        94,
        96,
        90,
        110,
        50,
        75,
        55,
        54,
        32,
        105,
        57,
        64,
        136,
        30,
        91,
        88,
        143
      ],
      "started": "2026-10-16T13:30:45Z",
      "updated": "2026-10-16T13:39:57Z",
      "format_of": null,
      "group": "mn",
      "latency": {
        "source": "run",
        "runs": [
          199
        ],
        "concurrency": 1,
        "median_request_seconds": 0.22344711299228948,
        "p90_request_seconds": 0.350752345994988
      },
      "single_question_latency": null
    },
    {
      "run": 314,
      "model": "z-ai/glm-5.3",
      "name": "GLM-5.3",
      "variant": null,
      "reasoning": "low",
      "provider": "openrouter-chat",
      "note": "The Fireworks pin admits all of Fireworks' endpoints: OpenRouter billed 135 of the 138 requests at the spot endpoint's rates, 2 at the standard rates and 1 at the fast endpoint's rates ($1.82 in all); the cost shown uses the spot rates.",
      "parameters": null,
      "active_parameters": null,
      "access": "OpenRouter chat API, Fireworks endpoint, fallbacks off",
      "questions_per_request": 16,
      "concurrency": 4,
      "correct": 1638,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 575
        },
        "grammar": {
          "questions": 1542,
          "correct": 1063
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 1636
        }
      },
      "tyt_mix": 0.84377140275537,
      "questions_per_minute": 143.52350440961675,
      "request_seconds": 918.8738844030304,
      "median_request_seconds": 17.073907982499804,
      "p90_request_seconds": 54.499747568386375,
      "billing": "api",
      "api_equivalent_usd": 1.7894617860000002,
      "cost_basis": "api",
      "requests": 138,
      "hardware": null,
      "input_tokens": 557949,
      "cached_input_tokens": 25563,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 410088,
      "confidence": {
        "source": "stated",
        "label": "Written estimate",
        "bins": [
          {
            "questions": 1,
            "correct": 0
          },
          {
            "questions": 120,
            "correct": 32
          },
          {
            "questions": 666,
            "correct": 411
          },
          {
            "questions": 930,
            "correct": 748
          },
          {
            "questions": 481,
            "correct": 447
          }
        ],
        "auroc": 0.7394808564451422,
        "auroc_interval": [
          0.7150982243016225,
          0.7646914005760851
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.8166398282340311,
            "auroc_interval": [
              0.7625340108834827,
              0.8653415783274441
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.7011255418056982,
            "auroc_interval": [
              0.6712726160786618,
              0.7290480729946888
            ]
          }
        },
        "aurc": 0.13178450855210244,
        "ece": 0.025787079162879617,
        "brier": 0.16031992720655142,
        "coverage_accuracy": {
          "0.20": 0.9304601585857273,
          "0.50": 0.8652042803750969,
          "0.80": 0.820509431178221
        },
        "coverage_intervals": {
          "0.20": [
            0.9071163222217556,
            0.9532287618314017
          ],
          "0.50": [
            0.8457762013927937,
            0.8834071590602115
          ],
          "0.80": [
            0.8008626137873532,
            0.8376105373817326
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0382
          ],
          [
            0.04,
            0.0386
          ],
          [
            0.06,
            0.0385
          ],
          [
            0.08,
            0.0496
          ],
          [
            0.1,
            0.0562
          ],
          [
            0.12,
            0.0607
          ],
          [
            0.14,
            0.0638
          ],
          [
            0.16,
            0.0662
          ],
          [
            0.18,
            0.0681
          ],
          [
            0.2,
            0.0695
          ],
          [
            0.22,
            0.0709
          ],
          [
            0.24,
            0.08
          ],
          [
            0.26,
            0.0877
          ],
          [
            0.28,
            0.0943
          ],
          [
            0.3,
            0.1
          ],
          [
            0.32,
            0.105
          ],
          [
            0.34,
            0.1094
          ],
          [
            0.36,
            0.1134
          ],
          [
            0.38,
            0.1169
          ],
          [
            0.4,
            0.12
          ],
          [
            0.42,
            0.1232
          ],
          [
            0.44,
            0.1265
          ],
          [
            0.46,
            0.1295
          ],
          [
            0.48,
            0.1323
          ],
          [
            0.5,
            0.1348
          ],
          [
            0.52,
            0.1371
          ],
          [
            0.54,
            0.1393
          ],
          [
            0.56,
            0.1413
          ],
          [
            0.58,
            0.1435
          ],
          [
            0.6,
            0.1468
          ],
          [
            0.62,
            0.1499
          ],
          [
            0.64,
            0.1528
          ],
          [
            0.66,
            0.1561
          ],
          [
            0.68,
            0.1593
          ],
          [
            0.7,
            0.1624
          ],
          [
            0.72,
            0.1652
          ],
          [
            0.74,
            0.1679
          ],
          [
            0.76,
            0.1705
          ],
          [
            0.78,
            0.1735
          ],
          [
            0.8,
            0.1795
          ],
          [
            0.82,
            0.1852
          ],
          [
            0.84,
            0.1907
          ],
          [
            0.86,
            0.1959
          ],
          [
            0.88,
            0.202
          ],
          [
            0.9,
            0.2106
          ],
          [
            0.92,
            0.218
          ],
          [
            0.94,
            0.2251
          ],
          [
            0.96,
            0.2327
          ],
          [
            0.98,
            0.2436
          ],
          [
            1.0,
            0.2548
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 2,
            "mean_confidence": 0.225,
            "accuracy": 0.0
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 39,
            "mean_confidence": 0.3294871794871794,
            "accuracy": 0.20512820512820512
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 80,
            "mean_confidence": 0.4181249999999994,
            "accuracy": 0.3
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 156,
            "mean_confidence": 0.5179487179487176,
            "accuracy": 0.4358974358974359
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 242,
            "mean_confidence": 0.6041322314049564,
            "accuracy": 0.6033057851239669
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 412,
            "mean_confidence": 0.7174757281553362,
            "accuracy": 0.7427184466019418
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 786,
            "mean_confidence": 0.8269465648855023,
            "accuracy": 0.8129770992366412
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 481,
            "mean_confidence": 0.9136174636174559,
            "accuracy": 0.9293139293139293
          }
        ],
        "median_right": 0.8,
        "median_wrong": 0.7,
        "sure": {
          "questions": 1,
          "correct": 1,
          "interval": [
            0.20654329147389294,
            1.0
          ]
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 519,
          "correct": 246,
          "interval": [
            0.43137827087413855,
            0.5169808497467148
          ]
        },
        "ece_refit": 0.02609673046326124,
        "brier_refit": 0.1601983406385367,
        "brier_base": 0.1900157187221895,
        "refit_slopes": [
          1.0966059784547428,
          1.247734517791726,
          1.200588591140709,
          1.1445448501878384,
          1.1033529873299028
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        111,
        144,
        52,
        94,
        86,
        88,
        109,
        42,
        69,
        65,
        62,
        38,
        106,
        52,
        64,
        138,
        26,
        89,
        73,
        130
      ],
      "started": "2026-10-28T14:52:06Z",
      "updated": "2026-10-28T15:07:25Z",
      "format_of": null,
      "group": "no",
      "latency": {
        "source": "run",
        "runs": [
          314
        ],
        "concurrency": 4,
        "median_request_seconds": 17.073907982499804,
        "p90_request_seconds": 54.499747568386375
      },
      "single_question_latency": null
    },
    {
      "run": 318,
      "model": "qwen38-27b-bf16",
      "name": "Qwen3.8-27B",
      "variant": null,
      "reasoning": "low",
      "provider": "vllm",
      "note": "Open-Jev's base model Reasoning on at the chat template's low effort. Five of the 138 replies never left the thinking part, so their 80 questions have no answer and count as wrong. The GPU was shared with another program during the run, so its request times are longer than on a free GPU.",
      "parameters": 27781427952,
      "active_parameters": null,
      "access": "vLLM, self-hosted",
      "questions_per_request": 16,
      "concurrency": 4,
      "correct": 1625,
      "invalid": 80,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 586
        },
        "grammar": {
          "questions": 1542,
          "correct": 1039
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 1623
        }
      },
      "tyt_mix": 0.8548815088102243,
      "questions_per_minute": 5.486924736625842,
      "request_seconds": 24035.321483395994,
      "median_request_seconds": 648.3749564884638,
      "p90_request_seconds": 1337.1598235789686,
      "billing": "local",
      "api_equivalent_usd": 6.4412905,
      "cost_basis": "maker",
      "requests": 138,
      "hardware": "1 × RTX PRO 6000 Blackwell (96 GB); BF16",
      "input_tokens": 524063,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 2059753,
      "confidence": {
        "source": "probability",
        "label": "Answer letter probability",
        "bins": [
          {
            "questions": 1,
            "correct": 1
          },
          {
            "questions": 0,
            "correct": 0
          },
          {
            "questions": 0,
            "correct": 0
          },
          {
            "questions": 0,
            "correct": 0
          },
          {
            "questions": 2117,
            "correct": 1624
          }
        ],
        "auroc": 0.6345358090185677,
        "auroc_interval": [
          0.6063124461826351,
          0.6609779686702764
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.7565090199902487,
            "auroc_interval": [
              0.6967818333797715,
              0.8091418820087762
            ]
          },
          "form": {
            "questions": 1462,
            "auroc": 0.6027083233787717,
            "auroc_interval": [
              0.5703697534976683,
              0.63182114369172
            ]
          }
        },
        "aurc": 0.16039801989462252,
        "ece": 0.23300113126490113,
        "brier": 0.2329188557813457,
        "coverage_accuracy": {
          "0.20": 0.8805476864966949,
          "0.50": 0.8385269121813032,
          "0.80": 0.796152030217186
        },
        "coverage_intervals": {
          "0.20": [
            0.8441190363722249,
            0.9102079395085066
          ],
          "0.50": [
            0.8166351606805293,
            0.8611764705882353
          ],
          "0.80": [
            0.7765330188679245,
            0.8150377714825306
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.1029
          ],
          [
            0.04,
            0.0911
          ],
          [
            0.06,
            0.0944
          ],
          [
            0.08,
            0.1003
          ],
          [
            0.1,
            0.0944
          ],
          [
            0.12,
            0.0984
          ],
          [
            0.14,
            0.1063
          ],
          [
            0.16,
            0.1151
          ],
          [
            0.18,
            0.1233
          ],
          [
            0.2,
            0.1195
          ],
          [
            0.22,
            0.1288
          ],
          [
            0.24,
            0.1279
          ],
          [
            0.26,
            0.138
          ],
          [
            0.28,
            0.1383
          ],
          [
            0.3,
            0.136
          ],
          [
            0.32,
            0.1461
          ],
          [
            0.34,
            0.15
          ],
          [
            0.36,
            0.1475
          ],
          [
            0.38,
            0.1479
          ],
          [
            0.4,
            0.1487
          ],
          [
            0.42,
            0.1484
          ],
          [
            0.44,
            0.1567
          ],
          [
            0.46,
            0.1581
          ],
          [
            0.48,
            0.1584
          ],
          [
            0.5,
            0.1615
          ],
          [
            0.52,
            0.1647
          ],
          [
            0.54,
            0.1653
          ],
          [
            0.56,
            0.1661
          ],
          [
            0.58,
            0.1672
          ],
          [
            0.6,
            0.1715
          ],
          [
            0.62,
            0.1737
          ],
          [
            0.64,
            0.1763
          ],
          [
            0.66,
            0.1788
          ],
          [
            0.68,
            0.1819
          ],
          [
            0.7,
            0.1879
          ],
          [
            0.72,
            0.1889
          ],
          [
            0.74,
            0.194
          ],
          [
            0.76,
            0.1932
          ],
          [
            0.78,
            0.1986
          ],
          [
            0.8,
            0.2038
          ],
          [
            0.82,
            0.2071
          ],
          [
            0.84,
            0.2113
          ],
          [
            0.86,
            0.2169
          ],
          [
            0.88,
            0.2188
          ],
          [
            0.9,
            0.2194
          ],
          [
            0.92,
            0.2191
          ],
          [
            0.94,
            0.222
          ],
          [
            0.96,
            0.2257
          ],
          [
            0.98,
            0.2293
          ],
          [
            1.0,
            0.2328
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 1,
            "mean_confidence": 0.2018019629088712,
            "accuracy": 1.0
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 2117,
            "mean_confidence": 0.9998574388200139,
            "accuracy": 0.7671232876712328
          }
        ],
        "median_right": 0.9999941178422007,
        "median_wrong": 0.9999857536908587,
        "sure": {
          "questions": 2113,
          "correct": 1623,
          "interval": [
            0.7496299235686176,
            0.7856014324289212
          ]
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 1,
          "correct": 1,
          "interval": [
            0.20654329147389294,
            1.0
          ]
        },
        "ece_refit": 0.022750741839562657,
        "brier_refit": 0.1732184892154537,
        "brier_base": 0.17889320573282208,
        "refit_slopes": [
          0.21826028809717343,
          0.20399784805870902,
          0.22627791913890047,
          0.2082578053342295,
          0.2430936440206276
        ],
        "refit_questions": 2118,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        113,
        148,
        55,
        94,
        86,
        90,
        109,
        52,
        73,
        61,
        63,
        26,
        115,
        59,
        66,
        96,
        20,
        87,
        77,
        135
      ],
      "started": "2026-10-28T22:04:39Z",
      "updated": "2026-10-29T04:45:15Z",
      "format_of": null,
      "group": "no",
      "latency": {
        "source": "run",
        "runs": [
          318
        ],
        "concurrency": 4,
        "median_request_seconds": 648.3749564884638,
        "p90_request_seconds": 1337.1598235789686
      },
      "single_question_latency": null
    },
    {
      "run": 119,
      "model": "gemma-4-31B-it",
      "name": "Gemma 4 31B",
      "variant": null,
      "reasoning": "off",
      "provider": "vllm",
      "note": null,
      "parameters": 31273088876,
      "active_parameters": null,
      "access": "vLLM, self-hosted, FP8",
      "questions_per_request": 16,
      "concurrency": 1,
      "correct": 1607,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 596
        },
        "grammar": {
          "questions": 1542,
          "correct": 1011
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 1604
        }
      },
      "tyt_mix": 0.8642800370124324,
      "questions_per_minute": 245.32171674964297,
      "request_seconds": 537.5798023400712,
      "median_request_seconds": 3.972206312988419,
      "p90_request_seconds": 4.256375251116697,
      "billing": "local",
      "api_equivalent_usd": null,
      "cost_basis": null,
      "requests": 138,
      "hardware": "1 × RTX PRO 6000 Blackwell (96 GB); FP8 weights and KV cache",
      "input_tokens": 489825,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 20046,
      "confidence": {
        "source": "probability",
        "label": "Answer letter probability",
        "bins": [
          {
            "questions": 0,
            "correct": 0
          },
          {
            "questions": 12,
            "correct": 5
          },
          {
            "questions": 78,
            "correct": 30
          },
          {
            "questions": 94,
            "correct": 29
          },
          {
            "questions": 2014,
            "correct": 1543
          }
        ],
        "auroc": 0.8330400942576734,
        "auroc_interval": [
          0.8159328986887407,
          0.8511324555869844
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.8842841163310962,
            "auroc_interval": [
              0.84754279873946,
              0.9165817626082526
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.7885779960919528,
            "auroc_interval": [
              0.7658726105355319,
              0.8103122447464172
            ]
          }
        },
        "aurc": 0.09064879467975782,
        "ece": 0.24156780736968866,
        "brier": 0.24203788326889283,
        "coverage_accuracy": {
          "0.20": 0.986351228389445,
          "0.50": 0.9426751592356688,
          "0.80": 0.8180163785259327
        },
        "coverage_intervals": {
          "0.20": [
            0.9749544626593807,
            0.9977231329690346
          ],
          "0.50": [
            0.9245454545454546,
            0.9585798816568047
          ],
          "0.80": [
            0.7982954545454546,
            0.8375
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0
          ],
          [
            0.04,
            0.0
          ],
          [
            0.06,
            0.0
          ],
          [
            0.08,
            0.0
          ],
          [
            0.1,
            0.0
          ],
          [
            0.12,
            0.0
          ],
          [
            0.14,
            0.0032
          ],
          [
            0.16,
            0.0057
          ],
          [
            0.18,
            0.0101
          ],
          [
            0.2,
            0.0136
          ],
          [
            0.22,
            0.0124
          ],
          [
            0.24,
            0.0114
          ],
          [
            0.26,
            0.0122
          ],
          [
            0.28,
            0.0146
          ],
          [
            0.3,
            0.0167
          ],
          [
            0.32,
            0.0213
          ],
          [
            0.34,
            0.0281
          ],
          [
            0.36,
            0.0341
          ],
          [
            0.38,
            0.0347
          ],
          [
            0.4,
            0.0353
          ],
          [
            0.42,
            0.0368
          ],
          [
            0.44,
            0.0403
          ],
          [
            0.46,
            0.0446
          ],
          [
            0.48,
            0.0484
          ],
          [
            0.5,
            0.0573
          ],
          [
            0.52,
            0.0656
          ],
          [
            0.54,
            0.0741
          ],
          [
            0.56,
            0.0845
          ],
          [
            0.58,
            0.0941
          ],
          [
            0.6,
            0.0971
          ],
          [
            0.62,
            0.1086
          ],
          [
            0.64,
            0.1164
          ],
          [
            0.66,
            0.1266
          ],
          [
            0.68,
            0.1342
          ],
          [
            0.7,
            0.1421
          ],
          [
            0.72,
            0.151
          ],
          [
            0.74,
            0.1562
          ],
          [
            0.76,
            0.1634
          ],
          [
            0.78,
            0.17
          ],
          [
            0.8,
            0.182
          ],
          [
            0.82,
            0.19
          ],
          [
            0.84,
            0.2
          ],
          [
            0.86,
            0.2121
          ],
          [
            0.88,
            0.2182
          ],
          [
            0.9,
            0.2265
          ],
          [
            0.92,
            0.236
          ],
          [
            0.94,
            0.2479
          ],
          [
            0.96,
            0.2545
          ],
          [
            0.98,
            0.2609
          ],
          [
            1.0,
            0.2689
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 1,
            "mean_confidence": 0.2617147737506787,
            "accuracy": 0.0
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 2,
            "mean_confidence": 0.36520061426339834,
            "accuracy": 0.0
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 9,
            "mean_confidence": 0.4750944265186299,
            "accuracy": 0.5555555555555556
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 26,
            "mean_confidence": 0.5520949019450557,
            "accuracy": 0.34615384615384615
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 39,
            "mean_confidence": 0.6501353062005182,
            "accuracy": 0.38461538461538464
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 43,
            "mean_confidence": 0.761164279599376,
            "accuracy": 0.4186046511627907
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 64,
            "mean_confidence": 0.8542942679028498,
            "accuracy": 0.265625
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 2014,
            "mean_confidence": 0.9951018534627699,
            "accuracy": 0.766137040714995
          }
        ],
        "median_right": 0.9999995466782146,
        "median_wrong": 0.9946603553455324,
        "sure": {
          "questions": 1782,
          "correct": 1450,
          "interval": [
            0.7949466515577485,
            0.831088714624784
          ]
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 77,
          "correct": 29,
          "interval": [
            0.27669846411806526,
            0.48827402529332703
          ]
        },
        "ece_refit": 0.06368929885066907,
        "brier_refit": 0.1485754844409127,
        "brier_base": 0.19670143349770386,
        "refit_slopes": [
          0.2648119671644789,
          0.2694078193522273,
          0.25407510664998084,
          0.2690658697342816,
          0.2863609999794513
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        113,
        144,
        62,
        95,
        89,
        93,
        94,
        42,
        69,
        57,
        54,
        38,
        93,
        57,
        58,
        109,
        38,
        98,
        82,
        122
      ],
      "started": "2026-10-05T18:23:31Z",
      "updated": "2026-10-05T18:32:29Z",
      "format_of": null,
      "group": "o",
      "latency": {
        "source": "run",
        "runs": [
          119
        ],
        "concurrency": 1,
        "median_request_seconds": 3.972206312988419,
        "p90_request_seconds": 4.256375251116697
      },
      "single_question_latency": {
        "run": 136,
        "questions": 200,
        "median_request_seconds": 0.29488859651610255,
        "p90_request_seconds": 0.30137023626593873
      }
    },
    {
      "run": 315,
      "model": "moonshotai/kimi-k3.1-pro",
      "name": "Kimi K3.1 Pro",
      "variant": null,
      "reasoning": "none",
      "provider": "openrouter-chat",
      "note": null,
      "parameters": null,
      "active_parameters": null,
      "access": "OpenRouter chat API, Moonshot AI's endpoint (mxfp4), fallbacks off",
      "questions_per_request": 16,
      "concurrency": 4,
      "correct": 1598,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 607
        },
        "grammar": {
          "questions": 1542,
          "correct": 991
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 1593
        }
      },
      "tyt_mix": 0.8758440989687134,
      "questions_per_minute": 625.5590601469394,
      "request_seconds": 210.8194228200009,
      "median_request_seconds": 5.745670074480586,
      "p90_request_seconds": 7.645184530690313,
      "billing": "api",
      "api_equivalent_usd": 1.875844,
      "cost_basis": "api",
      "requests": 138,
      "hardware": null,
      "input_tokens": 778642,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 31856,
      "confidence": {
        "source": "stated",
        "label": "Written estimate",
        "bins": [
          {
            "questions": 0,
            "correct": 0
          },
          {
            "questions": 2,
            "correct": 0
          },
          {
            "questions": 69,
            "correct": 25
          },
          {
            "questions": 366,
            "correct": 171
          },
          {
            "questions": 1761,
            "correct": 1402
          }
        ],
        "auroc": 0.7283588861076346,
        "auroc_interval": [
          0.705745176575303,
          0.7513716891095612
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.7708200248798036,
            "auroc_interval": [
              0.6992383512544803,
              0.8457739156268568
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.676060955129743,
            "auroc_interval": [
              0.6471181368722352,
              0.7015418217535153
            ]
          }
        },
        "aurc": 0.1500412380690483,
        "ece": 0.20101000909919464,
        "brier": 0.22080664240218378,
        "coverage_accuracy": {
          "0.20": 0.918552036199095,
          "0.50": 0.852722104568111,
          "0.80": 0.7963712258292192
        },
        "coverage_intervals": {
          "0.20": [
            0.8905799575178274,
            0.9422330160986717
          ],
          "0.50": [
            0.8306611614563784,
            0.8757044125465178
          ],
          "0.80": [
            0.7734458124335005,
            0.8155044250883269
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0814
          ],
          [
            0.04,
            0.0814
          ],
          [
            0.06,
            0.0814
          ],
          [
            0.08,
            0.0814
          ],
          [
            0.1,
            0.0814
          ],
          [
            0.12,
            0.0814
          ],
          [
            0.14,
            0.0814
          ],
          [
            0.16,
            0.0814
          ],
          [
            0.18,
            0.0814
          ],
          [
            0.2,
            0.0814
          ],
          [
            0.22,
            0.0875
          ],
          [
            0.24,
            0.093
          ],
          [
            0.26,
            0.0975
          ],
          [
            0.28,
            0.1015
          ],
          [
            0.3,
            0.1049
          ],
          [
            0.32,
            0.1078
          ],
          [
            0.34,
            0.1105
          ],
          [
            0.36,
            0.1151
          ],
          [
            0.38,
            0.1216
          ],
          [
            0.4,
            0.1274
          ],
          [
            0.42,
            0.1327
          ],
          [
            0.44,
            0.1373
          ],
          [
            0.46,
            0.141
          ],
          [
            0.48,
            0.1443
          ],
          [
            0.5,
            0.1473
          ],
          [
            0.52,
            0.15
          ],
          [
            0.54,
            0.1525
          ],
          [
            0.56,
            0.1549
          ],
          [
            0.58,
            0.1571
          ],
          [
            0.6,
            0.1591
          ],
          [
            0.62,
            0.161
          ],
          [
            0.64,
            0.1632
          ],
          [
            0.66,
            0.1684
          ],
          [
            0.68,
            0.1763
          ],
          [
            0.7,
            0.1811
          ],
          [
            0.72,
            0.1861
          ],
          [
            0.74,
            0.1909
          ],
          [
            0.76,
            0.1953
          ],
          [
            0.78,
            0.1996
          ],
          [
            0.8,
            0.2036
          ],
          [
            0.82,
            0.2132
          ],
          [
            0.84,
            0.2192
          ],
          [
            0.86,
            0.2251
          ],
          [
            0.88,
            0.2308
          ],
          [
            0.9,
            0.2362
          ],
          [
            0.92,
            0.2431
          ],
          [
            0.94,
            0.2503
          ],
          [
            0.96,
            0.2579
          ],
          [
            0.98,
            0.2645
          ],
          [
            1.0,
            0.273
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 2,
            "mean_confidence": 0.4,
            "accuracy": 0.0
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 3,
            "mean_confidence": 0.5333333333333333,
            "accuracy": 0.3333333333333333
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 24,
            "mean_confidence": 0.6258333333333334,
            "accuracy": 0.25
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 86,
            "mean_confidence": 0.729302325581396,
            "accuracy": 0.4069767441860465
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 322,
            "mean_confidence": 0.836925465838509,
            "accuracy": 0.4782608695652174
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 1761,
            "mean_confidence": 0.9597898921067745,
            "accuracy": 0.7961385576377058
          }
        ],
        "median_right": 0.97,
        "median_wrong": 0.9,
        "sure": {
          "questions": 442,
          "correct": 406,
          "interval": [
            0.8893010060494675,
            0.9405901481182908
          ]
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 29,
          "correct": 7,
          "interval": [
            0.12218268185062131,
            0.4210795221832869
          ]
        },
        "ece_refit": 0.02190034249992485,
        "brier_refit": 0.17188160268310715,
        "brier_base": 0.19868910146045646,
        "refit_slopes": [
          0.7703560594212969,
          0.7771814380295238,
          0.810411136401663,
          0.7682462559234052,
          0.8199194109713996
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        119,
        148,
        58,
        96,
        91,
        95,
        94,
        36,
        69,
        51,
        51,
        35,
        89,
        61,
        58,
        111,
        24,
        97,
        76,
        139
      ],
      "started": "2026-10-28T14:52:06Z",
      "updated": "2026-10-28T14:55:37Z",
      "format_of": null,
      "group": "o",
      "latency": {
        "source": "run",
        "runs": [
          315
        ],
        "concurrency": 4,
        "median_request_seconds": 5.745670074480586,
        "p90_request_seconds": 7.645184530690313
      },
      "single_question_latency": null
    },
    {
      "run": 248,
      "model": "glide-v1.2",
      "name": "Fastino GLiDE v1.2",
      "variant": "local",
      "reasoning": null,
      "provider": "glide",
      "note": "Fastino's released checkpoint, a modified Qwen3.8-27B, run on our hardware through the server it ships. Fastino's hosted GLiDE gave the same answers on 20 sample questions on 22 October 2026; the other GLiDE row is the hosted model as it answered on 6 October 2026, before a change on Fastino's side.",
      "parameters": 26086635760,
      "active_parameters": null,
      "access": "GLiDE server (Fastino's release code), self-hosted",
      "questions_per_request": 1,
      "concurrency": 1,
      "correct": 1505,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 588
        },
        "grammar": {
          "questions": 1542,
          "correct": 917
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 1501
        }
      },
      "tyt_mix": 0.8435510977191484,
      "questions_per_minute": 139.71066492704924,
      "request_seconds": 943.9508434725576,
      "median_request_seconds": 0.41326581648900174,
      "p90_request_seconds": 0.5194234423222952,
      "billing": "local",
      "api_equivalent_usd": 0.03158416,
      "cost_basis": "maker",
      "requests": 2198,
      "hardware": "1 × DGX Spark, GB10 (128 GB unified memory); BF16",
      "input_tokens": 789604,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 0,
      "confidence": {
        "source": "probability",
        "label": "Option score",
        "bins": [
          {
            "questions": 13,
            "correct": 4
          },
          {
            "questions": 547,
            "correct": 183
          },
          {
            "questions": 617,
            "correct": 372
          },
          {
            "questions": 362,
            "correct": 297
          },
          {
            "questions": 659,
            "correct": 649
          }
        ],
        "auroc": 0.8393632576356829,
        "auroc_interval": [
          0.8246874200090575,
          0.8553071448413269
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.937625050020008,
            "auroc_interval": [
              0.915505533494678,
              0.9576436453576864
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.7719494002181025,
            "auroc_interval": [
              0.7496901181940123,
              0.7963970313715775
            ]
          }
        },
        "aurc": 0.11059325331501722,
        "ece": 0.029152681148497294,
        "brier": 0.15129155311587794,
        "coverage_accuracy": {
          "0.20": 1.0,
          "0.50": 0.9153776160145587,
          "0.80": 0.774567788898999
        },
        "coverage_intervals": {
          "0.20": [
            0.9954462659380692,
            1.0
          ],
          "0.50": [
            0.8970856102003643,
            0.9330296127562643
          ],
          "0.80": [
            0.7535291438979964,
            0.7977438468550593
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0
          ],
          [
            0.04,
            0.0
          ],
          [
            0.06,
            0.0
          ],
          [
            0.08,
            0.0
          ],
          [
            0.1,
            0.0
          ],
          [
            0.12,
            0.0
          ],
          [
            0.14,
            0.0
          ],
          [
            0.16,
            0.0
          ],
          [
            0.18,
            0.0
          ],
          [
            0.2,
            0.0
          ],
          [
            0.22,
            0.0041
          ],
          [
            0.24,
            0.0057
          ],
          [
            0.26,
            0.0122
          ],
          [
            0.28,
            0.013
          ],
          [
            0.3,
            0.0152
          ],
          [
            0.32,
            0.0185
          ],
          [
            0.34,
            0.0232
          ],
          [
            0.36,
            0.0291
          ],
          [
            0.38,
            0.0362
          ],
          [
            0.4,
            0.0434
          ],
          [
            0.42,
            0.0531
          ],
          [
            0.44,
            0.06
          ],
          [
            0.46,
            0.0663
          ],
          [
            0.48,
            0.0796
          ],
          [
            0.5,
            0.0846
          ],
          [
            0.52,
            0.0901
          ],
          [
            0.54,
            0.1003
          ],
          [
            0.56,
            0.112
          ],
          [
            0.58,
            0.1269
          ],
          [
            0.6,
            0.1371
          ],
          [
            0.62,
            0.146
          ],
          [
            0.64,
            0.15
          ],
          [
            0.66,
            0.1579
          ],
          [
            0.68,
            0.1666
          ],
          [
            0.7,
            0.1742
          ],
          [
            0.72,
            0.183
          ],
          [
            0.74,
            0.1946
          ],
          [
            0.76,
            0.2029
          ],
          [
            0.78,
            0.2137
          ],
          [
            0.8,
            0.2254
          ],
          [
            0.82,
            0.2343
          ],
          [
            0.84,
            0.2461
          ],
          [
            0.86,
            0.256
          ],
          [
            0.88,
            0.2638
          ],
          [
            0.9,
            0.2735
          ],
          [
            0.92,
            0.2829
          ],
          [
            0.94,
            0.2894
          ],
          [
            0.96,
            0.2967
          ],
          [
            0.98,
            0.3055
          ],
          [
            1.0,
            0.3153
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 67,
            "mean_confidence": 0.2686772106044082,
            "accuracy": 0.22388059701492538
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 220,
            "mean_confidence": 0.35420065652416755,
            "accuracy": 0.33181818181818185
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 273,
            "mean_confidence": 0.45337279014004245,
            "accuracy": 0.3626373626373626
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 252,
            "mean_confidence": 0.5485137829881164,
            "accuracy": 0.5396825396825397
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 234,
            "mean_confidence": 0.6507659354996671,
            "accuracy": 0.5811965811965812
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 244,
            "mean_confidence": 0.7482371512527779,
            "accuracy": 0.7540983606557377
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 249,
            "mean_confidence": 0.8509263191451586,
            "accuracy": 0.8554216867469879
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 659,
            "mean_confidence": 0.9691545977795931,
            "accuracy": 0.9848254931714719
          }
        ],
        "median_right": 0.8540157939819392,
        "median_wrong": 0.48690194377589996,
        "sure": {
          "questions": 240,
          "correct": 240,
          "interval": [
            0.9842455102000642,
            1.0
          ]
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 1046,
          "correct": 459,
          "interval": [
            0.4090191633672684,
            0.46905768174917595
          ]
        },
        "ece_refit": 0.024502222047147578,
        "brier_refit": 0.15059467228887072,
        "brier_base": 0.21597335585393268,
        "refit_slopes": [
          1.12431765175443,
          1.1959691653034052,
          1.1802944297830875,
          1.1935095218394345,
          1.1901214004345908
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        117,
        144,
        60,
        91,
        83,
        93,
        90,
        39,
        56,
        48,
        52,
        28,
        84,
        52,
        53,
        104,
        27,
        80,
        76,
        128
      ],
      "started": "2026-10-22T18:39:36Z",
      "updated": "2026-10-22T18:55:25Z",
      "format_of": null,
      "group": "p",
      "latency": {
        "source": "run",
        "runs": [
          248
        ],
        "concurrency": 1,
        "median_request_seconds": 0.41326581648900174,
        "p90_request_seconds": 0.5194234423222952
      },
      "single_question_latency": null
    },
    {
      "run": 133,
      "model": "gpt-6-luna",
      "name": "GPT-6 Luna",
      "variant": null,
      "reasoning": "low",
      "provider": "openai",
      "note": null,
      "parameters": null,
      "active_parameters": null,
      "access": "Codex CLI, ChatGPT subscription",
      "questions_per_request": 16,
      "concurrency": 1,
      "correct": 1501,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 573
        },
        "grammar": {
          "questions": 1542,
          "correct": 928
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 1499
        }
      },
      "tyt_mix": 0.8259351471797792,
      "questions_per_minute": 106.27926346966372,
      "request_seconds": 1240.8817646505777,
      "median_request_seconds": 7.705671253963374,
      "p90_request_seconds": 11.16688626020914,
      "billing": "subscription",
      "api_equivalent_usd": 0.10553408,
      "cost_basis": "subscription",
      "requests": 138,
      "hardware": null,
      "input_tokens": 1197871,
      "cached_input_tokens": 316928,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 28541,
      "confidence": {
        "source": "stated",
        "label": "Written estimate",
        "bins": [
          {
            "questions": 0,
            "correct": 0
          },
          {
            "questions": 2,
            "correct": 1
          },
          {
            "questions": 85,
            "correct": 29
          },
          {
            "questions": 321,
            "correct": 157
          },
          {
            "questions": 1790,
            "correct": 1314
          }
        ],
        "auroc": 0.6851137978793669,
        "auroc_interval": [
          0.6600100900103961,
          0.7067208024931074
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.7189175550369016,
            "auroc_interval": [
              0.6625968992248062,
              0.7739693757361602
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.6522213369089072,
            "auroc_interval": [
              0.6249880660662572,
              0.6778510105352353
            ]
          }
        },
        "aurc": 0.19950706787609154,
        "ece": 0.2565468607825413,
        "brier": 0.26862033666969976,
        "coverage_accuracy": {
          "0.20": 0.8457488696651195,
          "0.50": 0.8039505611161661,
          "0.80": 0.7384321066841689
        },
        "coverage_intervals": {
          "0.20": [
            0.8210651940610787,
            0.8686369787181695
          ],
          "0.50": [
            0.7797339995591153,
            0.8265441298228183
          ],
          "0.80": [
            0.7168841071909161,
            0.7594877741433856
          ]
        },
        "risk_coverage": [
          [
            0.02,
            -0.0
          ],
          [
            0.04,
            0.0623
          ],
          [
            0.06,
            0.1006
          ],
          [
            0.08,
            0.1198
          ],
          [
            0.1,
            0.1313
          ],
          [
            0.12,
            0.1389
          ],
          [
            0.14,
            0.1444
          ],
          [
            0.16,
            0.1485
          ],
          [
            0.18,
            0.1517
          ],
          [
            0.2,
            0.1543
          ],
          [
            0.22,
            0.1563
          ],
          [
            0.24,
            0.1581
          ],
          [
            0.26,
            0.1596
          ],
          [
            0.28,
            0.1608
          ],
          [
            0.3,
            0.1619
          ],
          [
            0.32,
            0.1629
          ],
          [
            0.34,
            0.1637
          ],
          [
            0.36,
            0.1645
          ],
          [
            0.38,
            0.1659
          ],
          [
            0.4,
            0.1721
          ],
          [
            0.42,
            0.1778
          ],
          [
            0.44,
            0.183
          ],
          [
            0.46,
            0.1877
          ],
          [
            0.48,
            0.1921
          ],
          [
            0.5,
            0.196
          ],
          [
            0.52,
            0.1998
          ],
          [
            0.54,
            0.2056
          ],
          [
            0.56,
            0.211
          ],
          [
            0.58,
            0.2161
          ],
          [
            0.6,
            0.2219
          ],
          [
            0.62,
            0.2276
          ],
          [
            0.64,
            0.2329
          ],
          [
            0.66,
            0.2376
          ],
          [
            0.68,
            0.2419
          ],
          [
            0.7,
            0.2446
          ],
          [
            0.72,
            0.248
          ],
          [
            0.74,
            0.2538
          ],
          [
            0.76,
            0.2576
          ],
          [
            0.78,
            0.2575
          ],
          [
            0.8,
            0.2616
          ],
          [
            0.82,
            0.2678
          ],
          [
            0.84,
            0.2717
          ],
          [
            0.86,
            0.2764
          ],
          [
            0.88,
            0.2811
          ],
          [
            0.9,
            0.2866
          ],
          [
            0.92,
            0.2907
          ],
          [
            0.94,
            0.2958
          ],
          [
            0.96,
            0.303
          ],
          [
            0.98,
            0.309
          ],
          [
            1.0,
            0.3171
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 2,
            "mean_confidence": 0.44999999999999996,
            "accuracy": 0.5
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 12,
            "mean_confidence": 0.5641666666666666,
            "accuracy": 0.4166666666666667
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 32,
            "mean_confidence": 0.6521875000000001,
            "accuracy": 0.21875
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 111,
            "mean_confidence": 0.7541441441441444,
            "accuracy": 0.40540540540540543
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 251,
            "mean_confidence": 0.8499601593625498,
            "accuracy": 0.5139442231075697
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 1790,
            "mean_confidence": 0.9715642458100702,
            "accuracy": 0.7340782122905027
          }
        ],
        "median_right": 0.98,
        "median_wrong": 0.95,
        "sure": {
          "questions": 830,
          "correct": 693,
          "interval": [
            0.8081516217664547,
            0.8586416876582609
          ]
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 46,
          "correct": 13,
          "interval": [
            0.1732404154491407,
            0.42548835730293777
          ]
        },
        "ece_refit": 0.015729877768250293,
        "brier_refit": 0.1963742004501457,
        "brier_base": 0.21668908170357778,
        "refit_slopes": [
          0.5071607445878575,
          0.5046288580741005,
          0.5034945376017976,
          0.5142980184318481,
          0.544396897375931
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        114,
        144,
        52,
        93,
        79,
        91,
        77,
        31,
        64,
        49,
        51,
        36,
        78,
        55,
        55,
        91,
        29,
        109,
        76,
        127
      ],
      "started": "2026-10-06T12:36:07Z",
      "updated": "2026-10-06T13:02:06Z",
      "format_of": null,
      "group": "pq",
      "latency": {
        "source": "run",
        "runs": [
          133
        ],
        "concurrency": 1,
        "median_request_seconds": 7.705671253963374,
        "p90_request_seconds": 11.16688626020914
      },
      "single_question_latency": null
    },
    {
      "run": 129,
      "model": "fastino/GLiDE",
      "name": "Fastino GLiDE",
      "variant": null,
      "reasoning": null,
      "provider": "fastino",
      "note": "Hosted decision model; Fastino describes it as reasoning on uncertain decisions",
      "parameters": null,
      "active_parameters": null,
      "access": "Fastino System One API",
      "questions_per_request": 1,
      "concurrency": 4,
      "correct": 1492,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 576
        },
        "grammar": {
          "questions": 1542,
          "correct": 916
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 1489
        }
      },
      "tyt_mix": 0.8283461453291576,
      "questions_per_minute": 152.131021033152,
      "request_seconds": 866.8843415654264,
      "median_request_seconds": 2.088375358027406,
      "p90_request_seconds": 2.2788841759203935,
      "billing": "api",
      "api_equivalent_usd": 0.35185395,
      "cost_basis": "api",
      "requests": 2198,
      "hardware": null,
      "input_tokens": 2345693,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 558873,
      "confidence": {
        "source": "probability",
        "label": "Option score",
        "bins": [
          {
            "questions": 15,
            "correct": 8
          },
          {
            "questions": 449,
            "correct": 157
          },
          {
            "questions": 860,
            "correct": 510
          },
          {
            "questions": 463,
            "correct": 406
          },
          {
            "questions": 411,
            "correct": 411
          }
        ],
        "auroc": 0.8104432326515733,
        "auroc_interval": [
          0.793241449320631,
          0.828115829826555
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.9104600694444445,
            "auroc_interval": [
              0.8832774049217003,
              0.9343247196489517
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.7410605912635853,
            "auroc_interval": [
              0.7154466847690026,
              0.764084241293191
            ]
          }
        },
        "aurc": 0.12585535521113503,
        "ece": 0.04508679278866653,
        "brier": 0.16507941304957036,
        "coverage_accuracy": {
          "0.20": 0.9954504094631483,
          "0.50": 0.8908098271155596,
          "0.80": 0.7592129208371247
        },
        "coverage_intervals": {
          "0.20": [
            0.9840619307832422,
            1.0
          ],
          "0.50": [
            0.8671519563239308,
            0.9107874374146564
          ],
          "0.80": [
            0.7381710646041856,
            0.7809512972234866
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0
          ],
          [
            0.04,
            0.0
          ],
          [
            0.06,
            0.0
          ],
          [
            0.08,
            0.0
          ],
          [
            0.1,
            0.0
          ],
          [
            0.12,
            0.0
          ],
          [
            0.14,
            0.0
          ],
          [
            0.16,
            0.0
          ],
          [
            0.18,
            0.0
          ],
          [
            0.2,
            0.0045
          ],
          [
            0.22,
            0.0124
          ],
          [
            0.24,
            0.0246
          ],
          [
            0.26,
            0.0315
          ],
          [
            0.28,
            0.0341
          ],
          [
            0.3,
            0.0349
          ],
          [
            0.32,
            0.0455
          ],
          [
            0.34,
            0.0549
          ],
          [
            0.36,
            0.0632
          ],
          [
            0.38,
            0.0623
          ],
          [
            0.4,
            0.066
          ],
          [
            0.42,
            0.0737
          ],
          [
            0.44,
            0.0807
          ],
          [
            0.46,
            0.094
          ],
          [
            0.48,
            0.0986
          ],
          [
            0.5,
            0.1092
          ],
          [
            0.52,
            0.119
          ],
          [
            0.54,
            0.1263
          ],
          [
            0.56,
            0.1373
          ],
          [
            0.58,
            0.142
          ],
          [
            0.6,
            0.1539
          ],
          [
            0.62,
            0.1627
          ],
          [
            0.64,
            0.1706
          ],
          [
            0.66,
            0.1792
          ],
          [
            0.68,
            0.1873
          ],
          [
            0.7,
            0.1973
          ],
          [
            0.72,
            0.2057
          ],
          [
            0.74,
            0.2137
          ],
          [
            0.76,
            0.2233
          ],
          [
            0.78,
            0.231
          ],
          [
            0.8,
            0.2408
          ],
          [
            0.82,
            0.2493
          ],
          [
            0.84,
            0.258
          ],
          [
            0.86,
            0.2684
          ],
          [
            0.88,
            0.2776
          ],
          [
            0.9,
            0.2851
          ],
          [
            0.92,
            0.2933
          ],
          [
            0.94,
            0.2992
          ],
          [
            0.96,
            0.3061
          ],
          [
            0.98,
            0.3162
          ],
          [
            1.0,
            0.3212
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 73,
            "mean_confidence": 0.26845270906220975,
            "accuracy": 0.3698630136986301
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 195,
            "mean_confidence": 0.3510300694351157,
            "accuracy": 0.35384615384615387
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 196,
            "mean_confidence": 0.4533001515542841,
            "accuracy": 0.3520408163265306
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 336,
            "mean_confidence": 0.5539630017608486,
            "accuracy": 0.49404761904761907
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 309,
            "mean_confidence": 0.6446963949948158,
            "accuracy": 0.6148867313915858
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 420,
            "mean_confidence": 0.7479242216339844,
            "accuracy": 0.7738095238095238
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 258,
            "mean_confidence": 0.8414252609494551,
            "accuracy": 0.9108527131782945
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 411,
            "mean_confidence": 0.9679440683395915,
            "accuracy": 1.0
          }
        ],
        "median_right": 0.7674958348331262,
        "median_wrong": 0.5319454748081022,
        "sure": {
          "questions": 121,
          "correct": 121,
          "interval": [
            0.969228205982621,
            0.9999999999999999
          ]
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 1109,
          "correct": 521,
          "interval": [
            0.44057322753832523,
            0.4992205400562723
          ]
        },
        "ece_refit": 0.03428655734434904,
        "brier_refit": 0.164958127089605,
        "brier_base": 0.21805159117505948,
        "refit_slopes": [
          1.229261435702072,
          1.2731899127885113,
          1.2217939062875345,
          1.3516333626890171,
          1.3131072777219572
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        108,
        144,
        58,
        89,
        88,
        89,
        87,
        40,
        59,
        50,
        49,
        31,
        98,
        51,
        54,
        100,
        25,
        80,
        64,
        128
      ],
      "started": "2026-10-06T10:53:16Z",
      "updated": "2026-10-06T11:07:43Z",
      "format_of": null,
      "group": "pqr",
      "latency": {
        "source": "run",
        "runs": [
          129
        ],
        "concurrency": 4,
        "median_request_seconds": 2.088375358027406,
        "p90_request_seconds": 2.2788841759203935
      },
      "single_question_latency": null
    },
    {
      "run": 259,
      "model": "vllm-sr/d3",
      "name": "vLLM-SR d3",
      "variant": null,
      "reasoning": null,
      "provider": "vllm-sr",
      "note": "The vLLM Semantic Router's released Decision 3.0 checkpoint (vllm-sr/d3 at 20f37b9), fine-tuned from Qwen3.8-27B (Qwen3.5 architecture), served on our hardware through vLLM's System One endpoint by a nightly build (0.32.1rc1.dev613+g8977b13a1) with --max-num-seqs 256: with the default, vLLM's startup memory check did not fit the GPU. One request at a time, as for the other setups.",
      "parameters": 26086635760,
      "active_parameters": null,
      "access": "vLLM System One endpoint (nightly), self-hosted, BF16",
      "questions_per_request": 1,
      "concurrency": 1,
      "correct": 1490,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 585
        },
        "grammar": {
          "questions": 1542,
          "correct": 905
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 1486
        }
      },
      "tyt_mix": 0.8384163641612098,
      "questions_per_minute": 591.0559151650041,
      "request_seconds": 223.12609791441355,
      "median_request_seconds": 0.0924945070000831,
      "p90_request_seconds": 0.14396354870405048,
      "billing": "local",
      "api_equivalent_usd": null,
      "cost_basis": null,
      "requests": 2198,
      "hardware": "1 × RTX PRO 6000 Blackwell (96 GB); BF16",
      "input_tokens": 864336,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 0,
      "confidence": {
        "source": "probability",
        "label": "Option score",
        "bins": [
          {
            "questions": 17,
            "correct": 5
          },
          {
            "questions": 613,
            "correct": 225
          },
          {
            "questions": 583,
            "correct": 330
          },
          {
            "questions": 335,
            "correct": 286
          },
          {
            "questions": 650,
            "correct": 644
          }
        ],
        "auroc": 0.8354955826034202,
        "auroc_interval": [
          0.8189457959227394,
          0.8511914536206034
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.9279643674009871,
            "auroc_interval": [
              0.9087998073913567,
              0.9479375786289783
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.7714996920995343,
            "auroc_interval": [
              0.7480481519276337,
              0.7946117648694299
            ]
          }
        },
        "aurc": 0.1146593559464455,
        "ece": 0.0260618026254459,
        "brier": 0.15273599003721192,
        "coverage_accuracy": {
          "0.20": 1.0,
          "0.50": 0.910828025477707,
          "0.80": 0.7611464968152867
        },
        "coverage_intervals": {
          "0.20": [
            0.9954483386436049,
            1.0
          ],
          "0.50": [
            0.8889899909008189,
            0.9290586630286494
          ],
          "0.80": [
            0.7393327280980481,
            0.782316795630405
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0
          ],
          [
            0.04,
            0.0
          ],
          [
            0.06,
            0.0
          ],
          [
            0.08,
            0.0
          ],
          [
            0.1,
            0.0
          ],
          [
            0.12,
            0.0
          ],
          [
            0.14,
            0.0
          ],
          [
            0.16,
            0.0
          ],
          [
            0.18,
            0.0
          ],
          [
            0.2,
            0.0
          ],
          [
            0.22,
            0.0041
          ],
          [
            0.24,
            0.0095
          ],
          [
            0.26,
            0.0105
          ],
          [
            0.28,
            0.0097
          ],
          [
            0.3,
            0.0091
          ],
          [
            0.32,
            0.0128
          ],
          [
            0.34,
            0.0187
          ],
          [
            0.36,
            0.0265
          ],
          [
            0.38,
            0.0338
          ],
          [
            0.4,
            0.04
          ],
          [
            0.42,
            0.0477
          ],
          [
            0.44,
            0.0527
          ],
          [
            0.46,
            0.0623
          ],
          [
            0.48,
            0.0768
          ],
          [
            0.5,
            0.0892
          ],
          [
            0.52,
            0.1023
          ],
          [
            0.54,
            0.1087
          ],
          [
            0.56,
            0.1211
          ],
          [
            0.58,
            0.1271
          ],
          [
            0.6,
            0.1371
          ],
          [
            0.62,
            0.1475
          ],
          [
            0.64,
            0.1612
          ],
          [
            0.66,
            0.1696
          ],
          [
            0.68,
            0.178
          ],
          [
            0.7,
            0.1889
          ],
          [
            0.72,
            0.1988
          ],
          [
            0.74,
            0.2109
          ],
          [
            0.76,
            0.22
          ],
          [
            0.78,
            0.2289
          ],
          [
            0.8,
            0.2389
          ],
          [
            0.82,
            0.2488
          ],
          [
            0.84,
            0.2567
          ],
          [
            0.86,
            0.2652
          ],
          [
            0.88,
            0.2715
          ],
          [
            0.9,
            0.2812
          ],
          [
            0.92,
            0.2903
          ],
          [
            0.94,
            0.2981
          ],
          [
            0.96,
            0.3052
          ],
          [
            0.98,
            0.3129
          ],
          [
            1.0,
            0.3221
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 95,
            "mean_confidence": 0.27089039568219175,
            "accuracy": 0.2736842105263158
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 271,
            "mean_confidence": 0.35465385250305076,
            "accuracy": 0.36162361623616235
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 264,
            "mean_confidence": 0.4492525496476692,
            "accuracy": 0.4015151515151515
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 259,
            "mean_confidence": 0.5452962437356342,
            "accuracy": 0.4942084942084942
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 216,
            "mean_confidence": 0.6451352213845302,
            "accuracy": 0.6203703703703703
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 215,
            "mean_confidence": 0.7494669953220469,
            "accuracy": 0.7209302325581395
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 228,
            "mean_confidence": 0.8519612069289955,
            "accuracy": 0.8728070175438597
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 650,
            "mean_confidence": 0.9706803585832869,
            "accuracy": 0.9907692307692307
          }
        ],
        "median_right": 0.8577558716700046,
        "median_wrong": 0.46500011652971796,
        "sure": {
          "questions": 250,
          "correct": 250,
          "interval": [
            0.9848661527503766,
            1.0
          ]
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 1105,
          "correct": 492,
          "interval": [
            0.4161848446510179,
            0.4746922652986809
          ]
        },
        "ece_refit": 0.027003523206855205,
        "brier_refit": 0.1528786032830721,
        "brier_base": 0.21851928478515936,
        "refit_slopes": [
          1.1135672676179305,
          1.1538884466581394,
          1.1806608634835851,
          1.1338607010265647,
          1.1419739907602016
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        112,
        142,
        57,
        94,
        89,
        91,
        92,
        39,
        57,
        51,
        49,
        29,
        85,
        50,
        53,
        100,
        24,
        79,
        72,
        125
      ],
      "started": "2026-10-27T09:16:30Z",
      "updated": "2026-10-27T09:20:19Z",
      "format_of": null,
      "group": "pqr",
      "latency": {
        "source": "run",
        "runs": [
          259
        ],
        "concurrency": 1,
        "median_request_seconds": 0.0924945070000831,
        "p90_request_seconds": 0.14396354870405048
      },
      "single_question_latency": null
    },
    {
      "run": 269,
      "model": "vllm-sr/d3-ultra",
      "name": "vLLM-SR d3-ultra",
      "variant": "transformers, rev 52ba9e7",
      "reasoning": null,
      "provider": "d3-server",
      "note": "The vLLM Semantic Router's released d3-ultra checkpoint (vllm-sr/d3-ultra at 52ba9e7), fine-tuned from Qwen3.8-27B (Qwen3.5 architecture), served on our hardware through the server its repository ships (d3_server.py, the package's transformers runtime as in the model card's Quickstart; transformers 5.17.0). vLLM cannot serve it yet: it rejects the checkpoint's attention mode (prefix_bidir). One request at a time, as for the other setups.",
      "parameters": 26086635760,
      "active_parameters": null,
      "access": "d3 package server (transformers, the release code), self-hosted, BF16",
      "questions_per_request": 1,
      "concurrency": 1,
      "correct": 1490,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 589
        },
        "grammar": {
          "questions": 1542,
          "correct": 901
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 1488
        }
      },
      "tyt_mix": 0.842992896064661,
      "questions_per_minute": 192.26901396874433,
      "request_seconds": 685.9139560649055,
      "median_request_seconds": 0.3334329740027897,
      "p90_request_seconds": 0.4403526291134767,
      "billing": "local",
      "api_equivalent_usd": null,
      "cost_basis": null,
      "requests": 2198,
      "hardware": "1 × RTX PRO 6000 Blackwell (96 GB); BF16",
      "input_tokens": 1126925,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 0,
      "confidence": {
        "source": "probability",
        "label": "Option score",
        "bins": [
          {
            "questions": 1,
            "correct": 1
          },
          {
            "questions": 281,
            "correct": 93
          },
          {
            "questions": 495,
            "correct": 196
          },
          {
            "questions": 366,
            "correct": 228
          },
          {
            "questions": 1055,
            "correct": 972
          }
        ],
        "auroc": 0.8334499298525007,
        "auroc_interval": [
          0.8164771628239232,
          0.8493394320871955
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.9156931809543116,
            "auroc_interval": [
              0.8889541577574661,
              0.9389048793859649
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.768293506435041,
            "auroc_interval": [
              0.7434554060396069,
              0.7922017542953373
            ]
          }
        },
        "aurc": 0.11554804085045396,
        "ece": 0.12470444406965506,
        "brier": 0.17686164787503558,
        "coverage_accuracy": {
          "0.20": 0.9977252047315741,
          "0.50": 0.908098271155596,
          "0.80": 0.7586442220200181
        },
        "coverage_intervals": {
          "0.20": [
            0.988615664845173,
            1.0
          ],
          "0.50": [
            0.891057648660917,
            0.9312072892938497
          ],
          "0.80": [
            0.7372669395179627,
            0.7800409649522075
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0
          ],
          [
            0.04,
            0.0
          ],
          [
            0.06,
            0.0
          ],
          [
            0.08,
            0.0
          ],
          [
            0.1,
            0.0
          ],
          [
            0.12,
            0.0
          ],
          [
            0.14,
            0.0032
          ],
          [
            0.16,
            0.0028
          ],
          [
            0.18,
            0.0025
          ],
          [
            0.2,
            0.0023
          ],
          [
            0.22,
            0.0062
          ],
          [
            0.24,
            0.0057
          ],
          [
            0.26,
            0.0105
          ],
          [
            0.28,
            0.0114
          ],
          [
            0.3,
            0.0136
          ],
          [
            0.32,
            0.0213
          ],
          [
            0.34,
            0.0214
          ],
          [
            0.36,
            0.0278
          ],
          [
            0.38,
            0.0359
          ],
          [
            0.4,
            0.0432
          ],
          [
            0.42,
            0.05
          ],
          [
            0.44,
            0.0589
          ],
          [
            0.46,
            0.0673
          ],
          [
            0.48,
            0.0787
          ],
          [
            0.5,
            0.0919
          ],
          [
            0.52,
            0.0971
          ],
          [
            0.54,
            0.1061
          ],
          [
            0.56,
            0.1194
          ],
          [
            0.58,
            0.1255
          ],
          [
            0.6,
            0.1333
          ],
          [
            0.62,
            0.1414
          ],
          [
            0.64,
            0.1533
          ],
          [
            0.66,
            0.1652
          ],
          [
            0.68,
            0.1753
          ],
          [
            0.7,
            0.1876
          ],
          [
            0.72,
            0.199
          ],
          [
            0.74,
            0.2078
          ],
          [
            0.76,
            0.2194
          ],
          [
            0.78,
            0.2301
          ],
          [
            0.8,
            0.2414
          ],
          [
            0.82,
            0.251
          ],
          [
            0.84,
            0.2589
          ],
          [
            0.86,
            0.2662
          ],
          [
            0.88,
            0.2747
          ],
          [
            0.9,
            0.2842
          ],
          [
            0.92,
            0.2937
          ],
          [
            0.94,
            0.303
          ],
          [
            0.96,
            0.3109
          ],
          [
            0.98,
            0.3157
          ],
          [
            1.0,
            0.3221
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 19,
            "mean_confidence": 0.28121051892721544,
            "accuracy": 0.3684210526315789
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 111,
            "mean_confidence": 0.36311672631502084,
            "accuracy": 0.38738738738738737
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 152,
            "mean_confidence": 0.4518019789104278,
            "accuracy": 0.2894736842105263
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 205,
            "mean_confidence": 0.5512797562268145,
            "accuracy": 0.375609756097561
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 176,
            "mean_confidence": 0.6508201939184306,
            "accuracy": 0.4034090909090909
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 195,
            "mean_confidence": 0.7465808718851927,
            "accuracy": 0.4666666666666667
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 285,
            "mean_confidence": 0.8493221987134827,
            "accuracy": 0.6491228070175439
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 1055,
            "mean_confidence": 0.9723967989992467,
            "accuracy": 0.9213270142180094
          }
        ],
        "median_right": 0.9612619598258114,
        "median_wrong": 0.6387847235460056,
        "sure": {
          "questions": 384,
          "correct": 383,
          "interval": [
            0.9853980164858338,
            0.999540164462265
          ]
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 663,
          "correct": 242,
          "interval": [
            0.3292359853215557,
            0.4023344505960595
          ]
        },
        "ece_refit": 0.0329092068512694,
        "brier_refit": 0.15448776532044248,
        "brier_base": 0.21852157266113936,
        "refit_slopes": [
          0.8173932433924662,
          0.873328131003129,
          0.8477289430659002,
          0.8576800876681072,
          0.8348190266546786
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        112,
        143,
        58,
        93,
        91,
        92,
        91,
        35,
        55,
        49,
        49,
        27,
        85,
        56,
        59,
        98,
        21,
        81,
        72,
        123
      ],
      "started": "2026-10-27T14:45:17Z",
      "updated": "2026-10-27T14:56:49Z",
      "format_of": null,
      "group": "pqr",
      "latency": {
        "source": "run",
        "runs": [
          269
        ],
        "concurrency": 1,
        "median_request_seconds": 0.3334329740027897,
        "p90_request_seconds": 0.4403526291134767
      },
      "single_question_latency": null
    },
    {
      "run": 322,
      "model": "drex-max-v1",
      "name": "Drex Max v1",
      "variant": null,
      "reasoning": null,
      "provider": "drex-max",
      "note": "Nace.AI's released Drex Max weights (nace-ai/drex-max-v1 at cac65e1), the original Drex Max, a fine-tune of Qwen3.8-27B that also reads images. It ran on our hardware through Nace's own loader (drex_max, from nace-ai/drex-decision-models at e206b27), which makes 16 of the 64 layers attend non-causally, behind our HTTP server (servers/drex_max_server.py). One request at a time, as for the other setups.",
      "parameters": 26086635760,
      "active_parameters": null,
      "access": "Drex Max server (Nace's loader, drex_max), self-hosted, BF16",
      "questions_per_request": 1,
      "concurrency": 1,
      "correct": 1489,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 588
        },
        "grammar": {
          "questions": 1542,
          "correct": 901
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 1485
        }
      },
      "tyt_mix": 0.8417352741134415,
      "questions_per_minute": 673.5294239576491,
      "request_seconds": 195.804363267569,
      "median_request_seconds": 0.08219127899792511,
      "p90_request_seconds": 0.11037726640497567,
      "billing": "local",
      "api_equivalent_usd": 0.05120076,
      "cost_basis": "maker",
      "requests": 2198,
      "hardware": "1 × RTX PRO 6000 Blackwell (96 GB); BF16",
      "input_tokens": 853346,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 0,
      "confidence": {
        "source": "probability",
        "label": "Option score",
        "bins": [
          {
            "questions": 16,
            "correct": 1
          },
          {
            "questions": 487,
            "correct": 170
          },
          {
            "questions": 494,
            "correct": 266
          },
          {
            "questions": 307,
            "correct": 207
          },
          {
            "questions": 894,
            "correct": 845
          }
        ],
        "auroc": 0.8257925302713552,
        "auroc_interval": [
          0.8094250932494766,
          0.842538682707222
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.9073629451780713,
            "auroc_interval": [
              0.8812348065489266,
              0.9324088341037493
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.7613173783333131,
            "auroc_interval": [
              0.7359517825299429,
              0.7840773307467387
            ]
          }
        },
        "aurc": 0.11988140277373124,
        "ece": 0.06325311335192765,
        "brier": 0.16303781296526573,
        "coverage_accuracy": {
          "0.20": 0.9977252047315741,
          "0.50": 0.8962693357597816,
          "0.80": 0.7645586897179254
        },
        "coverage_intervals": {
          "0.20": [
            0.986351228389445,
            1.0
          ],
          "0.50": [
            0.8744313011828936,
            0.9208010923987255
          ],
          "0.80": [
            0.7425968109339408,
            0.7854223433242506
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0
          ],
          [
            0.04,
            0.0
          ],
          [
            0.06,
            0.0
          ],
          [
            0.08,
            0.0
          ],
          [
            0.1,
            0.0
          ],
          [
            0.12,
            0.0
          ],
          [
            0.14,
            0.0032
          ],
          [
            0.16,
            0.0028
          ],
          [
            0.18,
            0.0025
          ],
          [
            0.2,
            0.0023
          ],
          [
            0.22,
            0.0083
          ],
          [
            0.24,
            0.0095
          ],
          [
            0.26,
            0.0157
          ],
          [
            0.28,
            0.0195
          ],
          [
            0.3,
            0.0227
          ],
          [
            0.32,
            0.027
          ],
          [
            0.34,
            0.0335
          ],
          [
            0.36,
            0.0392
          ],
          [
            0.38,
            0.0479
          ],
          [
            0.4,
            0.0523
          ],
          [
            0.42,
            0.0596
          ],
          [
            0.44,
            0.0682
          ],
          [
            0.46,
            0.0723
          ],
          [
            0.48,
            0.0863
          ],
          [
            0.5,
            0.1037
          ],
          [
            0.52,
            0.112
          ],
          [
            0.54,
            0.1213
          ],
          [
            0.56,
            0.1276
          ],
          [
            0.58,
            0.1387
          ],
          [
            0.6,
            0.1441
          ],
          [
            0.62,
            0.1534
          ],
          [
            0.64,
            0.1635
          ],
          [
            0.66,
            0.1751
          ],
          [
            0.68,
            0.1873
          ],
          [
            0.7,
            0.1956
          ],
          [
            0.72,
            0.2028
          ],
          [
            0.74,
            0.2118
          ],
          [
            0.76,
            0.2167
          ],
          [
            0.78,
            0.2272
          ],
          [
            0.8,
            0.2354
          ],
          [
            0.82,
            0.2463
          ],
          [
            0.84,
            0.2564
          ],
          [
            0.86,
            0.2662
          ],
          [
            0.88,
            0.2721
          ],
          [
            0.9,
            0.2837
          ],
          [
            0.92,
            0.2923
          ],
          [
            0.94,
            0.2996
          ],
          [
            0.96,
            0.3057
          ],
          [
            0.98,
            0.3129
          ],
          [
            1.0,
            0.3226
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 74,
            "mean_confidence": 0.2698540538754287,
            "accuracy": 0.2702702702702703
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 215,
            "mean_confidence": 0.3548622220926534,
            "accuracy": 0.3395348837209302
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 214,
            "mean_confidence": 0.45084987177442515,
            "accuracy": 0.3644859813084112
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 208,
            "mean_confidence": 0.5515679403748031,
            "accuracy": 0.5096153846153846
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 192,
            "mean_confidence": 0.6487832855937934,
            "accuracy": 0.5208333333333334
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 177,
            "mean_confidence": 0.7484555119209089,
            "accuracy": 0.655367231638418
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 224,
            "mean_confidence": 0.8491007283658923,
            "accuracy": 0.6741071428571429
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 894,
            "mean_confidence": 0.9767948340366254,
            "accuracy": 0.9451901565995525
          }
        ],
        "median_right": 0.9409136908147775,
        "median_wrong": 0.5215864181518555,
        "sure": {
          "questions": 443,
          "correct": 442,
          "interval": [
            0.9873254872500569,
            0.9996014237627048
          ]
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 903,
          "correct": 377,
          "interval": [
            0.385747704031804,
            0.4499457621700111
          ]
        },
        "ece_refit": 0.02073126888705256,
        "brier_refit": 0.1568762977852593,
        "brier_base": 0.21866857324088124,
        "refit_slopes": [
          0.7726312675820702,
          0.7964006951809445,
          0.7827790971302763,
          0.7992167169327984,
          0.7717933632123798
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        110,
        143,
        57,
        92,
        93,
        93,
        93,
        39,
        58,
        50,
        51,
        28,
        75,
        50,
        55,
        102,
        25,
        79,
        72,
        124
      ],
      "started": "2026-10-31T15:17:59Z",
      "updated": "2026-10-31T15:21:22Z",
      "format_of": null,
      "group": "pqr",
      "latency": {
        "source": "run",
        "runs": [
          322
        ],
        "concurrency": 1,
        "median_request_seconds": 0.08219127899792511,
        "p90_request_seconds": 0.11037726640497567
      },
      "single_question_latency": null
    },
    {
      "run": 326,
      "model": "drex-max-v1.1",
      "name": "Drex Max 1.1",
      "variant": null,
      "reasoning": null,
      "provider": "nace",
      "note": "Nace.AI's hosted Drex Max v1.1 (drex-max-v1.1, released 28 October 2026), a newer checkpoint than the open Drex Max v1 weights; Nace publishes no weights for it. Nace bills $0.03 per million input tokens during its launch, half the $0.06 list price the cost here uses.",
      "parameters": null,
      "active_parameters": null,
      "access": "Nace.AI System One API",
      "questions_per_request": 1,
      "concurrency": 1,
      "correct": 1481,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 581
        },
        "grammar": {
          "questions": 1542,
          "correct": 900
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 1478
        }
      },
      "tyt_mix": 0.8328184314795483,
      "questions_per_minute": 118.9534211632787,
      "request_seconds": 1108.6692480998754,
      "median_request_seconds": 0.3127196345012635,
      "p90_request_seconds": 0.448678748808743,
      "billing": "api",
      "api_equivalent_usd": 0.050805119999999995,
      "cost_basis": "api",
      "requests": 2198,
      "hardware": null,
      "input_tokens": 846752,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 357765,
      "confidence": {
        "source": "probability",
        "label": "Option score",
        "bins": [
          {
            "questions": 30,
            "correct": 4
          },
          {
            "questions": 612,
            "correct": 231
          },
          {
            "questions": 581,
            "correct": 333
          },
          {
            "questions": 325,
            "correct": 269
          },
          {
            "questions": 650,
            "correct": 644
          }
        ],
        "auroc": 0.8342209125915714,
        "auroc_interval": [
          0.8178667106215447,
          0.8509248893864279
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.9241537578886977,
            "auroc_interval": [
              0.9022992656795473,
              0.9468384326355341
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.7697213568708896,
            "auroc_interval": [
              0.7457432835820895,
              0.7914382777244506
            ]
          }
        },
        "aurc": 0.11795241001774175,
        "ece": 0.02920837835252908,
        "brier": 0.1538822896945315,
        "coverage_accuracy": {
          "0.20": 0.9954504094631483,
          "0.50": 0.9017288444040037,
          "0.80": 0.7600090991810737
        },
        "coverage_intervals": {
          "0.20": [
            0.9863574351978172,
            1.0
          ],
          "0.50": [
            0.8828337874659401,
            0.924476797088262
          ],
          "0.80": [
            0.7365725989986345,
            0.7805404178019981
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0
          ],
          [
            0.04,
            0.0
          ],
          [
            0.06,
            0.0
          ],
          [
            0.08,
            0.0
          ],
          [
            0.1,
            0.0
          ],
          [
            0.12,
            0.0
          ],
          [
            0.14,
            0.0
          ],
          [
            0.16,
            0.0028
          ],
          [
            0.18,
            0.0025
          ],
          [
            0.2,
            0.0045
          ],
          [
            0.22,
            0.0062
          ],
          [
            0.24,
            0.0095
          ],
          [
            0.26,
            0.0105
          ],
          [
            0.28,
            0.0097
          ],
          [
            0.3,
            0.0106
          ],
          [
            0.32,
            0.0156
          ],
          [
            0.34,
            0.0241
          ],
          [
            0.36,
            0.0341
          ],
          [
            0.38,
            0.0335
          ],
          [
            0.4,
            0.0432
          ],
          [
            0.42,
            0.0511
          ],
          [
            0.44,
            0.061
          ],
          [
            0.46,
            0.0723
          ],
          [
            0.48,
            0.0844
          ],
          [
            0.5,
            0.0983
          ],
          [
            0.52,
            0.1058
          ],
          [
            0.54,
            0.1095
          ],
          [
            0.56,
            0.1169
          ],
          [
            0.58,
            0.1332
          ],
          [
            0.6,
            0.1454
          ],
          [
            0.62,
            0.1569
          ],
          [
            0.64,
            0.1683
          ],
          [
            0.66,
            0.1799
          ],
          [
            0.68,
            0.1911
          ],
          [
            0.7,
            0.1976
          ],
          [
            0.72,
            0.2032
          ],
          [
            0.74,
            0.2121
          ],
          [
            0.76,
            0.2227
          ],
          [
            0.78,
            0.2324
          ],
          [
            0.8,
            0.24
          ],
          [
            0.82,
            0.2493
          ],
          [
            0.84,
            0.258
          ],
          [
            0.86,
            0.2662
          ],
          [
            0.88,
            0.2747
          ],
          [
            0.9,
            0.2837
          ],
          [
            0.92,
            0.2914
          ],
          [
            0.94,
            0.2972
          ],
          [
            0.96,
            0.3067
          ],
          [
            0.98,
            0.3157
          ],
          [
            1.0,
            0.3262
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 127,
            "mean_confidence": 0.26800227416603983,
            "accuracy": 0.2125984251968504
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 246,
            "mean_confidence": 0.3542756127477776,
            "accuracy": 0.37398373983739835
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 269,
            "mean_confidence": 0.4511783093148202,
            "accuracy": 0.4312267657992565
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 247,
            "mean_confidence": 0.5459953504447651,
            "accuracy": 0.5020242914979757
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 223,
            "mean_confidence": 0.6476426004811321,
            "accuracy": 0.6098654708520179
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 224,
            "mean_confidence": 0.7516058531031441,
            "accuracy": 0.7053571428571429
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 212,
            "mean_confidence": 0.8508556654153413,
            "accuracy": 0.8679245283018868
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 650,
            "mean_confidence": 0.9697152294833274,
            "accuracy": 0.9907692307692307
          }
        ],
        "median_right": 0.8492686289757672,
        "median_wrong": 0.4657882943945759,
        "sure": {
          "questions": 220,
          "correct": 220,
          "interval": [
            0.9828378639180564,
            1.0
          ]
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 1112,
          "correct": 495,
          "interval": [
            0.41617161589714946,
            0.47449386923980896
          ]
        },
        "ece_refit": 0.0297182811814658,
        "brier_refit": 0.15416223100387794,
        "brier_base": 0.2198646827772548,
        "refit_slopes": [
          1.072459508503225,
          1.1716091192594293,
          1.092826370243513,
          1.1403789663635684,
          1.1419488184108768
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        111,
        144,
        59,
        92,
        86,
        89,
        92,
        37,
        57,
        49,
        51,
        30,
        78,
        45,
        56,
        102,
        26,
        82,
        72,
        123
      ],
      "started": "2026-10-31T15:41:43Z",
      "updated": "2026-10-31T16:00:19Z",
      "format_of": null,
      "group": "pqr",
      "latency": {
        "source": "run",
        "runs": [
          326
        ],
        "concurrency": 1,
        "median_request_seconds": 0.3127196345012635,
        "p90_request_seconds": 0.448678748808743
      },
      "single_question_latency": null
    },
    {
      "run": 374,
      "model": "baselabs/mammoth-1",
      "name": "Mammoth-1",
      "variant": null,
      "reasoning": null,
      "provider": "baselabs",
      "note": "Base Labs' released mammoth-1 weights (baselabs/mammoth-1 at 61cec44), a decision model post-trained from Qwen3.5-27B that also reads images. It ran on our hardware through its model card's loader (qev's DecisionEngine from OneJev at 81ce62f with the checkpoint's serve.patch, as the card's setup.sh installs it) behind our HTTP server (servers/baselabs_server.py), one request at a time. That runtime reads images only through an engine the patch leaves without the attention the model was trained with, so it is not on the Page images tab. The card mentions a trained \"I don't know\" option, but its answers have no such output, and our questions offer five options. Probabilities rounded to 6 decimals by the model's own loader. They sum to 1 only within that rounding, and the check before its runs allowed for it. Its card lists English only.",
      "parameters": 27356728560,
      "active_parameters": null,
      "access": "Base Labs server (qev, the model card's loader), self-hosted, BF16",
      "questions_per_request": 1,
      "concurrency": 1,
      "correct": 1475,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 593
        },
        "grammar": {
          "questions": 1542,
          "correct": 882
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 1472
        }
      },
      "tyt_mix": 0.8458670933377622,
      "questions_per_minute": 355.75443930545117,
      "request_seconds": 370.70514216905576,
      "median_request_seconds": 0.1691462635062635,
      "p90_request_seconds": 0.19061791689891833,
      "billing": "local",
      "api_equivalent_usd": 0.087093,
      "cost_basis": "maker",
      "requests": 2198,
      "hardware": "1 × RTX PRO 6000 Blackwell (96 GB); BF16",
      "input_tokens": 870930,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 2198,
      "confidence": {
        "source": "probability",
        "label": "Option score",
        "bins": [
          {
            "questions": 7,
            "correct": 1
          },
          {
            "questions": 537,
            "correct": 187
          },
          {
            "questions": 512,
            "correct": 273
          },
          {
            "questions": 318,
            "correct": 223
          },
          {
            "questions": 824,
            "correct": 791
          }
        ],
        "auroc": 0.8327102234099913,
        "auroc_interval": [
          0.8152869049831065,
          0.8504099717489185
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.9225086324580423,
            "auroc_interval": [
              0.8960750853242321,
              0.9469380802316668
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.7620336013193156,
            "auroc_interval": [
              0.7378503508628222,
              0.7854264870167026
            ]
          }
        },
        "aurc": 0.12041925681288687,
        "ece": 0.05571809872611476,
        "brier": 0.15971927846674477,
        "coverage_accuracy": {
          "0.20": 0.9931756141947224,
          "0.50": 0.8953594176524113,
          "0.80": 0.7575068243858052
        },
        "coverage_intervals": {
          "0.20": [
            0.9863325740318907,
            1.0
          ],
          "0.50": [
            0.8762511373976342,
            0.9162112932604736
          ],
          "0.80": [
            0.7340909090909091,
            0.7808234758871702
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0
          ],
          [
            0.04,
            0.0
          ],
          [
            0.06,
            0.0
          ],
          [
            0.08,
            0.0
          ],
          [
            0.1,
            0.0
          ],
          [
            0.12,
            0.0
          ],
          [
            0.14,
            0.0
          ],
          [
            0.16,
            0.0
          ],
          [
            0.18,
            0.0025
          ],
          [
            0.2,
            0.0068
          ],
          [
            0.22,
            0.0062
          ],
          [
            0.24,
            0.0076
          ],
          [
            0.26,
            0.007
          ],
          [
            0.28,
            0.0097
          ],
          [
            0.3,
            0.0136
          ],
          [
            0.32,
            0.0227
          ],
          [
            0.34,
            0.0272
          ],
          [
            0.36,
            0.0329
          ],
          [
            0.38,
            0.0443
          ],
          [
            0.4,
            0.0514
          ],
          [
            0.42,
            0.0617
          ],
          [
            0.44,
            0.0765
          ],
          [
            0.46,
            0.0881
          ],
          [
            0.48,
            0.0948
          ],
          [
            0.5,
            0.1046
          ],
          [
            0.52,
            0.112
          ],
          [
            0.54,
            0.1205
          ],
          [
            0.56,
            0.1284
          ],
          [
            0.58,
            0.1341
          ],
          [
            0.6,
            0.1395
          ],
          [
            0.62,
            0.1519
          ],
          [
            0.64,
            0.1635
          ],
          [
            0.66,
            0.1728
          ],
          [
            0.68,
            0.1817
          ],
          [
            0.7,
            0.198
          ],
          [
            0.72,
            0.2038
          ],
          [
            0.74,
            0.2161
          ],
          [
            0.76,
            0.2254
          ],
          [
            0.78,
            0.233
          ],
          [
            0.8,
            0.2425
          ],
          [
            0.82,
            0.2541
          ],
          [
            0.84,
            0.2601
          ],
          [
            0.86,
            0.2687
          ],
          [
            0.88,
            0.2767
          ],
          [
            0.9,
            0.2852
          ],
          [
            0.92,
            0.2957
          ],
          [
            0.94,
            0.3025
          ],
          [
            0.96,
            0.3123
          ],
          [
            0.98,
            0.3203
          ],
          [
            1.0,
            0.3289
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 62,
            "mean_confidence": 0.27671475806451623,
            "accuracy": 0.24193548387096775
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 225,
            "mean_confidence": 0.3520465955555558,
            "accuracy": 0.3111111111111111
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 257,
            "mean_confidence": 0.45387767704280163,
            "accuracy": 0.40077821011673154
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 222,
            "mean_confidence": 0.5533114189189194,
            "accuracy": 0.44594594594594594
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 192,
            "mean_confidence": 0.6515547083333331,
            "accuracy": 0.5625
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 187,
            "mean_confidence": 0.7503166256684491,
            "accuracy": 0.679144385026738
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 229,
            "mean_confidence": 0.8529821135371178,
            "accuracy": 0.7074235807860262
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 824,
            "mean_confidence": 0.9719408580097089,
            "accuracy": 0.9599514563106796
          }
        ],
        "median_right": 0.921594,
        "median_wrong": 0.505521,
        "sure": {
          "questions": 285,
          "correct": 285,
          "interval": [
            0.9866999767346533,
            1.0
          ]
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 958,
          "correct": 395,
          "interval": [
            0.3815561692581953,
            0.443778896307694
          ]
        },
        "ece_refit": 0.020895586306424477,
        "brier_refit": 0.15563579929695506,
        "brier_base": 0.22090571706835546,
        "refit_slopes": [
          0.9234793792447812,
          0.9480904839332088,
          0.8990763342488687,
          0.9352123345470247,
          0.9218226344170842
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        111,
        142,
        58,
        94,
        92,
        96,
        97,
        38,
        53,
        42,
        51,
        34,
        76,
        52,
        44,
        88,
        30,
        77,
        75,
        125
      ],
      "started": "2026-11-01T18:14:08Z",
      "updated": "2026-11-01T18:20:26Z",
      "format_of": null,
      "group": "pqrs",
      "latency": {
        "source": "run",
        "runs": [
          374
        ],
        "concurrency": 1,
        "median_request_seconds": 0.1691462635062635,
        "p90_request_seconds": 0.19061791689891833
      },
      "single_question_latency": null
    },
    {
      "run": 217,
      "model": "pplx-decider-v1.2",
      "name": "Perplexity Decider 27B v1.2",
      "variant": "local",
      "reasoning": null,
      "provider": "pplx-decider",
      "note": "Perplexity's released checkpoint, run on our hardware. An API run is pending; this is an additional local setup.",
      "parameters": 26086635760,
      "active_parameters": null,
      "access": "Perplexity Decider server, self-hosted",
      "questions_per_request": 1,
      "concurrency": 1,
      "correct": 1474,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 585
        },
        "grammar": {
          "questions": 1542,
          "correct": 889
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 1471
        }
      },
      "tyt_mix": 0.8366005405555029,
      "questions_per_minute": 97.394776360673,
      "request_seconds": 1354.0767269860662,
      "median_request_seconds": 0.5795541804982349,
      "p90_request_seconds": 0.7866766697028652,
      "billing": "local",
      "api_equivalent_usd": 0.017066920000000003,
      "cost_basis": "maker",
      "requests": 2198,
      "hardware": "1 × DGX Spark, GB10 (128 GB unified memory); BF16",
      "input_tokens": 846752,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 0,
      "confidence": {
        "source": "probability",
        "label": "Option score",
        "bins": [
          {
            "questions": 21,
            "correct": 1
          },
          {
            "questions": 537,
            "correct": 194
          },
          {
            "questions": 538,
            "correct": 281
          },
          {
            "questions": 315,
            "correct": 233
          },
          {
            "questions": 787,
            "correct": 765
          }
        ],
        "auroc": 0.8367035990314625,
        "auroc_interval": [
          0.8201133487194758,
          0.8517750445028236
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.9248826291079812,
            "auroc_interval": [
              0.9039046377756055,
              0.9452028409774889
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.7727387828435688,
            "auroc_interval": [
              0.7490899788785221,
              0.7952706678434833
            ]
          }
        },
        "aurc": 0.11914230666460575,
        "ece": 0.0443262835543908,
        "brier": 0.15589100038042744,
        "coverage_accuracy": {
          "0.20": 0.9931756141947224,
          "0.50": 0.905368516833485,
          "0.80": 0.7558007279344858
        },
        "coverage_intervals": {
          "0.20": [
            0.9840764331210191,
            1.0
          ],
          "0.50": [
            0.8857532999544834,
            0.9262295081967213
          ],
          "0.80": [
            0.7338195077484048,
            0.7768425841674249
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0
          ],
          [
            0.04,
            0.0
          ],
          [
            0.06,
            0.0
          ],
          [
            0.08,
            0.0
          ],
          [
            0.1,
            0.0
          ],
          [
            0.12,
            0.0
          ],
          [
            0.14,
            0.0
          ],
          [
            0.16,
            0.0
          ],
          [
            0.18,
            0.0
          ],
          [
            0.2,
            0.0068
          ],
          [
            0.22,
            0.0083
          ],
          [
            0.24,
            0.0133
          ],
          [
            0.26,
            0.014
          ],
          [
            0.28,
            0.0179
          ],
          [
            0.3,
            0.0167
          ],
          [
            0.32,
            0.0199
          ],
          [
            0.34,
            0.0227
          ],
          [
            0.36,
            0.0291
          ],
          [
            0.38,
            0.035
          ],
          [
            0.4,
            0.0478
          ],
          [
            0.42,
            0.0596
          ],
          [
            0.44,
            0.0682
          ],
          [
            0.46,
            0.0781
          ],
          [
            0.48,
            0.0844
          ],
          [
            0.5,
            0.0946
          ],
          [
            0.52,
            0.1032
          ],
          [
            0.54,
            0.1104
          ],
          [
            0.56,
            0.1227
          ],
          [
            0.58,
            0.1324
          ],
          [
            0.6,
            0.1403
          ],
          [
            0.62,
            0.1504
          ],
          [
            0.64,
            0.1633
          ],
          [
            0.66,
            0.1742
          ],
          [
            0.68,
            0.1824
          ],
          [
            0.7,
            0.1956
          ],
          [
            0.72,
            0.2076
          ],
          [
            0.74,
            0.2164
          ],
          [
            0.76,
            0.2266
          ],
          [
            0.78,
            0.2365
          ],
          [
            0.8,
            0.2442
          ],
          [
            0.82,
            0.2526
          ],
          [
            0.84,
            0.2589
          ],
          [
            0.86,
            0.2672
          ],
          [
            0.88,
            0.275
          ],
          [
            0.9,
            0.2851
          ],
          [
            0.92,
            0.2938
          ],
          [
            0.94,
            0.3015
          ],
          [
            0.96,
            0.31
          ],
          [
            0.98,
            0.319
          ],
          [
            1.0,
            0.3294
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 95,
            "mean_confidence": 0.26555401554473923,
            "accuracy": 0.22105263157894736
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 223,
            "mean_confidence": 0.3471734936690066,
            "accuracy": 0.32286995515695066
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 240,
            "mean_confidence": 0.45074156154635336,
            "accuracy": 0.425
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 243,
            "mean_confidence": 0.5481161982133999,
            "accuracy": 0.4403292181069959
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 188,
            "mean_confidence": 0.6506931566368025,
            "accuracy": 0.5531914893617021
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 199,
            "mean_confidence": 0.746914866017765,
            "accuracy": 0.6834170854271356
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 223,
            "mean_confidence": 0.8504062930852067,
            "accuracy": 0.7488789237668162
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 787,
            "mean_confidence": 0.9743388368581654,
            "accuracy": 0.9720457433290979
          }
        ],
        "median_right": 0.9120518105191167,
        "median_wrong": 0.4981026235648757,
        "sure": {
          "questions": 360,
          "correct": 360,
          "interval": [
            0.9894415591839967,
            1.0
          ]
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 989,
          "correct": 406,
          "interval": [
            0.38026026056212714,
            0.4414635672871491
          ]
        },
        "ece_refit": 0.025984092711145913,
        "brier_refit": 0.15352428153042574,
        "brier_base": 0.22091402135118196,
        "refit_slopes": [
          0.8985967334816428,
          0.9730945445992665,
          0.9288062936279472,
          0.9727544078775495,
          0.9691618210337565
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        109,
        144,
        59,
        92,
        90,
        91,
        93,
        38,
        55,
        47,
        49,
        28,
        79,
        49,
        51,
        97,
        21,
        81,
        74,
        127
      ],
      "started": "2026-10-17T18:04:03Z",
      "updated": "2026-10-17T18:26:45Z",
      "format_of": null,
      "group": "pqr",
      "latency": {
        "source": "run",
        "runs": [
          217
        ],
        "concurrency": 1,
        "median_request_seconds": 0.5795541804982349,
        "p90_request_seconds": 0.7866766697028652
      },
      "single_question_latency": null
    },
    {
      "run": 145,
      "model": "pplx-decider-v1.1-27b",
      "name": "Perplexity Decider 27B v1.1",
      "variant": null,
      "reasoning": null,
      "provider": "perplexity",
      "note": null,
      "parameters": 26086635760,
      "active_parameters": null,
      "access": "Perplexity Decisions API",
      "questions_per_request": 1,
      "concurrency": 1,
      "correct": 1470,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 584
        },
        "grammar": {
          "questions": 1542,
          "correct": 886
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 1467
        }
      },
      "tyt_mix": 0.8350024516782133,
      "questions_per_minute": 230.1414509049513,
      "request_seconds": 573.0388831800083,
      "median_request_seconds": 0.23294744099257514,
      "p90_request_seconds": 0.29029369752388445,
      "billing": "api",
      "api_equivalent_usd": 0.017066920000000003,
      "cost_basis": "api",
      "requests": 2198,
      "hardware": null,
      "input_tokens": 853346,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 2198,
      "confidence": {
        "source": "probability",
        "label": "Option score",
        "bins": [
          {
            "questions": 13,
            "correct": 2
          },
          {
            "questions": 581,
            "correct": 196
          },
          {
            "questions": 548,
            "correct": 301
          },
          {
            "questions": 289,
            "correct": 229
          },
          {
            "questions": 767,
            "correct": 742
          }
        ],
        "auroc": 0.84085744187785,
        "auroc_interval": [
          0.8257913831780878,
          0.8569916377573434
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.9166428843226788,
            "auroc_interval": [
              0.8924109798589275,
              0.9389083390117249
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.7815528134118813,
            "auroc_interval": [
              0.7589892896174864,
              0.8030251000106206
            ]
          }
        },
        "aurc": 0.11864539432837173,
        "ece": 0.03844291588122355,
        "brier": 0.15376309789004014,
        "coverage_accuracy": {
          "0.20": 0.9931756141947224,
          "0.50": 0.9099181073703366,
          "0.80": 0.7571656050955414
        },
        "coverage_intervals": {
          "0.20": [
            0.9863263445761167,
            1.0
          ],
          "0.50": [
            0.8915717539863326,
            0.929968167348795
          ],
          "0.80": [
            0.7358962693357597,
            0.7806818181818181
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0
          ],
          [
            0.04,
            0.0
          ],
          [
            0.06,
            0.0
          ],
          [
            0.08,
            0.0
          ],
          [
            0.1,
            0.0
          ],
          [
            0.12,
            0.0
          ],
          [
            0.14,
            0.0
          ],
          [
            0.16,
            0.0
          ],
          [
            0.18,
            0.0
          ],
          [
            0.2,
            0.0068
          ],
          [
            0.22,
            0.0062
          ],
          [
            0.24,
            0.0076
          ],
          [
            0.26,
            0.0105
          ],
          [
            0.28,
            0.013
          ],
          [
            0.3,
            0.0152
          ],
          [
            0.32,
            0.0199
          ],
          [
            0.34,
            0.0281
          ],
          [
            0.36,
            0.0379
          ],
          [
            0.38,
            0.0419
          ],
          [
            0.4,
            0.0512
          ],
          [
            0.42,
            0.0563
          ],
          [
            0.44,
            0.0632
          ],
          [
            0.46,
            0.0703
          ],
          [
            0.48,
            0.0797
          ],
          [
            0.5,
            0.0901
          ],
          [
            0.52,
            0.0989
          ],
          [
            0.54,
            0.1121
          ],
          [
            0.56,
            0.1251
          ],
          [
            0.58,
            0.1341
          ],
          [
            0.6,
            0.1447
          ],
          [
            0.62,
            0.1556
          ],
          [
            0.64,
            0.1642
          ],
          [
            0.66,
            0.1758
          ],
          [
            0.68,
            0.1831
          ],
          [
            0.7,
            0.1934
          ],
          [
            0.72,
            0.2009
          ],
          [
            0.74,
            0.2118
          ],
          [
            0.76,
            0.2188
          ],
          [
            0.78,
            0.231
          ],
          [
            0.8,
            0.2428
          ],
          [
            0.82,
            0.2504
          ],
          [
            0.84,
            0.26
          ],
          [
            0.86,
            0.2682
          ],
          [
            0.88,
            0.2766
          ],
          [
            0.9,
            0.2857
          ],
          [
            0.92,
            0.2942
          ],
          [
            0.94,
            0.3035
          ],
          [
            0.96,
            0.3109
          ],
          [
            0.98,
            0.3213
          ],
          [
            1.0,
            0.3312
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 90,
            "mean_confidence": 0.27396246234511235,
            "accuracy": 0.17777777777777778
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 234,
            "mean_confidence": 0.3531100341373387,
            "accuracy": 0.3333333333333333
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 270,
            "mean_confidence": 0.451745568000649,
            "accuracy": 0.3851851851851852
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 241,
            "mean_confidence": 0.5473221389345523,
            "accuracy": 0.5020746887966805
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 200,
            "mean_confidence": 0.6483083987605979,
            "accuracy": 0.535
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 193,
            "mean_confidence": 0.7467035127093767,
            "accuracy": 0.7046632124352331
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 203,
            "mean_confidence": 0.8518054351975634,
            "accuracy": 0.8177339901477833
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 767,
            "mean_confidence": 0.973461876941584,
            "accuracy": 0.9674054758800521
          }
        ],
        "median_right": 0.9025795337471803,
        "median_wrong": 0.47858612711299375,
        "sure": {
          "questions": 349,
          "correct": 349,
          "interval": [
            0.9891123949103505,
            0.9999999999999999
          ]
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 1035,
          "correct": 426,
          "interval": [
            0.3819929371258632,
            0.4418493117790709
          ]
        },
        "ece_refit": 0.01559104193890272,
        "brier_refit": 0.15195856252968057,
        "brier_base": 0.22175639016012455,
        "refit_slopes": [
          1.0030744638028724,
          1.0394874577966549,
          1.0077649062166523,
          1.0096077526153153,
          0.9690820941829514
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        110,
        143,
        59,
        93,
        88,
        91,
        89,
        38,
        59,
        47,
        50,
        27,
        77,
        53,
        52,
        93,
        19,
        83,
        73,
        126
      ],
      "started": "2026-10-09T08:15:06Z",
      "updated": "2026-10-09T08:24:44Z",
      "format_of": null,
      "group": "pqrs",
      "latency": {
        "source": "run",
        "runs": [
          145
        ],
        "concurrency": 1,
        "median_request_seconds": 0.23294744099257514,
        "p90_request_seconds": 0.29029369752388445
      },
      "single_question_latency": null
    },
    {
      "run": 189,
      "model": "gemma-4-31B-it",
      "name": "Gemma 4 31B",
      "variant": null,
      "reasoning": "off",
      "provider": "vllm",
      "note": null,
      "parameters": 31273088876,
      "active_parameters": null,
      "access": "vLLM, self-hosted, FP8",
      "questions_per_request": 1,
      "concurrency": 1,
      "correct": 1468,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 584
        },
        "grammar": {
          "questions": 1542,
          "correct": 884
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 1464
        }
      },
      "tyt_mix": 0.8347754737275,
      "questions_per_minute": 195.4762313352229,
      "request_seconds": 674.6600295042444,
      "median_request_seconds": 0.29770804748113733,
      "p90_request_seconds": 0.3055964823171962,
      "billing": "local",
      "api_equivalent_usd": null,
      "cost_basis": null,
      "requests": 2198,
      "hardware": "1 × RTX PRO 6000 Blackwell (96 GB); FP8 weights and KV cache",
      "input_tokens": 698986,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 25184,
      "confidence": {
        "source": "probability",
        "label": "Answer letter probability",
        "bins": [
          {
            "questions": 0,
            "correct": 0
          },
          {
            "questions": 6,
            "correct": 3
          },
          {
            "questions": 57,
            "correct": 15
          },
          {
            "questions": 65,
            "correct": 16
          },
          {
            "questions": 2070,
            "correct": 1434
          }
        ],
        "auroc": 0.820290396028517,
        "auroc_interval": [
          0.8023188871973229,
          0.8378121484087059
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.884560502283105,
            "auroc_interval": [
              0.8387033059279559,
              0.9235870818915801
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.7532595689667029,
            "auroc_interval": [
              0.730494905385735,
              0.7790830603330603
            ]
          }
        },
        "aurc": 0.1290815051869733,
        "ece": 0.31180072836162526,
        "brier": 0.30873282999039137,
        "coverage_accuracy": {
          "0.20": 0.9727024567788899,
          "0.50": 0.8944494995450409,
          "0.80": 0.7467015468607825
        },
        "coverage_intervals": {
          "0.20": [
            0.958997722095672,
            0.9886208466090123
          ],
          "0.50": [
            0.8706150341685649,
            0.9170086639306886
          ],
          "0.80": [
            0.7255681818181818,
            0.7688979963570127
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0
          ],
          [
            0.04,
            0.0
          ],
          [
            0.06,
            0.0076
          ],
          [
            0.08,
            0.0057
          ],
          [
            0.1,
            0.0127
          ],
          [
            0.12,
            0.0152
          ],
          [
            0.14,
            0.0195
          ],
          [
            0.16,
            0.0227
          ],
          [
            0.18,
            0.0227
          ],
          [
            0.2,
            0.0273
          ],
          [
            0.22,
            0.028
          ],
          [
            0.24,
            0.0303
          ],
          [
            0.26,
            0.0297
          ],
          [
            0.28,
            0.0276
          ],
          [
            0.3,
            0.0273
          ],
          [
            0.32,
            0.0256
          ],
          [
            0.34,
            0.0281
          ],
          [
            0.36,
            0.0341
          ],
          [
            0.38,
            0.0419
          ],
          [
            0.4,
            0.0489
          ],
          [
            0.42,
            0.0552
          ],
          [
            0.44,
            0.0672
          ],
          [
            0.46,
            0.0792
          ],
          [
            0.48,
            0.0938
          ],
          [
            0.5,
            0.1056
          ],
          [
            0.52,
            0.1181
          ],
          [
            0.54,
            0.1264
          ],
          [
            0.56,
            0.1341
          ],
          [
            0.58,
            0.1467
          ],
          [
            0.6,
            0.1591
          ],
          [
            0.62,
            0.1644
          ],
          [
            0.64,
            0.1754
          ],
          [
            0.66,
            0.1838
          ],
          [
            0.68,
            0.1958
          ],
          [
            0.7,
            0.2071
          ],
          [
            0.72,
            0.2186
          ],
          [
            0.74,
            0.2272
          ],
          [
            0.76,
            0.2373
          ],
          [
            0.78,
            0.2447
          ],
          [
            0.8,
            0.2533
          ],
          [
            0.82,
            0.2613
          ],
          [
            0.84,
            0.2708
          ],
          [
            0.86,
            0.2774
          ],
          [
            0.88,
            0.2854
          ],
          [
            0.9,
            0.2917
          ],
          [
            0.92,
            0.2987
          ],
          [
            0.94,
            0.3064
          ],
          [
            0.96,
            0.3161
          ],
          [
            0.98,
            0.3236
          ],
          [
            1.0,
            0.3321
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 6,
            "mean_confidence": 0.49392629054354686,
            "accuracy": 0.5
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 21,
            "mean_confidence": 0.5605667780804391,
            "accuracy": 0.23809523809523808
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 23,
            "mean_confidence": 0.6509198102049357,
            "accuracy": 0.21739130434782608
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 34,
            "mean_confidence": 0.7593847441845082,
            "accuracy": 0.29411764705882354
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 44,
            "mean_confidence": 0.8610995315906476,
            "accuracy": 0.25
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 2070,
            "mean_confidence": 0.9950971207804102,
            "accuracy": 0.6927536231884058
          }
        ],
        "median_right": 0.9999999355963465,
        "median_wrong": 0.9984214825072699,
        "sure": {
          "questions": 1826,
          "correct": 1339,
          "interval": [
            0.7125381376763197,
            0.7530759351483446
          ]
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 50,
          "correct": 13,
          "interval": [
            0.15871379651367384,
            0.3955342198899992
          ]
        },
        "ece_refit": 0.06206751415407556,
        "brier_refit": 0.17029715372929774,
        "brier_base": 0.22194104002884293,
        "refit_slopes": [
          0.25992673686132545,
          0.261561928788477,
          0.2500130719874258,
          0.26339541325335497,
          0.25868844756742193
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        113,
        141,
        61,
        94,
        84,
        91,
        91,
        34,
        60,
        45,
        48,
        33,
        70,
        48,
        43,
        88,
        32,
        93,
        74,
        125
      ],
      "started": "2026-10-13T16:20:09Z",
      "updated": "2026-10-13T19:45:13Z",
      "format_of": 119,
      "group": "pqrs",
      "latency": {
        "source": "run",
        "runs": [
          189
        ],
        "concurrency": 1,
        "median_request_seconds": 0.29770804748113733,
        "p90_request_seconds": 0.3055964823171962
      },
      "single_question_latency": null
    },
    {
      "run": 120,
      "model": "deepseek-v4.1-flash",
      "name": "DeepSeek-V4.1-Flash, first release",
      "variant": null,
      "reasoning": "off",
      "provider": "vllm",
      "note": null,
      "parameters": 552000000000,
      "active_parameters": [
        8000000000,
        16000000000
      ],
      "access": "SGLang, self-hosted, published mixed precision; first-release weights (revision fb2764a), not the 1017 update",
      "questions_per_request": 16,
      "concurrency": 1,
      "correct": 1449,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 576
        },
        "grammar": {
          "questions": 1542,
          "correct": 873
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 1447
        }
      },
      "tyt_mix": 0.8234661193888204,
      "questions_per_minute": 177.54848553740285,
      "request_seconds": 742.783018401009,
      "median_request_seconds": 4.361823945015203,
      "p90_request_seconds": 7.860654652415542,
      "billing": "local",
      "api_equivalent_usd": null,
      "cost_basis": null,
      "requests": 138,
      "hardware": "4 × RTX PRO 6000 Blackwell (96 GB); FP4 experts, FP8 and BF16 layers",
      "input_tokens": 665866,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 13878,
      "confidence": {
        "source": "probability",
        "label": "Answer letter probability",
        "bins": [
          {
            "questions": 3,
            "correct": 1
          },
          {
            "questions": 307,
            "correct": 105
          },
          {
            "questions": 506,
            "correct": 207
          },
          {
            "questions": 313,
            "correct": 179
          },
          {
            "questions": 1069,
            "correct": 957
          }
        ],
        "auroc": 0.8247840921550795,
        "auroc_interval": [
          0.8090117907065028,
          0.8409324220465525
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.9104817708333334,
            "auroc_interval": [
              0.8840836509735243,
              0.934260429835651
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.7573287308852008,
            "auroc_interval": [
              0.7346649153024596,
              0.7807149057050422
            ]
          }
        },
        "aurc": 0.12959397834920494,
        "ece": 0.13539692525320116,
        "brier": 0.18661813604410746,
        "coverage_accuracy": {
          "0.20": 0.9954504094631483,
          "0.50": 0.8862602365787079,
          "0.80": 0.741810737033667
        },
        "coverage_intervals": {
          "0.20": [
            0.9886208466090123,
            1.0
          ],
          "0.50": [
            0.8650250797993616,
            0.9088422971741112
          ],
          "0.80": [
            0.7191624943104232,
            0.7636363636363637
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0
          ],
          [
            0.04,
            0.0
          ],
          [
            0.06,
            0.0
          ],
          [
            0.08,
            0.0
          ],
          [
            0.1,
            0.0
          ],
          [
            0.12,
            0.0038
          ],
          [
            0.14,
            0.0032
          ],
          [
            0.16,
            0.0028
          ],
          [
            0.18,
            0.0025
          ],
          [
            0.2,
            0.0045
          ],
          [
            0.22,
            0.0041
          ],
          [
            0.24,
            0.0057
          ],
          [
            0.26,
            0.0087
          ],
          [
            0.28,
            0.0179
          ],
          [
            0.3,
            0.0188
          ],
          [
            0.32,
            0.0313
          ],
          [
            0.34,
            0.0361
          ],
          [
            0.36,
            0.0392
          ],
          [
            0.38,
            0.047
          ],
          [
            0.4,
            0.0591
          ],
          [
            0.42,
            0.0628
          ],
          [
            0.44,
            0.0775
          ],
          [
            0.46,
            0.091
          ],
          [
            0.48,
            0.1005
          ],
          [
            0.5,
            0.1137
          ],
          [
            0.52,
            0.1207
          ],
          [
            0.54,
            0.1297
          ],
          [
            0.56,
            0.1414
          ],
          [
            0.58,
            0.149
          ],
          [
            0.6,
            0.1621
          ],
          [
            0.62,
            0.1717
          ],
          [
            0.64,
            0.1825
          ],
          [
            0.66,
            0.19
          ],
          [
            0.68,
            0.2014
          ],
          [
            0.7,
            0.2116
          ],
          [
            0.72,
            0.219
          ],
          [
            0.74,
            0.2296
          ],
          [
            0.76,
            0.2421
          ],
          [
            0.78,
            0.252
          ],
          [
            0.8,
            0.2582
          ],
          [
            0.82,
            0.2652
          ],
          [
            0.84,
            0.2769
          ],
          [
            0.86,
            0.2894
          ],
          [
            0.88,
            0.2942
          ],
          [
            0.9,
            0.3019
          ],
          [
            0.92,
            0.3116
          ],
          [
            0.94,
            0.3176
          ],
          [
            0.96,
            0.3247
          ],
          [
            0.98,
            0.3329
          ],
          [
            1.0,
            0.3408
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 21,
            "mean_confidence": 0.2720217646474662,
            "accuracy": 0.23809523809523808
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 102,
            "mean_confidence": 0.35871668653728334,
            "accuracy": 0.3137254901960784
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 187,
            "mean_confidence": 0.45084843878086894,
            "accuracy": 0.3689839572192513
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 209,
            "mean_confidence": 0.5512095121489691,
            "accuracy": 0.3492822966507177
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 198,
            "mean_confidence": 0.6443963171907059,
            "accuracy": 0.42424242424242425
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 187,
            "mean_confidence": 0.7476381027622825,
            "accuracy": 0.5133689839572193
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 225,
            "mean_confidence": 0.8529687096085111,
            "accuracy": 0.5911111111111111
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 1069,
            "mean_confidence": 0.9779912823655359,
            "accuracy": 0.8952291861552854
          }
        ],
        "median_right": 0.9714270404055687,
        "median_wrong": 0.6195863390284381,
        "sure": {
          "questions": 576,
          "correct": 571,
          "interval": [
            0.9798419495618019,
            0.9962867035048287
          ]
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 717,
          "correct": 263,
          "interval": [
            0.33232662618967,
            0.4027053142532788
          ]
        },
        "ece_refit": 0.021718273913082113,
        "brier_refit": 0.16003599065854884,
        "brier_base": 0.22509459344626093,
        "refit_slopes": [
          0.6946005643892169,
          0.691605231367296,
          0.6863304271779487,
          0.6787721474717973,
          0.6890351950129143
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        111,
        140,
        56,
        94,
        85,
        90,
        84,
        34,
        60,
        46,
        46,
        28,
        69,
        53,
        46,
        82,
        31,
        102,
        65,
        127
      ],
      "started": "2026-10-05T18:49:31Z",
      "updated": "2026-10-05T19:01:55Z",
      "format_of": null,
      "group": "pqrs",
      "latency": {
        "source": "run",
        "runs": [
          120
        ],
        "concurrency": 1,
        "median_request_seconds": 4.361823945015203,
        "p90_request_seconds": 7.860654652415542
      },
      "single_question_latency": null
    },
    {
      "run": 177,
      "model": "subq-snap-preview",
      "name": "SubQ Snap Preview",
      "variant": null,
      "reasoning": null,
      "provider": "subq",
      "note": null,
      "parameters": null,
      "active_parameters": null,
      "access": "SubQ Decisions API (public preview)",
      "questions_per_request": 1,
      "concurrency": 1,
      "correct": 1434,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 575
        },
        "grammar": {
          "questions": 1542,
          "correct": 859
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 1430
        }
      },
      "tyt_mix": 0.8206196517826074,
      "questions_per_minute": 150.8851709604276,
      "request_seconds": 874.042155107396,
      "median_request_seconds": 0.38925252700573765,
      "p90_request_seconds": 0.4138357652031118,
      "billing": "api",
      "api_equivalent_usd": 0.042667300000000005,
      "cost_basis": "api",
      "requests": 2198,
      "hardware": null,
      "input_tokens": 853346,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 0,
      "confidence": {
        "source": "probability",
        "label": "Option score",
        "bins": [
          {
            "questions": 10,
            "correct": 2
          },
          {
            "questions": 608,
            "correct": 207
          },
          {
            "questions": 558,
            "correct": 297
          },
          {
            "questions": 428,
            "correct": 350
          },
          {
            "questions": 594,
            "correct": 578
          }
        ],
        "auroc": 0.8247049953631697,
        "auroc_interval": [
          0.8091328269236069,
          0.8416059539513897
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.9040472356414385,
            "auroc_interval": [
              0.8743510494313476,
              0.9298336017176597
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.7583948784466258,
            "auroc_interval": [
              0.7344124577399856,
              0.7850450629232795
            ]
          }
        },
        "aurc": 0.13491098831141704,
        "ece": 0.048948969200484,
        "brier": 0.16338679079365506,
        "coverage_accuracy": {
          "0.20": 0.986351228389445,
          "0.50": 0.8880800727934486,
          "0.80": 0.7327115559599636
        },
        "coverage_intervals": {
          "0.20": [
            0.9727396637891867,
            0.9954627949183303
          ],
          "0.50": [
            0.8632634457611669,
            0.9079307201458523
          ],
          "0.80": [
            0.7119318181818182,
            0.7535179300953245
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0
          ],
          [
            0.04,
            0.0
          ],
          [
            0.06,
            0.0
          ],
          [
            0.08,
            0.0
          ],
          [
            0.1,
            0.0
          ],
          [
            0.12,
            0.0
          ],
          [
            0.14,
            0.0
          ],
          [
            0.16,
            0.0028
          ],
          [
            0.18,
            0.0076
          ],
          [
            0.2,
            0.0136
          ],
          [
            0.22,
            0.0145
          ],
          [
            0.24,
            0.019
          ],
          [
            0.26,
            0.0245
          ],
          [
            0.28,
            0.0292
          ],
          [
            0.3,
            0.0364
          ],
          [
            0.32,
            0.0441
          ],
          [
            0.34,
            0.0508
          ],
          [
            0.36,
            0.0556
          ],
          [
            0.38,
            0.0599
          ],
          [
            0.4,
            0.0639
          ],
          [
            0.42,
            0.0726
          ],
          [
            0.44,
            0.0796
          ],
          [
            0.46,
            0.091
          ],
          [
            0.48,
            0.1043
          ],
          [
            0.5,
            0.1119
          ],
          [
            0.52,
            0.1225
          ],
          [
            0.54,
            0.1306
          ],
          [
            0.56,
            0.1405
          ],
          [
            0.58,
            0.1506
          ],
          [
            0.6,
            0.1629
          ],
          [
            0.62,
            0.1715
          ],
          [
            0.64,
            0.1862
          ],
          [
            0.66,
            0.1962
          ],
          [
            0.68,
            0.2072
          ],
          [
            0.7,
            0.2164
          ],
          [
            0.72,
            0.2253
          ],
          [
            0.74,
            0.237
          ],
          [
            0.76,
            0.2493
          ],
          [
            0.78,
            0.2601
          ],
          [
            0.8,
            0.2673
          ],
          [
            0.82,
            0.2726
          ],
          [
            0.84,
            0.2813
          ],
          [
            0.86,
            0.2911
          ],
          [
            0.88,
            0.301
          ],
          [
            0.9,
            0.309
          ],
          [
            0.92,
            0.3165
          ],
          [
            0.94,
            0.3239
          ],
          [
            0.96,
            0.3294
          ],
          [
            0.98,
            0.338
          ],
          [
            1.0,
            0.3476
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 66,
            "mean_confidence": 0.27440997913028253,
            "accuracy": 0.19696969696969696
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 244,
            "mean_confidence": 0.3556488243038537,
            "accuracy": 0.3319672131147541
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 308,
            "mean_confidence": 0.4534817435718202,
            "accuracy": 0.37337662337662336
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 214,
            "mean_confidence": 0.5467024322982147,
            "accuracy": 0.4439252336448598
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 237,
            "mean_confidence": 0.6495107673894503,
            "accuracy": 0.5822784810126582
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 230,
            "mean_confidence": 0.7509541179822838,
            "accuracy": 0.6652173913043479
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 305,
            "mean_confidence": 0.8571867628175704,
            "accuracy": 0.8557377049180328
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 594,
            "mean_confidence": 0.9495988159067301,
            "accuracy": 0.9730639730639731
          }
        ],
        "median_right": 0.8589162528514862,
        "median_wrong": 0.48866577446460724,
        "sure": {
          "questions": 0,
          "correct": 0,
          "interval": null
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 1069,
          "correct": 442,
          "interval": [
            0.3843104203460428,
            0.44325033046749773
          ]
        },
        "ece_refit": 0.026071818802750085,
        "brier_refit": 0.16107894426549876,
        "brier_base": 0.22690258076879777,
        "refit_slopes": [
          1.1446870682340602,
          1.1650813519454442,
          1.1708060760615557,
          1.190277863873604,
          1.1284973300186134
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        106,
        146,
        53,
        90,
        87,
        93,
        86,
        35,
        52,
        49,
        48,
        25,
        79,
        50,
        48,
        95,
        20,
        74,
        71,
        127
      ],
      "started": "2026-10-13T06:41:20Z",
      "updated": "2026-10-13T06:55:58Z",
      "format_of": null,
      "group": "qrst",
      "latency": {
        "source": "run",
        "runs": [
          177
        ],
        "concurrency": 1,
        "median_request_seconds": 0.38925252700573765,
        "p90_request_seconds": 0.4138357652031118
      },
      "single_question_latency": null
    },
    {
      "run": 346,
      "model": "subq-snap-1.1",
      "name": "SubQ Snap 1.1",
      "variant": null,
      "reasoning": null,
      "provider": "subq",
      "note": "SubQ's subq-snap-1.1, in public preview, through its Decisions API, which also takes images. Its option probabilities arrive as 32-bit floats, so they sum to 1 only within 32-bit rounding. Its probability check used an experimental 32-bit rule of ours. For this model only, the check before its runs accepted a sum within one 32-bit epsilon (2^-23) per option when every value is an exact 32-bit float, instead of 7 decimal places; this is our tolerance, not an error bound SubQ states. The values are stored as SubQ returned them. It was paused twice while the approval of its probability check was confirmed, for 12 minutes and then for 3 hours 24 minutes, and resumed from its stored answers. The two requests in flight count those pauses in their times (706 and 12,236 seconds; the first after one retry, when SubQ had closed the connection), so the run's total request time and questions per minute include them. Without them, its median request time stays 0.378 s and its 90th percentile moves from 0.388 s to 0.387 s.",
      "parameters": null,
      "active_parameters": null,
      "access": "SubQ Decisions API (public preview)",
      "questions_per_request": 1,
      "concurrency": 1,
      "correct": 1404,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 571
        },
        "grammar": {
          "questions": 1542,
          "correct": 833
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 1401
        }
      },
      "tyt_mix": 0.8126384506184555,
      "questions_per_minute": 9.571646632418748,
      "request_seconds": 13778.193561108725,
      "median_request_seconds": 0.378216398996301,
      "p90_request_seconds": 0.38751173970231323,
      "billing": "api",
      "api_equivalent_usd": 0.033870080000000004,
      "cost_basis": "api",
      "requests": 2198,
      "hardware": null,
      "input_tokens": 846752,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 0,
      "confidence": {
        "source": "probability",
        "label": "Option score",
        "bins": [
          {
            "questions": 4,
            "correct": 0
          },
          {
            "questions": 403,
            "correct": 112
          },
          {
            "questions": 551,
            "correct": 234
          },
          {
            "questions": 401,
            "correct": 271
          },
          {
            "questions": 839,
            "correct": 787
          }
        ],
        "auroc": 0.8360531622496358,
        "auroc_interval": [
          0.8198638231237263,
          0.852454588063557
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.9201195013907489,
            "auroc_interval": [
              0.8941062801932367,
              0.94552871712292
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.7699463424297787,
            "auroc_interval": [
              0.7451723727401058,
              0.7933346570341973
            ]
          }
        },
        "aurc": 0.13827301786283627,
        "ece": 0.11150360240298471,
        "brier": 0.17485511408561052,
        "coverage_accuracy": {
          "0.20": 0.9954504094631483,
          "0.50": 0.8817106460418562,
          "0.80": 0.7275932666060054
        },
        "coverage_intervals": {
          "0.20": [
            0.9863822060826146,
            1.0
          ],
          "0.50": [
            0.8584433318161129,
            0.9018181818181819
          ],
          "0.80": [
            0.7051136363636363,
            0.7501135847342117
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0
          ],
          [
            0.04,
            0.0
          ],
          [
            0.06,
            0.0
          ],
          [
            0.08,
            0.0
          ],
          [
            0.1,
            0.0
          ],
          [
            0.12,
            0.0038
          ],
          [
            0.14,
            0.0032
          ],
          [
            0.16,
            0.0028
          ],
          [
            0.18,
            0.0051
          ],
          [
            0.2,
            0.0045
          ],
          [
            0.22,
            0.0103
          ],
          [
            0.24,
            0.0171
          ],
          [
            0.26,
            0.021
          ],
          [
            0.28,
            0.0244
          ],
          [
            0.3,
            0.0273
          ],
          [
            0.32,
            0.037
          ],
          [
            0.34,
            0.0415
          ],
          [
            0.36,
            0.0468
          ],
          [
            0.38,
            0.0611
          ],
          [
            0.4,
            0.0694
          ],
          [
            0.42,
            0.0758
          ],
          [
            0.44,
            0.09
          ],
          [
            0.46,
            0.096
          ],
          [
            0.48,
            0.109
          ],
          [
            0.5,
            0.1183
          ],
          [
            0.52,
            0.1304
          ],
          [
            0.54,
            0.1356
          ],
          [
            0.56,
            0.1462
          ],
          [
            0.58,
            0.1585
          ],
          [
            0.6,
            0.172
          ],
          [
            0.62,
            0.1825
          ],
          [
            0.64,
            0.1912
          ],
          [
            0.66,
            0.2024
          ],
          [
            0.68,
            0.2105
          ],
          [
            0.7,
            0.2227
          ],
          [
            0.72,
            0.2281
          ],
          [
            0.74,
            0.2419
          ],
          [
            0.76,
            0.2544
          ],
          [
            0.78,
            0.2631
          ],
          [
            0.8,
            0.2724
          ],
          [
            0.82,
            0.2807
          ],
          [
            0.84,
            0.2925
          ],
          [
            0.86,
            0.3017
          ],
          [
            0.88,
            0.3114
          ],
          [
            0.9,
            0.3176
          ],
          [
            0.92,
            0.326
          ],
          [
            0.94,
            0.3335
          ],
          [
            0.96,
            0.3408
          ],
          [
            0.98,
            0.3515
          ],
          [
            1.0,
            0.3612
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 50,
            "mean_confidence": 0.2789399653673172,
            "accuracy": 0.14
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 150,
            "mean_confidence": 0.35455246110757194,
            "accuracy": 0.26
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 207,
            "mean_confidence": 0.4506799140990068,
            "accuracy": 0.3188405797101449
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 241,
            "mean_confidence": 0.5488332650968148,
            "accuracy": 0.3651452282157676
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 227,
            "mean_confidence": 0.6508642303786089,
            "accuracy": 0.4845814977973568
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 206,
            "mean_confidence": 0.7554469776963725,
            "accuracy": 0.558252427184466
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 278,
            "mean_confidence": 0.8552818113951375,
            "accuracy": 0.6906474820143885
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 839,
            "mean_confidence": 0.9717030873315695,
            "accuracy": 0.9380214541120382
          }
        ],
        "median_right": 0.9333043694496155,
        "median_wrong": 0.5619476139545441,
        "sure": {
          "questions": 299,
          "correct": 298,
          "interval": [
            0.9813013202671312,
            0.9994093879050621
          ]
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 875,
          "correct": 310,
          "interval": [
            0.3232936323339767,
            0.38655169132844447
          ]
        },
        "ece_refit": 0.020014521554760217,
        "brier_refit": 0.15802679812199902,
        "brier_base": 0.23081482880129198,
        "refit_slopes": [
          0.9051873593153977,
          0.9615529020808709,
          0.9142125977964988,
          0.9704722818773116,
          0.9248016780498077
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        108,
        142,
        55,
        90,
        85,
        91,
        90,
        36,
        50,
        46,
        47,
        24,
        73,
        44,
        48,
        90,
        15,
        79,
        70,
        121
      ],
      "started": "2026-11-01T15:44:28Z",
      "updated": "2026-11-01T19:34:16Z",
      "format_of": null,
      "group": "stu",
      "latency": {
        "source": "run",
        "runs": [
          346
        ],
        "concurrency": 1,
        "median_request_seconds": 0.378216398996301,
        "p90_request_seconds": 0.38751173970231323
      },
      "single_question_latency": null
    },
    {
      "run": 143,
      "model": "gpt-6-luna",
      "name": "GPT-6 Luna",
      "variant": "Decisions API, beta",
      "reasoning": null,
      "provider": "openai-decisions",
      "note": null,
      "parameters": null,
      "active_parameters": null,
      "access": "OpenAI Decisions API (public beta)",
      "questions_per_request": 1,
      "concurrency": 4,
      "correct": 1398,
      "invalid": 4,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 561
        },
        "grammar": {
          "questions": 1542,
          "correct": 837
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 1396
        }
      },
      "tyt_mix": 0.8005161870076872,
      "questions_per_minute": 1214.8216473277553,
      "request_seconds": 108.55914552567992,
      "median_request_seconds": 0.19129549904027954,
      "p90_request_seconds": 0.24994003138272092,
      "billing": "api",
      "api_equivalent_usd": 0.08887830000000001,
      "cost_basis": "api",
      "requests": 2198,
      "hardware": null,
      "input_tokens": 888783,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 0,
      "confidence": {
        "source": "probability",
        "label": "Option score",
        "bins": [
          {
            "questions": 37,
            "correct": 11
          },
          {
            "questions": 820,
            "correct": 304
          },
          {
            "questions": 464,
            "correct": 288
          },
          {
            "questions": 227,
            "correct": 177
          },
          {
            "questions": 646,
            "correct": 618
          }
        ],
        "auroc": 0.820190005823107,
        "auroc_interval": [
          0.8038249823570924,
          0.8361226321063436
        ],
        "parts": {
          "meaning": {
            "questions": 655,
            "auroc": 0.8569044639132248,
            "auroc_interval": [
              0.8196527229676401,
              0.8935670995670996
            ]
          },
          "form": {
            "questions": 1539,
            "auroc": 0.76564994366667,
            "auroc_interval": [
              0.7417799967047904,
              0.7902609521870441
            ]
          }
        },
        "aurc": 0.1482095318278282,
        "ece": 0.023755697356427245,
        "brier": 0.1656602552415679,
        "coverage_accuracy": {
          "0.20": 0.9859313278638712,
          "0.50": 0.8662786620854077,
          "0.80": 0.7283728350045579
        },
        "coverage_intervals": {
          "0.20": [
            0.9745806585214329,
            0.9954462659380692
          ],
          "0.50": [
            0.8412408759124088,
            0.8865709074327406
          ],
          "0.80": [
            0.7049705540809555,
            0.7499344804010939
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0072
          ],
          [
            0.04,
            0.0072
          ],
          [
            0.06,
            0.0072
          ],
          [
            0.08,
            0.0072
          ],
          [
            0.1,
            0.0072
          ],
          [
            0.12,
            0.0072
          ],
          [
            0.14,
            0.0076
          ],
          [
            0.16,
            0.008
          ],
          [
            0.18,
            0.0108
          ],
          [
            0.2,
            0.0141
          ],
          [
            0.22,
            0.0162
          ],
          [
            0.24,
            0.0249
          ],
          [
            0.26,
            0.0305
          ],
          [
            0.28,
            0.0375
          ],
          [
            0.3,
            0.0445
          ],
          [
            0.32,
            0.0558
          ],
          [
            0.34,
            0.0641
          ],
          [
            0.36,
            0.0721
          ],
          [
            0.38,
            0.0861
          ],
          [
            0.4,
            0.0898
          ],
          [
            0.42,
            0.0954
          ],
          [
            0.44,
            0.108
          ],
          [
            0.46,
            0.1168
          ],
          [
            0.48,
            0.1268
          ],
          [
            0.5,
            0.1337
          ],
          [
            0.52,
            0.1485
          ],
          [
            0.54,
            0.1592
          ],
          [
            0.56,
            0.1651
          ],
          [
            0.58,
            0.1767
          ],
          [
            0.6,
            0.1869
          ],
          [
            0.62,
            0.1939
          ],
          [
            0.64,
            0.2005
          ],
          [
            0.66,
            0.2096
          ],
          [
            0.68,
            0.22
          ],
          [
            0.7,
            0.227
          ],
          [
            0.72,
            0.2349
          ],
          [
            0.74,
            0.244
          ],
          [
            0.76,
            0.2539
          ],
          [
            0.78,
            0.2617
          ],
          [
            0.8,
            0.2716
          ],
          [
            0.82,
            0.2836
          ],
          [
            0.84,
            0.2944
          ],
          [
            0.86,
            0.3014
          ],
          [
            0.88,
            0.3118
          ],
          [
            0.9,
            0.3219
          ],
          [
            0.92,
            0.3299
          ],
          [
            0.94,
            0.3381
          ],
          [
            0.96,
            0.3468
          ],
          [
            0.98,
            0.3555
          ],
          [
            1.0,
            0.3628
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 212,
            "mean_confidence": 0.26646226415094343,
            "accuracy": 0.2688679245283019
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 355,
            "mean_confidence": 0.34549295774647837,
            "accuracy": 0.3154929577464789
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 290,
            "mean_confidence": 0.44244827586206925,
            "accuracy": 0.503448275862069
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 214,
            "mean_confidence": 0.5438317757009347,
            "accuracy": 0.5560747663551402
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 181,
            "mean_confidence": 0.6459116022099448,
            "accuracy": 0.6464088397790055
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 140,
            "mean_confidence": 0.7457857142857145,
            "accuracy": 0.7428571428571429
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 156,
            "mean_confidence": 0.8454487179487173,
            "accuracy": 0.8012820512820513
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 646,
            "mean_confidence": 0.9771826625387025,
            "accuracy": 0.9566563467492261
          }
        ],
        "median_right": 0.83,
        "median_wrong": 0.395,
        "sure": {
          "questions": 368,
          "correct": 365,
          "interval": [
            0.9763090368954989,
            0.9972237792283561
          ]
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 1252,
          "correct": 551,
          "interval": [
            0.4128236282731903,
            0.46773455677180276
          ]
        },
        "ece_refit": 0.024476812913109897,
        "brier_refit": 0.1663163401109067,
        "brier_base": 0.2317333722348638,
        "refit_slopes": [
          0.8274631076325162,
          0.7991664438420922,
          0.7633222683531202,
          0.8730070272762679,
          0.7820886154857185
        ],
        "refit_questions": 2194,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        114,
        136,
        51,
        93,
        79,
        88,
        70,
        27,
        60,
        45,
        38,
        31,
        74,
        48,
        49,
        80,
        33,
        101,
        62,
        119
      ],
      "started": "2026-10-08T12:06:17Z",
      "updated": "2026-10-08T12:08:06Z",
      "format_of": null,
      "group": "rstuv",
      "latency": {
        "source": "run",
        "runs": [
          143
        ],
        "concurrency": 4,
        "median_request_seconds": 0.19129549904027954,
        "p90_request_seconds": 0.24994003138272092
      },
      "single_question_latency": null
    },
    {
      "run": 254,
      "model": "d1.1",
      "name": "Liquid d1.1",
      "variant": null,
      "reasoning": null,
      "provider": "liquid",
      "note": null,
      "parameters": null,
      "active_parameters": null,
      "access": "Liquid AI decisions API",
      "questions_per_request": 1,
      "concurrency": 1,
      "correct": 1352,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 555
        },
        "grammar": {
          "questions": 1542,
          "correct": 797
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 1351
        }
      },
      "tyt_mix": 0.7884308962861029,
      "questions_per_minute": 274.85478875697214,
      "request_seconds": 479.81699935600864,
      "median_request_seconds": 0.19458774350005115,
      "p90_request_seconds": 0.2346541320999677,
      "billing": "api",
      "api_equivalent_usd": 0.01500756,
      "cost_basis": "api",
      "requests": 2198,
      "hardware": null,
      "input_tokens": 750378,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 0,
      "confidence": {
        "source": "probability",
        "label": "Option score",
        "bins": [
          {
            "questions": 47,
            "correct": 12
          },
          {
            "questions": 899,
            "correct": 332
          },
          {
            "questions": 513,
            "correct": 318
          },
          {
            "questions": 270,
            "correct": 229
          },
          {
            "questions": 469,
            "correct": 461
          }
        ],
        "auroc": 0.8055704183977507,
        "auroc_interval": [
          0.787256874930486,
          0.8224344760930127
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.8834359111586835,
            "auroc_interval": [
              0.8528771143894338,
              0.9117162926384067
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.7306678568120384,
            "auroc_interval": [
              0.7052810126582278,
              0.7576310917944384
            ]
          }
        },
        "aurc": 0.16525777142201,
        "ece": 0.019863492797650786,
        "brier": 0.1716804311494476,
        "coverage_accuracy": {
          "0.20": 0.986351228389445,
          "0.50": 0.8416742493175614,
          "0.80": 0.6872156505914467
        },
        "coverage_intervals": {
          "0.20": [
            0.9704545454545455,
            0.9954504094631483
          ],
          "0.50": [
            0.8159927305770105,
            0.8671519563239308
          ],
          "0.80": [
            0.6625,
            0.7093366651522035
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0
          ],
          [
            0.04,
            0.0
          ],
          [
            0.06,
            0.0
          ],
          [
            0.08,
            0.0
          ],
          [
            0.1,
            0.0
          ],
          [
            0.12,
            0.0038
          ],
          [
            0.14,
            0.0032
          ],
          [
            0.16,
            0.0114
          ],
          [
            0.18,
            0.0101
          ],
          [
            0.2,
            0.0136
          ],
          [
            0.22,
            0.0186
          ],
          [
            0.24,
            0.0246
          ],
          [
            0.26,
            0.0271
          ],
          [
            0.28,
            0.0406
          ],
          [
            0.3,
            0.0455
          ],
          [
            0.32,
            0.0611
          ],
          [
            0.34,
            0.0669
          ],
          [
            0.36,
            0.0774
          ],
          [
            0.38,
            0.0958
          ],
          [
            0.4,
            0.1035
          ],
          [
            0.42,
            0.115
          ],
          [
            0.44,
            0.1241
          ],
          [
            0.46,
            0.1335
          ],
          [
            0.48,
            0.146
          ],
          [
            0.5,
            0.1583
          ],
          [
            0.52,
            0.1688
          ],
          [
            0.54,
            0.1786
          ],
          [
            0.56,
            0.1909
          ],
          [
            0.58,
            0.2
          ],
          [
            0.6,
            0.2093
          ],
          [
            0.62,
            0.2194
          ],
          [
            0.64,
            0.2294
          ],
          [
            0.66,
            0.2383
          ],
          [
            0.68,
            0.2496
          ],
          [
            0.7,
            0.261
          ],
          [
            0.72,
            0.2746
          ],
          [
            0.74,
            0.2868
          ],
          [
            0.76,
            0.2987
          ],
          [
            0.78,
            0.308
          ],
          [
            0.8,
            0.3128
          ],
          [
            0.82,
            0.3229
          ],
          [
            0.84,
            0.3331
          ],
          [
            0.86,
            0.3391
          ],
          [
            0.88,
            0.346
          ],
          [
            0.9,
            0.3518
          ],
          [
            0.92,
            0.3565
          ],
          [
            0.94,
            0.364
          ],
          [
            0.96,
            0.3687
          ],
          [
            0.98,
            0.3779
          ],
          [
            1.0,
            0.3849
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 234,
            "mean_confidence": 0.26950145696139044,
            "accuracy": 0.3247863247863248
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 398,
            "mean_confidence": 0.3482328611962827,
            "accuracy": 0.3241206030150754
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 314,
            "mean_confidence": 0.44591858091984543,
            "accuracy": 0.4426751592356688
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 243,
            "mean_confidence": 0.5446695223492299,
            "accuracy": 0.551440329218107
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 194,
            "mean_confidence": 0.6472734818844487,
            "accuracy": 0.6649484536082474
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 164,
            "mean_confidence": 0.7521572883140277,
            "accuracy": 0.75
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 182,
            "mean_confidence": 0.8555041016980522,
            "accuracy": 0.8846153846153846
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 469,
            "mean_confidence": 0.9629384102873333,
            "accuracy": 0.9829424307036247
          }
        ],
        "median_right": 0.7599662811063806,
        "median_wrong": 0.39771640338701664,
        "sure": {
          "questions": 87,
          "correct": 87,
          "interval": [
            0.957711004649852,
            1.0
          ]
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 1383,
          "correct": 607,
          "interval": [
            0.41295129239608563,
            0.46518908066454356
          ]
        },
        "ece_refit": 0.019884464180768346,
        "brier_refit": 0.1719582558480069,
        "brier_base": 0.23691827416625139,
        "refit_slopes": [
          1.0758980536845688,
          1.1116219103417928,
          1.117339534229297,
          1.073224354599052,
          1.0828207115718915
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        101,
        141,
        53,
        93,
        80,
        87,
        70,
        31,
        53,
        46,
        41,
        23,
        83,
        43,
        43,
        90,
        25,
        67,
        70,
        112
      ],
      "started": "2026-10-24T07:34:52Z",
      "updated": "2026-10-24T07:42:55Z",
      "format_of": null,
      "group": "tuvw",
      "latency": {
        "source": "run",
        "runs": [
          254
        ],
        "concurrency": 1,
        "median_request_seconds": 0.19458774350005115,
        "p90_request_seconds": 0.2346541320999677
      },
      "single_question_latency": null
    },
    {
      "run": 54,
      "model": "claude-sonnet-5-5",
      "name": "Claude Sonnet 5.5",
      "variant": null,
      "reasoning": "low",
      "provider": "claude",
      "note": null,
      "parameters": null,
      "active_parameters": null,
      "access": "Claude Code CLI, Claude subscription",
      "questions_per_request": 16,
      "concurrency": 1,
      "correct": 1349,
      "invalid": 0,
      "answered_questions": 2210,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 544
        },
        "grammar": {
          "questions": 1542,
          "correct": 805
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 1346
        }
      },
      "tyt_mix": 0.7755049666255418,
      "questions_per_minute": 230.28394322199546,
      "request_seconds": 575.8108800150803,
      "median_request_seconds": null,
      "p90_request_seconds": null,
      "billing": "subscription",
      "api_equivalent_usd": 4.1900397,
      "cost_basis": "subscription",
      "requests": 139,
      "hardware": null,
      "input_tokens": 1215056,
      "cached_input_tokens": 298797,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 915981,
      "output_tokens": 49568,
      "confidence": {
        "source": "stated",
        "label": "Written estimate",
        "bins": [
          {
            "questions": 3,
            "correct": 1
          },
          {
            "questions": 912,
            "correct": 334
          },
          {
            "questions": 823,
            "correct": 581
          },
          {
            "questions": 372,
            "correct": 346
          },
          {
            "questions": 88,
            "correct": 87
          }
        ],
        "auroc": 0.7860889844678386,
        "auroc_interval": [
          0.7678298917238497,
          0.8035783272165347
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.8243910845588235,
            "auroc_interval": [
              0.7823407074451134,
              0.8639108832669112
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.7241924201690587,
            "auroc_interval": [
              0.7000665778961385,
              0.7469150107178031
            ]
          }
        },
        "aurc": 0.18194892138889104,
        "ece": 0.07909463148316544,
        "brier": 0.1897033212010919,
        "coverage_accuracy": {
          "0.20": 0.943000129988301,
          "0.50": 0.8234421864995113,
          "0.80": 0.6899102276671388
        },
        "coverage_intervals": {
          "0.20": [
            0.9193531623294471,
            0.9638424356908717
          ],
          "0.50": [
            0.7998646055540944,
            0.8464240102171137
          ],
          "0.80": [
            0.6662982642882895,
            0.7136591267635682
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0076
          ],
          [
            0.04,
            0.0114
          ],
          [
            0.06,
            0.0131
          ],
          [
            0.08,
            0.0153
          ],
          [
            0.1,
            0.0167
          ],
          [
            0.12,
            0.0257
          ],
          [
            0.14,
            0.0369
          ],
          [
            0.16,
            0.0453
          ],
          [
            0.18,
            0.0518
          ],
          [
            0.2,
            0.057
          ],
          [
            0.22,
            0.062
          ],
          [
            0.24,
            0.0673
          ],
          [
            0.26,
            0.0718
          ],
          [
            0.28,
            0.082
          ],
          [
            0.3,
            0.0921
          ],
          [
            0.32,
            0.1009
          ],
          [
            0.34,
            0.1086
          ],
          [
            0.36,
            0.1155
          ],
          [
            0.38,
            0.1217
          ],
          [
            0.4,
            0.1273
          ],
          [
            0.42,
            0.1333
          ],
          [
            0.44,
            0.1451
          ],
          [
            0.46,
            0.1565
          ],
          [
            0.48,
            0.1669
          ],
          [
            0.5,
            0.1766
          ],
          [
            0.52,
            0.1854
          ],
          [
            0.54,
            0.1937
          ],
          [
            0.56,
            0.2013
          ],
          [
            0.58,
            0.2084
          ],
          [
            0.6,
            0.221
          ],
          [
            0.62,
            0.2325
          ],
          [
            0.64,
            0.2433
          ],
          [
            0.66,
            0.2534
          ],
          [
            0.68,
            0.2629
          ],
          [
            0.7,
            0.2719
          ],
          [
            0.72,
            0.2804
          ],
          [
            0.74,
            0.2884
          ],
          [
            0.76,
            0.296
          ],
          [
            0.78,
            0.3032
          ],
          [
            0.8,
            0.3101
          ],
          [
            0.82,
            0.3177
          ],
          [
            0.84,
            0.3257
          ],
          [
            0.86,
            0.3334
          ],
          [
            0.88,
            0.3412
          ],
          [
            0.9,
            0.3495
          ],
          [
            0.92,
            0.3575
          ],
          [
            0.94,
            0.3652
          ],
          [
            0.96,
            0.3725
          ],
          [
            0.98,
            0.3796
          ],
          [
            1.0,
            0.3863
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 3,
            "mean_confidence": 0.20000000000000004,
            "accuracy": 0.3333333333333333
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 418,
            "mean_confidence": 0.3169856459330124,
            "accuracy": 0.3038277511961722
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 494,
            "mean_confidence": 0.40091093117409227,
            "accuracy": 0.4190283400809717
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 369,
            "mean_confidence": 0.5024390243902442,
            "accuracy": 0.5962059620596206
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 327,
            "mean_confidence": 0.6007645259938804,
            "accuracy": 0.764525993883792
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 148,
            "mean_confidence": 0.7070945945945961,
            "accuracy": 0.8783783783783784
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 351,
            "mean_confidence": 0.821994301994304,
            "accuracy": 0.9316239316239316
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 88,
            "mean_confidence": 0.9088636363636371,
            "accuracy": 0.9886363636363636
          }
        ],
        "median_right": 0.6,
        "median_wrong": 0.4,
        "sure": {
          "questions": 0,
          "correct": 0,
          "interval": null
        },
        "sure_before_audit": {
          "questions": 0,
          "correct": 0,
          "interval": null
        },
        "doubtful": {
          "questions": 1611,
          "correct": 805,
          "interval": [
            0.47530322332618785,
            0.5240775208904584
          ]
        },
        "ece_refit": 0.014900599006025547,
        "brier_refit": 0.1806297130001828,
        "brier_base": 0.23741334808931258,
        "refit_slopes": [
          1.6293091533848973,
          1.6256963120779857,
          1.6619719126200811,
          1.6527700483823924,
          1.555247235160037
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": {
        "run": 38,
        "changed": {
          "questions": 372,
          "older_correct": 155,
          "newer_correct": 229,
          "flipped": 134,
          "changed_answer": 205,
          "newer_wrong": 143,
          "repeated_mistakes": 42,
          "p": 9.319307580879302e-11
        },
        "unchanged": {
          "questions": 1814,
          "older_correct": 1095,
          "newer_correct": 1113,
          "flipped": 276,
          "changed_answer": 424,
          "newer_wrong": 701,
          "repeated_mistakes": 424,
          "p": 0.3061693933230553
        },
        "added": {
          "questions": 12,
          "newer_correct": 7
        }
      },
      "units": [
        107,
        133,
        50,
        91,
        76,
        87,
        75,
        31,
        61,
        44,
        42,
        28,
        64,
        53,
        41,
        68,
        27,
        93,
        62,
        116
      ],
      "started": "2026-10-04T09:02:05Z",
      "updated": "2026-10-04T09:11:41Z",
      "format_of": null,
      "group": "tuvwx",
      "latency": {
        "source": "sample",
        "runs": [
          229,
          230,
          231,
          232,
          233
        ],
        "repeats": [
          1,
          2,
          3,
          4,
          5
        ],
        "questions": 200,
        "concurrency": 1,
        "median_request_seconds": 3.873064635001356,
        "p90_request_seconds": 5.84579606440384
      },
      "single_question_latency": null
    },
    {
      "run": 150,
      "model": "celeris-1-decision",
      "name": "Celeris-1 Decision",
      "variant": null,
      "reasoning": null,
      "provider": "celeris",
      "note": null,
      "parameters": null,
      "active_parameters": null,
      "access": "Celeris System One API",
      "questions_per_request": 1,
      "concurrency": 1,
      "correct": 1329,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 559
        },
        "grammar": {
          "questions": 1542,
          "correct": 770
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 1326
        }
      },
      "tyt_mix": 0.7903971817563507,
      "questions_per_minute": 148.99183210646953,
      "request_seconds": 885.1491933179168,
      "median_request_seconds": 0.4010543754975515,
      "p90_request_seconds": 0.4175183980994916,
      "billing": "api",
      "api_equivalent_usd": 0.033577800000000005,
      "cost_basis": "api",
      "requests": 2198,
      "hardware": null,
      "input_tokens": 839445,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 0,
      "confidence": {
        "source": "probability",
        "label": "Option score",
        "bins": [
          {
            "questions": 14,
            "correct": 3
          },
          {
            "questions": 822,
            "correct": 284
          },
          {
            "questions": 582,
            "correct": 328
          },
          {
            "questions": 348,
            "correct": 288
          },
          {
            "questions": 432,
            "correct": 426
          }
        ],
        "auroc": 0.8051863319886293,
        "auroc_interval": [
          0.7873404410369045,
          0.8214388310096522
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.8757999372959814,
            "auroc_interval": [
              0.8465765746001799,
              0.9033230658909044
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.7323329520220712,
            "auroc_interval": [
              0.7061765380232111,
              0.7581695292567547
            ]
          }
        },
        "aurc": 0.17325778776677944,
        "ece": 0.04257051865332119,
        "brier": 0.1750853958143767,
        "coverage_accuracy": {
          "0.20": 0.986351228389445,
          "0.50": 0.8183375260932398,
          "0.80": 0.6843851553360197
        },
        "coverage_intervals": {
          "0.20": [
            0.9750227066303361,
            0.9959194214876033
          ],
          "0.50": [
            0.7940411137132449,
            0.842896174863388
          ],
          "0.80": [
            0.6609443686524178,
            0.7063733515234197
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0
          ],
          [
            0.04,
            0.0
          ],
          [
            0.06,
            0.0008
          ],
          [
            0.08,
            0.0034
          ],
          [
            0.1,
            0.0051
          ],
          [
            0.12,
            0.0063
          ],
          [
            0.14,
            0.0076
          ],
          [
            0.16,
            0.009
          ],
          [
            0.18,
            0.0103
          ],
          [
            0.2,
            0.0136
          ],
          [
            0.22,
            0.0149
          ],
          [
            0.24,
            0.0212
          ],
          [
            0.26,
            0.0296
          ],
          [
            0.28,
            0.041
          ],
          [
            0.3,
            0.0524
          ],
          [
            0.32,
            0.0599
          ],
          [
            0.34,
            0.0735
          ],
          [
            0.36,
            0.0875
          ],
          [
            0.38,
            0.0992
          ],
          [
            0.4,
            0.1133
          ],
          [
            0.42,
            0.1318
          ],
          [
            0.44,
            0.1455
          ],
          [
            0.46,
            0.1599
          ],
          [
            0.48,
            0.1725
          ],
          [
            0.5,
            0.1817
          ],
          [
            0.52,
            0.1922
          ],
          [
            0.54,
            0.2012
          ],
          [
            0.56,
            0.2077
          ],
          [
            0.58,
            0.2163
          ],
          [
            0.6,
            0.2245
          ],
          [
            0.62,
            0.2351
          ],
          [
            0.64,
            0.2432
          ],
          [
            0.66,
            0.2562
          ],
          [
            0.68,
            0.2639
          ],
          [
            0.7,
            0.2731
          ],
          [
            0.72,
            0.2847
          ],
          [
            0.74,
            0.2932
          ],
          [
            0.76,
            0.3001
          ],
          [
            0.78,
            0.308
          ],
          [
            0.8,
            0.3156
          ],
          [
            0.82,
            0.3254
          ],
          [
            0.84,
            0.3348
          ],
          [
            0.86,
            0.3404
          ],
          [
            0.88,
            0.3472
          ],
          [
            0.9,
            0.3541
          ],
          [
            0.92,
            0.3621
          ],
          [
            0.94,
            0.3722
          ],
          [
            0.96,
            0.3802
          ],
          [
            0.98,
            0.3877
          ],
          [
            1.0,
            0.3954
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 116,
            "mean_confidence": 0.26991379310344815,
            "accuracy": 0.25
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 354,
            "mean_confidence": 0.3467796610169486,
            "accuracy": 0.3022598870056497
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 366,
            "mean_confidence": 0.4440437158469945,
            "accuracy": 0.412568306010929
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 287,
            "mean_confidence": 0.5419512195121959,
            "accuracy": 0.5470383275261324
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 194,
            "mean_confidence": 0.6424742268041237,
            "accuracy": 0.5360824742268041
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 209,
            "mean_confidence": 0.7439234449760762,
            "accuracy": 0.6889952153110048
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 240,
            "mean_confidence": 0.8488333333333333,
            "accuracy": 0.8791666666666667
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 432,
            "mean_confidence": 0.9325925925925914,
            "accuracy": 0.9861111111111112
          }
        ],
        "median_right": 0.77,
        "median_wrong": 0.44,
        "sure": {
          "questions": 0,
          "correct": 0,
          "interval": null
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 1317,
          "correct": 548,
          "interval": [
            0.3897574412395216,
            0.4429249939593698
          ]
        },
        "ece_refit": 0.035657971879972675,
        "brier_refit": 0.17471930343604664,
        "brier_base": 0.23925115236417377,
        "refit_slopes": [
          1.2156374289555851,
          1.2107775552529414,
          1.2070745124033397,
          1.2276157608330067,
          1.223639086566504
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        104,
        140,
        57,
        87,
        83,
        88,
        77,
        34,
        56,
        36,
        41,
        26,
        66,
        41,
        48,
        78,
        32,
        60,
        60,
        115
      ],
      "started": "2026-10-10T18:37:11Z",
      "updated": "2026-10-10T18:52:00Z",
      "format_of": null,
      "group": "uvwxy",
      "latency": {
        "source": "run",
        "runs": [
          150
        ],
        "concurrency": 1,
        "median_request_seconds": 0.4010543754975515,
        "p90_request_seconds": 0.4175183980994916
      },
      "single_question_latency": null
    },
    {
      "run": 44,
      "model": "pplx-decider-v1-27b",
      "name": "Perplexity Decider 27B v1",
      "variant": null,
      "reasoning": null,
      "provider": "perplexity",
      "note": null,
      "parameters": 26085330160,
      "active_parameters": null,
      "access": "Perplexity Decisions API",
      "questions_per_request": 1,
      "concurrency": 1,
      "correct": 1327,
      "invalid": 0,
      "answered_questions": 2210,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 564
        },
        "grammar": {
          "questions": 1542,
          "correct": 763
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 1324
        }
      },
      "tyt_mix": 0.7958908686849514,
      "questions_per_minute": 284.5404418373233,
      "request_seconds": 466.01459934405284,
      "median_request_seconds": null,
      "p90_request_seconds": null,
      "billing": "api",
      "api_equivalent_usd": 0.034337480000000004,
      "cost_basis": "api",
      "requests": 2210,
      "hardware": null,
      "input_tokens": 858437,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 2210,
      "confidence": {
        "source": "probability",
        "label": "Option score",
        "bins": [
          {
            "questions": 119,
            "correct": 24
          },
          {
            "questions": 1002,
            "correct": 390
          },
          {
            "questions": 437,
            "correct": 299
          },
          {
            "questions": 235,
            "correct": 210
          },
          {
            "questions": 405,
            "correct": 404
          }
        ],
        "auroc": 0.8272624472559237,
        "auroc_interval": [
          0.8102045264332542,
          0.843935920421831
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.9090541165587419,
            "auroc_interval": [
              0.8849522790116849,
              0.9309925415949513
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.7514910570227313,
            "auroc_interval": [
              0.7269312861180243,
              0.775486531986532
            ]
          }
        },
        "aurc": 0.16357576140263955,
        "ece": 0.049916517793603135,
        "brier": 0.16688403705981944,
        "coverage_accuracy": {
          "0.20": 0.9931756141947224,
          "0.50": 0.8416742493175614,
          "0.80": 0.6934713375796179
        },
        "coverage_intervals": {
          "0.20": [
            0.9817850637522769,
            1.0
          ],
          "0.50": [
            0.8142076502732241,
            0.8660592255125285
          ],
          "0.80": [
            0.6698455949137148,
            0.7162346521145976
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0
          ],
          [
            0.04,
            0.0
          ],
          [
            0.06,
            0.0
          ],
          [
            0.08,
            0.0
          ],
          [
            0.1,
            0.0
          ],
          [
            0.12,
            0.0
          ],
          [
            0.14,
            0.0
          ],
          [
            0.16,
            0.0
          ],
          [
            0.18,
            0.0025
          ],
          [
            0.2,
            0.0068
          ],
          [
            0.22,
            0.0124
          ],
          [
            0.24,
            0.019
          ],
          [
            0.26,
            0.0227
          ],
          [
            0.28,
            0.0341
          ],
          [
            0.3,
            0.0431
          ],
          [
            0.32,
            0.0569
          ],
          [
            0.34,
            0.0682
          ],
          [
            0.36,
            0.0758
          ],
          [
            0.38,
            0.085
          ],
          [
            0.4,
            0.0955
          ],
          [
            0.42,
            0.1063
          ],
          [
            0.44,
            0.1179
          ],
          [
            0.46,
            0.1365
          ],
          [
            0.48,
            0.146
          ],
          [
            0.5,
            0.1583
          ],
          [
            0.52,
            0.1706
          ],
          [
            0.54,
            0.1778
          ],
          [
            0.56,
            0.1869
          ],
          [
            0.58,
            0.1983
          ],
          [
            0.6,
            0.21
          ],
          [
            0.62,
            0.2209
          ],
          [
            0.64,
            0.233
          ],
          [
            0.66,
            0.2454
          ],
          [
            0.68,
            0.2576
          ],
          [
            0.7,
            0.2669
          ],
          [
            0.72,
            0.2717
          ],
          [
            0.74,
            0.284
          ],
          [
            0.76,
            0.293
          ],
          [
            0.78,
            0.3018
          ],
          [
            0.8,
            0.3065
          ],
          [
            0.82,
            0.3142
          ],
          [
            0.84,
            0.3246
          ],
          [
            0.86,
            0.3328
          ],
          [
            0.88,
            0.3434
          ],
          [
            0.9,
            0.3519
          ],
          [
            0.92,
            0.3605
          ],
          [
            0.94,
            0.3703
          ],
          [
            0.96,
            0.3792
          ],
          [
            0.98,
            0.3891
          ],
          [
            1.0,
            0.3963
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 380,
            "mean_confidence": 0.26378993079791785,
            "accuracy": 0.22894736842105262
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 428,
            "mean_confidence": 0.3456194893336824,
            "accuracy": 0.3901869158878505
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 313,
            "mean_confidence": 0.44942247914590755,
            "accuracy": 0.5111821086261981
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 189,
            "mean_confidence": 0.5451189982136457,
            "accuracy": 0.5978835978835979
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 171,
            "mean_confidence": 0.6501243797086762,
            "accuracy": 0.7368421052631579
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 153,
            "mean_confidence": 0.7498241112656384,
            "accuracy": 0.7973856209150327
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 159,
            "mean_confidence": 0.8540340393436772,
            "accuracy": 0.9308176100628931
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 405,
            "mean_confidence": 0.9634963686753925,
            "accuracy": 0.9975308641975309
          }
        ],
        "median_right": 0.7079715624511713,
        "median_wrong": 0.3444417726744183,
        "sure": {
          "questions": 21,
          "correct": 21,
          "interval": [
            0.8453561767357979,
            0.9999999999999999
          ]
        },
        "sure_before_audit": {
          "questions": 21,
          "correct": 21,
          "interval": [
            0.8453561767357979,
            0.9999999999999999
          ]
        },
        "doubtful": {
          "questions": 1481,
          "correct": 653,
          "interval": [
            0.4158165454715931,
            0.4663257651964269
          ]
        },
        "ece_refit": 0.022306911799509107,
        "brier_refit": 0.16506312022237138,
        "brier_base": 0.2392670940267875,
        "refit_slopes": [
          1.2530834625770766,
          1.3265731788593536,
          1.2981866433062688,
          1.348260428819055,
          1.346757935168172
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": {
        "run": 43,
        "changed": {
          "questions": 372,
          "older_correct": 147,
          "newer_correct": 192,
          "flipped": 111,
          "changed_answer": 183,
          "newer_wrong": 180,
          "repeated_mistakes": 75,
          "p": 2.32626705010725e-05
        },
        "unchanged": {
          "questions": 1814,
          "older_correct": 1127,
          "newer_correct": 1127,
          "flipped": 0,
          "changed_answer": 0,
          "newer_wrong": 687,
          "repeated_mistakes": 687,
          "p": 1.0
        },
        "added": {
          "questions": 12,
          "newer_correct": 8
        }
      },
      "units": [
        103,
        137,
        56,
        92,
        83,
        93,
        76,
        33,
        55,
        33,
        45,
        20,
        64,
        49,
        49,
        80,
        24,
        67,
        59,
        109
      ],
      "started": "2026-10-04T08:31:15Z",
      "updated": "2026-10-04T08:39:05Z",
      "format_of": null,
      "group": "uvwxy",
      "latency": {
        "source": "repeats",
        "runs": [
          64,
          84,
          96,
          100
        ],
        "repeats": [
          2,
          3,
          4,
          5
        ],
        "questions": 2198,
        "concurrency": 1,
        "median_request_seconds": 0.21641260248725303,
        "p90_request_seconds": 0.2719852286914829
      },
      "single_question_latency": null
    },
    {
      "run": 264,
      "model": "OmniJev/OneJev-27B",
      "name": "OneJev 27B",
      "variant": null,
      "reasoning": null,
      "provider": "onejev",
      "note": "OmniJev's released OneJev checkpoint (OmniJev/OneJev-27B at a5e5f55), a full fine-tune of Qwen3.8-27B that also reads images, served on our hardware through the server its package ships (qev 0.1.4, qev serve) with its defaults. One request at a time, as for the other setups. The five runs are identical because decoding is deterministic: each run sent all 2,198 questions again as separate requests.",
      "parameters": 27356728560,
      "active_parameters": null,
      "access": "OneJev server (qev, OmniJev's release code), self-hosted, BF16",
      "questions_per_request": 1,
      "concurrency": 1,
      "correct": 1310,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 559
        },
        "grammar": {
          "questions": 1542,
          "correct": 751
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 1306
        }
      },
      "tyt_mix": 0.7882408912245737,
      "questions_per_minute": 418.8018794259747,
      "request_seconds": 314.8983003150788,
      "median_request_seconds": 0.1249308105034288,
      "p90_request_seconds": 0.1888540582935093,
      "billing": "local",
      "api_equivalent_usd": null,
      "cost_basis": null,
      "requests": 2198,
      "hardware": "1 × RTX PRO 6000 Blackwell (96 GB); BF16",
      "input_tokens": 870930,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 2198,
      "confidence": {
        "source": "probability",
        "label": "Option score",
        "bins": [
          {
            "questions": 143,
            "correct": 41
          },
          {
            "questions": 996,
            "correct": 369
          },
          {
            "questions": 459,
            "correct": 320
          },
          {
            "questions": 277,
            "correct": 257
          },
          {
            "questions": 323,
            "correct": 323
          }
        ],
        "auroc": 0.8182277697544873,
        "auroc_interval": [
          0.7989093048321231,
          0.8352211212274316
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.9109234088855283,
            "auroc_interval": [
              0.8848028273809524,
              0.9352144082332762
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.737617437180262,
            "auroc_interval": [
              0.7131894850094459,
              0.7630338585316795
            ]
          }
        },
        "aurc": 0.17137802838466898,
        "ece": 0.05534345950864414,
        "brier": 0.17049210655903185,
        "coverage_accuracy": {
          "0.20": 0.9886260236578708,
          "0.50": 0.83803457688808,
          "0.80": 0.680732484076433
        },
        "coverage_intervals": {
          "0.20": [
            0.9749886311959982,
            0.9977303676804358
          ],
          "0.50": [
            0.8076398362892224,
            0.8606445755787563
          ],
          "0.80": [
            0.6554669703872438,
            0.7051136363636363
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0
          ],
          [
            0.04,
            0.0
          ],
          [
            0.06,
            0.0
          ],
          [
            0.08,
            0.0
          ],
          [
            0.1,
            0.0
          ],
          [
            0.12,
            0.0
          ],
          [
            0.14,
            0.0
          ],
          [
            0.16,
            0.0028
          ],
          [
            0.18,
            0.0051
          ],
          [
            0.2,
            0.0114
          ],
          [
            0.22,
            0.0207
          ],
          [
            0.24,
            0.0209
          ],
          [
            0.26,
            0.0245
          ],
          [
            0.28,
            0.0325
          ],
          [
            0.3,
            0.047
          ],
          [
            0.32,
            0.0554
          ],
          [
            0.34,
            0.0656
          ],
          [
            0.36,
            0.0724
          ],
          [
            0.38,
            0.0826
          ],
          [
            0.4,
            0.0978
          ],
          [
            0.42,
            0.1094
          ],
          [
            0.44,
            0.121
          ],
          [
            0.46,
            0.1356
          ],
          [
            0.48,
            0.1507
          ],
          [
            0.5,
            0.162
          ],
          [
            0.52,
            0.1811
          ],
          [
            0.54,
            0.1954
          ],
          [
            0.56,
            0.2023
          ],
          [
            0.58,
            0.2179
          ],
          [
            0.6,
            0.2319
          ],
          [
            0.62,
            0.2449
          ],
          [
            0.64,
            0.2507
          ],
          [
            0.66,
            0.2599
          ],
          [
            0.68,
            0.271
          ],
          [
            0.7,
            0.2799
          ],
          [
            0.72,
            0.2875
          ],
          [
            0.74,
            0.2988
          ],
          [
            0.76,
            0.3038
          ],
          [
            0.78,
            0.3106
          ],
          [
            0.8,
            0.3193
          ],
          [
            0.82,
            0.3303
          ],
          [
            0.84,
            0.3409
          ],
          [
            0.86,
            0.3509
          ],
          [
            0.88,
            0.3594
          ],
          [
            0.9,
            0.3685
          ],
          [
            0.92,
            0.3769
          ],
          [
            0.94,
            0.3848
          ],
          [
            0.96,
            0.3915
          ],
          [
            0.98,
            0.3969
          ],
          [
            1.0,
            0.404
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 424,
            "mean_confidence": 0.2618709716981131,
            "accuracy": 0.25471698113207547
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 427,
            "mean_confidence": 0.34599103981264656,
            "accuracy": 0.41920374707259955
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 288,
            "mean_confidence": 0.4466268333333333,
            "accuracy": 0.4270833333333333
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 215,
            "mean_confidence": 0.5442800186046511,
            "accuracy": 0.5906976744186047
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 159,
            "mean_confidence": 0.6483798679245286,
            "accuracy": 0.779874213836478
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 169,
            "mean_confidence": 0.7501690769230765,
            "accuracy": 0.8520710059171598
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 193,
            "mean_confidence": 0.8525109999999999,
            "accuracy": 0.9430051813471503
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 323,
            "mean_confidence": 0.9500090464396288,
            "accuracy": 1.0
          }
        ],
        "median_right": 0.6954834999999999,
        "median_wrong": 0.343397,
        "sure": {
          "questions": 2,
          "correct": 2,
          "interval": [
            0.34237195288961925,
            1.0
          ]
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 1513,
          "correct": 661,
          "interval": [
            0.41207839656171885,
            0.46200206114724696
          ]
        },
        "ece_refit": 0.026126721434686515,
        "brier_refit": 0.16791994671454427,
        "brier_base": 0.24081708552045458,
        "refit_slopes": [
          1.324722946262829,
          1.382542366760224,
          1.3616946747940821,
          1.378997897617812,
          1.3734128164584745
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        106,
        133,
        56,
        91,
        83,
        90,
        71,
        31,
        53,
        37,
        47,
        23,
        64,
        43,
        44,
        77,
        25,
        66,
        61,
        109
      ],
      "started": "2026-10-27T13:44:13Z",
      "updated": "2026-10-27T13:49:37Z",
      "format_of": null,
      "group": "vwxyz",
      "latency": {
        "source": "run",
        "runs": [
          264
        ],
        "concurrency": 1,
        "median_request_seconds": 0.1249308105034288,
        "p90_request_seconds": 0.1888540582935093
      },
      "single_question_latency": null
    },
    {
      "run": 194,
      "model": "jev-1.14.0",
      "name": "Jev 1.14.0",
      "variant": null,
      "reasoning": null,
      "provider": "jev",
      "note": null,
      "parameters": null,
      "active_parameters": null,
      "access": "TypeSafe System One API",
      "questions_per_request": 1,
      "concurrency": 1,
      "correct": 1301,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 558
        },
        "grammar": {
          "questions": 1542,
          "correct": 743
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 1301
        }
      },
      "tyt_mix": 0.7860753574705008,
      "questions_per_minute": 265.6886711846118,
      "request_seconds": 496.3704301428952,
      "median_request_seconds": 0.2105813504967955,
      "p90_request_seconds": 0.2774909528983699,
      "billing": "api",
      "api_equivalent_usd": 0.063384972,
      "cost_basis": "api",
      "requests": 2198,
      "hardware": null,
      "input_tokens": 1509166,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 114296,
      "confidence": {
        "source": "probability",
        "label": "Option score",
        "bins": [
          {
            "questions": 24,
            "correct": 11
          },
          {
            "questions": 948,
            "correct": 340
          },
          {
            "questions": 489,
            "correct": 278
          },
          {
            "questions": 247,
            "correct": 194
          },
          {
            "questions": 490,
            "correct": 478
          }
        ],
        "auroc": 0.7900731535728027,
        "auroc_interval": [
          0.7719382531510404,
          0.8085741297808337
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.8771395655036208,
            "auroc_interval": [
              0.8492152776497058,
              0.9059482184482185
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.697428818324386,
            "auroc_interval": [
              0.669101450162149,
              0.7238458172641519
            ]
          }
        },
        "aurc": 0.1875196998364388,
        "ece": 0.025313921747042376,
        "brier": 0.17966988171064605,
        "coverage_accuracy": {
          "0.20": 0.985787945751549,
          "0.50": 0.8129768320851123,
          "0.80": 0.6602866242038217
        },
        "coverage_intervals": {
          "0.20": [
            0.9681058586384029,
            0.9954566106315311
          ],
          "0.50": [
            0.7850538123389419,
            0.8377215849843587
          ],
          "0.80": [
            0.637464724624488,
            0.6838379242837841
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0
          ],
          [
            0.04,
            0.0
          ],
          [
            0.06,
            0.0
          ],
          [
            0.08,
            0.0
          ],
          [
            0.1,
            0.0014
          ],
          [
            0.12,
            0.0041
          ],
          [
            0.14,
            0.0065
          ],
          [
            0.16,
            0.0071
          ],
          [
            0.18,
            0.0104
          ],
          [
            0.2,
            0.0142
          ],
          [
            0.22,
            0.0248
          ],
          [
            0.24,
            0.0353
          ],
          [
            0.26,
            0.0414
          ],
          [
            0.28,
            0.0504
          ],
          [
            0.3,
            0.065
          ],
          [
            0.32,
            0.0781
          ],
          [
            0.34,
            0.0927
          ],
          [
            0.36,
            0.1095
          ],
          [
            0.38,
            0.119
          ],
          [
            0.4,
            0.1343
          ],
          [
            0.42,
            0.1463
          ],
          [
            0.44,
            0.1571
          ],
          [
            0.46,
            0.1674
          ],
          [
            0.48,
            0.1792
          ],
          [
            0.5,
            0.187
          ],
          [
            0.52,
            0.2025
          ],
          [
            0.54,
            0.2136
          ],
          [
            0.56,
            0.2268
          ],
          [
            0.58,
            0.2393
          ],
          [
            0.6,
            0.2488
          ],
          [
            0.62,
            0.2586
          ],
          [
            0.64,
            0.2697
          ],
          [
            0.66,
            0.2801
          ],
          [
            0.68,
            0.2908
          ],
          [
            0.7,
            0.3003
          ],
          [
            0.72,
            0.3069
          ],
          [
            0.74,
            0.317
          ],
          [
            0.76,
            0.3232
          ],
          [
            0.78,
            0.3313
          ],
          [
            0.8,
            0.3397
          ],
          [
            0.82,
            0.347
          ],
          [
            0.84,
            0.3535
          ],
          [
            0.86,
            0.3619
          ],
          [
            0.88,
            0.3701
          ],
          [
            0.9,
            0.3768
          ],
          [
            0.92,
            0.385
          ],
          [
            0.94,
            0.3905
          ],
          [
            0.96,
            0.396
          ],
          [
            0.98,
            0.4031
          ],
          [
            1.0,
            0.4081
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 174,
            "mean_confidence": 0.26850574712643677,
            "accuracy": 0.3275862068965517
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 437,
            "mean_confidence": 0.3477574370709381,
            "accuracy": 0.3318077803203661
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 361,
            "mean_confidence": 0.4421329639889194,
            "accuracy": 0.41274238227146814
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 234,
            "mean_confidence": 0.5442735042735045,
            "accuracy": 0.5128205128205128
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 182,
            "mean_confidence": 0.6428571428571429,
            "accuracy": 0.6098901098901099
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 163,
            "mean_confidence": 0.7464417177914113,
            "accuracy": 0.6871165644171779
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 157,
            "mean_confidence": 0.8480254777070063,
            "accuracy": 0.821656050955414
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 490,
            "mean_confidence": 0.9742653061224507,
            "accuracy": 0.9755102040816327
          }
        ],
        "median_right": 0.76,
        "median_wrong": 0.41000000000000003,
        "sure": {
          "questions": 260,
          "correct": 259,
          "interval": [
            0.978538689004996,
            0.9993207539334944
          ]
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 1388,
          "correct": 582,
          "interval": [
            0.39360619404503616,
            0.4454559521072271
          ]
        },
        "ece_refit": 0.01577745939058952,
        "brier_refit": 0.179274232142994,
        "brier_base": 0.24174024691167653,
        "refit_slopes": [
          0.9061177962788771,
          0.9097932963861856,
          0.9442781512558797,
          0.9198175072302215,
          0.9662763287648225
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        104,
        138,
        52,
        93,
        82,
        89,
        78,
        31,
        53,
        38,
        38,
        24,
        66,
        43,
        43,
        81,
        26,
        55,
        60,
        107
      ],
      "started": "2026-10-16T09:52:25Z",
      "updated": "2026-10-16T10:06:22Z",
      "format_of": null,
      "group": "wxyzA",
      "latency": {
        "source": "run",
        "runs": [
          194
        ],
        "concurrency": 1,
        "median_request_seconds": 0.2105813504967955,
        "p90_request_seconds": 0.2774909528983699
      },
      "single_question_latency": null
    },
    {
      "run": 204,
      "model": "inception/mercury-decide",
      "name": "Inception Mercury Decide",
      "variant": "via OpenRouter",
      "reasoning": null,
      "provider": "openrouter",
      "note": "Inception's decision model, called through OpenRouter's System One route (paid listing). It returns a probability for each option.",
      "parameters": null,
      "active_parameters": null,
      "access": "OpenRouter System One API",
      "questions_per_request": 1,
      "concurrency": 1,
      "correct": 1281,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 554
        },
        "grammar": {
          "questions": 1542,
          "correct": 727
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 1275
        }
      },
      "tyt_mix": 0.7792290460599158,
      "questions_per_minute": 140.8778626326503,
      "request_seconds": 936.1300458105834,
      "median_request_seconds": 0.31287493899435503,
      "p90_request_seconds": 0.7718596629958483,
      "billing": "api",
      "api_equivalent_usd": 0.01694982,
      "cost_basis": "api",
      "requests": 2198,
      "hardware": null,
      "input_tokens": 847491,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 2198,
      "confidence": {
        "source": "probability",
        "label": "Option score",
        "bins": [
          {
            "questions": 37,
            "correct": 14
          },
          {
            "questions": 488,
            "correct": 147
          },
          {
            "questions": 463,
            "correct": 166
          },
          {
            "questions": 290,
            "correct": 149
          },
          {
            "questions": 920,
            "correct": 805
          }
        ],
        "auroc": 0.8077326788555492,
        "auroc_interval": [
          0.7909348830431913,
          0.824551168178533
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.9051107807744037,
            "auroc_interval": [
              0.8774695359087373,
              0.930428863172226
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.7179720002362849,
            "auroc_interval": [
              0.6931119415661627,
              0.7421193423935691
            ]
          }
        },
        "aurc": 0.18612302828875937,
        "ece": 0.15503569618513982,
        "brier": 0.2081494977500661,
        "coverage_accuracy": {
          "0.20": 0.9727024567788899,
          "0.50": 0.8198362147406734,
          "0.80": 0.6540036396724295
        },
        "coverage_intervals": {
          "0.20": [
            0.954524783992724,
            0.9863387978142076
          ],
          "0.50": [
            0.7931662870159454,
            0.8461538461538461
          ],
          "0.80": [
            0.6284121929026387,
            0.6780045351473923
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0
          ],
          [
            0.04,
            0.0
          ],
          [
            0.06,
            0.0
          ],
          [
            0.08,
            0.0
          ],
          [
            0.1,
            0.0091
          ],
          [
            0.12,
            0.0114
          ],
          [
            0.14,
            0.013
          ],
          [
            0.16,
            0.0142
          ],
          [
            0.18,
            0.0227
          ],
          [
            0.2,
            0.0273
          ],
          [
            0.22,
            0.031
          ],
          [
            0.24,
            0.0341
          ],
          [
            0.26,
            0.042
          ],
          [
            0.28,
            0.0487
          ],
          [
            0.3,
            0.0576
          ],
          [
            0.32,
            0.0716
          ],
          [
            0.34,
            0.0789
          ],
          [
            0.36,
            0.0935
          ],
          [
            0.38,
            0.1042
          ],
          [
            0.4,
            0.1137
          ],
          [
            0.42,
            0.1246
          ],
          [
            0.44,
            0.1417
          ],
          [
            0.46,
            0.1533
          ],
          [
            0.48,
            0.1669
          ],
          [
            0.5,
            0.1802
          ],
          [
            0.52,
            0.1934
          ],
          [
            0.54,
            0.2072
          ],
          [
            0.56,
            0.2168
          ],
          [
            0.58,
            0.2274
          ],
          [
            0.6,
            0.2425
          ],
          [
            0.62,
            0.2532
          ],
          [
            0.64,
            0.2609
          ],
          [
            0.66,
            0.2741
          ],
          [
            0.68,
            0.2854
          ],
          [
            0.7,
            0.2955
          ],
          [
            0.72,
            0.3081
          ],
          [
            0.74,
            0.3213
          ],
          [
            0.76,
            0.3304
          ],
          [
            0.78,
            0.3395
          ],
          [
            0.8,
            0.346
          ],
          [
            0.82,
            0.3553
          ],
          [
            0.84,
            0.3603
          ],
          [
            0.86,
            0.3694
          ],
          [
            0.88,
            0.3774
          ],
          [
            0.9,
            0.3847
          ],
          [
            0.92,
            0.3902
          ],
          [
            0.94,
            0.3974
          ],
          [
            0.96,
            0.4043
          ],
          [
            0.98,
            0.4118
          ],
          [
            1.0,
            0.4172
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 127,
            "mean_confidence": 0.26332911080820337,
            "accuracy": 0.2755905511811024
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 193,
            "mean_confidence": 0.35215325849708934,
            "accuracy": 0.3005181347150259
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 205,
            "mean_confidence": 0.4462241393299637,
            "accuracy": 0.33170731707317075
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 207,
            "mean_confidence": 0.5466126646064091,
            "accuracy": 0.2995169082125604
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 177,
            "mean_confidence": 0.652740046517218,
            "accuracy": 0.384180790960452
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 176,
            "mean_confidence": 0.7523700085817816,
            "accuracy": 0.4772727272727273
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 193,
            "mean_confidence": 0.8551015285757928,
            "accuracy": 0.5233160621761658
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 920,
            "mean_confidence": 0.9778632646765245,
            "accuracy": 0.875
          }
        ],
        "median_right": 0.9664165926978489,
        "median_wrong": 0.5573042430351277,
        "sure": {
          "questions": 451,
          "correct": 438,
          "interval": [
            0.9513124663540422,
            0.9830787590747665
          ]
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 909,
          "correct": 291,
          "interval": [
            0.2906148855782108,
            0.3511630526752605
          ]
        },
        "ece_refit": 0.036387055620216234,
        "brier_refit": 0.17316428604268613,
        "brier_base": 0.243306123799318,
        "refit_slopes": [
          0.6130592896293957,
          0.6455068360406191,
          0.6306868920339721,
          0.6308480931848867,
          0.6308437875490661
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        103,
        137,
        52,
        92,
        81,
        89,
        79,
        30,
        46,
        31,
        41,
        22,
        56,
        44,
        39,
        76,
        27,
        68,
        59,
        109
      ],
      "started": "2026-10-17T15:08:47Z",
      "updated": "2026-10-17T15:24:29Z",
      "format_of": null,
      "group": "wxyzA",
      "latency": {
        "source": "run",
        "runs": [
          204
        ],
        "concurrency": 1,
        "median_request_seconds": 0.31287493899435503,
        "p90_request_seconds": 0.7718596629958483
      },
      "single_question_latency": null
    },
    {
      "run": 159,
      "model": "glm-5.3-flash",
      "name": "GLM-5.3-Flash",
      "variant": null,
      "reasoning": "off",
      "provider": "vllm-stated",
      "note": null,
      "parameters": 320000000000,
      "active_parameters": [
        18000000000
      ],
      "access": "TensorFold, self-hosted, EXL3 4-bit experts",
      "questions_per_request": 16,
      "concurrency": 1,
      "correct": 1270,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 539
        },
        "grammar": {
          "questions": 1542,
          "correct": 731
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 1269
        }
      },
      "tyt_mix": 0.7608186726930499,
      "questions_per_minute": 879.5015970459278,
      "request_seconds": 149.94856227999912,
      "median_request_seconds": 1.065357404999986,
      "p90_request_seconds": 1.2150850257999992,
      "billing": "local",
      "api_equivalent_usd": null,
      "cost_basis": null,
      "requests": 138,
      "hardware": "4 × RTX PRO 6000 Blackwell (96 GB); EXL3 4-bit experts, FP8 dense layers and KV cache",
      "input_tokens": 557121,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 40702,
      "confidence": {
        "source": "stated",
        "label": "Written estimate",
        "bins": [
          {
            "questions": 0,
            "correct": 0
          },
          {
            "questions": 13,
            "correct": 1
          },
          {
            "questions": 705,
            "correct": 241
          },
          {
            "questions": 795,
            "correct": 476
          },
          {
            "questions": 685,
            "correct": 552
          }
        ],
        "auroc": 0.7326457711105078,
        "auroc_interval": [
          0.7124153172680522,
          0.7527593352401034
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.776794633937491,
            "auroc_interval": [
              0.7290483914571506,
              0.8217620664564638
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.6678013160358343,
            "auroc_interval": [
              0.6388584102569265,
              0.6963156106675726
            ]
          }
        },
        "aurc": 0.2505219805943148,
        "ece": 0.20956778889899869,
        "brier": 0.25416892629663335,
        "coverage_accuracy": {
          "0.20": 0.8420310157697317,
          "0.50": 0.7486437613019892,
          "0.80": 0.64571270579217
        },
        "coverage_intervals": {
          "0.20": [
            0.8132519561534002,
            0.8705475677046333
          ],
          "0.50": [
            0.7222535211267606,
            0.7721607942378821
          ],
          "0.80": [
            0.6227987311767986,
            0.6675112612612613
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0749
          ],
          [
            0.04,
            0.0859
          ],
          [
            0.06,
            0.0896
          ],
          [
            0.08,
            0.0914
          ],
          [
            0.1,
            0.0925
          ],
          [
            0.12,
            0.0932
          ],
          [
            0.14,
            0.1147
          ],
          [
            0.16,
            0.1327
          ],
          [
            0.18,
            0.1467
          ],
          [
            0.2,
            0.158
          ],
          [
            0.22,
            0.1672
          ],
          [
            0.24,
            0.1748
          ],
          [
            0.26,
            0.1813
          ],
          [
            0.28,
            0.1868
          ],
          [
            0.3,
            0.1916
          ],
          [
            0.32,
            0.1981
          ],
          [
            0.34,
            0.2068
          ],
          [
            0.36,
            0.2146
          ],
          [
            0.38,
            0.2215
          ],
          [
            0.4,
            0.2277
          ],
          [
            0.42,
            0.2333
          ],
          [
            0.44,
            0.2385
          ],
          [
            0.46,
            0.2431
          ],
          [
            0.48,
            0.2474
          ],
          [
            0.5,
            0.2514
          ],
          [
            0.52,
            0.255
          ],
          [
            0.54,
            0.2609
          ],
          [
            0.56,
            0.2677
          ],
          [
            0.58,
            0.2741
          ],
          [
            0.6,
            0.2801
          ],
          [
            0.62,
            0.2872
          ],
          [
            0.64,
            0.2944
          ],
          [
            0.66,
            0.3011
          ],
          [
            0.68,
            0.3084
          ],
          [
            0.7,
            0.3172
          ],
          [
            0.72,
            0.3254
          ],
          [
            0.74,
            0.3332
          ],
          [
            0.76,
            0.3406
          ],
          [
            0.78,
            0.3476
          ],
          [
            0.8,
            0.3543
          ],
          [
            0.82,
            0.3611
          ],
          [
            0.84,
            0.3683
          ],
          [
            0.86,
            0.3752
          ],
          [
            0.88,
            0.3817
          ],
          [
            0.9,
            0.3879
          ],
          [
            0.92,
            0.3939
          ],
          [
            0.94,
            0.401
          ],
          [
            0.96,
            0.409
          ],
          [
            0.98,
            0.4151
          ],
          [
            1.0,
            0.4222
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 1,
            "mean_confidence": 0.35,
            "accuracy": 0.0
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 12,
            "mean_confidence": 0.4125000000000001,
            "accuracy": 0.08333333333333333
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 138,
            "mean_confidence": 0.5177536231884055,
            "accuracy": 0.2536231884057971
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 256,
            "mean_confidence": 0.6007812499999973,
            "accuracy": 0.3359375
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 462,
            "mean_confidence": 0.7163419913419873,
            "accuracy": 0.41774891774891776
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 644,
            "mean_confidence": 0.8368012422360338,
            "accuracy": 0.6257763975155279
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 685,
            "mean_confidence": 0.920043795620432,
            "accuracy": 0.8058394160583942
          }
        ],
        "median_right": 0.85,
        "median_wrong": 0.7,
        "sure": {
          "questions": 4,
          "correct": 4,
          "interval": [
            0.5100999795960008,
            1.0
          ]
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 407,
          "correct": 122,
          "interval": [
            0.2572848053794851,
            0.34596861394319295
          ]
        },
        "ece_refit": 0.017285732129880334,
        "brier_refit": 0.20449592312306472,
        "brier_base": 0.24410564254575748,
        "refit_slopes": [
          1.115680077702488,
          1.0774269946382875,
          1.1153950065551088,
          1.094551167463906,
          1.0970179710127186
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        100,
        130,
        44,
        91,
        83,
        91,
        73,
        27,
        49,
        35,
        38,
        21,
        56,
        44,
        38,
        75,
        28,
        76,
        58,
        113
      ],
      "started": "2026-10-11T10:20:58Z",
      "updated": "2026-10-11T10:23:29Z",
      "format_of": null,
      "group": "wxyzAB",
      "latency": {
        "source": "run",
        "runs": [
          159
        ],
        "concurrency": 1,
        "median_request_seconds": 1.065357404999986,
        "p90_request_seconds": 1.2150850257999992
      },
      "single_question_latency": null
    },
    {
      "run": 49,
      "model": "jev-1.13.0",
      "name": "Jev 1.13.0",
      "variant": null,
      "reasoning": null,
      "provider": "jev",
      "note": null,
      "parameters": null,
      "active_parameters": null,
      "access": "TypeSafe System One API",
      "questions_per_request": 1,
      "concurrency": 1,
      "correct": 1264,
      "invalid": 0,
      "answered_questions": 2210,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 549
        },
        "grammar": {
          "questions": 1542,
          "correct": 715
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 1263
        }
      },
      "tyt_mix": 0.7715790685995382,
      "questions_per_minute": 251.69948821856477,
      "request_seconds": 526.8187112277956,
      "median_request_seconds": null,
      "p90_request_seconds": null,
      "billing": "api",
      "api_equivalent_usd": 0.06375499200000001,
      "cost_basis": "api",
      "requests": 2210,
      "hardware": null,
      "input_tokens": 1517976,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 114920,
      "confidence": {
        "source": "probability",
        "label": "Option score",
        "bins": [
          {
            "questions": 16,
            "correct": 3
          },
          {
            "questions": 861,
            "correct": 272
          },
          {
            "questions": 549,
            "correct": 297
          },
          {
            "questions": 238,
            "correct": 183
          },
          {
            "questions": 534,
            "correct": 509
          }
        ],
        "auroc": 0.8071648923915106,
        "auroc_interval": [
          0.789331726540943,
          0.8258374679328807
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.8880887935583814,
            "auroc_interval": [
              0.8614403953431967,
              0.914153748141856
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.7219091670119482,
            "auroc_interval": [
              0.6962876163304124,
              0.7483772084954441
            ]
          }
        },
        "aurc": 0.19182447863902138,
        "ece": 0.051401273885350714,
        "brier": 0.1769368516833485,
        "coverage_accuracy": {
          "0.20": 0.9819416252537272,
          "0.50": 0.8142467177953985,
          "0.80": 0.6468070146414096
        },
        "coverage_intervals": {
          "0.20": [
            0.9572727272727273,
            0.9956439393939394
          ],
          "0.50": [
            0.7878925807919891,
            0.8390092879256966
          ],
          "0.80": [
            0.6233470132238943,
            0.6716007053291535
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0
          ],
          [
            0.04,
            0.0
          ],
          [
            0.06,
            0.0
          ],
          [
            0.08,
            0.0
          ],
          [
            0.1,
            0.0
          ],
          [
            0.12,
            0.0012
          ],
          [
            0.14,
            0.0031
          ],
          [
            0.16,
            0.0053
          ],
          [
            0.18,
            0.0067
          ],
          [
            0.2,
            0.0181
          ],
          [
            0.22,
            0.0376
          ],
          [
            0.24,
            0.0462
          ],
          [
            0.26,
            0.0507
          ],
          [
            0.28,
            0.062
          ],
          [
            0.3,
            0.0689
          ],
          [
            0.32,
            0.0826
          ],
          [
            0.34,
            0.0954
          ],
          [
            0.36,
            0.1072
          ],
          [
            0.38,
            0.1194
          ],
          [
            0.4,
            0.1302
          ],
          [
            0.42,
            0.1449
          ],
          [
            0.44,
            0.153
          ],
          [
            0.46,
            0.1627
          ],
          [
            0.48,
            0.1734
          ],
          [
            0.5,
            0.1858
          ],
          [
            0.52,
            0.1968
          ],
          [
            0.54,
            0.208
          ],
          [
            0.56,
            0.2215
          ],
          [
            0.58,
            0.2354
          ],
          [
            0.6,
            0.2504
          ],
          [
            0.62,
            0.2631
          ],
          [
            0.64,
            0.2724
          ],
          [
            0.66,
            0.2822
          ],
          [
            0.68,
            0.2922
          ],
          [
            0.7,
            0.3041
          ],
          [
            0.72,
            0.3156
          ],
          [
            0.74,
            0.3259
          ],
          [
            0.76,
            0.338
          ],
          [
            0.78,
            0.3446
          ],
          [
            0.8,
            0.3532
          ],
          [
            0.82,
            0.3592
          ],
          [
            0.84,
            0.3698
          ],
          [
            0.86,
            0.378
          ],
          [
            0.88,
            0.3836
          ],
          [
            0.9,
            0.3913
          ],
          [
            0.92,
            0.3973
          ],
          [
            0.94,
            0.4043
          ],
          [
            0.96,
            0.4109
          ],
          [
            0.98,
            0.4181
          ],
          [
            1.0,
            0.4249
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 142,
            "mean_confidence": 0.2684507042253521,
            "accuracy": 0.2605633802816901
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 379,
            "mean_confidence": 0.34928759894459077,
            "accuracy": 0.316622691292876
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 356,
            "mean_confidence": 0.44401685393258383,
            "accuracy": 0.33146067415730335
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 279,
            "mean_confidence": 0.5423655913978499,
            "accuracy": 0.44086021505376344
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 183,
            "mean_confidence": 0.6432240437158472,
            "accuracy": 0.6229508196721312
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 171,
            "mean_confidence": 0.7433918128654977,
            "accuracy": 0.6842105263157895
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 154,
            "mean_confidence": 0.8488961038961033,
            "accuracy": 0.8181818181818182
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 534,
            "mean_confidence": 0.9766479400749082,
            "accuracy": 0.9531835205992509
          }
        ],
        "median_right": 0.8,
        "median_wrong": 0.43,
        "sure": {
          "questions": 312,
          "correct": 311,
          "interval": [
            0.9820706464658654,
            0.9994340064994186
          ]
        },
        "sure_before_audit": {
          "questions": 314,
          "correct": 311,
          "interval": [
            0.97229057656989,
            0.9967455722539266
          ]
        },
        "doubtful": {
          "questions": 1339,
          "correct": 512,
          "interval": [
            0.35671656371940946,
            0.40870625308936387
          ]
        },
        "ece_refit": 0.021633445258244212,
        "brier_refit": 0.17392242653604426,
        "brier_base": 0.24461851531423254,
        "refit_slopes": [
          0.8940733626719859,
          0.9339733496041935,
          0.9553313191705414,
          0.9096259616038113,
          0.9412733178232202
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": {
        "run": 29,
        "changed": {
          "questions": 372,
          "older_correct": 135,
          "newer_correct": 186,
          "flipped": 121,
          "changed_answer": 201,
          "newer_wrong": 186,
          "repeated_mistakes": 71,
          "p": 4.004113903707686e-06
        },
        "unchanged": {
          "questions": 1814,
          "older_correct": 1079,
          "newer_correct": 1071,
          "flipped": 70,
          "changed_answer": 114,
          "newer_wrong": 743,
          "repeated_mistakes": 660,
          "p": 0.40296307504607976
        },
        "added": {
          "questions": 12,
          "newer_correct": 7
        }
      },
      "units": [
        105,
        137,
        50,
        93,
        78,
        86,
        76,
        32,
        51,
        41,
        39,
        24,
        65,
        44,
        41,
        68,
        21,
        52,
        55,
        106
      ],
      "started": "2026-10-04T08:39:08Z",
      "updated": "2026-10-04T08:47:59Z",
      "format_of": null,
      "group": "xyzAB",
      "latency": {
        "source": "repeats",
        "runs": [
          65,
          85,
          97,
          101
        ],
        "repeats": [
          2,
          3,
          4,
          5
        ],
        "questions": 2198,
        "concurrency": 1,
        "median_request_seconds": 0.24339387452346273,
        "p90_request_seconds": 0.2807294782833196
      },
      "single_question_latency": null
    },
    {
      "run": 125,
      "model": "clef",
      "name": "Clef",
      "variant": null,
      "reasoning": null,
      "provider": "clef",
      "note": "Cloudflare's decision model, post-trained from Qwen3.8-27B",
      "parameters": 27484784884,
      "active_parameters": null,
      "access": "Clef server, self-hosted",
      "questions_per_request": 1,
      "concurrency": 1,
      "correct": 1259,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 558
        },
        "grammar": {
          "questions": 1542,
          "correct": 701
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 1256
        }
      },
      "tyt_mix": 0.7813088205055203,
      "questions_per_minute": 285.0197317975998,
      "request_seconds": 462.7048070259625,
      "median_request_seconds": 0.20457697298843414,
      "p90_request_seconds": 0.24825825718580746,
      "billing": "local",
      "api_equivalent_usd": 0.24108839999999998,
      "cost_basis": "maker",
      "requests": 2198,
      "hardware": "1 × RTX PRO 6000 Blackwell (96 GB); BF16",
      "input_tokens": 1004535,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 0,
      "confidence": {
        "source": "probability",
        "label": "Option score",
        "bins": [
          {
            "questions": 3,
            "correct": 0
          },
          {
            "questions": 611,
            "correct": 188
          },
          {
            "questions": 644,
            "correct": 294
          },
          {
            "questions": 363,
            "correct": 233
          },
          {
            "questions": 577,
            "correct": 544
          }
        ],
        "auroc": 0.7850073718428592,
        "auroc_interval": [
          0.7664443128121592,
          0.8040524558700295
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.8756857581742374,
            "auroc_interval": [
              0.8456854345165239,
              0.9043910806174957
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.6873287862930653,
            "auroc_interval": [
              0.6597310357465234,
              0.7150232129741613
            ]
          }
        },
        "aurc": 0.20484180390491813,
        "ece": 0.1069694722474975,
        "brier": 0.19947404868516835,
        "coverage_accuracy": {
          "0.20": 0.9727024567788899,
          "0.50": 0.7816196542311192,
          "0.80": 0.6426296633303002
        },
        "coverage_intervals": {
          "0.20": [
            0.9567395264116576,
            0.9863822060826146
          ],
          "0.50": [
            0.7531876138433515,
            0.8098495212038304
          ],
          "0.80": [
            0.6184510250569476,
            0.6662112932604736
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0
          ],
          [
            0.04,
            0.0
          ],
          [
            0.06,
            0.0
          ],
          [
            0.08,
            0.0057
          ],
          [
            0.1,
            0.0045
          ],
          [
            0.12,
            0.0076
          ],
          [
            0.14,
            0.0097
          ],
          [
            0.16,
            0.0142
          ],
          [
            0.18,
            0.0253
          ],
          [
            0.2,
            0.0273
          ],
          [
            0.22,
            0.0352
          ],
          [
            0.24,
            0.0398
          ],
          [
            0.26,
            0.0542
          ],
          [
            0.28,
            0.0796
          ],
          [
            0.3,
            0.0886
          ],
          [
            0.32,
            0.0986
          ],
          [
            0.34,
            0.1155
          ],
          [
            0.36,
            0.1289
          ],
          [
            0.38,
            0.1428
          ],
          [
            0.4,
            0.1554
          ],
          [
            0.42,
            0.1712
          ],
          [
            0.44,
            0.1846
          ],
          [
            0.46,
            0.1929
          ],
          [
            0.48,
            0.2057
          ],
          [
            0.5,
            0.2184
          ],
          [
            0.52,
            0.2345
          ],
          [
            0.54,
            0.2435
          ],
          [
            0.56,
            0.2534
          ],
          [
            0.58,
            0.2627
          ],
          [
            0.6,
            0.2705
          ],
          [
            0.62,
            0.2779
          ],
          [
            0.64,
            0.2882
          ],
          [
            0.66,
            0.2996
          ],
          [
            0.68,
            0.3069
          ],
          [
            0.7,
            0.3159
          ],
          [
            0.72,
            0.3237
          ],
          [
            0.74,
            0.3363
          ],
          [
            0.76,
            0.3436
          ],
          [
            0.78,
            0.3485
          ],
          [
            0.8,
            0.3574
          ],
          [
            0.82,
            0.3667
          ],
          [
            0.84,
            0.3721
          ],
          [
            0.86,
            0.3816
          ],
          [
            0.88,
            0.3888
          ],
          [
            0.9,
            0.3951
          ],
          [
            0.92,
            0.4016
          ],
          [
            0.94,
            0.4095
          ],
          [
            0.96,
            0.4157
          ],
          [
            0.98,
            0.422
          ],
          [
            1.0,
            0.4272
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 48,
            "mean_confidence": 0.278475,
            "accuracy": 0.2916666666666667
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 239,
            "mean_confidence": 0.356336820083682,
            "accuracy": 0.301255230125523
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 327,
            "mean_confidence": 0.44891987767584085,
            "accuracy": 0.3119266055045872
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 282,
            "mean_confidence": 0.5460460992907799,
            "accuracy": 0.41134751773049644
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 254,
            "mean_confidence": 0.6488141732283463,
            "accuracy": 0.48031496062992124
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 215,
            "mean_confidence": 0.7499934883720922,
            "accuracy": 0.5534883720930233
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 256,
            "mean_confidence": 0.852619140625,
            "accuracy": 0.6640625
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 577,
            "mean_confidence": 0.9518570190641246,
            "accuracy": 0.9428076256499134
          }
        ],
        "median_right": 0.8571,
        "median_wrong": 0.5215,
        "sure": {
          "questions": 0,
          "correct": 0,
          "interval": null
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 1150,
          "correct": 426,
          "interval": [
            0.3429978072936793,
            0.39873450674318683
          ]
        },
        "ece_refit": 0.037366558649174826,
        "brier_refit": 0.1848633813700203,
        "brier_base": 0.24479424644816136,
        "refit_slopes": [
          0.9140366196677421,
          0.9541055113082861,
          0.9594733287424564,
          0.9392647973648997,
          0.9671586232141537
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        101,
        142,
        54,
        93,
        80,
        88,
        71,
        27,
        47,
        30,
        41,
        25,
        50,
        38,
        42,
        75,
        24,
        70,
        55,
        106
      ],
      "started": "2026-10-05T20:59:17Z",
      "updated": "2026-10-05T21:07:04Z",
      "format_of": null,
      "group": "xyzAB",
      "latency": {
        "source": "run",
        "runs": [
          125
        ],
        "concurrency": 1,
        "median_request_seconds": 0.20457697298843414,
        "p90_request_seconds": 0.24825825718580746
      },
      "single_question_latency": null
    },
    {
      "run": 52,
      "model": "d1:free",
      "name": "Liquid d1",
      "variant": null,
      "reasoning": null,
      "provider": "liquid",
      "note": null,
      "parameters": null,
      "active_parameters": null,
      "access": "Liquid AI decisions API",
      "questions_per_request": 1,
      "concurrency": 1,
      "correct": 1253,
      "invalid": 0,
      "answered_questions": 2210,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 543
        },
        "grammar": {
          "questions": 1542,
          "correct": 710
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 1251
        }
      },
      "tyt_mix": 0.7634658920154377,
      "questions_per_minute": 150.12319443415166,
      "request_seconds": 883.2745699277148,
      "median_request_seconds": null,
      "p90_request_seconds": null,
      "billing": "free",
      "api_equivalent_usd": 0.016426200000000002,
      "cost_basis": "free tier",
      "requests": 2210,
      "hardware": null,
      "input_tokens": 821310,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 0,
      "confidence": {
        "source": "probability",
        "label": "Option score",
        "bins": [
          {
            "questions": 13,
            "correct": 2
          },
          {
            "questions": 717,
            "correct": 226
          },
          {
            "questions": 541,
            "correct": 254
          },
          {
            "questions": 295,
            "correct": 183
          },
          {
            "questions": 632,
            "correct": 588
          }
        ],
        "auroc": 0.7916340465422668,
        "auroc_interval": [
          0.7722874257989321,
          0.8090258990574328
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.8834237846118744,
            "auroc_interval": [
              0.8551205600207415,
              0.9098257794292606
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.7024935671722644,
            "auroc_interval": [
              0.6747661719784824,
              0.7296222427998196
            ]
          }
        },
        "aurc": 0.2030287774937924,
        "ece": 0.10208650451299976,
        "brier": 0.19436615238682217,
        "coverage_accuracy": {
          "0.20": 0.9749772520473158,
          "0.50": 0.7816196542311192,
          "0.80": 0.6422884440400364
        },
        "coverage_intervals": {
          "0.20": [
            0.9554140127388534,
            0.9908966772872099
          ],
          "0.50": [
            0.756486117432863,
            0.8089171974522293
          ],
          "0.80": [
            0.6193130118289354,
            0.6647005444646098
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0
          ],
          [
            0.04,
            0.0
          ],
          [
            0.06,
            0.0
          ],
          [
            0.08,
            0.0
          ],
          [
            0.1,
            0.0045
          ],
          [
            0.12,
            0.0038
          ],
          [
            0.14,
            0.0032
          ],
          [
            0.16,
            0.0057
          ],
          [
            0.18,
            0.0177
          ],
          [
            0.2,
            0.025
          ],
          [
            0.22,
            0.0331
          ],
          [
            0.24,
            0.0427
          ],
          [
            0.26,
            0.0542
          ],
          [
            0.28,
            0.0673
          ],
          [
            0.3,
            0.0819
          ],
          [
            0.32,
            0.0943
          ],
          [
            0.34,
            0.1111
          ],
          [
            0.36,
            0.1238
          ],
          [
            0.38,
            0.1401
          ],
          [
            0.4,
            0.1538
          ],
          [
            0.42,
            0.1659
          ],
          [
            0.44,
            0.1821
          ],
          [
            0.46,
            0.1969
          ],
          [
            0.48,
            0.2085
          ],
          [
            0.5,
            0.2184
          ],
          [
            0.52,
            0.224
          ],
          [
            0.54,
            0.2333
          ],
          [
            0.56,
            0.2454
          ],
          [
            0.58,
            0.2572
          ],
          [
            0.6,
            0.2662
          ],
          [
            0.62,
            0.2765
          ],
          [
            0.64,
            0.2884
          ],
          [
            0.66,
            0.2983
          ],
          [
            0.68,
            0.3075
          ],
          [
            0.7,
            0.3176
          ],
          [
            0.72,
            0.3264
          ],
          [
            0.74,
            0.3354
          ],
          [
            0.76,
            0.343
          ],
          [
            0.78,
            0.3502
          ],
          [
            0.8,
            0.3577
          ],
          [
            0.82,
            0.3645
          ],
          [
            0.84,
            0.3732
          ],
          [
            0.86,
            0.3837
          ],
          [
            0.88,
            0.3929
          ],
          [
            0.9,
            0.3989
          ],
          [
            0.92,
            0.4051
          ],
          [
            0.94,
            0.4124
          ],
          [
            0.96,
            0.4175
          ],
          [
            0.98,
            0.4229
          ],
          [
            1.0,
            0.4299
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 119,
            "mean_confidence": 0.2752297090240214,
            "accuracy": 0.3025210084033613
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 298,
            "mean_confidence": 0.3517646086306328,
            "accuracy": 0.2684563758389262
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 313,
            "mean_confidence": 0.4484922384775263,
            "accuracy": 0.35782747603833864
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 242,
            "mean_confidence": 0.5471528749257146,
            "accuracy": 0.4090909090909091
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 205,
            "mean_confidence": 0.6470857402475192,
            "accuracy": 0.5414634146341464
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 194,
            "mean_confidence": 0.7513873534806637,
            "accuracy": 0.5154639175257731
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 195,
            "mean_confidence": 0.8533268327982725,
            "accuracy": 0.6512820512820513
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 632,
            "mean_confidence": 0.9742137607980402,
            "accuracy": 0.930379746835443
          }
        ],
        "median_right": 0.8746144451881748,
        "median_wrong": 0.4830418527453172,
        "sure": {
          "questions": 274,
          "correct": 273,
          "interval": [
            0.9796195950792701,
            0.9993554756084886
          ]
        },
        "sure_before_audit": {
          "questions": 276,
          "correct": 273,
          "interval": [
            0.9685348547189621,
            0.9962966785484291
          ]
        },
        "doubtful": {
          "questions": 1177,
          "correct": 438,
          "interval": [
            0.34497499042876506,
            0.4001220657725069
          ]
        },
        "ece_refit": 0.019447904935036918,
        "brier_refit": 0.18078639530511723,
        "brier_base": 0.24544141522771967,
        "refit_slopes": [
          0.7607547418287448,
          0.7562267021901162,
          0.7768707132043405,
          0.7354082115201845,
          0.7696250673292939
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": {
        "run": 40,
        "changed": {
          "questions": 372,
          "older_correct": 138,
          "newer_correct": 201,
          "flipped": 115,
          "changed_answer": 193,
          "newer_wrong": 171,
          "repeated_mistakes": 67,
          "p": 2.952378329881734e-09
        },
        "unchanged": {
          "questions": 1814,
          "older_correct": 1041,
          "newer_correct": 1045,
          "flipped": 58,
          "changed_answer": 91,
          "newer_wrong": 769,
          "repeated_mistakes": 709,
          "p": 0.6940040940761316
        },
        "added": {
          "questions": 12,
          "newer_correct": 7
        }
      },
      "units": [
        103,
        133,
        53,
        93,
        76,
        85,
        69,
        33,
        46,
        40,
        39,
        24,
        66,
        40,
        40,
        63,
        22,
        61,
        57,
        110
      ],
      "started": "2026-10-04T08:48:02Z",
      "updated": "2026-10-04T09:02:49Z",
      "format_of": null,
      "group": "yzAB",
      "latency": {
        "source": "repeats",
        "runs": [
          66,
          94,
          105,
          108
        ],
        "repeats": [
          2,
          3,
          4,
          5
        ],
        "questions": 2198,
        "concurrency": 1,
        "median_request_seconds": 0.3054448555049021,
        "p90_request_seconds": 0.78636241707718
      },
      "single_question_latency": null
    },
    {
      "run": 334,
      "model": "drex-v2.0",
      "name": "Drex 2.0",
      "variant": null,
      "reasoning": null,
      "provider": "nace",
      "note": "Nace.AI's hosted Drex 2.0 (drex-v2.0, released 6 October 2026), built on Qwen3.6-35B-A3B. It reads text only, so it is not on the Page images tab. Nace's API gives its option probabilities rounded to 4 decimal places, so they sum to 1 only within that rounding; the check before its runs allowed for it.",
      "parameters": 35000000000,
      "active_parameters": [
        3000000000
      ],
      "access": "Nace.AI System One API",
      "questions_per_request": 1,
      "concurrency": 1,
      "correct": 1225,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 534
        },
        "grammar": {
          "questions": 1542,
          "correct": 691
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 1224
        }
      },
      "tyt_mix": 0.7499910039226851,
      "questions_per_minute": 120.51272554278829,
      "request_seconds": 1094.324266636686,
      "median_request_seconds": 0.32470757849660004,
      "p90_request_seconds": 0.39495088000403483,
      "billing": "api",
      "api_equivalent_usd": 0.04210104,
      "cost_basis": "api",
      "requests": 2198,
      "hardware": null,
      "input_tokens": 701684,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 185394,
      "confidence": {
        "source": "probability",
        "label": "Option score",
        "bins": [
          {
            "questions": 106,
            "correct": 19
          },
          {
            "questions": 1190,
            "correct": 455
          },
          {
            "questions": 411,
            "correct": 274
          },
          {
            "questions": 219,
            "correct": 206
          },
          {
            "questions": 272,
            "correct": 271
          }
        ],
        "auroc": 0.8038744048492984,
        "auroc_interval": [
          0.785982736598816,
          0.8222696854149998
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.8807100755203536,
            "auroc_interval": [
              0.8513476410561479,
              0.9059544297839166
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.7204667701741886,
            "auroc_interval": [
              0.6931319097379979,
              0.7476841998581129
            ]
          }
        },
        "aurc": 0.20674176675442435,
        "ece": 0.042306960873521385,
        "brier": 0.17906242492720653,
        "coverage_accuracy": {
          "0.20": 0.9795268425841674,
          "0.50": 0.7852593266606005,
          "0.80": 0.6302320291173794
        },
        "coverage_intervals": {
          "0.20": [
            0.9658780709736123,
            0.9931787175989086
          ],
          "0.50": [
            0.75591985428051,
            0.8123861566484517
          ],
          "0.80": [
            0.6062329390354868,
            0.653732362312244
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0
          ],
          [
            0.04,
            0.0
          ],
          [
            0.06,
            0.0
          ],
          [
            0.08,
            0.0
          ],
          [
            0.1,
            0.0
          ],
          [
            0.12,
            0.0
          ],
          [
            0.14,
            0.0065
          ],
          [
            0.16,
            0.0114
          ],
          [
            0.18,
            0.0152
          ],
          [
            0.2,
            0.0205
          ],
          [
            0.22,
            0.026
          ],
          [
            0.24,
            0.0436
          ],
          [
            0.26,
            0.0577
          ],
          [
            0.28,
            0.0731
          ],
          [
            0.3,
            0.0883
          ],
          [
            0.32,
            0.0986
          ],
          [
            0.34,
            0.1084
          ],
          [
            0.36,
            0.1305
          ],
          [
            0.38,
            0.144
          ],
          [
            0.4,
            0.1583
          ],
          [
            0.42,
            0.1701
          ],
          [
            0.44,
            0.179
          ],
          [
            0.46,
            0.1958
          ],
          [
            0.48,
            0.2057
          ],
          [
            0.5,
            0.2147
          ],
          [
            0.52,
            0.2231
          ],
          [
            0.54,
            0.2358
          ],
          [
            0.56,
            0.2494
          ],
          [
            0.58,
            0.2579
          ],
          [
            0.6,
            0.266
          ],
          [
            0.62,
            0.2809
          ],
          [
            0.64,
            0.2934
          ],
          [
            0.66,
            0.3033
          ],
          [
            0.68,
            0.3165
          ],
          [
            0.7,
            0.3276
          ],
          [
            0.72,
            0.3336
          ],
          [
            0.74,
            0.3414
          ],
          [
            0.76,
            0.3493
          ],
          [
            0.78,
            0.3601
          ],
          [
            0.8,
            0.3698
          ],
          [
            0.82,
            0.3791
          ],
          [
            0.84,
            0.3849
          ],
          [
            0.86,
            0.3915
          ],
          [
            0.88,
            0.3972
          ],
          [
            0.9,
            0.4055
          ],
          [
            0.92,
            0.4125
          ],
          [
            0.94,
            0.4197
          ],
          [
            0.96,
            0.4266
          ],
          [
            0.98,
            0.4348
          ],
          [
            1.0,
            0.4427
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 482,
            "mean_confidence": 0.26640684647302904,
            "accuracy": 0.26348547717842324
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 502,
            "mean_confidence": 0.3466396414342633,
            "accuracy": 0.36254980079681276
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 312,
            "mean_confidence": 0.44399807692307647,
            "accuracy": 0.5288461538461539
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 199,
            "mean_confidence": 0.5477603015075377,
            "accuracy": 0.5879396984924623
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 147,
            "mean_confidence": 0.6461095238095239,
            "accuracy": 0.7346938775510204
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 121,
            "mean_confidence": 0.7466148760330575,
            "accuracy": 0.8264462809917356
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 163,
            "mean_confidence": 0.8513809815950922,
            "accuracy": 0.950920245398773
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 272,
            "mean_confidence": 0.9587463235294116,
            "accuracy": 0.9963235294117647
          }
        ],
        "median_right": 0.6172,
        "median_wrong": 0.3347,
        "sure": {
          "questions": 33,
          "correct": 33,
          "interval": [
            0.8957265699643882,
            1.0
          ]
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 1642,
          "correct": 699,
          "interval": [
            0.401984965552444,
            0.4497626145305966
          ]
        },
        "ece_refit": 0.019986659684974317,
        "brier_refit": 0.17661197419392236,
        "brier_base": 0.24695320260354714,
        "refit_slopes": [
          1.3740857088428582,
          1.3081665381500927,
          1.362834627055941,
          1.3809552866427175,
          1.4191179001966205
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        96,
        130,
        53,
        92,
        79,
        84,
        68,
        26,
        52,
        35,
        33,
        25,
        61,
        45,
        36,
        73,
        20,
        60,
        58,
        99
      ],
      "started": "2026-10-31T21:26:44Z",
      "updated": "2026-10-31T21:45:07Z",
      "format_of": null,
      "group": "zABC",
      "latency": {
        "source": "run",
        "runs": [
          334
        ],
        "concurrency": 1,
        "median_request_seconds": 0.32470757849660004,
        "p90_request_seconds": 0.39495088000403483
      },
      "single_question_latency": null
    },
    {
      "run": 127,
      "model": "cygnet",
      "name": "Cygnet",
      "variant": null,
      "reasoning": null,
      "provider": "cygnet",
      "note": "Gemma 4 12B IT, unchanged, read out by option letter",
      "parameters": 11959730224,
      "active_parameters": null,
      "access": "Cygnet decision server on vLLM, self-hosted",
      "questions_per_request": 1,
      "concurrency": 1,
      "correct": 1224,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 543
        },
        "grammar": {
          "questions": 1542,
          "correct": 681
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 1222
        }
      },
      "tyt_mix": 0.760174711730094,
      "questions_per_minute": 1619.6114236317753,
      "request_seconds": 81.4269386321539,
      "median_request_seconds": 0.0359221679973416,
      "p90_request_seconds": 0.04518266418017447,
      "billing": "local",
      "api_equivalent_usd": null,
      "cost_basis": null,
      "requests": 2198,
      "hardware": "1 × RTX PRO 6000 Blackwell (96 GB); BF16",
      "input_tokens": 859227,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 2198,
      "confidence": {
        "source": "probability",
        "label": "Option score",
        "bins": [
          {
            "questions": 4,
            "correct": 0
          },
          {
            "questions": 433,
            "correct": 113
          },
          {
            "questions": 854,
            "correct": 351
          },
          {
            "questions": 396,
            "correct": 277
          },
          {
            "questions": 511,
            "correct": 483
          }
        ],
        "auroc": 0.8020107769322651,
        "auroc_interval": [
          0.7854736983360172,
          0.8217569417400272
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.8839779005524862,
            "auroc_interval": [
              0.8567941542288557,
              0.9121513407227693
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.7157950748796349,
            "auroc_interval": [
              0.6897354681755437,
              0.7417689184251237
            ]
          }
        },
        "aurc": 0.2082964460444915,
        "ece": 0.137566046929174,
        "brier": 0.2034251381583377,
        "coverage_accuracy": {
          "0.20": 0.9727024567788899,
          "0.50": 0.7843494085532302,
          "0.80": 0.6318243858052774
        },
        "coverage_intervals": {
          "0.20": [
            0.9498861047835991,
            0.988631195998181
          ],
          "0.50": [
            0.758306781975421,
            0.8126421100500227
          ],
          "0.80": [
            0.6082004555808656,
            0.6579845313921747
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0
          ],
          [
            0.04,
            0.0
          ],
          [
            0.06,
            0.0
          ],
          [
            0.08,
            0.0
          ],
          [
            0.1,
            0.0045
          ],
          [
            0.12,
            0.0038
          ],
          [
            0.14,
            0.0032
          ],
          [
            0.16,
            0.0114
          ],
          [
            0.18,
            0.0177
          ],
          [
            0.2,
            0.0273
          ],
          [
            0.22,
            0.0434
          ],
          [
            0.24,
            0.0569
          ],
          [
            0.26,
            0.0665
          ],
          [
            0.28,
            0.0747
          ],
          [
            0.3,
            0.0864
          ],
          [
            0.32,
            0.0981
          ],
          [
            0.34,
            0.1084
          ],
          [
            0.36,
            0.1188
          ],
          [
            0.38,
            0.1416
          ],
          [
            0.4,
            0.1558
          ],
          [
            0.42,
            0.1659
          ],
          [
            0.44,
            0.183
          ],
          [
            0.46,
            0.1899
          ],
          [
            0.48,
            0.2038
          ],
          [
            0.5,
            0.2157
          ],
          [
            0.52,
            0.2266
          ],
          [
            0.54,
            0.2426
          ],
          [
            0.56,
            0.2502
          ],
          [
            0.58,
            0.2627
          ],
          [
            0.6,
            0.273
          ],
          [
            0.62,
            0.2832
          ],
          [
            0.64,
            0.2891
          ],
          [
            0.66,
            0.3024
          ],
          [
            0.68,
            0.3135
          ],
          [
            0.7,
            0.3228
          ],
          [
            0.72,
            0.3353
          ],
          [
            0.74,
            0.3458
          ],
          [
            0.76,
            0.3517
          ],
          [
            0.78,
            0.3596
          ],
          [
            0.8,
            0.3682
          ],
          [
            0.82,
            0.3775
          ],
          [
            0.84,
            0.388
          ],
          [
            0.86,
            0.3941
          ],
          [
            0.88,
            0.4008
          ],
          [
            0.9,
            0.4086
          ],
          [
            0.92,
            0.4165
          ],
          [
            0.94,
            0.4221
          ],
          [
            0.96,
            0.4308
          ],
          [
            0.98,
            0.4364
          ],
          [
            1.0,
            0.4431
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 38,
            "mean_confidence": 0.28024519967218425,
            "accuracy": 0.21052631578947367
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 145,
            "mean_confidence": 0.35628180090661954,
            "accuracy": 0.25517241379310346
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 254,
            "mean_confidence": 0.4511311330576579,
            "accuracy": 0.2677165354330709
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 327,
            "mean_confidence": 0.553803756722205,
            "accuracy": 0.3180428134556575
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 368,
            "mean_confidence": 0.6495956286828283,
            "accuracy": 0.43478260869565216
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 300,
            "mean_confidence": 0.7471879223594707,
            "accuracy": 0.56
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 255,
            "mean_confidence": 0.8471681004584712,
            "accuracy": 0.7686274509803922
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 511,
            "mean_confidence": 0.9572278326937051,
            "accuracy": 0.9452054794520548
          }
        ],
        "median_right": 0.8323890018069826,
        "median_wrong": 0.5778010945804937,
        "sure": {
          "questions": 2,
          "correct": 2,
          "interval": [
            0.34237195288961925,
            1.0
          ]
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 1132,
          "correct": 377,
          "interval": [
            0.30618863397814017,
            0.3610184836340338
          ]
        },
        "ece_refit": 0.029542411162906398,
        "brier_refit": 0.17823512225501617,
        "brier_base": 0.24684346021692383,
        "refit_slopes": [
          1.135521338648259,
          1.115633909829227,
          1.1331829776409128,
          1.1743442660799894,
          1.1558614014717787
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        100,
        140,
        48,
        94,
        76,
        85,
        70,
        27,
        54,
        37,
        39,
        25,
        54,
        38,
        34,
        63,
        24,
        61,
        54,
        101
      ],
      "started": "2026-10-06T10:05:32Z",
      "updated": "2026-10-06T10:06:56Z",
      "format_of": null,
      "group": "zABC",
      "latency": {
        "source": "run",
        "runs": [
          127
        ],
        "concurrency": 1,
        "median_request_seconds": 0.0359221679973416,
        "p90_request_seconds": 0.04518266418017447
      },
      "single_question_latency": null
    },
    {
      "run": 45,
      "model": "open-jev-27b-v1.1",
      "name": "Open-Jev 27B v1.1",
      "variant": null,
      "reasoning": null,
      "provider": "open-jev",
      "note": "Fine-tune of Qwen3.8-27B (LoRA and a scoring head)",
      "parameters": 27796899569,
      "active_parameters": null,
      "access": "Open-Jev server, self-hosted",
      "questions_per_request": 1,
      "concurrency": 1,
      "correct": 1224,
      "invalid": 0,
      "answered_questions": 2210,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 544
        },
        "grammar": {
          "questions": 1542,
          "correct": 680
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 1221
        }
      },
      "tyt_mix": 0.7613188447059568,
      "questions_per_minute": 153.40666379539002,
      "request_seconds": 864.3692309015896,
      "median_request_seconds": null,
      "p90_request_seconds": null,
      "billing": "local",
      "api_equivalent_usd": null,
      "cost_basis": null,
      "requests": 2210,
      "hardware": "1 × RTX PRO 6000 Blackwell (96 GB); BF16",
      "input_tokens": 3043367,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 0,
      "confidence": {
        "source": "probability",
        "label": "Option score",
        "bins": [
          {
            "questions": 800,
            "correct": 216
          },
          {
            "questions": 1218,
            "correct": 828
          },
          {
            "questions": 180,
            "correct": 180
          },
          {
            "questions": 0,
            "correct": 0
          },
          {
            "questions": 0,
            "correct": 0
          }
        ],
        "auroc": 0.8230571660560185,
        "auroc_interval": [
          0.8058129497373231,
          0.8402808764276913
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.8857832195378151,
            "auroc_interval": [
              0.8559752702790185,
              0.9156152053363337
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.7498003957963696,
            "auroc_interval": [
              0.724566745545657,
              0.774410137675715
            ]
          }
        },
        "aurc": 0.19828234697206562,
        "ece": 0.24515588984478145,
        "brier": 0.26499084792220756,
        "coverage_accuracy": {
          "0.20": 0.9727024567788899,
          "0.50": 0.7998180163785259,
          "0.80": 0.6397861692447679
        },
        "coverage_intervals": {
          "0.20": [
            0.9566605839416058,
            0.9886208466090123
          ],
          "0.50": [
            0.7704918032786885,
            0.8296903460837887
          ],
          "0.80": [
            0.6138850889192886,
            0.6628356850250341
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0
          ],
          [
            0.04,
            0.0
          ],
          [
            0.06,
            0.0
          ],
          [
            0.08,
            0.0
          ],
          [
            0.1,
            0.0
          ],
          [
            0.12,
            0.0038
          ],
          [
            0.14,
            0.0065
          ],
          [
            0.16,
            0.0114
          ],
          [
            0.18,
            0.0193
          ],
          [
            0.2,
            0.0273
          ],
          [
            0.22,
            0.0363
          ],
          [
            0.24,
            0.0379
          ],
          [
            0.26,
            0.0472
          ],
          [
            0.28,
            0.0666
          ],
          [
            0.3,
            0.0743
          ],
          [
            0.32,
            0.081
          ],
          [
            0.34,
            0.0963
          ],
          [
            0.36,
            0.1087
          ],
          [
            0.38,
            0.1224
          ],
          [
            0.4,
            0.1299
          ],
          [
            0.42,
            0.1462
          ],
          [
            0.44,
            0.1604
          ],
          [
            0.46,
            0.1701
          ],
          [
            0.48,
            0.182
          ],
          [
            0.5,
            0.2002
          ],
          [
            0.52,
            0.2126
          ],
          [
            0.54,
            0.2207
          ],
          [
            0.56,
            0.2355
          ],
          [
            0.58,
            0.2485
          ],
          [
            0.6,
            0.2578
          ],
          [
            0.62,
            0.2706
          ],
          [
            0.64,
            0.2792
          ],
          [
            0.66,
            0.2879
          ],
          [
            0.68,
            0.3015
          ],
          [
            0.7,
            0.31
          ],
          [
            0.72,
            0.3188
          ],
          [
            0.74,
            0.332
          ],
          [
            0.76,
            0.3409
          ],
          [
            0.78,
            0.352
          ],
          [
            0.8,
            0.3602
          ],
          [
            0.82,
            0.3692
          ],
          [
            0.84,
            0.3791
          ],
          [
            0.86,
            0.3863
          ],
          [
            0.88,
            0.3961
          ],
          [
            0.9,
            0.4054
          ],
          [
            0.92,
            0.4135
          ],
          [
            0.94,
            0.4216
          ],
          [
            0.96,
            0.428
          ],
          [
            0.98,
            0.435
          ],
          [
            1.0,
            0.4431
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 1332,
            "mean_confidence": 0.2465196590412351,
            "accuracy": 0.35285285285285284
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 486,
            "mean_confidence": 0.3436312727236108,
            "accuracy": 0.7818930041152263
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 200,
            "mean_confidence": 0.44650386014577154,
            "accuracy": 0.97
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 144,
            "mean_confidence": 0.5429269200138777,
            "accuracy": 1.0
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 36,
            "mean_confidence": 0.6193367006504896,
            "accuracy": 1.0
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          }
        ],
        "median_right": 0.33237373883121457,
        "median_wrong": 0.2415857873006363,
        "sure": {
          "questions": 0,
          "correct": 0,
          "interval": null
        },
        "sure_before_audit": {
          "questions": 0,
          "correct": 0,
          "interval": null
        },
        "doubtful": {
          "questions": 2198,
          "correct": 1224,
          "interval": [
            0.5360210227955665,
            0.5775202964346632
          ]
        },
        "ece_refit": 0.01015020504548234,
        "brier_refit": 0.16829893651718528,
        "brier_base": 0.24700252692614721,
        "refit_slopes": [
          4.3444882364018484,
          4.334657327299751,
          4.4710736269798455,
          4.476495945402257,
          4.50608809644886
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": {
        "run": 31,
        "changed": {
          "questions": 372,
          "older_correct": 137,
          "newer_correct": 179,
          "flipped": 108,
          "changed_answer": 192,
          "newer_wrong": 193,
          "repeated_mistakes": 76,
          "p": 6.550370792480044e-05
        },
        "unchanged": {
          "questions": 1814,
          "older_correct": 1037,
          "newer_correct": 1037,
          "flipped": 0,
          "changed_answer": 0,
          "newer_wrong": 777,
          "repeated_mistakes": 777,
          "p": 1.0
        },
        "added": {
          "questions": 12,
          "newer_correct": 8
        }
      },
      "units": [
        101,
        131,
        50,
        92,
        80,
        90,
        71,
        34,
        52,
        33,
        47,
        19,
        58,
        40,
        35,
        40,
        18,
        69,
        55,
        109
      ],
      "started": "2026-10-04T08:31:15Z",
      "updated": "2026-10-04T08:45:43Z",
      "format_of": null,
      "group": "ABC",
      "latency": {
        "source": "repeats",
        "runs": [
          60,
          92,
          102,
          106
        ],
        "repeats": [
          2,
          3,
          4,
          5
        ],
        "questions": 2198,
        "concurrency": 1,
        "median_request_seconds": 0.386393806518754,
        "p90_request_seconds": 0.5727336546988226
      },
      "single_question_latency": null
    },
    {
      "run": 128,
      "model": "winnow-12b",
      "name": "Winnow-12B",
      "variant": null,
      "reasoning": null,
      "provider": "winnow",
      "note": "Fine-tune of Gemma 4 12B IT (Q8_0 GGUF)",
      "parameters": 11907350576,
      "active_parameters": null,
      "access": "Winnow server (llama.cpp), self-hosted",
      "questions_per_request": 1,
      "concurrency": 1,
      "correct": 1219,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 538
        },
        "grammar": {
          "questions": 1542,
          "correct": 681
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 1217
        }
      },
      "tyt_mix": 0.7538866019739964,
      "questions_per_minute": 1002.408676880479,
      "request_seconds": 131.56310698587913,
      "median_request_seconds": 0.05853745195781812,
      "p90_request_seconds": 0.0698976724059321,
      "billing": "local",
      "api_equivalent_usd": null,
      "cost_basis": null,
      "requests": 2198,
      "hardware": "1 × RTX PRO 6000 Blackwell (96 GB); Q8_0",
      "input_tokens": 887137,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 0,
      "confidence": {
        "source": "probability",
        "label": "Option score",
        "bins": [
          {
            "questions": 7,
            "correct": 0
          },
          {
            "questions": 436,
            "correct": 128
          },
          {
            "questions": 452,
            "correct": 158
          },
          {
            "questions": 414,
            "correct": 207
          },
          {
            "questions": 889,
            "correct": 726
          }
        ],
        "auroc": 0.7886904736966033,
        "auroc_interval": [
          0.7690403940425866,
          0.8072120208709612
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.8539789553273266,
            "auroc_interval": [
              0.8196089624661054,
              0.8867903152715738
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.7005599130881177,
            "auroc_interval": [
              0.6733671335038659,
              0.725885104392567
            ]
          }
        },
        "aurc": 0.21675540882268893,
        "ece": 0.1970922224223394,
        "brier": 0.23695187291984546,
        "coverage_accuracy": {
          "0.20": 0.9704276615104641,
          "0.50": 0.7688808007279345,
          "0.80": 0.6210191082802548
        },
        "coverage_intervals": {
          "0.20": [
            0.9522292993630573,
            0.9863636363636363
          ],
          "0.50": [
            0.740673339399454,
            0.7969034608378871
          ],
          "0.80": [
            0.5977272727272728,
            0.6454731574158326
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0
          ],
          [
            0.04,
            0.0
          ],
          [
            0.06,
            0.0
          ],
          [
            0.08,
            0.0057
          ],
          [
            0.1,
            0.0091
          ],
          [
            0.12,
            0.0114
          ],
          [
            0.14,
            0.0097
          ],
          [
            0.16,
            0.0171
          ],
          [
            0.18,
            0.0202
          ],
          [
            0.2,
            0.0296
          ],
          [
            0.22,
            0.0393
          ],
          [
            0.24,
            0.055
          ],
          [
            0.26,
            0.0708
          ],
          [
            0.28,
            0.0845
          ],
          [
            0.3,
            0.0895
          ],
          [
            0.32,
            0.1123
          ],
          [
            0.34,
            0.1231
          ],
          [
            0.36,
            0.1415
          ],
          [
            0.38,
            0.1616
          ],
          [
            0.4,
            0.1811
          ],
          [
            0.42,
            0.1874
          ],
          [
            0.44,
            0.1987
          ],
          [
            0.46,
            0.2098
          ],
          [
            0.48,
            0.2189
          ],
          [
            0.5,
            0.2311
          ],
          [
            0.52,
            0.2441
          ],
          [
            0.54,
            0.2586
          ],
          [
            0.56,
            0.2672
          ],
          [
            0.58,
            0.276
          ],
          [
            0.6,
            0.2874
          ],
          [
            0.62,
            0.2985
          ],
          [
            0.64,
            0.3076
          ],
          [
            0.66,
            0.3141
          ],
          [
            0.68,
            0.3238
          ],
          [
            0.7,
            0.3299
          ],
          [
            0.72,
            0.3412
          ],
          [
            0.74,
            0.3508
          ],
          [
            0.76,
            0.3619
          ],
          [
            0.78,
            0.3712
          ],
          [
            0.8,
            0.379
          ],
          [
            0.82,
            0.3869
          ],
          [
            0.84,
            0.3891
          ],
          [
            0.86,
            0.3962
          ],
          [
            0.88,
            0.4013
          ],
          [
            0.9,
            0.4096
          ],
          [
            0.92,
            0.417
          ],
          [
            0.94,
            0.4231
          ],
          [
            0.96,
            0.4313
          ],
          [
            0.98,
            0.4387
          ],
          [
            1.0,
            0.4454
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 54,
            "mean_confidence": 0.27570746588423467,
            "accuracy": 0.24074074074074073
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 205,
            "mean_confidence": 0.35191796241065953,
            "accuracy": 0.23414634146341465
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 184,
            "mean_confidence": 0.4445220380299274,
            "accuracy": 0.3641304347826087
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 205,
            "mean_confidence": 0.5473235674878659,
            "accuracy": 0.2780487804878049
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 148,
            "mean_confidence": 0.6522315912053411,
            "accuracy": 0.4189189189189189
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 213,
            "mean_confidence": 0.7518490636190589,
            "accuracy": 0.4225352112676056
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 300,
            "mean_confidence": 0.8570440447215989,
            "accuracy": 0.52
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 889,
            "mean_confidence": 0.9644506115119451,
            "accuracy": 0.8166479190101237
          }
        ],
        "median_right": 0.9445495387223898,
        "median_wrong": 0.6346326114389067,
        "sure": {
          "questions": 163,
          "correct": 162,
          "interval": [
            0.9660709164826251,
            0.9989162330052723
          ]
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 796,
          "correct": 247,
          "interval": [
            0.27913878040238144,
            0.3432864597951775
          ]
        },
        "ece_refit": 0.06294285283414827,
        "brier_refit": 0.18640593553293328,
        "brier_base": 0.24725261655475175,
        "refit_slopes": [
          0.7150224197162878,
          0.7104462636712428,
          0.6792083702676536,
          0.7232457874270419,
          0.7169341941651167
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        99,
        138,
        47,
        91,
        79,
        84,
        64,
        28,
        46,
        31,
        38,
        26,
        58,
        32,
        33,
        74,
        26,
        63,
        57,
        105
      ],
      "started": "2026-10-06T10:25:55Z",
      "updated": "2026-10-06T10:28:09Z",
      "format_of": null,
      "group": "ABC",
      "latency": {
        "source": "run",
        "runs": [
          128
        ],
        "concurrency": 1,
        "median_request_seconds": 0.05853745195781812,
        "p90_request_seconds": 0.0698976724059321
      },
      "single_question_latency": null
    },
    {
      "run": 83,
      "model": "qwen38-27b-bf16",
      "name": "Qwen3.8-27B",
      "variant": "vote share",
      "reasoning": "off",
      "provider": "vllm-vote",
      "note": "Open-Jev's base model",
      "parameters": 27781427952,
      "active_parameters": null,
      "access": "vLLM, self-hosted, BF16",
      "questions_per_request": 1,
      "concurrency": 1,
      "correct": 1194,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 551
        },
        "grammar": {
          "questions": 1542,
          "correct": 643
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 1190
        }
      },
      "tyt_mix": 0.7659231062762962,
      "questions_per_minute": 36.42730498097688,
      "request_seconds": 3620.3611567990156,
      "median_request_seconds": 1.6220232204941567,
      "p90_request_seconds": 1.8360200999944936,
      "billing": "local",
      "api_equivalent_usd": 1.814203,
      "cost_basis": "maker",
      "requests": 2198,
      "hardware": "1 × RTX PRO 6000 Blackwell (96 GB); BF16",
      "input_tokens": 737600,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 481801,
      "confidence": {
        "source": "votes",
        "label": "Agreement across ten answers",
        "bins": [
          {
            "questions": 0,
            "correct": 0
          },
          {
            "questions": 201,
            "correct": 49
          },
          {
            "questions": 830,
            "correct": 269
          },
          {
            "questions": 155,
            "correct": 71
          },
          {
            "questions": 1012,
            "correct": 805
          }
        ],
        "auroc": 0.7774655148251216,
        "auroc_interval": [
          0.7590938390202316,
          0.7966106093636458
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.8031630801140783,
            "auroc_interval": [
              0.7532553956834532,
              0.8463760504201681
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.6930294071345905,
            "auroc_interval": [
              0.6668708085756976,
              0.7180165699536367
            ]
          }
        },
        "aurc": 0.25747343636994446,
        "ece": 0.22493175614194588,
        "brier": 0.24103730664240217,
        "coverage_accuracy": {
          "0.20": 0.8525714285714286,
          "0.50": 0.7687457806216795,
          "0.80": 0.6128316917386526
        },
        "coverage_intervals": {
          "0.20": [
            0.8296639629200463,
            0.8764568764568764
          ],
          "0.50": [
            0.7417853103391985,
            0.7943292684627982
          ],
          "0.80": [
            0.5899474824866682,
            0.6366341216141759
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.1474
          ],
          [
            0.04,
            0.1474
          ],
          [
            0.06,
            0.1474
          ],
          [
            0.08,
            0.1474
          ],
          [
            0.1,
            0.1474
          ],
          [
            0.12,
            0.1474
          ],
          [
            0.14,
            0.1474
          ],
          [
            0.16,
            0.1474
          ],
          [
            0.18,
            0.1474
          ],
          [
            0.2,
            0.1474
          ],
          [
            0.22,
            0.1474
          ],
          [
            0.24,
            0.1474
          ],
          [
            0.26,
            0.1474
          ],
          [
            0.28,
            0.1474
          ],
          [
            0.3,
            0.1474
          ],
          [
            0.32,
            0.1474
          ],
          [
            0.34,
            0.1474
          ],
          [
            0.36,
            0.1474
          ],
          [
            0.38,
            0.1474
          ],
          [
            0.4,
            0.1494
          ],
          [
            0.42,
            0.1694
          ],
          [
            0.44,
            0.1876
          ],
          [
            0.46,
            0.2042
          ],
          [
            0.48,
            0.2183
          ],
          [
            0.5,
            0.2313
          ],
          [
            0.52,
            0.2432
          ],
          [
            0.54,
            0.2552
          ],
          [
            0.56,
            0.2676
          ],
          [
            0.58,
            0.279
          ],
          [
            0.6,
            0.2897
          ],
          [
            0.62,
            0.2997
          ],
          [
            0.64,
            0.3109
          ],
          [
            0.66,
            0.3223
          ],
          [
            0.68,
            0.333
          ],
          [
            0.7,
            0.3431
          ],
          [
            0.72,
            0.3526
          ],
          [
            0.74,
            0.3616
          ],
          [
            0.76,
            0.3701
          ],
          [
            0.78,
            0.3787
          ],
          [
            0.8,
            0.3872
          ],
          [
            0.82,
            0.3952
          ],
          [
            0.84,
            0.4029
          ],
          [
            0.86,
            0.4102
          ],
          [
            0.88,
            0.4172
          ],
          [
            0.9,
            0.4239
          ],
          [
            0.92,
            0.4307
          ],
          [
            0.94,
            0.4376
          ],
          [
            0.96,
            0.4443
          ],
          [
            0.98,
            0.4506
          ],
          [
            1.0,
            0.4568
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 25,
            "mean_confidence": 0.2999999999999999,
            "accuracy": 0.24
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 176,
            "mean_confidence": 0.3999999999999995,
            "accuracy": 0.24431818181818182
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 308,
            "mean_confidence": 0.5,
            "accuracy": 0.2824675324675325
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 312,
            "mean_confidence": 0.5999999999999966,
            "accuracy": 0.3141025641025641
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 210,
            "mean_confidence": 0.7000000000000003,
            "accuracy": 0.4
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 155,
            "mean_confidence": 0.7999999999999979,
            "accuracy": 0.45806451612903226
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 1012,
            "mean_confidence": 0.9864624505928838,
            "accuracy": 0.7954545454545454
          }
        ],
        "median_right": 1.0,
        "median_wrong": 0.6,
        "sure": {
          "questions": 875,
          "correct": 746,
          "interval": [
            0.8275396883011108,
            0.8745208422882469
          ]
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 821,
          "correct": 234,
          "interval": [
            0.2551958862106012,
            0.31684315740698554
          ]
        },
        "ece_refit": 0.013108500101185768,
        "brier_refit": 0.1830104244854263,
        "brier_base": 0.24842840906179325,
        "refit_slopes": [
          0.18520683806643243,
          0.185590967984637,
          0.1822760154678159,
          0.18825151373618362,
          0.18663781832224743
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        106,
        133,
        52,
        91,
        82,
        87,
        59,
        22,
        42,
        27,
        33,
        25,
        59,
        33,
        37,
        52,
        24,
        67,
        59,
        104
      ],
      "started": "2026-10-04T20:32:46Z",
      "updated": "2026-10-04T21:33:38Z",
      "format_of": null,
      "group": "BC",
      "latency": {
        "source": "run",
        "runs": [
          83
        ],
        "concurrency": 1,
        "median_request_seconds": 1.6220232204941567,
        "p90_request_seconds": 1.8360200999944936
      },
      "single_question_latency": null
    },
    {
      "run": 53,
      "model": "qwen38-27b-bf16",
      "name": "Qwen3.8-27B",
      "variant": "stated confidence",
      "reasoning": "off",
      "provider": "vllm-verbal",
      "note": "Open-Jev's base model",
      "parameters": 27781427952,
      "active_parameters": null,
      "access": "vLLM, self-hosted, BF16",
      "questions_per_request": 1,
      "concurrency": 1,
      "correct": 1191,
      "invalid": 0,
      "answered_questions": 2210,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 547
        },
        "grammar": {
          "questions": 1542,
          "correct": 644
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 1187
        }
      },
      "tyt_mix": 0.7610061074467749,
      "questions_per_minute": 147.0429555938642,
      "request_seconds": 901.7773035400896,
      "median_request_seconds": null,
      "p90_request_seconds": null,
      "billing": "local",
      "api_equivalent_usd": 0.516798,
      "cost_basis": "maker",
      "requests": 2210,
      "hardware": "1 × RTX PRO 6000 Blackwell (96 GB); BF16",
      "input_tokens": 741942,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 48609,
      "confidence": {
        "source": "stated",
        "label": "Written estimate",
        "bins": [
          {
            "questions": 0,
            "correct": 0
          },
          {
            "questions": 0,
            "correct": 0
          },
          {
            "questions": 2,
            "correct": 1
          },
          {
            "questions": 2,
            "correct": 0
          },
          {
            "questions": 2194,
            "correct": 1190
          }
        ],
        "auroc": 0.5147923394342041,
        "auroc_interval": [
          0.5097210481825867,
          0.5206051082322689
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.5377622729483589,
            "auroc_interval": [
              0.522202486678508,
              0.556028746340165
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.5040998630497032,
            "auroc_interval": [
              0.4996402865286011,
              0.508755028840095
            ]
          }
        },
        "aurc": 0.4296303914450874,
        "ece": 0.40830755232031896,
        "brier": 0.41452948134667883,
        "coverage_accuracy": {
          "0.20": 0.5701460884268448,
          "0.50": 0.5493182778904229,
          "0.80": 0.5441113252563176
        },
        "coverage_intervals": {
          "0.20": [
            0.5479706684856753,
            0.594939003255209
          ],
          "0.50": [
            0.5295949781058125,
            0.5718040394951814
          ],
          "0.80": [
            0.5250641854627058,
            0.5659673738412266
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.1174
          ],
          [
            0.04,
            0.291
          ],
          [
            0.06,
            0.3489
          ],
          [
            0.08,
            0.3778
          ],
          [
            0.1,
            0.3951
          ],
          [
            0.12,
            0.4067
          ],
          [
            0.14,
            0.415
          ],
          [
            0.16,
            0.4212
          ],
          [
            0.18,
            0.426
          ],
          [
            0.2,
            0.4299
          ],
          [
            0.22,
            0.433
          ],
          [
            0.24,
            0.4356
          ],
          [
            0.26,
            0.4379
          ],
          [
            0.28,
            0.4398
          ],
          [
            0.3,
            0.4414
          ],
          [
            0.32,
            0.4429
          ],
          [
            0.34,
            0.4441
          ],
          [
            0.36,
            0.4453
          ],
          [
            0.38,
            0.4463
          ],
          [
            0.4,
            0.4472
          ],
          [
            0.42,
            0.448
          ],
          [
            0.44,
            0.4488
          ],
          [
            0.46,
            0.4495
          ],
          [
            0.48,
            0.4501
          ],
          [
            0.5,
            0.4507
          ],
          [
            0.52,
            0.4512
          ],
          [
            0.54,
            0.4517
          ],
          [
            0.56,
            0.4522
          ],
          [
            0.58,
            0.4526
          ],
          [
            0.6,
            0.453
          ],
          [
            0.62,
            0.4534
          ],
          [
            0.64,
            0.4537
          ],
          [
            0.66,
            0.454
          ],
          [
            0.68,
            0.4544
          ],
          [
            0.7,
            0.4546
          ],
          [
            0.72,
            0.4549
          ],
          [
            0.74,
            0.4552
          ],
          [
            0.76,
            0.4554
          ],
          [
            0.78,
            0.4557
          ],
          [
            0.8,
            0.4559
          ],
          [
            0.82,
            0.4561
          ],
          [
            0.84,
            0.4563
          ],
          [
            0.86,
            0.4565
          ],
          [
            0.88,
            0.4567
          ],
          [
            0.9,
            0.4569
          ],
          [
            0.92,
            0.457
          ],
          [
            0.94,
            0.4572
          ],
          [
            0.96,
            0.4573
          ],
          [
            0.98,
            0.4575
          ],
          [
            1.0,
            0.4581
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 2,
            "mean_confidence": 0.625,
            "accuracy": 0.5
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 2,
            "mean_confidence": 0.85,
            "accuracy": 0.0
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 2194,
            "mean_confidence": 0.9505515041021245,
            "accuracy": 0.5423883318140383
          }
        ],
        "median_right": 0.95,
        "median_wrong": 0.95,
        "sure": {
          "questions": 8,
          "correct": 7,
          "interval": [
            0.5291051942301386,
            0.9775830911367038
          ]
        },
        "sure_before_audit": {
          "questions": 8,
          "correct": 7,
          "interval": [
            0.5291051942301386,
            0.9775830911367038
          ]
        },
        "doubtful": {
          "questions": 2,
          "correct": 1,
          "interval": [
            0.09452865480086614,
            0.9054713451991339
          ]
        },
        "ece_refit": 0.008613030258276554,
        "brier_refit": 0.24720969143412783,
        "brier_base": 0.2484045525652157,
        "refit_slopes": [
          0.3355015929290412,
          0.2831172369775395,
          9.620245187064572,
          0.2815050096383593,
          0.227820292571765
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": {
        "run": 35,
        "changed": {
          "questions": 372,
          "older_correct": 135,
          "newer_correct": 181,
          "flipped": 112,
          "changed_answer": 181,
          "newer_wrong": 191,
          "repeated_mistakes": 89,
          "p": 1.6356858141740275e-05
        },
        "unchanged": {
          "questions": 1814,
          "older_correct": 996,
          "newer_correct": 1000,
          "flipped": 18,
          "changed_answer": 31,
          "newer_wrong": 814,
          "repeated_mistakes": 794,
          "p": 0.480682373046875
        },
        "added": {
          "questions": 12,
          "newer_correct": 10
        }
      },
      "units": [
        102,
        133,
        52,
        91,
        79,
        90,
        55,
        22,
        41,
        28,
        39,
        19,
        62,
        34,
        35,
        52,
        29,
        64,
        59,
        105
      ],
      "started": "2026-10-04T08:54:01Z",
      "updated": "2026-10-04T09:09:07Z",
      "format_of": null,
      "group": "BC",
      "latency": {
        "source": "repeats",
        "runs": [
          74,
          95,
          103,
          107
        ],
        "repeats": [
          2,
          3,
          4,
          5
        ],
        "questions": 2198,
        "concurrency": 1,
        "median_request_seconds": 0.43298505601705983,
        "p90_request_seconds": 0.4745744730113074
      },
      "single_question_latency": null
    },
    {
      "run": 373,
      "model": "baselabs/elephant-1",
      "name": "Elephant-1",
      "variant": null,
      "reasoning": null,
      "provider": "baselabs",
      "note": "Base Labs' released elephant-1 weights (baselabs/elephant-1 at b3f8a93), a decision model post-trained from Qwen3.5-9B that also reads images. It ran on our hardware through its model card's loader (qev's DecisionEngine from OneJev at 81ce62f with the checkpoint's serve.patch, as the card's setup.sh installs it) behind our HTTP server (servers/baselabs_server.py), one request at a time. That runtime reads images only through an engine the patch leaves without the attention the model was trained with, so it is not on the Page images tab. The card mentions a trained \"I don't know\" option, but its answers have no such output, and our questions offer five options. Probabilities rounded to 6 decimals by the model's own loader. They sum to 1 only within that rounding, and the check before its runs allowed for it. Its card lists English only.",
      "parameters": 9409813744,
      "active_parameters": null,
      "access": "Base Labs server (qev, the model card's loader), self-hosted, BF16",
      "questions_per_request": 1,
      "concurrency": 1,
      "correct": 1151,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 528
        },
        "grammar": {
          "questions": 1542,
          "correct": 623
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 1149
        }
      },
      "tyt_mix": 0.7347280218911139,
      "questions_per_minute": 783.5925546206023,
      "request_seconds": 168.30175225931453,
      "median_request_seconds": 0.07427877650479786,
      "p90_request_seconds": 0.08908395380713045,
      "billing": "local",
      "api_equivalent_usd": 0.030482550000000004,
      "cost_basis": "maker",
      "requests": 2198,
      "hardware": "1 × RTX PRO 6000 Blackwell (96 GB); BF16",
      "input_tokens": 870930,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 2198,
      "confidence": {
        "source": "probability",
        "label": "Option score",
        "bins": [
          {
            "questions": 9,
            "correct": 3
          },
          {
            "questions": 819,
            "correct": 233
          },
          {
            "questions": 548,
            "correct": 245
          },
          {
            "questions": 312,
            "correct": 195
          },
          {
            "questions": 510,
            "correct": 475
          }
        ],
        "auroc": 0.7881597912865106,
        "auroc_interval": [
          0.769079016282778,
          0.8063506761228061
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.8617424242424242,
            "auroc_interval": [
              0.830323040072506,
              0.8909016492845016
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.6937507968917292,
            "auroc_interval": [
              0.666987693953986,
              0.720105147864184
            ]
          }
        },
        "aurc": 0.24104311314866456,
        "ece": 0.11820176478616924,
        "brier": 0.20041272484378936,
        "coverage_accuracy": {
          "0.20": 0.9522292993630573,
          "0.50": 0.7288444040036397,
          "0.80": 0.5950864422202002
        },
        "coverage_intervals": {
          "0.20": [
            0.9272065514103731,
            0.9708958617553433
          ],
          "0.50": [
            0.6982680036463081,
            0.7585421412300684
          ],
          "0.80": [
            0.5725843208751139,
            0.6184748070812528
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0
          ],
          [
            0.04,
            0.0
          ],
          [
            0.06,
            0.0
          ],
          [
            0.08,
            0.0
          ],
          [
            0.1,
            0.0045
          ],
          [
            0.12,
            0.0152
          ],
          [
            0.14,
            0.0218
          ],
          [
            0.16,
            0.0247
          ],
          [
            0.18,
            0.0329
          ],
          [
            0.2,
            0.0478
          ],
          [
            0.22,
            0.06
          ],
          [
            0.24,
            0.0758
          ],
          [
            0.26,
            0.0866
          ],
          [
            0.28,
            0.104
          ],
          [
            0.3,
            0.1128
          ],
          [
            0.32,
            0.1285
          ],
          [
            0.34,
            0.149
          ],
          [
            0.36,
            0.1719
          ],
          [
            0.38,
            0.1907
          ],
          [
            0.4,
            0.2004
          ],
          [
            0.42,
            0.2156
          ],
          [
            0.44,
            0.2275
          ],
          [
            0.46,
            0.2414
          ],
          [
            0.48,
            0.2569
          ],
          [
            0.5,
            0.2712
          ],
          [
            0.52,
            0.2843
          ],
          [
            0.54,
            0.2932
          ],
          [
            0.56,
            0.2973
          ],
          [
            0.58,
            0.3081
          ],
          [
            0.6,
            0.3183
          ],
          [
            0.62,
            0.3308
          ],
          [
            0.64,
            0.3382
          ],
          [
            0.66,
            0.3451
          ],
          [
            0.68,
            0.3559
          ],
          [
            0.7,
            0.365
          ],
          [
            0.72,
            0.3703
          ],
          [
            0.74,
            0.3806
          ],
          [
            0.76,
            0.3894
          ],
          [
            0.78,
            0.3975
          ],
          [
            0.8,
            0.4049
          ],
          [
            0.82,
            0.413
          ],
          [
            0.84,
            0.4205
          ],
          [
            0.86,
            0.4292
          ],
          [
            0.88,
            0.4391
          ],
          [
            0.9,
            0.4449
          ],
          [
            0.92,
            0.4545
          ],
          [
            0.94,
            0.4599
          ],
          [
            0.96,
            0.4649
          ],
          [
            0.98,
            0.4694
          ],
          [
            1.0,
            0.4763
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 100,
            "mean_confidence": 0.2759115900000001,
            "accuracy": 0.28
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 352,
            "mean_confidence": 0.3534122642045453,
            "accuracy": 0.23295454545454544
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 376,
            "mean_confidence": 0.44810209042553173,
            "accuracy": 0.3351063829787234
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 254,
            "mean_confidence": 0.548179255905512,
            "accuracy": 0.4251968503937008
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 199,
            "mean_confidence": 0.6493466180904522,
            "accuracy": 0.4271356783919598
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 191,
            "mean_confidence": 0.7495748691099479,
            "accuracy": 0.5130890052356021
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 216,
            "mean_confidence": 0.8500924583333338,
            "accuracy": 0.6898148148148148
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 510,
            "mean_confidence": 0.9691467764705881,
            "accuracy": 0.9313725490196079
          }
        ],
        "median_right": 0.836393,
        "median_wrong": 0.470385,
        "sure": {
          "questions": 155,
          "correct": 155,
          "interval": [
            0.9758148998750957,
            1.0
          ]
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 1281,
          "correct": 429,
          "interval": [
            0.3095769596147314,
            0.36119957813124426
          ]
        },
        "ece_refit": 0.01868978180158881,
        "brier_refit": 0.18346000921426453,
        "brier_base": 0.2494723121659429,
        "refit_slopes": [
          0.8641490562312143,
          0.8744507526732945,
          0.852218475167967,
          0.8195245590792527,
          0.8653924042626965
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        96,
        134,
        48,
        92,
        75,
        83,
        65,
        25,
        33,
        28,
        34,
        20,
        57,
        43,
        27,
        65,
        16,
        51,
        55,
        104
      ],
      "started": "2026-11-01T18:13:53Z",
      "updated": "2026-11-01T18:16:49Z",
      "format_of": null,
      "group": "C",
      "latency": {
        "source": "run",
        "runs": [
          373
        ],
        "concurrency": 1,
        "median_request_seconds": 0.07427877650479786,
        "p90_request_seconds": 0.08908395380713045
      },
      "single_question_latency": null
    },
    {
      "run": 50,
      "model": "qwen38-27b-bf16",
      "name": "Qwen3.8-27B",
      "variant": "yes/no scoring",
      "reasoning": null,
      "provider": "vllm-yes-no",
      "note": "Open-Jev's base model",
      "parameters": 27781427952,
      "active_parameters": null,
      "access": "vLLM, self-hosted, BF16",
      "questions_per_request": 1,
      "concurrency": 1,
      "correct": 1141,
      "invalid": 0,
      "answered_questions": 2210,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 528
        },
        "grammar": {
          "questions": 1542,
          "correct": 613
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 1135
        }
      },
      "tyt_mix": 0.7335931321375471,
      "questions_per_minute": 193.206449833369,
      "request_seconds": 686.3124917121604,
      "median_request_seconds": null,
      "p90_request_seconds": null,
      "billing": "local",
      "api_equivalent_usd": 1.5548335,
      "cost_basis": "maker",
      "requests": 2210,
      "hardware": "1 × RTX PRO 6000 Blackwell (96 GB); BF16",
      "input_tokens": 3043367,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 11050,
      "confidence": {
        "source": "probability",
        "label": "Option score",
        "bins": [
          {
            "questions": 23,
            "correct": 5
          },
          {
            "questions": 986,
            "correct": 280
          },
          {
            "questions": 454,
            "correct": 211
          },
          {
            "questions": 183,
            "correct": 127
          },
          {
            "questions": 552,
            "correct": 518
          }
        ],
        "auroc": 0.8016462181508528,
        "auroc_interval": [
          0.783210332103321,
          0.8198229620384987
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.9113399621212122,
            "auroc_interval": [
              0.8869505678519863,
              0.9331013655462185
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.6925354316328842,
            "auroc_interval": [
              0.6636908796996402,
              0.7203838846601407
            ]
          }
        },
        "aurc": 0.23471013783527137,
        "ece": 0.08917885793346515,
        "brier": 0.18363261426976416,
        "coverage_accuracy": {
          "0.20": 0.956778889899909,
          "0.50": 0.7488626023657871,
          "0.80": 0.5851910828025477
        },
        "coverage_intervals": {
          "0.20": [
            0.9363057324840764,
            0.9817850637522769
          ],
          "0.50": [
            0.7222222222222222,
            0.7778282598818719
          ],
          "0.80": [
            0.5607954545454545,
            0.6093536640873919
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0
          ],
          [
            0.04,
            0.0
          ],
          [
            0.06,
            0.0
          ],
          [
            0.08,
            0.0
          ],
          [
            0.1,
            0.0
          ],
          [
            0.12,
            0.0076
          ],
          [
            0.14,
            0.0065
          ],
          [
            0.16,
            0.0114
          ],
          [
            0.18,
            0.0278
          ],
          [
            0.2,
            0.0432
          ],
          [
            0.22,
            0.0496
          ],
          [
            0.24,
            0.0569
          ],
          [
            0.26,
            0.0673
          ],
          [
            0.28,
            0.0861
          ],
          [
            0.3,
            0.0961
          ],
          [
            0.32,
            0.1143
          ],
          [
            0.34,
            0.1275
          ],
          [
            0.36,
            0.134
          ],
          [
            0.38,
            0.1521
          ],
          [
            0.4,
            0.1708
          ],
          [
            0.42,
            0.1876
          ],
          [
            0.44,
            0.2027
          ],
          [
            0.46,
            0.2166
          ],
          [
            0.48,
            0.2332
          ],
          [
            0.5,
            0.2511
          ],
          [
            0.52,
            0.2633
          ],
          [
            0.54,
            0.2788
          ],
          [
            0.56,
            0.2924
          ],
          [
            0.58,
            0.3019
          ],
          [
            0.6,
            0.3145
          ],
          [
            0.62,
            0.3278
          ],
          [
            0.64,
            0.3389
          ],
          [
            0.66,
            0.3486
          ],
          [
            0.68,
            0.36
          ],
          [
            0.7,
            0.3728
          ],
          [
            0.72,
            0.3789
          ],
          [
            0.74,
            0.3883
          ],
          [
            0.76,
            0.3969
          ],
          [
            0.78,
            0.4068
          ],
          [
            0.8,
            0.4148
          ],
          [
            0.82,
            0.4241
          ],
          [
            0.84,
            0.4318
          ],
          [
            0.86,
            0.4401
          ],
          [
            0.88,
            0.4458
          ],
          [
            0.9,
            0.4529
          ],
          [
            0.92,
            0.459
          ],
          [
            0.94,
            0.4652
          ],
          [
            0.96,
            0.4687
          ],
          [
            0.98,
            0.4749
          ],
          [
            1.0,
            0.4809
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 187,
            "mean_confidence": 0.2740844531352885,
            "accuracy": 0.2727272727272727
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 497,
            "mean_confidence": 0.3485550710304312,
            "accuracy": 0.2595573440643863
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 325,
            "mean_confidence": 0.4450073111641953,
            "accuracy": 0.3230769230769231
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 223,
            "mean_confidence": 0.5452885947354849,
            "accuracy": 0.38565022421524664
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 150,
            "mean_confidence": 0.6482703972036756,
            "accuracy": 0.4866666666666667
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 135,
            "mean_confidence": 0.7444350097305018,
            "accuracy": 0.6370370370370371
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 129,
            "mean_confidence": 0.8483890448105549,
            "accuracy": 0.7209302325581395
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 552,
            "mean_confidence": 0.9766689515764961,
            "accuracy": 0.9384057971014492
          }
        ],
        "median_right": 0.8421285979532981,
        "median_wrong": 0.40781324947263337,
        "sure": {
          "questions": 289,
          "correct": 287,
          "interval": [
            0.9751222274562773,
            0.99810015624877
          ]
        },
        "sure_before_audit": {
          "questions": 295,
          "correct": 287,
          "interval": [
            0.9474089683498349,
            0.9861959916021933
          ]
        },
        "doubtful": {
          "questions": 1382,
          "correct": 444,
          "interval": [
            0.2971781730087698,
            0.34635973204473713
          ]
        },
        "ece_refit": 0.01523421929559486,
        "brier_refit": 0.17369258169585255,
        "brier_base": 0.24978905554317182,
        "refit_slopes": [
          0.7979493921876868,
          0.8149470423259706,
          0.8185432647421391,
          0.8189973545328548,
          0.8280187023720046
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": {
        "run": 34,
        "changed": {
          "questions": 372,
          "older_correct": 123,
          "newer_correct": 166,
          "flipped": 103,
          "changed_answer": 174,
          "newer_wrong": 206,
          "repeated_mistakes": 105,
          "p": 2.717641312570181e-05
        },
        "unchanged": {
          "questions": 1814,
          "older_correct": 967,
          "newer_correct": 969,
          "flipped": 68,
          "changed_answer": 115,
          "newer_wrong": 845,
          "repeated_mistakes": 765,
          "p": 0.9035973456835125
        },
        "added": {
          "questions": 12,
          "newer_correct": 6
        }
      },
      "units": [
        91,
        135,
        51,
        89,
        79,
        83,
        59,
        19,
        41,
        27,
        29,
        21,
        55,
        33,
        27,
        66,
        25,
        62,
        47,
        102
      ],
      "started": "2026-10-04T08:42:28Z",
      "updated": "2026-10-04T08:53:57Z",
      "format_of": null,
      "group": "CD",
      "latency": {
        "source": "repeats",
        "runs": [
          75,
          89,
          99,
          104
        ],
        "repeats": [
          2,
          3,
          4,
          5
        ],
        "questions": 2198,
        "concurrency": 1,
        "median_request_seconds": 0.2798689884948544,
        "p90_request_seconds": 0.40955076351528985
      },
      "single_question_latency": null
    },
    {
      "run": 223,
      "model": "pplx-decider-flash-v1.2",
      "name": "Perplexity Decider Flash v1.2",
      "variant": "local",
      "reasoning": null,
      "provider": "pplx-decider",
      "note": "Perplexity's released Flash checkpoint, based on Qwen3.5-4B, run on our hardware. An API run is pending; this is an additional local setup.",
      "parameters": 4539918336,
      "active_parameters": null,
      "access": "Perplexity Decider server, self-hosted",
      "questions_per_request": 1,
      "concurrency": 1,
      "correct": 1050,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 477
        },
        "grammar": {
          "questions": 1542,
          "correct": 573
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 1048
        }
      },
      "tyt_mix": 0.6649148536110847,
      "questions_per_minute": 419.92277077942566,
      "request_seconds": 314.05774865510466,
      "median_request_seconds": 0.13165274250059156,
      "p90_request_seconds": 0.186934632298653,
      "billing": "local",
      "api_equivalent_usd": 0.008533460000000001,
      "cost_basis": "maker",
      "requests": 2198,
      "hardware": "1 × DGX Spark, GB10 (128 GB unified memory); BF16",
      "input_tokens": 846752,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 0,
      "confidence": {
        "source": "probability",
        "label": "Option score",
        "bins": [
          {
            "questions": 41,
            "correct": 8
          },
          {
            "questions": 1070,
            "correct": 330
          },
          {
            "questions": 521,
            "correct": 243
          },
          {
            "questions": 281,
            "correct": 198
          },
          {
            "questions": 285,
            "correct": 271
          }
        ],
        "auroc": 0.7565032354405177,
        "auroc_interval": [
          0.735475234270415,
          0.776817564478615
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.8399095838750102,
            "auroc_interval": [
              0.8084617180205416,
              0.8688261667253927
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.6557325970711606,
            "auroc_interval": [
              0.6261288023913305,
              0.6850308182045904
            ]
          }
        },
        "aurc": 0.30045313418068686,
        "ece": 0.08269965655542914,
        "brier": 0.2028494577943973,
        "coverage_accuracy": {
          "0.20": 0.8817106460418562,
          "0.50": 0.654231119199272,
          "0.80": 0.5370791628753413
        },
        "coverage_intervals": {
          "0.20": [
            0.8522055479763528,
            0.9135184342284934
          ],
          "0.50": [
            0.6256830601092896,
            0.6850250341374602
          ],
          "0.80": [
            0.5146791078743742,
            0.5636094674556213
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0
          ],
          [
            0.04,
            0.0
          ],
          [
            0.06,
            0.0
          ],
          [
            0.08,
            0.0114
          ],
          [
            0.1,
            0.0273
          ],
          [
            0.12,
            0.0417
          ],
          [
            0.14,
            0.065
          ],
          [
            0.16,
            0.0881
          ],
          [
            0.18,
            0.1112
          ],
          [
            0.2,
            0.1183
          ],
          [
            0.22,
            0.1344
          ],
          [
            0.24,
            0.1526
          ],
          [
            0.26,
            0.1732
          ],
          [
            0.28,
            0.1892
          ],
          [
            0.3,
            0.2062
          ],
          [
            0.32,
            0.2209
          ],
          [
            0.34,
            0.2395
          ],
          [
            0.36,
            0.2477
          ],
          [
            0.38,
            0.2658
          ],
          [
            0.4,
            0.2798
          ],
          [
            0.42,
            0.2948
          ],
          [
            0.44,
            0.3123
          ],
          [
            0.46,
            0.3245
          ],
          [
            0.48,
            0.3355
          ],
          [
            0.5,
            0.3458
          ],
          [
            0.52,
            0.3534
          ],
          [
            0.54,
            0.3588
          ],
          [
            0.56,
            0.3671
          ],
          [
            0.58,
            0.3756
          ],
          [
            0.6,
            0.3875
          ],
          [
            0.62,
            0.3977
          ],
          [
            0.64,
            0.4073
          ],
          [
            0.66,
            0.4148
          ],
          [
            0.68,
            0.4222
          ],
          [
            0.7,
            0.432
          ],
          [
            0.72,
            0.437
          ],
          [
            0.74,
            0.4436
          ],
          [
            0.76,
            0.4481
          ],
          [
            0.78,
            0.4552
          ],
          [
            0.8,
            0.4629
          ],
          [
            0.82,
            0.4663
          ],
          [
            0.84,
            0.4719
          ],
          [
            0.86,
            0.4789
          ],
          [
            0.88,
            0.4851
          ],
          [
            0.9,
            0.4899
          ],
          [
            0.92,
            0.4981
          ],
          [
            0.94,
            0.5034
          ],
          [
            0.96,
            0.5099
          ],
          [
            0.98,
            0.5163
          ],
          [
            1.0,
            0.5223
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 241,
            "mean_confidence": 0.27179888478170927,
            "accuracy": 0.1950207468879668
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 483,
            "mean_confidence": 0.3506362382665138,
            "accuracy": 0.30434782608695654
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 387,
            "mean_confidence": 0.44825815823522913,
            "accuracy": 0.37209302325581395
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 241,
            "mean_confidence": 0.5454125873706175,
            "accuracy": 0.38589211618257263
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 204,
            "mean_confidence": 0.6469999204892455,
            "accuracy": 0.5147058823529411
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 163,
            "mean_confidence": 0.7505358890298272,
            "accuracy": 0.6012269938650306
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 194,
            "mean_confidence": 0.8567289681714597,
            "accuracy": 0.7474226804123711
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 285,
            "mean_confidence": 0.9492564333914681,
            "accuracy": 0.9508771929824561
          }
        ],
        "median_right": 0.6893534022093815,
        "median_wrong": 0.41513940383418946,
        "sure": {
          "questions": 0,
          "correct": 0,
          "interval": null
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 1556,
          "correct": 536,
          "interval": [
            0.32127049283187603,
            0.36844159075404714
          ]
        },
        "ece_refit": 0.028481814362844797,
        "brier_refit": 0.1949905077574606,
        "brier_base": 0.2495990743804777,
        "refit_slopes": [
          0.9214599726825536,
          0.9489744184733389,
          0.8990222947611821,
          0.9634133441040607,
          0.9629822440797038
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        81,
        123,
        49,
        85,
        61,
        78,
        63,
        25,
        37,
        38,
        33,
        18,
        48,
        28,
        31,
        60,
        12,
        42,
        46,
        92
      ],
      "started": "2026-10-17T20:36:02Z",
      "updated": "2026-10-17T20:41:22Z",
      "format_of": null,
      "group": "DE",
      "latency": {
        "source": "run",
        "runs": [
          223
        ],
        "concurrency": 1,
        "median_request_seconds": 0.13165274250059156,
        "p90_request_seconds": 0.186934632298653
      },
      "single_question_latency": null
    },
    {
      "run": 205,
      "model": "microsoft/microsoft-decision-1",
      "name": "Microsoft-Decision-1",
      "variant": "via OpenRouter",
      "reasoning": null,
      "provider": "openrouter",
      "note": "Microsoft's decision model, called through OpenRouter's System One route.",
      "parameters": null,
      "active_parameters": null,
      "access": "OpenRouter System One API",
      "questions_per_request": 1,
      "concurrency": 1,
      "correct": 1021,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 482
        },
        "grammar": {
          "questions": 1542,
          "correct": 539
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 1021
        }
      },
      "tyt_mix": 0.6673443382050552,
      "questions_per_minute": 202.12929603415938,
      "request_seconds": 652.4536649932852,
      "median_request_seconds": 0.28612651149887824,
      "p90_request_seconds": 0.3269303315973957,
      "billing": "api",
      "api_equivalent_usd": 0.030368352,
      "cost_basis": "api",
      "requests": 2198,
      "hardware": null,
      "input_tokens": 723056,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 2198,
      "confidence": {
        "source": "probability",
        "label": "Option score",
        "bins": [
          {
            "questions": 18,
            "correct": 2
          },
          {
            "questions": 678,
            "correct": 184
          },
          {
            "questions": 604,
            "correct": 206
          },
          {
            "questions": 330,
            "correct": 148
          },
          {
            "questions": 568,
            "correct": 481
          }
        ],
        "auroc": 0.7487078904600667,
        "auroc_interval": [
          0.7279528099135762,
          0.7701815532648807
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.8461809033242714,
            "auroc_interval": [
              0.8139438801859185,
              0.8774229061991263
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.6370101199185375,
            "auroc_interval": [
              0.6050796584100456,
              0.6674084249084249
            ]
          }
        },
        "aurc": 0.31457506287394066,
        "ece": 0.19786253783505095,
        "brier": 0.2447841719756419,
        "coverage_accuracy": {
          "0.20": 0.8962693357597816,
          "0.50": 0.629663330300273,
          "0.80": 0.5118289353958143
        },
        "coverage_intervals": {
          "0.20": [
            0.8566226672735548,
            0.9250681198910082
          ],
          "0.50": [
            0.5963636363636363,
            0.6627272727272727
          ],
          "0.80": [
            0.4868001820664543,
            0.5390482695810564
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0455
          ],
          [
            0.04,
            0.0341
          ],
          [
            0.06,
            0.0227
          ],
          [
            0.08,
            0.0341
          ],
          [
            0.1,
            0.0455
          ],
          [
            0.12,
            0.0569
          ],
          [
            0.14,
            0.0585
          ],
          [
            0.16,
            0.0739
          ],
          [
            0.18,
            0.085
          ],
          [
            0.2,
            0.1037
          ],
          [
            0.22,
            0.1261
          ],
          [
            0.24,
            0.1384
          ],
          [
            0.26,
            0.154
          ],
          [
            0.28,
            0.1657
          ],
          [
            0.3,
            0.1887
          ],
          [
            0.32,
            0.2109
          ],
          [
            0.34,
            0.2355
          ],
          [
            0.36,
            0.2528
          ],
          [
            0.38,
            0.2709
          ],
          [
            0.4,
            0.2914
          ],
          [
            0.42,
            0.3098
          ],
          [
            0.44,
            0.3267
          ],
          [
            0.46,
            0.3413
          ],
          [
            0.48,
            0.3573
          ],
          [
            0.5,
            0.3703
          ],
          [
            0.52,
            0.3832
          ],
          [
            0.54,
            0.3909
          ],
          [
            0.56,
            0.4005
          ],
          [
            0.58,
            0.4063
          ],
          [
            0.6,
            0.4155
          ],
          [
            0.62,
            0.4262
          ],
          [
            0.64,
            0.432
          ],
          [
            0.66,
            0.4354
          ],
          [
            0.68,
            0.4433
          ],
          [
            0.7,
            0.4498
          ],
          [
            0.72,
            0.4591
          ],
          [
            0.74,
            0.4676
          ],
          [
            0.76,
            0.4714
          ],
          [
            0.78,
            0.48
          ],
          [
            0.8,
            0.4882
          ],
          [
            0.82,
            0.4949
          ],
          [
            0.84,
            0.4995
          ],
          [
            0.86,
            0.5022
          ],
          [
            0.88,
            0.5047
          ],
          [
            0.9,
            0.5101
          ],
          [
            0.92,
            0.5138
          ],
          [
            0.94,
            0.5189
          ],
          [
            0.96,
            0.5237
          ],
          [
            0.98,
            0.5288
          ],
          [
            1.0,
            0.5355
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 91,
            "mean_confidence": 0.2707932630263674,
            "accuracy": 0.18681318681318682
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 321,
            "mean_confidence": 0.35110951295081755,
            "accuracy": 0.30218068535825543
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 284,
            "mean_confidence": 0.450514057005876,
            "accuracy": 0.2535211267605634
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 276,
            "mean_confidence": 0.5484646639165213,
            "accuracy": 0.358695652173913
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 224,
            "mean_confidence": 0.6515062073771557,
            "accuracy": 0.33035714285714285
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 214,
            "mean_confidence": 0.7511663953586485,
            "accuracy": 0.3411214953271028
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 220,
            "mean_confidence": 0.8504548183715953,
            "accuracy": 0.4909090909090909
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 568,
            "mean_confidence": 0.9602891180232691,
            "accuracy": 0.846830985915493
          }
        ],
        "median_right": 0.8811299828957022,
        "median_wrong": 0.5450694732462988,
        "sure": {
          "questions": 61,
          "correct": 58,
          "interval": [
            0.8650865720629857,
            0.9831343231647693
          ]
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 1196,
          "correct": 359,
          "interval": [
            0.27486494681566265,
            0.3267491329095279
          ]
        },
        "ece_refit": 0.050618674992628995,
        "brier_refit": 0.1945657235399729,
        "brier_base": 0.2489232384239895,
        "refit_slopes": [
          0.7085993405913765,
          0.6851586823404104,
          0.689792283807666,
          0.6899546739318855,
          0.7122078459576054
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        86,
        121,
        49,
        86,
        67,
        73,
        61,
        19,
        40,
        24,
        29,
        21,
        47,
        30,
        25,
        56,
        18,
        44,
        42,
        83
      ],
      "started": "2026-10-17T15:08:47Z",
      "updated": "2026-10-17T15:19:46Z",
      "format_of": null,
      "group": "E",
      "latency": {
        "source": "run",
        "runs": [
          205
        ],
        "concurrency": 1,
        "median_request_seconds": 0.28612651149887824,
        "p90_request_seconds": 0.3269303315973957
      },
      "single_question_latency": null
    },
    {
      "run": 139,
      "model": "clef-flash",
      "name": "Clef-flash",
      "variant": null,
      "reasoning": null,
      "provider": "clef",
      "note": "Cloudflare's 9B decision model, post-trained from Qwen3.5-9B",
      "parameters": 9531576564,
      "active_parameters": null,
      "access": "Clef server, self-hosted",
      "questions_per_request": 1,
      "concurrency": 1,
      "correct": 1005,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 479
        },
        "grammar": {
          "questions": 1542,
          "correct": 526
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 1001
        }
      },
      "tyt_mix": 0.6620961156717599,
      "questions_per_minute": 795.294880781204,
      "request_seconds": 165.8252846673131,
      "median_request_seconds": 0.07536213693674654,
      "p90_request_seconds": 0.08099431772716344,
      "billing": "local",
      "api_equivalent_usd": 0.038172330000000004,
      "cost_basis": "maker",
      "requests": 2198,
      "hardware": "1 × RTX PRO 6000 Blackwell (96 GB); BF16",
      "input_tokens": 1004535,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 0,
      "confidence": {
        "source": "probability",
        "label": "Option score",
        "bins": [
          {
            "questions": 16,
            "correct": 4
          },
          {
            "questions": 803,
            "correct": 197
          },
          {
            "questions": 656,
            "correct": 286
          },
          {
            "questions": 450,
            "correct": 286
          },
          {
            "questions": 273,
            "correct": 232
          }
        ],
        "auroc": 0.7398777278736244,
        "auroc_interval": [
          0.7199551147189751,
          0.7604508757525998
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.7726902798910159,
            "auroc_interval": [
              0.7333251349800312,
              0.8133634481460569
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.640966962067004,
            "auroc_interval": [
              0.6122295765152908,
              0.6712306651496048
            ]
          }
        },
        "aurc": 0.3452392438545273,
        "ece": 0.15476878980891715,
        "brier": 0.229707022411283,
        "coverage_accuracy": {
          "0.20": 0.7861692447679709,
          "0.50": 0.6360327570518653,
          "0.80": 0.5175159235668789
        },
        "coverage_intervals": {
          "0.20": [
            0.740909090909091,
            0.8202093764223942
          ],
          "0.50": [
            0.6056466302367942,
            0.6639418710263397
          ],
          "0.80": [
            0.4935109289617486,
            0.5439972714870396
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.091
          ],
          [
            0.04,
            0.091
          ],
          [
            0.06,
            0.091
          ],
          [
            0.08,
            0.091
          ],
          [
            0.1,
            0.1224
          ],
          [
            0.12,
            0.1365
          ],
          [
            0.14,
            0.1795
          ],
          [
            0.16,
            0.1877
          ],
          [
            0.18,
            0.2098
          ],
          [
            0.2,
            0.2138
          ],
          [
            0.22,
            0.2275
          ],
          [
            0.24,
            0.2407
          ],
          [
            0.26,
            0.2533
          ],
          [
            0.28,
            0.2632
          ],
          [
            0.3,
            0.2657
          ],
          [
            0.32,
            0.2801
          ],
          [
            0.34,
            0.2946
          ],
          [
            0.36,
            0.3008
          ],
          [
            0.38,
            0.3068
          ],
          [
            0.4,
            0.3176
          ],
          [
            0.42,
            0.3262
          ],
          [
            0.44,
            0.3382
          ],
          [
            0.46,
            0.3482
          ],
          [
            0.48,
            0.3564
          ],
          [
            0.5,
            0.364
          ],
          [
            0.52,
            0.3727
          ],
          [
            0.54,
            0.385
          ],
          [
            0.56,
            0.3892
          ],
          [
            0.58,
            0.3977
          ],
          [
            0.6,
            0.4086
          ],
          [
            0.62,
            0.4144
          ],
          [
            0.64,
            0.4232
          ],
          [
            0.66,
            0.4306
          ],
          [
            0.68,
            0.436
          ],
          [
            0.7,
            0.4462
          ],
          [
            0.72,
            0.4524
          ],
          [
            0.74,
            0.4588
          ],
          [
            0.76,
            0.4642
          ],
          [
            0.78,
            0.4736
          ],
          [
            0.8,
            0.4825
          ],
          [
            0.82,
            0.4896
          ],
          [
            0.84,
            0.4976
          ],
          [
            0.86,
            0.5027
          ],
          [
            0.88,
            0.5098
          ],
          [
            0.9,
            0.5167
          ],
          [
            0.92,
            0.5232
          ],
          [
            0.94,
            0.5266
          ],
          [
            0.96,
            0.5327
          ],
          [
            0.98,
            0.5385
          ],
          [
            1.0,
            0.5428
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 123,
            "mean_confidence": 0.2734837398373984,
            "accuracy": 0.2032520325203252
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 375,
            "mean_confidence": 0.3499536,
            "accuracy": 0.216
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 321,
            "mean_confidence": 0.4517993769470404,
            "accuracy": 0.29595015576323985
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 290,
            "mean_confidence": 0.546690344827586,
            "accuracy": 0.38620689655172413
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 247,
            "mean_confidence": 0.651218623481781,
            "accuracy": 0.44534412955465585
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 228,
            "mean_confidence": 0.7484285087719302,
            "accuracy": 0.5701754385964912
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 341,
            "mean_confidence": 0.8534926686217009,
            "accuracy": 0.6451612903225806
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 273,
            "mean_confidence": 0.9311692307692304,
            "accuracy": 0.8498168498168498
          }
        ],
        "median_right": 0.7606,
        "median_wrong": 0.491,
        "sure": {
          "questions": 0,
          "correct": 0,
          "interval": null
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 1356,
          "correct": 423,
          "interval": [
            0.2878481669397402,
            0.3371081515020548
          ]
        },
        "ece_refit": 0.022918863859525514,
        "brier_refit": 0.20485639786603901,
        "brier_base": 0.24865932819251338,
        "refit_slopes": [
          0.8501864965153166,
          0.7913363703418413,
          0.8310482206996037,
          0.8339913521558626,
          0.9079656523874993
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        77,
        132,
        42,
        86,
        65,
        77,
        52,
        25,
        33,
        25,
        30,
        17,
        47,
        26,
        26,
        57,
        22,
        40,
        45,
        81
      ],
      "started": "2026-10-07T19:44:42Z",
      "updated": "2026-10-07T19:47:57Z",
      "format_of": null,
      "group": "E",
      "latency": {
        "source": "run",
        "runs": [
          139
        ],
        "concurrency": 1,
        "median_request_seconds": 0.07536213693674654,
        "p90_request_seconds": 0.08099431772716344
      },
      "single_question_latency": null
    },
    {
      "run": 131,
      "model": "qwen3-14b",
      "name": "Qwen3-14B",
      "variant": null,
      "reasoning": "off",
      "provider": "vllm",
      "note": "Erk-14B's base model",
      "parameters": 14768307200,
      "active_parameters": null,
      "access": "vLLM, self-hosted, BF16",
      "questions_per_request": 16,
      "concurrency": 1,
      "correct": 1005,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 473
        },
        "grammar": {
          "questions": 1542,
          "correct": 532
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 1003
        }
      },
      "tyt_mix": 0.6552313178165827,
      "questions_per_minute": 331.1300910910813,
      "request_seconds": 398.27247220405843,
      "median_request_seconds": 2.8455167649663053,
      "p90_request_seconds": 3.1145364760770464,
      "billing": "local",
      "api_equivalent_usd": null,
      "cost_basis": null,
      "requests": 138,
      "hardware": "1 × RTX PRO 6000 Blackwell (96 GB); BF16",
      "input_tokens": 556424,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 17716,
      "confidence": {
        "source": "probability",
        "label": "Answer letter probability",
        "bins": [
          {
            "questions": 0,
            "correct": 0
          },
          {
            "questions": 138,
            "correct": 39
          },
          {
            "questions": 395,
            "correct": 104
          },
          {
            "questions": 287,
            "correct": 81
          },
          {
            "questions": 1378,
            "correct": 781
          }
        ],
        "auroc": 0.7336857206006847,
        "auroc_interval": [
          0.7098968834321366,
          0.7538051312338585
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.8069467068704583,
            "auroc_interval": [
              0.7680431218522581,
              0.84231053019922
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.6189384351969032,
            "auroc_interval": [
              0.5875231781257358,
              0.6475021854942953
            ]
          }
        },
        "aurc": 0.33405093708542116,
        "ece": 0.41002744844910793,
        "brier": 0.40025856324402215,
        "coverage_accuracy": {
          "0.20": 0.8680618744313012,
          "0.50": 0.6232939035486806,
          "0.80": 0.5069381255686989
        },
        "coverage_intervals": {
          "0.20": [
            0.8295350957155879,
            0.9020947176684881
          ],
          "0.50": [
            0.5901639344262295,
            0.6542649727767695
          ],
          "0.80": [
            0.4822565969062784,
            0.5303961748633881
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0682
          ],
          [
            0.04,
            0.0569
          ],
          [
            0.06,
            0.0531
          ],
          [
            0.08,
            0.0626
          ],
          [
            0.1,
            0.0637
          ],
          [
            0.12,
            0.072
          ],
          [
            0.14,
            0.0715
          ],
          [
            0.16,
            0.0958
          ],
          [
            0.18,
            0.1163
          ],
          [
            0.2,
            0.1319
          ],
          [
            0.22,
            0.1572
          ],
          [
            0.24,
            0.1763
          ],
          [
            0.26,
            0.1916
          ],
          [
            0.28,
            0.2275
          ],
          [
            0.3,
            0.2433
          ],
          [
            0.32,
            0.2721
          ],
          [
            0.34,
            0.2917
          ],
          [
            0.36,
            0.297
          ],
          [
            0.38,
            0.3125
          ],
          [
            0.4,
            0.3255
          ],
          [
            0.42,
            0.3401
          ],
          [
            0.44,
            0.3455
          ],
          [
            0.46,
            0.358
          ],
          [
            0.48,
            0.3649
          ],
          [
            0.5,
            0.3767
          ],
          [
            0.52,
            0.3902
          ],
          [
            0.54,
            0.3993
          ],
          [
            0.56,
            0.4069
          ],
          [
            0.58,
            0.4164
          ],
          [
            0.6,
            0.4239
          ],
          [
            0.62,
            0.43
          ],
          [
            0.64,
            0.4386
          ],
          [
            0.66,
            0.4458
          ],
          [
            0.68,
            0.4527
          ],
          [
            0.7,
            0.4631
          ],
          [
            0.72,
            0.4711
          ],
          [
            0.74,
            0.4762
          ],
          [
            0.76,
            0.4834
          ],
          [
            0.78,
            0.489
          ],
          [
            0.8,
            0.4931
          ],
          [
            0.82,
            0.5012
          ],
          [
            0.84,
            0.5064
          ],
          [
            0.86,
            0.511
          ],
          [
            0.88,
            0.516
          ],
          [
            0.9,
            0.5212
          ],
          [
            0.92,
            0.5267
          ],
          [
            0.94,
            0.531
          ],
          [
            0.96,
            0.5341
          ],
          [
            0.98,
            0.5385
          ],
          [
            1.0,
            0.5428
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 4,
            "mean_confidence": 0.2905192759080034,
            "accuracy": 0.0
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 30,
            "mean_confidence": 0.3596873627205906,
            "accuracy": 0.3
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 104,
            "mean_confidence": 0.4637286027047631,
            "accuracy": 0.28846153846153844
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 131,
            "mean_confidence": 0.5510678808594073,
            "accuracy": 0.25190839694656486
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 194,
            "mean_confidence": 0.6498088818259937,
            "accuracy": 0.26804123711340205
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 168,
            "mean_confidence": 0.7561127168340297,
            "accuracy": 0.26785714285714285
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 189,
            "mean_confidence": 0.8529922570693944,
            "accuracy": 0.291005291005291
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 1378,
            "mean_confidence": 0.9866216041680235,
            "accuracy": 0.5667634252539913
          }
        ],
        "median_right": 0.9995688557992545,
        "median_wrong": 0.9003574982798129,
        "sure": {
          "questions": 975,
          "correct": 633,
          "interval": [
            0.6187435577014109,
            0.6785466269413343
          ]
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 463,
          "correct": 124,
          "interval": [
            0.22951384168030597,
            0.3099445099318897
          ]
        },
        "ece_refit": 0.04607191669191213,
        "brier_refit": 0.20259951265295978,
        "brier_base": 0.24843670034331894,
        "refit_slopes": [
          0.20027972641275513,
          0.19270274107254212,
          0.19354969316260534,
          0.19710827274983073,
          0.20139711475309527
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        80,
        115,
        46,
        84,
        72,
        76,
        45,
        19,
        27,
        24,
        24,
        25,
        37,
        25,
        27,
        61,
        24,
        50,
        53,
        91
      ],
      "started": "2026-10-06T12:25:34Z",
      "updated": "2026-10-06T12:32:15Z",
      "format_of": null,
      "group": "E",
      "latency": {
        "source": "run",
        "runs": [
          131
        ],
        "concurrency": 1,
        "median_request_seconds": 2.8455167649663053,
        "p90_request_seconds": 3.1145364760770464
      },
      "single_question_latency": null
    },
    {
      "run": 132,
      "model": "erk-14b",
      "name": "Erk-14B",
      "variant": null,
      "reasoning": "off",
      "provider": "vllm",
      "note": "Qwen3-14B with continued Turkish training (eCloud)",
      "parameters": 14768307200,
      "active_parameters": null,
      "access": "vLLM, self-hosted, BF16",
      "questions_per_request": 16,
      "concurrency": 1,
      "correct": 989,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 474
        },
        "grammar": {
          "questions": 1542,
          "correct": 515
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 988
        }
      },
      "tyt_mix": 0.6545596271867388,
      "questions_per_minute": 392.70250832565097,
      "request_seconds": 335.82673195109237,
      "median_request_seconds": 2.3896605125046335,
      "p90_request_seconds": 2.662659723334946,
      "billing": "local",
      "api_equivalent_usd": null,
      "cost_basis": null,
      "requests": 138,
      "hardware": "1 × RTX PRO 6000 Blackwell (96 GB); BF16",
      "input_tokens": 556424,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 14720,
      "confidence": {
        "source": "probability",
        "label": "Answer letter probability",
        "bins": [
          {
            "questions": 41,
            "correct": 6
          },
          {
            "questions": 983,
            "correct": 257
          },
          {
            "questions": 533,
            "correct": 230
          },
          {
            "questions": 235,
            "correct": 145
          },
          {
            "questions": 406,
            "correct": 351
          }
        ],
        "auroc": 0.75398239191905,
        "auroc_interval": [
          0.7329048323570657,
          0.7755322555835458
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.833344925117077,
            "auroc_interval": [
              0.7980460808674046,
              0.8628873011883819
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.637928361425965,
            "auroc_interval": [
              0.6055290599746399,
              0.6669121413934426
            ]
          }
        },
        "aurc": 0.33057398258050896,
        "ece": 0.13455620377261668,
        "brier": 0.21437003773396557,
        "coverage_accuracy": {
          "0.20": 0.8430391264786169,
          "0.50": 0.6342129208371247,
          "0.80": 0.5006824385805277
        },
        "coverage_intervals": {
          "0.20": [
            0.804456571168713,
            0.884090909090909
          ],
          "0.50": [
            0.6061987237921604,
            0.6643929058663028
          ],
          "0.80": [
            0.47655894401456533,
            0.5233878292461398
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0227
          ],
          [
            0.04,
            0.056
          ],
          [
            0.06,
            0.0455
          ],
          [
            0.08,
            0.0569
          ],
          [
            0.1,
            0.0673
          ],
          [
            0.12,
            0.0749
          ],
          [
            0.14,
            0.091
          ],
          [
            0.16,
            0.1043
          ],
          [
            0.18,
            0.1314
          ],
          [
            0.2,
            0.157
          ],
          [
            0.22,
            0.1716
          ],
          [
            0.24,
            0.1868
          ],
          [
            0.26,
            0.1995
          ],
          [
            0.28,
            0.2201
          ],
          [
            0.3,
            0.2342
          ],
          [
            0.32,
            0.2516
          ],
          [
            0.34,
            0.2636
          ],
          [
            0.36,
            0.2796
          ],
          [
            0.38,
            0.2957
          ],
          [
            0.4,
            0.3062
          ],
          [
            0.42,
            0.3197
          ],
          [
            0.44,
            0.3319
          ],
          [
            0.46,
            0.3443
          ],
          [
            0.48,
            0.3545
          ],
          [
            0.5,
            0.3658
          ],
          [
            0.52,
            0.3745
          ],
          [
            0.54,
            0.3866
          ],
          [
            0.56,
            0.3989
          ],
          [
            0.58,
            0.4078
          ],
          [
            0.6,
            0.417
          ],
          [
            0.62,
            0.424
          ],
          [
            0.64,
            0.432
          ],
          [
            0.66,
            0.4396
          ],
          [
            0.68,
            0.4476
          ],
          [
            0.7,
            0.4553
          ],
          [
            0.72,
            0.4648
          ],
          [
            0.74,
            0.4737
          ],
          [
            0.76,
            0.4828
          ],
          [
            0.78,
            0.4902
          ],
          [
            0.8,
            0.4993
          ],
          [
            0.82,
            0.5049
          ],
          [
            0.84,
            0.5098
          ],
          [
            0.86,
            0.5165
          ],
          [
            0.88,
            0.5213
          ],
          [
            0.9,
            0.5253
          ],
          [
            0.92,
            0.5319
          ],
          [
            0.94,
            0.5349
          ],
          [
            0.96,
            0.5414
          ],
          [
            0.98,
            0.5439
          ],
          [
            1.0,
            0.55
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 220,
            "mean_confidence": 0.27045114386202834,
            "accuracy": 0.22727272727272727
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 449,
            "mean_confidence": 0.3478851609341981,
            "accuracy": 0.23162583518930957
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 355,
            "mean_confidence": 0.4478541688013936,
            "accuracy": 0.30704225352112674
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 276,
            "mean_confidence": 0.5445133475362183,
            "accuracy": 0.38768115942028986
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 169,
            "mean_confidence": 0.6489070711168314,
            "accuracy": 0.46153846153846156
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 177,
            "mean_confidence": 0.7516227878410612,
            "accuracy": 0.5536723163841808
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 146,
            "mean_confidence": 0.8520778195097863,
            "accuracy": 0.6301369863013698
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 406,
            "mean_confidence": 0.9671799089250693,
            "accuracy": 0.8645320197044335
          }
        ],
        "median_right": 0.750993488829512,
        "median_wrong": 0.4316644753636113,
        "sure": {
          "questions": 101,
          "correct": 96,
          "interval": [
            0.8893042659293906,
            0.9786718046380176
          ]
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 1469,
          "correct": 448,
          "interval": [
            0.28195955672336465,
            0.328996570534312
          ]
        },
        "ece_refit": 0.016002441972930805,
        "brier_refit": 0.19346005803957653,
        "brier_base": 0.24782511468443535,
        "refit_slopes": [
          0.7190802808360722,
          0.6984693268996082,
          0.6959441772431677,
          0.7081916658232581,
          0.7206087796279323
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        84,
        114,
        46,
        89,
        64,
        77,
        45,
        20,
        27,
        28,
        30,
        16,
        38,
        21,
        25,
        60,
        20,
        48,
        40,
        97
      ],
      "started": "2026-10-06T12:25:34Z",
      "updated": "2026-10-06T12:31:11Z",
      "format_of": null,
      "group": "EF",
      "latency": {
        "source": "run",
        "runs": [
          132
        ],
        "concurrency": 1,
        "median_request_seconds": 2.3896605125046335,
        "p90_request_seconds": 2.662659723334946
      },
      "single_question_latency": null
    },
    {
      "run": 126,
      "model": "metask-jev-4b-policy-mix",
      "name": "Metask-Jev 4B",
      "variant": null,
      "reasoning": null,
      "provider": "metask",
      "note": "Fine-tune of Qwen3.5-4B (merged LoRA)",
      "parameters": 4539265536,
      "active_parameters": null,
      "access": "Metask-Jev server, self-hosted",
      "questions_per_request": 1,
      "concurrency": 1,
      "correct": 896,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 412
        },
        "grammar": {
          "questions": 1542,
          "correct": 484
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 895
        }
      },
      "tyt_mix": 0.5730689079750719,
      "questions_per_minute": 660.6373826690542,
      "request_seconds": 199.62539732037112,
      "median_request_seconds": 0.08993917948100716,
      "p90_request_seconds": 0.09815078696701676,
      "billing": "local",
      "api_equivalent_usd": null,
      "cost_basis": null,
      "requests": 2198,
      "hardware": "1 × RTX PRO 6000 Blackwell (96 GB); BF16",
      "input_tokens": 1137590,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 0,
      "confidence": {
        "source": "probability",
        "label": "Option score",
        "bins": [
          {
            "questions": 257,
            "correct": 64
          },
          {
            "questions": 1207,
            "correct": 337
          },
          {
            "questions": 383,
            "correct": 192
          },
          {
            "questions": 231,
            "correct": 187
          },
          {
            "questions": 120,
            "correct": 116
          }
        ],
        "auroc": 0.7250071147410577,
        "auroc_interval": [
          0.7014091923364106,
          0.74806807783493
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.8031195288874742,
            "auroc_interval": [
              0.7686549287339801,
              0.8370647394754132
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.6178935774656689,
            "auroc_interval": [
              0.5848908057780516,
              0.6472913248604646
            ]
          }
        },
        "aurc": 0.38259426954731085,
        "ece": 0.058126771586986974,
        "brier": 0.19768712319064544,
        "coverage_accuracy": {
          "0.20": 0.813466787989081,
          "0.50": 0.5577797998180164,
          "0.80": 0.45040946314831665
        },
        "coverage_intervals": {
          "0.20": [
            0.7612551159618008,
            0.8523398455247615
          ],
          "0.50": [
            0.5252617205279927,
            0.5896269335759782
          ],
          "0.80": [
            0.4265241128298453,
            0.47451069640418747
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.0227
          ],
          [
            0.04,
            0.0114
          ],
          [
            0.06,
            0.0379
          ],
          [
            0.08,
            0.0455
          ],
          [
            0.1,
            0.0728
          ],
          [
            0.12,
            0.091
          ],
          [
            0.14,
            0.1137
          ],
          [
            0.16,
            0.1365
          ],
          [
            0.18,
            0.1668
          ],
          [
            0.2,
            0.1865
          ],
          [
            0.22,
            0.2171
          ],
          [
            0.24,
            0.237
          ],
          [
            0.26,
            0.2616
          ],
          [
            0.28,
            0.2843
          ],
          [
            0.3,
            0.3094
          ],
          [
            0.32,
            0.3142
          ],
          [
            0.34,
            0.3323
          ],
          [
            0.36,
            0.3488
          ],
          [
            0.38,
            0.3652
          ],
          [
            0.4,
            0.379
          ],
          [
            0.42,
            0.3956
          ],
          [
            0.44,
            0.4075
          ],
          [
            0.46,
            0.4194
          ],
          [
            0.48,
            0.4304
          ],
          [
            0.5,
            0.4422
          ],
          [
            0.52,
            0.454
          ],
          [
            0.54,
            0.4667
          ],
          [
            0.56,
            0.4727
          ],
          [
            0.58,
            0.4816
          ],
          [
            0.6,
            0.4889
          ],
          [
            0.62,
            0.4966
          ],
          [
            0.64,
            0.5069
          ],
          [
            0.66,
            0.5154
          ],
          [
            0.68,
            0.5225
          ],
          [
            0.7,
            0.5279
          ],
          [
            0.72,
            0.5318
          ],
          [
            0.74,
            0.5352
          ],
          [
            0.76,
            0.5403
          ],
          [
            0.78,
            0.5471
          ],
          [
            0.8,
            0.5496
          ],
          [
            0.82,
            0.5545
          ],
          [
            0.84,
            0.5595
          ],
          [
            0.86,
            0.5655
          ],
          [
            0.88,
            0.5706
          ],
          [
            0.9,
            0.5758
          ],
          [
            0.92,
            0.5782
          ],
          [
            0.94,
            0.5809
          ],
          [
            0.96,
            0.5851
          ],
          [
            0.98,
            0.5896
          ],
          [
            1.0,
            0.5924
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 681,
            "mean_confidence": 0.2603964106495685,
            "accuracy": 0.25844346549192365
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 520,
            "mean_confidence": 0.34607558691730866,
            "accuracy": 0.2653846153846154
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 263,
            "mean_confidence": 0.44510871545896785,
            "accuracy": 0.33079847908745247
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 177,
            "mean_confidence": 0.5471665589822887,
            "accuracy": 0.4519774011299435
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 144,
            "mean_confidence": 0.649934284389019,
            "accuracy": 0.5069444444444444
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 133,
            "mean_confidence": 0.7513148892194705,
            "accuracy": 0.6616541353383458
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 160,
            "mean_confidence": 0.8544770009815693,
            "accuracy": 0.8625
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 120,
            "mean_confidence": 0.935314892232418,
            "accuracy": 0.9666666666666667
          }
        ],
        "median_right": 0.5518057644367218,
        "median_wrong": 0.33168134093284607,
        "sure": {
          "questions": 0,
          "correct": 0,
          "interval": null
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 1785,
          "correct": 554,
          "interval": [
            0.28932797028205753,
            0.33221482031494104
          ]
        },
        "ece_refit": 0.0350213200799688,
        "brier_refit": 0.19424822710161838,
        "brier_base": 0.24163397288878724,
        "refit_slopes": [
          0.9786204307125993,
          0.9649909038087302,
          0.9435965277059788,
          0.9570330248239476,
          1.0252834910298516
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        66,
        106,
        38,
        84,
        52,
        66,
        47,
        22,
        36,
        31,
        26,
        21,
        41,
        19,
        21,
        55,
        26,
        36,
        32,
        71
      ],
      "started": "2026-10-06T09:45:14Z",
      "updated": "2026-10-06T09:48:37Z",
      "format_of": null,
      "group": "F",
      "latency": {
        "source": "run",
        "runs": [
          126
        ],
        "concurrency": 1,
        "median_request_seconds": 0.08993917948100716,
        "p90_request_seconds": 0.09815078696701676
      },
      "single_question_latency": null
    },
    {
      "run": 340,
      "model": "LiquidAI/d1-2B",
      "name": "Liquid d1-2B (open weights)",
      "variant": null,
      "reasoning": null,
      "provider": "d1-server",
      "note": "Liquid AI's released d1-2B weights (LiquidAI/d1-2B at 10e7671), a decision model built on LFM3-VL-2B that also reads images. It ran on our hardware through its model card's own call (model.system_one, Liquid's model code in Transformers) behind our HTTP server (servers/d1_server.py), one request at a time. Liquid lists 16 languages for it; Turkish is not among them. Its request times come from one GPU with no other known work while its runs took place, though the GPU was not monitored throughout. Only this model's server showed on it at 11:55 UTC on 1 November 2026, before the runs, and nothing at 12:35 UTC, after them; the one other project with a claim on that GPU confirmed it ran nothing there in between.",
      "parameters": 1918181888,
      "active_parameters": null,
      "access": "d1 server (Liquid's model code in Transformers, the model card's call), self-hosted, BF16",
      "questions_per_request": 1,
      "concurrency": 1,
      "correct": 646,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 268
        },
        "grammar": {
          "questions": 1542,
          "correct": 378
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 643
        }
      },
      "tyt_mix": 0.3799415156116542,
      "questions_per_minute": 3150.5652888015998,
      "request_seconds": 41.85915475827642,
      "median_request_seconds": 0.019204194002668373,
      "p90_request_seconds": 0.021930550708202646,
      "billing": "local",
      "api_equivalent_usd": null,
      "cost_basis": null,
      "requests": 2198,
      "hardware": "1 × RTX PRO 6000 Blackwell (96 GB); BF16",
      "input_tokens": 745982,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 0,
      "confidence": {
        "source": "probability",
        "label": "Option score",
        "bins": [
          {
            "questions": 411,
            "correct": 84
          },
          {
            "questions": 1503,
            "correct": 405
          },
          {
            "questions": 218,
            "correct": 115
          },
          {
            "questions": 48,
            "correct": 26
          },
          {
            "questions": 18,
            "correct": 16
          }
        ],
        "auroc": 0.6369211005074846,
        "auroc_interval": [
          0.610274275060105,
          0.6616953087541323
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.6563798276657947,
            "auroc_interval": [
              0.6173693361382477,
              0.6975213472485768
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.5799457717413044,
            "auroc_interval": [
              0.5454097236116272,
              0.6133037921382022
            ]
          }
        },
        "aurc": 0.5898220011970087,
        "ece": 0.0579868390338947,
        "brier": 0.19800640617258627,
        "coverage_accuracy": {
          "0.20": 0.4777070063694267,
          "0.50": 0.37579617834394907,
          "0.80": 0.31790263876251135
        },
        "coverage_intervals": {
          "0.20": [
            0.432407828857533,
            0.527512505684402
          ],
          "0.50": [
            0.34849863512283896,
            0.40427467030468395
          ],
          "0.80": [
            0.29691049522944124,
            0.33997722095671984
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.273
          ],
          [
            0.04,
            0.3526
          ],
          [
            0.06,
            0.364
          ],
          [
            0.08,
            0.3972
          ],
          [
            0.1,
            0.404
          ],
          [
            0.12,
            0.4389
          ],
          [
            0.14,
            0.4605
          ],
          [
            0.16,
            0.4853
          ],
          [
            0.18,
            0.497
          ],
          [
            0.2,
            0.5223
          ],
          [
            0.22,
            0.5335
          ],
          [
            0.24,
            0.5469
          ],
          [
            0.26,
            0.5468
          ],
          [
            0.28,
            0.5613
          ],
          [
            0.3,
            0.5754
          ],
          [
            0.32,
            0.5829
          ],
          [
            0.34,
            0.5838
          ],
          [
            0.36,
            0.5893
          ],
          [
            0.38,
            0.5965
          ],
          [
            0.4,
            0.5974
          ],
          [
            0.42,
            0.6079
          ],
          [
            0.44,
            0.6112
          ],
          [
            0.46,
            0.6153
          ],
          [
            0.48,
            0.6171
          ],
          [
            0.5,
            0.6242
          ],
          [
            0.52,
            0.6317
          ],
          [
            0.54,
            0.6318
          ],
          [
            0.56,
            0.6352
          ],
          [
            0.58,
            0.6431
          ],
          [
            0.6,
            0.6489
          ],
          [
            0.62,
            0.6544
          ],
          [
            0.64,
            0.6588
          ],
          [
            0.66,
            0.6638
          ],
          [
            0.68,
            0.6655
          ],
          [
            0.7,
            0.6672
          ],
          [
            0.72,
            0.6702
          ],
          [
            0.74,
            0.6748
          ],
          [
            0.76,
            0.6765
          ],
          [
            0.78,
            0.678
          ],
          [
            0.8,
            0.6821
          ],
          [
            0.82,
            0.6871
          ],
          [
            0.84,
            0.6891
          ],
          [
            0.86,
            0.693
          ],
          [
            0.88,
            0.695
          ],
          [
            0.9,
            0.6947
          ],
          [
            0.92,
            0.6964
          ],
          [
            0.94,
            0.6999
          ],
          [
            0.96,
            0.7024
          ],
          [
            0.98,
            0.7033
          ],
          [
            1.0,
            0.7061
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 1081,
            "mean_confidence": 0.25773026934846593,
            "accuracy": 0.21091581868640147
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 600,
            "mean_confidence": 0.3422753826036209,
            "accuracy": 0.3016666666666667
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 233,
            "mean_confidence": 0.4425341294876587,
            "accuracy": 0.34334763948497854
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 126,
            "mean_confidence": 0.5462631205187128,
            "accuracy": 0.4603174603174603
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 65,
            "mean_confidence": 0.6426845031793499,
            "accuracy": 0.5846153846153846
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 45,
            "mean_confidence": 0.7473737743132968,
            "accuracy": 0.5777777777777777
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 30,
            "mean_confidence": 0.8463440390956548,
            "accuracy": 0.6333333333333333
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 18,
            "mean_confidence": 0.9303990216126837,
            "accuracy": 0.8888888888888888
          }
        ],
        "median_right": 0.3440956647429336,
        "median_wrong": 0.28890486535686877,
        "sure": {
          "questions": 0,
          "correct": 0,
          "interval": null
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 2105,
          "correct": 585,
          "interval": [
            0.25919030417560057,
            0.29743832075289084
          ]
        },
        "ece_refit": 0.013573358419411893,
        "brier_refit": 0.1948452529116085,
        "brier_base": 0.20768341514302066,
        "refit_slopes": [
          0.9279105550309306,
          0.764438835692689,
          0.7594780224143544,
          0.8544019355021975,
          0.7952122795766929
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        50,
        67,
        27,
        54,
        30,
        40,
        28,
        15,
        25,
        15,
        20,
        15,
        39,
        25,
        22,
        46,
        19,
        27,
        26,
        56
      ],
      "started": "2026-11-01T12:00:08Z",
      "updated": "2026-11-01T12:00:56Z",
      "format_of": null,
      "group": "G",
      "latency": {
        "source": "run",
        "runs": [
          340
        ],
        "concurrency": 1,
        "median_request_seconds": 0.019204194002668373,
        "p90_request_seconds": 0.021930550708202646
      },
      "single_question_latency": null
    },
    {
      "run": 141,
      "model": "strands-decider-2b",
      "name": "Strands Decider 2B",
      "variant": null,
      "reasoning": null,
      "provider": "strands",
      "note": "Amazon's decision model: a LoRA adapter and a decision head on Qwen3.5-2B-Base",
      "parameters": 2291942208,
      "active_parameters": null,
      "access": "Strands Decider server, self-hosted",
      "questions_per_request": 1,
      "concurrency": 1,
      "correct": 604,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 242
        },
        "grammar": {
          "questions": 1542,
          "correct": 362
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 602
        }
      },
      "tyt_mix": 0.34542752127424,
      "questions_per_minute": 2062.4513111246683,
      "request_seconds": 63.94332767452579,
      "median_request_seconds": 0.026102936477400362,
      "p90_request_seconds": 0.0304201258928515,
      "billing": "local",
      "api_equivalent_usd": null,
      "cost_basis": null,
      "requests": 2198,
      "hardware": "1 × RTX PRO 6000 Blackwell (96 GB); BF16",
      "input_tokens": 789516,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 2198,
      "confidence": {
        "source": "probability",
        "label": "Option score",
        "bins": [
          {
            "questions": 105,
            "correct": 21
          },
          {
            "questions": 1694,
            "correct": 385
          },
          {
            "questions": 299,
            "correct": 121
          },
          {
            "questions": 68,
            "correct": 48
          },
          {
            "questions": 32,
            "correct": 29
          }
        ],
        "auroc": 0.6465693993202988,
        "auroc_interval": [
          0.619592285776639,
          0.672889759441758
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.7376981275202619,
            "auroc_interval": [
              0.6982162183055041,
              0.777432238138346
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.5729117894933983,
            "auroc_interval": [
              0.5365585772874163,
              0.6068985068985069
            ]
          }
        },
        "aurc": 0.5959519243533254,
        "ece": 0.12245345768880803,
        "brier": 0.19831942687443133,
        "coverage_accuracy": {
          "0.20": 0.4868061874431301,
          "0.50": 0.359417652411283,
          "0.80": 0.29401728844404
        },
        "coverage_intervals": {
          "0.20": [
            0.43533697632058294,
            0.5345768880800728
          ],
          "0.50": [
            0.3314006366530241,
            0.3883583447021373
          ],
          "0.80": [
            0.27340349509517314,
            0.316196542311192
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.1128
          ],
          [
            0.04,
            0.2047
          ],
          [
            0.06,
            0.3033
          ],
          [
            0.08,
            0.3858
          ],
          [
            0.1,
            0.4268
          ],
          [
            0.12,
            0.4436
          ],
          [
            0.14,
            0.468
          ],
          [
            0.16,
            0.4862
          ],
          [
            0.18,
            0.5071
          ],
          [
            0.2,
            0.5132
          ],
          [
            0.22,
            0.5264
          ],
          [
            0.24,
            0.5441
          ],
          [
            0.26,
            0.5608
          ],
          [
            0.28,
            0.5644
          ],
          [
            0.3,
            0.5678
          ],
          [
            0.32,
            0.5806
          ],
          [
            0.34,
            0.5843
          ],
          [
            0.36,
            0.5952
          ],
          [
            0.38,
            0.6022
          ],
          [
            0.4,
            0.6144
          ],
          [
            0.42,
            0.623
          ],
          [
            0.44,
            0.6267
          ],
          [
            0.46,
            0.6321
          ],
          [
            0.48,
            0.6379
          ],
          [
            0.5,
            0.6406
          ],
          [
            0.52,
            0.6454
          ],
          [
            0.54,
            0.6537
          ],
          [
            0.56,
            0.6564
          ],
          [
            0.58,
            0.6675
          ],
          [
            0.6,
            0.6694
          ],
          [
            0.62,
            0.6757
          ],
          [
            0.64,
            0.6769
          ],
          [
            0.66,
            0.6808
          ],
          [
            0.68,
            0.6862
          ],
          [
            0.7,
            0.6893
          ],
          [
            0.72,
            0.6929
          ],
          [
            0.74,
            0.6953
          ],
          [
            0.76,
            0.6995
          ],
          [
            0.78,
            0.7008
          ],
          [
            0.8,
            0.706
          ],
          [
            0.82,
            0.7083
          ],
          [
            0.84,
            0.7119
          ],
          [
            0.86,
            0.7164
          ],
          [
            0.88,
            0.7182
          ],
          [
            0.9,
            0.7189
          ],
          [
            0.92,
            0.7176
          ],
          [
            0.94,
            0.7202
          ],
          [
            0.96,
            0.7218
          ],
          [
            0.98,
            0.7228
          ],
          [
            1.0,
            0.7252
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 599,
            "mean_confidence": 0.2692565943238732,
            "accuracy": 0.19198664440734559
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 842,
            "mean_confidence": 0.34266128266033236,
            "accuracy": 0.21021377672209027
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 358,
            "mean_confidence": 0.44295921787709525,
            "accuracy": 0.31843575418994413
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 160,
            "mean_confidence": 0.5429062500000004,
            "accuracy": 0.40625
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 101,
            "mean_confidence": 0.6430089108910892,
            "accuracy": 0.38613861386138615
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 62,
            "mean_confidence": 0.7416096774193548,
            "accuracy": 0.5
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 44,
            "mean_confidence": 0.8399590909090907,
            "accuracy": 0.7727272727272727
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 32,
            "mean_confidence": 0.9381531250000001,
            "accuracy": 0.90625
          }
        ],
        "median_right": 0.4064,
        "median_wrong": 0.33494999999999997,
        "sure": {
          "questions": 0,
          "correct": 0,
          "interval": null
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 2060,
          "correct": 510,
          "interval": [
            0.22941581186116547,
            0.2666695466275867
          ]
        },
        "ece_refit": 0.01068940230224292,
        "brier_refit": 0.18251983413296638,
        "brier_base": 0.19950126249032396,
        "refit_slopes": [
          0.9083396336162506,
          0.7963674725800406,
          0.8867454747584909,
          0.9418236754372598,
          0.8704809728973301
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        42,
        71,
        23,
        59,
        35,
        12,
        35,
        15,
        29,
        17,
        19,
        10,
        26,
        15,
        17,
        49,
        15,
        35,
        29,
        51
      ],
      "started": "2026-10-07T19:49:08Z",
      "updated": "2026-10-07T19:50:14Z",
      "format_of": null,
      "group": "G",
      "latency": {
        "source": "run",
        "runs": [
          141
        ],
        "concurrency": 1,
        "median_request_seconds": 0.026102936477400362,
        "p90_request_seconds": 0.0304201258928515
      },
      "single_question_latency": null
    },
    {
      "run": 243,
      "model": "decider-1",
      "name": "meraGPT Decider 1",
      "variant": null,
      "reasoning": null,
      "provider": "meragpt",
      "note": "Hosted decision model; meraGPT serves its current version under this name",
      "parameters": null,
      "active_parameters": null,
      "access": "meraGPT System One API",
      "questions_per_request": 1,
      "concurrency": 1,
      "correct": 596,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 227
        },
        "grammar": {
          "questions": 1542,
          "correct": 369
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 595
        }
      },
      "tyt_mix": 0.32735761483344405,
      "questions_per_minute": 153.69089032800073,
      "request_seconds": 858.085991424392,
      "median_request_seconds": 0.38244215249142144,
      "p90_request_seconds": 0.4169523003976792,
      "billing": "api",
      "api_equivalent_usd": 0.032449,
      "cost_basis": "api",
      "requests": 2198,
      "hardware": null,
      "input_tokens": 811225,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 0,
      "confidence": {
        "source": "probability",
        "label": "Option score",
        "bins": [
          {
            "questions": 65,
            "correct": 13
          },
          {
            "questions": 1671,
            "correct": 397
          },
          {
            "questions": 380,
            "correct": 137
          },
          {
            "questions": 72,
            "correct": 43
          },
          {
            "questions": 10,
            "correct": 6
          }
        ],
        "auroc": 0.5918592740617852,
        "auroc_interval": [
          0.566047797954498,
          0.6188783029792609
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.6781779160633786,
            "auroc_interval": [
              0.6326806783101451,
              0.7208041098501952
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.5309100192451199,
            "auroc_interval": [
              0.49730408950652766,
              0.5655014294136074
            ]
          }
        },
        "aurc": 0.6469230319168837,
        "ece": 0.1431224749772521,
        "brier": 0.21533119770700637,
        "coverage_accuracy": {
          "0.20": 0.4017288444040037,
          "0.50": 0.32393084622383983,
          "0.80": 0.2826433121019108
        },
        "coverage_intervals": {
          "0.20": [
            0.3624772313296904,
            0.4481346678798908
          ],
          "0.50": [
            0.2976786527082385,
            0.3498861047835991
          ],
          "0.80": [
            0.2634090909090909,
            0.3046957972993476
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.3412
          ],
          [
            0.04,
            0.4199
          ],
          [
            0.06,
            0.4474
          ],
          [
            0.08,
            0.4939
          ],
          [
            0.1,
            0.5086
          ],
          [
            0.12,
            0.527
          ],
          [
            0.14,
            0.5515
          ],
          [
            0.16,
            0.5763
          ],
          [
            0.18,
            0.594
          ],
          [
            0.2,
            0.5983
          ],
          [
            0.22,
            0.6029
          ],
          [
            0.24,
            0.6161
          ],
          [
            0.26,
            0.6264
          ],
          [
            0.28,
            0.6353
          ],
          [
            0.3,
            0.6406
          ],
          [
            0.32,
            0.6517
          ],
          [
            0.34,
            0.6539
          ],
          [
            0.36,
            0.663
          ],
          [
            0.38,
            0.6636
          ],
          [
            0.4,
            0.6679
          ],
          [
            0.42,
            0.6706
          ],
          [
            0.44,
            0.6712
          ],
          [
            0.46,
            0.6726
          ],
          [
            0.48,
            0.6721
          ],
          [
            0.5,
            0.6761
          ],
          [
            0.52,
            0.6807
          ],
          [
            0.54,
            0.6852
          ],
          [
            0.56,
            0.6905
          ],
          [
            0.58,
            0.6941
          ],
          [
            0.6,
            0.6999
          ],
          [
            0.62,
            0.6993
          ],
          [
            0.64,
            0.6979
          ],
          [
            0.66,
            0.7011
          ],
          [
            0.68,
            0.7029
          ],
          [
            0.7,
            0.7088
          ],
          [
            0.72,
            0.7077
          ],
          [
            0.74,
            0.7117
          ],
          [
            0.76,
            0.7124
          ],
          [
            0.78,
            0.7148
          ],
          [
            0.8,
            0.7174
          ],
          [
            0.82,
            0.7148
          ],
          [
            0.84,
            0.7157
          ],
          [
            0.86,
            0.717
          ],
          [
            0.88,
            0.7193
          ],
          [
            0.9,
            0.7219
          ],
          [
            0.92,
            0.7231
          ],
          [
            0.94,
            0.7265
          ],
          [
            0.96,
            0.7256
          ],
          [
            0.98,
            0.7275
          ],
          [
            1.0,
            0.7288
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 411,
            "mean_confidence": 0.2722491484184915,
            "accuracy": 0.2116788321167883
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 860,
            "mean_confidence": 0.34513639534883744,
            "accuracy": 0.23488372093023255
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 465,
            "mean_confidence": 0.44509440860215044,
            "accuracy": 0.26021505376344084
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 214,
            "mean_confidence": 0.5461060747663549,
            "accuracy": 0.3177570093457944
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 132,
            "mean_confidence": 0.6462083333333332,
            "accuracy": 0.4015151515151515
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 59,
            "mean_confidence": 0.7428966101694915,
            "accuracy": 0.4576271186440678
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 47,
            "mean_confidence": 0.8433680851063827,
            "accuracy": 0.6808510638297872
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 10,
            "mean_confidence": 0.9267200000000001,
            "accuracy": 0.6
          }
        ],
        "median_right": 0.40175,
        "median_wrong": 0.36575,
        "sure": {
          "questions": 0,
          "correct": 0,
          "interval": null
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 2082,
          "correct": 531,
          "interval": [
            0.23678263966845525,
            0.27420611422349883
          ]
        },
        "ece_refit": 0.0063402307180002995,
        "brier_refit": 0.19116126299943217,
        "brier_base": 0.1980123801587921,
        "refit_slopes": [
          0.6242587068559422,
          0.5881400632659745,
          0.6061268341981485,
          0.6312358259949776,
          0.5980719312432586
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        35,
        46,
        24,
        53,
        33,
        36,
        42,
        17,
        23,
        14,
        23,
        14,
        36,
        16,
        13,
        38,
        17,
        28,
        30,
        58
      ],
      "started": "2026-10-19T10:31:37Z",
      "updated": "2026-10-19T10:46:02Z",
      "format_of": null,
      "group": "G",
      "latency": {
        "source": "run",
        "runs": [
          243
        ],
        "concurrency": 1,
        "median_request_seconds": 0.38244215249142144,
        "p90_request_seconds": 0.4169523003976792
      },
      "single_question_latency": null
    },
    {
      "run": 135,
      "model": "gliner2.5-multi-decide",
      "name": "GLiNER2.5-multi-Decide",
      "variant": null,
      "reasoning": null,
      "provider": "gliner",
      "note": "Fastino's decision fine-tune of GLiNER2.5 Multi",
      "parameters": 287355159,
      "active_parameters": null,
      "access": "GLiNER2 server, self-hosted",
      "questions_per_request": 1,
      "concurrency": 1,
      "correct": 444,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 138
        },
        "grammar": {
          "questions": 1542,
          "correct": 306
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 440
        }
      },
      "tyt_mix": 0.20827945572743664,
      "questions_per_minute": 6615.931772680665,
      "request_seconds": 19.933700124383904,
      "median_request_seconds": 0.008824052987620234,
      "p90_request_seconds": 0.010334555187728255,
      "billing": "local",
      "api_equivalent_usd": null,
      "cost_basis": null,
      "requests": 2198,
      "hardware": "1 × RTX PRO 6000 Blackwell (96 GB); FP16",
      "input_tokens": 431150,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 0,
      "confidence": {
        "source": "probability",
        "label": "Option score",
        "bins": [
          {
            "questions": 87,
            "correct": 14
          },
          {
            "questions": 1486,
            "correct": 291
          },
          {
            "questions": 502,
            "correct": 105
          },
          {
            "questions": 112,
            "correct": 30
          },
          {
            "questions": 11,
            "correct": 4
          }
        ],
        "auroc": 0.5366202861926921,
        "auroc_interval": [
          0.5073844516943666,
          0.5661652273031712
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.5558306753958928,
            "auroc_interval": [
              0.505829583519027,
              0.6074425612052731
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.5270718319690336,
            "auroc_interval": [
              0.4912793703979482,
              0.566275
            ]
          }
        },
        "aurc": 0.7743075179555913,
        "ece": 0.23118755647792288,
        "brier": 0.23346121045805518,
        "coverage_accuracy": {
          "0.20": 0.23430391264786168,
          "0.50": 0.2156505914467698,
          "0.80": 0.21269335759781619
        },
        "coverage_intervals": {
          "0.20": [
            0.19581056466302366,
            0.2753755120619026
          ],
          "0.50": [
            0.19190541155070487,
            0.24101864483856297
          ],
          "0.80": [
            0.19362186788154898,
            0.23234624145785876
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.636
          ],
          [
            0.04,
            0.7157
          ],
          [
            0.06,
            0.7346
          ],
          [
            0.08,
            0.7441
          ],
          [
            0.1,
            0.768
          ],
          [
            0.12,
            0.7611
          ],
          [
            0.14,
            0.7628
          ],
          [
            0.16,
            0.7583
          ],
          [
            0.18,
            0.7633
          ],
          [
            0.2,
            0.7657
          ],
          [
            0.22,
            0.7725
          ],
          [
            0.24,
            0.7725
          ],
          [
            0.26,
            0.776
          ],
          [
            0.28,
            0.7767
          ],
          [
            0.3,
            0.778
          ],
          [
            0.32,
            0.7768
          ],
          [
            0.34,
            0.7832
          ],
          [
            0.36,
            0.7776
          ],
          [
            0.38,
            0.7797
          ],
          [
            0.4,
            0.7793
          ],
          [
            0.42,
            0.7736
          ],
          [
            0.44,
            0.7767
          ],
          [
            0.46,
            0.7814
          ],
          [
            0.48,
            0.7839
          ],
          [
            0.5,
            0.7843
          ],
          [
            0.52,
            0.7831
          ],
          [
            0.54,
            0.7826
          ],
          [
            0.56,
            0.7863
          ],
          [
            0.58,
            0.7859
          ],
          [
            0.6,
            0.7892
          ],
          [
            0.62,
            0.7894
          ],
          [
            0.64,
            0.7905
          ],
          [
            0.66,
            0.7891
          ],
          [
            0.68,
            0.7872
          ],
          [
            0.7,
            0.7888
          ],
          [
            0.72,
            0.7889
          ],
          [
            0.74,
            0.787
          ],
          [
            0.76,
            0.7866
          ],
          [
            0.78,
            0.7865
          ],
          [
            0.8,
            0.7873
          ],
          [
            0.82,
            0.7869
          ],
          [
            0.84,
            0.7899
          ],
          [
            0.86,
            0.7921
          ],
          [
            0.88,
            0.7915
          ],
          [
            0.9,
            0.7917
          ],
          [
            0.92,
            0.7953
          ],
          [
            0.94,
            0.7943
          ],
          [
            0.96,
            0.7962
          ],
          [
            0.98,
            0.7976
          ],
          [
            1.0,
            0.798
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 545,
            "mean_confidence": 0.27184134281556543,
            "accuracy": 0.1669724770642202
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 597,
            "mean_confidence": 0.34473503150133233,
            "accuracy": 0.20938023450586266
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 431,
            "mean_confidence": 0.4487578502387292,
            "accuracy": 0.20649651972157773
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 268,
            "mean_confidence": 0.5477809289942926,
            "accuracy": 0.19776119402985073
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 166,
            "mean_confidence": 0.6473905910928566,
            "accuracy": 0.23493975903614459
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 121,
            "mean_confidence": 0.7467575319542372,
            "accuracy": 0.2231404958677686
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 59,
            "mean_confidence": 0.8475825907820362,
            "accuracy": 0.2711864406779661
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 11,
            "mean_confidence": 0.9216462482105602,
            "accuracy": 0.36363636363636365
          }
        ],
        "median_right": 0.40689434111118317,
        "median_wrong": 0.3854098320007324,
        "sure": {
          "questions": 0,
          "correct": 0,
          "interval": null
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 2007,
          "correct": 397,
          "interval": [
            0.18096425565768304,
            0.21580573358463223
          ]
        },
        "ece_refit": 0.006728899635709991,
        "brier_refit": 0.1610456094262967,
        "brier_base": 0.16131412983563043,
        "refit_slopes": [
          0.19830420971737112,
          0.15421343582058536,
          0.2470240430466516,
          0.13312976863291556,
          0.2021099276105289
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        29,
        36,
        12,
        22,
        15,
        24,
        25,
        8,
        23,
        9,
        14,
        14,
        37,
        18,
        15,
        39,
        15,
        26,
        27,
        36
      ],
      "started": "2026-10-07T17:25:44Z",
      "updated": "2026-10-07T17:26:07Z",
      "format_of": null,
      "group": "H",
      "latency": {
        "source": "run",
        "runs": [
          135
        ],
        "concurrency": 1,
        "median_request_seconds": 0.008824052987620234,
        "p90_request_seconds": 0.010334555187728255
      },
      "single_question_latency": null
    },
    {
      "run": 51,
      "model": "laya",
      "name": "Laya",
      "variant": null,
      "reasoning": null,
      "provider": "laya",
      "note": "Encoder with a decision head",
      "parameters": 421293830,
      "active_parameters": null,
      "access": "Laya server, self-hosted",
      "questions_per_request": 1,
      "concurrency": 1,
      "correct": 418,
      "invalid": 0,
      "answered_questions": 2210,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 137
        },
        "grammar": {
          "questions": 1542,
          "correct": 281
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 413
        }
      },
      "tyt_mix": 0.20418460939230015,
      "questions_per_minute": 4612.493508939851,
      "request_seconds": 28.74800793605391,
      "median_request_seconds": null,
      "p90_request_seconds": null,
      "billing": "local",
      "api_equivalent_usd": null,
      "cost_basis": null,
      "requests": 2210,
      "hardware": "1 × RTX PRO 6000 Blackwell (96 GB)",
      "input_tokens": 891078,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 0,
      "confidence": {
        "source": "probability",
        "label": "Option score",
        "bins": [
          {
            "questions": 494,
            "correct": 97
          },
          {
            "questions": 1546,
            "correct": 304
          },
          {
            "questions": 151,
            "correct": 16
          },
          {
            "questions": 7,
            "correct": 1
          },
          {
            "questions": 0,
            "correct": 0
          }
        ],
        "auroc": 0.47937745282511696,
        "auroc_interval": [
          0.44869939002828857,
          0.5089547295923135
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.486681293335021,
            "auroc_interval": [
              0.43233602518276626,
              0.53786672223628
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.4802577178480616,
            "auroc_interval": [
              0.44412685508700317,
              0.515994501718213
            ]
          }
        },
        "aurc": 0.8303736198115464,
        "ece": 0.1321950409463146,
        "brier": 0.1859147789490446,
        "coverage_accuracy": {
          "0.20": 0.1439945404913558,
          "0.50": 0.18016378525932666,
          "0.80": 0.18767060964513194
        },
        "coverage_intervals": {
          "0.20": [
            0.11358473421172194,
            0.17946388005452066
          ],
          "0.50": [
            0.1582537517053206,
            0.20045558086560364
          ],
          "0.80": [
            0.17035095715587967,
            0.2053002729754322
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.8863
          ],
          [
            0.04,
            0.8749
          ],
          [
            0.06,
            0.8938
          ],
          [
            0.08,
            0.8919
          ],
          [
            0.1,
            0.8999
          ],
          [
            0.12,
            0.891
          ],
          [
            0.14,
            0.883
          ],
          [
            0.16,
            0.8739
          ],
          [
            0.18,
            0.8585
          ],
          [
            0.2,
            0.856
          ],
          [
            0.22,
            0.847
          ],
          [
            0.24,
            0.8408
          ],
          [
            0.26,
            0.846
          ],
          [
            0.28,
            0.8343
          ],
          [
            0.3,
            0.8317
          ],
          [
            0.32,
            0.8221
          ],
          [
            0.34,
            0.818
          ],
          [
            0.36,
            0.8193
          ],
          [
            0.38,
            0.8216
          ],
          [
            0.4,
            0.8169
          ],
          [
            0.42,
            0.8185
          ],
          [
            0.44,
            0.818
          ],
          [
            0.46,
            0.824
          ],
          [
            0.48,
            0.8199
          ],
          [
            0.5,
            0.8198
          ],
          [
            0.52,
            0.8192
          ],
          [
            0.54,
            0.8172
          ],
          [
            0.56,
            0.8148
          ],
          [
            0.58,
            0.8125
          ],
          [
            0.6,
            0.812
          ],
          [
            0.62,
            0.8107
          ],
          [
            0.64,
            0.8086
          ],
          [
            0.66,
            0.8077
          ],
          [
            0.68,
            0.8073
          ],
          [
            0.7,
            0.8056
          ],
          [
            0.72,
            0.8048
          ],
          [
            0.74,
            0.8069
          ],
          [
            0.76,
            0.8102
          ],
          [
            0.78,
            0.8124
          ],
          [
            0.8,
            0.8123
          ],
          [
            0.82,
            0.8125
          ],
          [
            0.84,
            0.8117
          ],
          [
            0.86,
            0.8103
          ],
          [
            0.88,
            0.809
          ],
          [
            0.9,
            0.8105
          ],
          [
            0.92,
            0.8096
          ],
          [
            0.94,
            0.8088
          ],
          [
            0.96,
            0.8071
          ],
          [
            0.98,
            0.8092
          ],
          [
            1.0,
            0.8098
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 1214,
            "mean_confidence": 0.2575158978583194,
            "accuracy": 0.19934102141680396
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 620,
            "mean_confidence": 0.3395666129032255,
            "accuracy": 0.20806451612903226
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 206,
            "mean_confidence": 0.4438194174757278,
            "accuracy": 0.14563106796116504
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 95,
            "mean_confidence": 0.5445484210526319,
            "accuracy": 0.09473684210526316
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 46,
            "mean_confidence": 0.6412499999999999,
            "accuracy": 0.13043478260869565
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 15,
            "mean_confidence": 0.7367533333333334,
            "accuracy": 0.13333333333333333
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 2,
            "mean_confidence": 0.8507,
            "accuracy": 0.0
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          }
        ],
        "median_right": 0.28569999999999995,
        "median_wrong": 0.2908,
        "sure": {
          "questions": 0,
          "correct": 0,
          "interval": null
        },
        "sure_before_audit": {
          "questions": 0,
          "correct": 0,
          "interval": null
        },
        "doubtful": {
          "questions": 2181,
          "correct": 416,
          "interval": [
            0.17479860176088247,
            0.20776533330882702
          ]
        },
        "ece_refit": 0.007788380033255677,
        "brier_refit": 0.15370760216253015,
        "brier_base": 0.15404741783233078,
        "refit_slopes": [
          -0.286005913811855,
          -0.2940459563028879,
          -0.3278617058571644,
          -0.31850586313318335,
          -0.3993246198158715
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": {
        "run": 41,
        "changed": {
          "questions": 372,
          "older_correct": 68,
          "newer_correct": 60,
          "flipped": 46,
          "changed_answer": 99,
          "newer_wrong": 312,
          "repeated_mistakes": 232,
          "p": 0.3019956131896606
        },
        "unchanged": {
          "questions": 1814,
          "older_correct": 356,
          "newer_correct": 356,
          "flipped": 0,
          "changed_answer": 0,
          "newer_wrong": 1458,
          "repeated_mistakes": 1458,
          "p": 1.0
        },
        "added": {
          "questions": 12,
          "newer_correct": 2
        }
      },
      "units": [
        33,
        35,
        8,
        18,
        28,
        15,
        18,
        10,
        15,
        10,
        13,
        10,
        33,
        12,
        23,
        47,
        17,
        22,
        17,
        34
      ],
      "started": "2026-10-04T08:46:03Z",
      "updated": "2026-10-04T08:46:34Z",
      "format_of": null,
      "group": "H",
      "latency": {
        "source": "repeats",
        "runs": [
          234,
          235,
          236,
          237
        ],
        "repeats": [
          2,
          3,
          4,
          5
        ],
        "questions": 2198,
        "concurrency": 1,
        "median_request_seconds": 0.011380975003703497,
        "p90_request_seconds": 0.012440272996900602
      },
      "single_question_latency": null
    },
    {
      "run": 46,
      "model": "laya-multilingual",
      "name": "Laya multilingual",
      "variant": null,
      "reasoning": null,
      "provider": "laya",
      "note": "Encoder with a decision head",
      "parameters": 321908998,
      "active_parameters": null,
      "access": "Laya server, self-hosted",
      "questions_per_request": 1,
      "concurrency": 1,
      "correct": 393,
      "invalid": 0,
      "answered_questions": 2210,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 112
        },
        "grammar": {
          "questions": 1542,
          "correct": 281
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 390
        }
      },
      "tyt_mix": 0.17274406061181233,
      "questions_per_minute": 5364.397759879482,
      "request_seconds": 24.71852497436339,
      "median_request_seconds": null,
      "p90_request_seconds": null,
      "billing": "local",
      "api_equivalent_usd": null,
      "cost_basis": null,
      "requests": 2210,
      "hardware": "1 × RTX PRO 6000 Blackwell (96 GB)",
      "input_tokens": 676255,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 0,
      "confidence": {
        "source": "probability",
        "label": "Option score",
        "bins": [
          {
            "questions": 101,
            "correct": 19
          },
          {
            "questions": 1369,
            "correct": 255
          },
          {
            "questions": 508,
            "correct": 88
          },
          {
            "questions": 179,
            "correct": 23
          },
          {
            "questions": 41,
            "correct": 8
          }
        ],
        "auroc": 0.45420129270544785,
        "auroc_interval": [
          0.4220193207821926,
          0.487179542561586
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.42513294380252103,
            "auroc_interval": [
              0.36807966231522676,
              0.4884639170353456
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.4681803686279601,
            "auroc_interval": [
              0.4293916408668731,
              0.5062273009123925
            ]
          }
        },
        "aurc": 0.8381078513067499,
        "ece": 0.2771874886260236,
        "brier": 0.26321356655595995,
        "coverage_accuracy": {
          "0.20": 0.15013648771610555,
          "0.50": 0.1610555050045496,
          "0.80": 0.16549135577797996
        },
        "coverage_intervals": {
          "0.20": [
            0.11812812358019081,
            0.18752844788347753
          ],
          "0.50": [
            0.14058956916099774,
            0.18248175182481752
          ],
          "0.80": [
            0.14829545454545454,
            0.18223234624145787
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.818
          ],
          [
            0.04,
            0.8066
          ],
          [
            0.06,
            0.8635
          ],
          [
            0.08,
            0.8578
          ],
          [
            0.1,
            0.859
          ],
          [
            0.12,
            0.8559
          ],
          [
            0.14,
            0.857
          ],
          [
            0.16,
            0.8465
          ],
          [
            0.18,
            0.8509
          ],
          [
            0.2,
            0.8499
          ],
          [
            0.22,
            0.8408
          ],
          [
            0.24,
            0.837
          ],
          [
            0.26,
            0.8425
          ],
          [
            0.28,
            0.8343
          ],
          [
            0.3,
            0.8362
          ],
          [
            0.32,
            0.8365
          ],
          [
            0.34,
            0.8394
          ],
          [
            0.36,
            0.8458
          ],
          [
            0.38,
            0.8432
          ],
          [
            0.4,
            0.8439
          ],
          [
            0.42,
            0.8408
          ],
          [
            0.44,
            0.8428
          ],
          [
            0.46,
            0.8397
          ],
          [
            0.48,
            0.837
          ],
          [
            0.5,
            0.8389
          ],
          [
            0.52,
            0.8381
          ],
          [
            0.54,
            0.838
          ],
          [
            0.56,
            0.8391
          ],
          [
            0.58,
            0.8397
          ],
          [
            0.6,
            0.8392
          ],
          [
            0.62,
            0.8378
          ],
          [
            0.64,
            0.8372
          ],
          [
            0.66,
            0.8371
          ],
          [
            0.68,
            0.8374
          ],
          [
            0.7,
            0.8356
          ],
          [
            0.72,
            0.8351
          ],
          [
            0.74,
            0.8358
          ],
          [
            0.76,
            0.836
          ],
          [
            0.78,
            0.8361
          ],
          [
            0.8,
            0.8345
          ],
          [
            0.82,
            0.835
          ],
          [
            0.84,
            0.8295
          ],
          [
            0.86,
            0.8281
          ],
          [
            0.88,
            0.8289
          ],
          [
            0.9,
            0.8281
          ],
          [
            0.92,
            0.8259
          ],
          [
            0.94,
            0.8238
          ],
          [
            0.96,
            0.8209
          ],
          [
            0.98,
            0.8203
          ],
          [
            1.0,
            0.8212
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 435,
            "mean_confidence": 0.2675068965517242,
            "accuracy": 0.22988505747126436
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 665,
            "mean_confidence": 0.345217894736842,
            "accuracy": 0.17443609022556392
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 370,
            "mean_confidence": 0.44455729729729754,
            "accuracy": 0.15675675675675677
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 262,
            "mean_confidence": 0.5472534351145036,
            "accuracy": 0.17557251908396945
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 172,
            "mean_confidence": 0.6442081395348832,
            "accuracy": 0.18604651162790697
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 149,
            "mean_confidence": 0.7479107382550335,
            "accuracy": 0.1342281879194631
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 104,
            "mean_confidence": 0.844316346153846,
            "accuracy": 0.125
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 41,
            "mean_confidence": 0.9367000000000003,
            "accuracy": 0.1951219512195122
          }
        ],
        "median_right": 0.3786,
        "median_wrong": 0.4041,
        "sure": {
          "questions": 1,
          "correct": 0,
          "interval": [
            0.0,
            0.7934567085261071
          ]
        },
        "sure_before_audit": {
          "questions": 1,
          "correct": 0,
          "interval": [
            0.0,
            0.7934567085261071
          ]
        },
        "doubtful": {
          "questions": 1904,
          "correct": 352,
          "interval": [
            0.1680774503954,
            0.2029395145350219
          ]
        },
        "ece_refit": 0.008489409472552098,
        "brier_refit": 0.14694875600714,
        "brier_base": 0.14715891419993068,
        "refit_slopes": [
          -0.12867335350759437,
          -0.11589743345880729,
          -0.121477813830177,
          -0.20338891518607716,
          -0.2023120970545653
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": {
        "run": 42,
        "changed": {
          "questions": 372,
          "older_correct": 71,
          "newer_correct": 69,
          "flipped": 40,
          "changed_answer": 91,
          "newer_wrong": 303,
          "repeated_mistakes": 231,
          "p": 0.8746293123804207
        },
        "unchanged": {
          "questions": 1814,
          "older_correct": 320,
          "newer_correct": 320,
          "flipped": 0,
          "changed_answer": 0,
          "newer_wrong": 1494,
          "repeated_mistakes": 1494,
          "p": 1.0
        },
        "added": {
          "questions": 12,
          "newer_correct": 4
        }
      },
      "units": [
        25,
        19,
        11,
        21,
        20,
        16,
        26,
        8,
        14,
        12,
        16,
        11,
        24,
        11,
        17,
        42,
        13,
        29,
        20,
        38
      ],
      "started": "2026-10-04T08:31:15Z",
      "updated": "2026-10-04T08:31:44Z",
      "format_of": null,
      "group": "H",
      "latency": {
        "source": "repeats",
        "runs": [
          238,
          239,
          240,
          241
        ],
        "repeats": [
          2,
          3,
          4,
          5
        ],
        "questions": 2198,
        "concurrency": 1,
        "median_request_seconds": 0.009499495485215448,
        "p90_request_seconds": 0.010441777898813598
      },
      "single_question_latency": null
    },
    {
      "run": 122,
      "model": "gliner2.5-multi-v1",
      "name": "GLiNER2.5 Multi (base)",
      "variant": null,
      "reasoning": null,
      "provider": "gliner",
      "note": "Multilingual extraction and classification model",
      "parameters": 287355159,
      "active_parameters": null,
      "access": "GLiNER2 server, self-hosted",
      "questions_per_request": 1,
      "concurrency": 1,
      "correct": 384,
      "invalid": 0,
      "answered_questions": 2198,
      "groups": {
        "reading": {
          "questions": 656,
          "correct": 113
        },
        "grammar": {
          "questions": 1542,
          "correct": 271
        },
        "without_suspect": {
          "questions": 2186,
          "correct": 382
        }
      },
      "tyt_mix": 0.17286679280946507,
      "questions_per_minute": 1922.5854452978283,
      "request_seconds": 68.59513075090945,
      "median_request_seconds": 0.03178766701603308,
      "p90_request_seconds": 0.03576318579725921,
      "billing": "local",
      "api_equivalent_usd": null,
      "cost_basis": null,
      "requests": 2198,
      "hardware": "1 × RTX PRO 6000 Blackwell (96 GB); FP16",
      "input_tokens": 431150,
      "cached_input_tokens": 0,
      "cache_write_input_tokens": 0,
      "cache_write_1h_input_tokens": 0,
      "output_tokens": 0,
      "confidence": {
        "source": "probability",
        "label": "Option score",
        "bins": [
          {
            "questions": 5,
            "correct": 1
          },
          {
            "questions": 968,
            "correct": 192
          },
          {
            "questions": 1056,
            "correct": 167
          },
          {
            "questions": 164,
            "correct": 23
          },
          {
            "questions": 5,
            "correct": 1
          }
        ],
        "auroc": 0.4506471655641308,
        "auroc_interval": [
          0.4206265425239493,
          0.48061901803791646
        ],
        "parts": {
          "meaning": {
            "questions": 656,
            "auroc": 0.4186508906598869,
            "auroc_interval": [
              0.3584979118000991,
              0.48041863039431093
            ]
          },
          "form": {
            "questions": 1542,
            "auroc": 0.4638849614302595,
            "auroc_interval": [
              0.42443125710057983,
              0.4995170347094935
            ]
          }
        },
        "aurc": 0.8429137711686909,
        "ece": 0.35619655110491094,
        "brier": 0.29778652443440456,
        "coverage_accuracy": {
          "0.20": 0.15468607825295722,
          "0.50": 0.15013648771610555,
          "0.80": 0.16492265696087352
        },
        "coverage_intervals": {
          "0.20": [
            0.12050932241928149,
            0.18425841674249316
          ],
          "0.50": [
            0.13,
            0.17114246700045516
          ],
          "0.80": [
            0.14722601182355616,
            0.18278837420526792
          ]
        },
        "risk_coverage": [
          [
            0.02,
            0.7498
          ],
          [
            0.04,
            0.7962
          ],
          [
            0.06,
            0.8341
          ],
          [
            0.08,
            0.8587
          ],
          [
            0.1,
            0.8553
          ],
          [
            0.12,
            0.8568
          ],
          [
            0.14,
            0.844
          ],
          [
            0.16,
            0.8465
          ],
          [
            0.18,
            0.8518
          ],
          [
            0.2,
            0.8453
          ],
          [
            0.22,
            0.8532
          ],
          [
            0.24,
            0.8597
          ],
          [
            0.26,
            0.8618
          ],
          [
            0.28,
            0.8644
          ],
          [
            0.3,
            0.865
          ],
          [
            0.32,
            0.8678
          ],
          [
            0.34,
            0.8662
          ],
          [
            0.36,
            0.8622
          ],
          [
            0.38,
            0.8563
          ],
          [
            0.4,
            0.8533
          ],
          [
            0.42,
            0.8559
          ],
          [
            0.44,
            0.8552
          ],
          [
            0.46,
            0.8457
          ],
          [
            0.48,
            0.8493
          ],
          [
            0.5,
            0.8499
          ],
          [
            0.52,
            0.8486
          ],
          [
            0.54,
            0.8492
          ],
          [
            0.56,
            0.844
          ],
          [
            0.58,
            0.8447
          ],
          [
            0.6,
            0.8408
          ],
          [
            0.62,
            0.8378
          ],
          [
            0.64,
            0.8386
          ],
          [
            0.66,
            0.8366
          ],
          [
            0.68,
            0.8381
          ],
          [
            0.7,
            0.8382
          ],
          [
            0.72,
            0.8351
          ],
          [
            0.74,
            0.8334
          ],
          [
            0.76,
            0.833
          ],
          [
            0.78,
            0.8349
          ],
          [
            0.8,
            0.8351
          ],
          [
            0.82,
            0.8341
          ],
          [
            0.84,
            0.8332
          ],
          [
            0.86,
            0.8318
          ],
          [
            0.88,
            0.8303
          ],
          [
            0.9,
            0.8271
          ],
          [
            0.92,
            0.8259
          ],
          [
            0.94,
            0.8262
          ],
          [
            0.96,
            0.8256
          ],
          [
            0.98,
            0.8264
          ],
          [
            1.0,
            0.8253
          ]
        ],
        "reliability": [
          {
            "low": 0.0,
            "high": 0.1,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.1,
            "high": 0.2,
            "questions": 0,
            "mean_confidence": null,
            "accuracy": null
          },
          {
            "low": 0.2,
            "high": 0.3,
            "questions": 67,
            "mean_confidence": 0.27965851506190514,
            "accuracy": 0.22388059701492538
          },
          {
            "low": 0.3,
            "high": 0.4,
            "questions": 403,
            "mean_confidence": 0.35655440578685504,
            "accuracy": 0.2109181141439206
          },
          {
            "low": 0.4,
            "high": 0.5,
            "questions": 503,
            "mean_confidence": 0.44966872460088486,
            "accuracy": 0.1848906560636183
          },
          {
            "low": 0.5,
            "high": 0.6,
            "questions": 514,
            "mean_confidence": 0.5504470308699033,
            "accuracy": 0.188715953307393
          },
          {
            "low": 0.6,
            "high": 0.7,
            "questions": 404,
            "mean_confidence": 0.6435851771937738,
            "accuracy": 0.11386138613861387
          },
          {
            "low": 0.7,
            "high": 0.8,
            "questions": 229,
            "mean_confidence": 0.7414461704841347,
            "accuracy": 0.14847161572052403
          },
          {
            "low": 0.8,
            "high": 0.9,
            "questions": 73,
            "mean_confidence": 0.8345194034380455,
            "accuracy": 0.1780821917808219
          },
          {
            "low": 0.9,
            "high": 1.0,
            "questions": 5,
            "mean_confidence": 0.9317659735679626,
            "accuracy": 0.2
          }
        ],
        "median_right": 0.49681758880615234,
        "median_wrong": 0.5310882925987244,
        "sure": {
          "questions": 0,
          "correct": 0,
          "interval": null
        },
        "sure_before_audit": null,
        "doubtful": {
          "questions": 1891,
          "correct": 336,
          "interval": [
            0.16111355800596627,
            0.1955608996373534
          ]
        },
        "ece_refit": 0.0045904645430750474,
        "brier_refit": 0.14392675395611587,
        "brier_base": 0.144311224283753,
        "refit_slopes": [
          -0.16855925803026378,
          -0.26678650876377413,
          -0.2616460293652979,
          -0.26835123538262884,
          -0.27508109713783746
        ],
        "refit_questions": 2198,
        "refit_converged": true
      },
      "previous": null,
      "units": [
        24,
        20,
        12,
        21,
        22,
        14,
        23,
        9,
        16,
        11,
        17,
        9,
        27,
        15,
        19,
        32,
        13,
        18,
        17,
        45
      ],
      "started": "2026-10-05T19:34:39Z",
      "updated": "2026-10-05T19:35:51Z",
      "format_of": null,
      "group": "H",
      "latency": {
        "source": "run",
        "runs": [
          122
        ],
        "concurrency": 1,
        "median_request_seconds": 0.03178766701603308,
        "p90_request_seconds": 0.03576318579725921
      },
      "single_question_latency": null
    }
  ],
  "group_pairs": 2701,
  "paired": [
    {
      "first": 140,
      "second": 277,
      "questions": 2198,
      "first_only": 17,
      "second_only": 14,
      "difference": 0.001364877161055505,
      "low": -0.0035996803540367765,
      "high": 0.006329434676147787,
      "p": 0.720100131817162,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 277,
      "second": 276,
      "questions": 2198,
      "first_only": 9,
      "second_only": 5,
      "difference": 0.0018198362147406734,
      "low": -0.0015158060296484976,
      "high": 0.005155478459129844,
      "p": 0.4239501953125,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 276,
      "second": 48,
      "questions": 2198,
      "first_only": 14,
      "second_only": 10,
      "difference": 0.0018198362147406734,
      "low": -0.0025480179749911794,
      "high": 0.006187690404472526,
      "p": 0.5412561893463135,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 48,
      "second": 124,
      "questions": 2198,
      "first_only": 28,
      "second_only": 8,
      "difference": 0.009099181073703366,
      "low": 0.0037624029350301572,
      "high": 0.014435959212376575,
      "p": 0.0011932429624721408,
      "p_holm": 0.07398106367327273,
      "p_holm_all": 0.3245620857924223
    },
    {
      "first": 124,
      "second": 557,
      "questions": 2198,
      "first_only": 46,
      "second_only": 39,
      "difference": 0.0031847133757961785,
      "low": -0.005035458292328023,
      "high": 0.011404885043920381,
      "p": 0.5154257318839004,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 557,
      "second": 310,
      "questions": 2198,
      "first_only": 45,
      "second_only": 40,
      "difference": 0.0022747952684258415,
      "low": -0.005945904498870782,
      "high": 0.010495495035722465,
      "p": 0.6646454904132143,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 310,
      "second": 278,
      "questions": 2198,
      "first_only": 50,
      "second_only": 25,
      "difference": 0.011373976342129208,
      "low": 0.0036661100122780893,
      "high": 0.019081842671980327,
      "p": 0.0052282430040142824,
      "p_holm": 0.3189228232448712,
      "p_holm_all": 1.0
    },
    {
      "first": 278,
      "second": 275,
      "questions": 2198,
      "first_only": 63,
      "second_only": 47,
      "difference": 0.007279344858962694,
      "low": -0.0020681381745884806,
      "high": 0.01662682789251387,
      "p": 0.15233780606025277,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 275,
      "second": 123,
      "questions": 2198,
      "first_only": 94,
      "second_only": 72,
      "difference": 0.010009099181073703,
      "low": -0.0014722834006096579,
      "high": 0.021490481762757065,
      "p": 0.10283252418776918,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 123,
      "second": 311,
      "questions": 2198,
      "first_only": 110,
      "second_only": 98,
      "difference": 0.00545950864422202,
      "low": -0.00739903089853552,
      "high": 0.01831804818697956,
      "p": 0.4457122209928542,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 311,
      "second": 121,
      "questions": 2198,
      "first_only": 91,
      "second_only": 76,
      "difference": 0.006824385805277525,
      "low": -0.004695640573687478,
      "high": 0.01834441218424253,
      "p": 0.27861206921355974,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 121,
      "second": 274,
      "questions": 2198,
      "first_only": 118,
      "second_only": 92,
      "difference": 0.011828935395814377,
      "low": -0.0010838453767028338,
      "high": 0.02474171616833159,
      "p": 0.08425386654908265,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 274,
      "second": 47,
      "questions": 2198,
      "first_only": 127,
      "second_only": 118,
      "difference": 0.004094631483166515,
      "low": -0.00986194052966092,
      "high": 0.01805120349599395,
      "p": 0.6093708144344228,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 47,
      "second": 55,
      "questions": 2198,
      "first_only": 216,
      "second_only": 97,
      "difference": 0.054140127388535034,
      "low": 0.03852720387825319,
      "high": 0.06975305089881688,
      "p": 1.4657942286341076e-11,
      "p_holm": 1.0407139023302164e-09,
      "p_holm_all": 7.871315007765157e-09
    },
    {
      "first": 55,
      "second": 320,
      "questions": 2198,
      "first_only": 233,
      "second_only": 183,
      "difference": 0.022747952684258416,
      "low": 0.0045852477782764335,
      "high": 0.0409106575902404,
      "p": 0.016184431866081966,
      "p_holm": 0.954881480098836,
      "p_holm_all": 1.0
    },
    {
      "first": 320,
      "second": 160,
      "questions": 2198,
      "first_only": 253,
      "second_only": 166,
      "difference": 0.03958143767060965,
      "low": 0.02140355153547491,
      "high": 0.05775932380574439,
      "p": 2.4921970529184058e-05,
      "p_holm": 0.001669772025455332,
      "p_holm_all": 0.008448548009393396
    },
    {
      "first": 160,
      "second": 199,
      "questions": 2198,
      "first_only": 261,
      "second_only": 223,
      "difference": 0.017288444040036398,
      "low": -0.0023160716296129376,
      "high": 0.03689295970968573,
      "p": 0.09250135329497819,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 199,
      "second": 314,
      "questions": 2198,
      "first_only": 298,
      "second_only": 242,
      "difference": 0.025477707006369428,
      "low": 0.004783405507661015,
      "high": 0.04617200850507784,
      "p": 0.017859908740936698,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 314,
      "second": 318,
      "questions": 2198,
      "first_only": 275,
      "second_only": 262,
      "difference": 0.005914467697907188,
      "low": -0.014748106808133416,
      "high": 0.02657704220394779,
      "p": 0.6046123371741172,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 318,
      "second": 119,
      "questions": 2198,
      "first_only": 285,
      "second_only": 267,
      "difference": 0.00818926296633303,
      "low": -0.012758609786674636,
      "high": 0.029137135719340696,
      "p": 0.4693630998766396,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 119,
      "second": 315,
      "questions": 2198,
      "first_only": 246,
      "second_only": 237,
      "difference": 0.004094631483166515,
      "low": -0.015502178438504523,
      "high": 0.023691441404837553,
      "p": 0.7158894222848251,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 119,
      "second": 129,
      "questions": 2198,
      "first_only": 355,
      "second_only": 240,
      "difference": 0.05232029117379436,
      "low": 0.030679166621566563,
      "high": 0.07396141572602216,
      "p": 2.7673699726562264e-06,
      "p_holm": 0.0001881811581406234,
      "p_holm_all": 0.0010211595199101475
    },
    {
      "first": 315,
      "second": 248,
      "questions": 2198,
      "first_only": 309,
      "second_only": 216,
      "difference": 0.042311191992720654,
      "low": 0.02195603955184863,
      "high": 0.06266634443359267,
      "p": 5.698612452286787e-05,
      "p_holm": 0.0037610842185092793,
      "p_holm_all": 0.01840651822088632
    },
    {
      "first": 248,
      "second": 133,
      "questions": 2198,
      "first_only": 297,
      "second_only": 293,
      "difference": 0.0018198362147406734,
      "low": -0.019839827519914373,
      "high": 0.023479499949395723,
      "p": 0.9017184395071192,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 133,
      "second": 129,
      "questions": 2198,
      "first_only": 322,
      "second_only": 313,
      "difference": 0.004094631483166515,
      "low": -0.018375346329844618,
      "high": 0.026564609296177648,
      "p": 0.750915254482066,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 143,
      "second": 133,
      "questions": 2198,
      "first_only": 181,
      "second_only": 284,
      "difference": -0.04686078252957234,
      "low": -0.06598964347625874,
      "high": -0.027731921582885947,
      "p": 2.0485146663561087e-06,
      "p_holm": 0.0001413475119785715,
      "p_holm_all": 0.0007640959705508285
    },
    {
      "first": 129,
      "second": 259,
      "questions": 2198,
      "first_only": 218,
      "second_only": 216,
      "difference": 0.0009099181073703367,
      "low": -0.01766694314733146,
      "high": 0.019486779362072135,
      "p": 0.9617223711296439,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 129,
      "second": 44,
      "questions": 2198,
      "first_only": 305,
      "second_only": 140,
      "difference": 0.07506824385805277,
      "low": 0.056521036069118194,
      "high": 0.09361545164698735,
      "p": 3.678091628881056e-15,
      "p_holm": 2.685006889083171e-13,
      "p_holm_all": 2.2583482601329687e-12
    },
    {
      "first": 259,
      "second": 269,
      "questions": 2198,
      "first_only": 115,
      "second_only": 115,
      "difference": 0.0,
      "low": -0.01352359951805372,
      "high": 0.01352359951805372,
      "p": 1.0,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 269,
      "second": 322,
      "questions": 2198,
      "first_only": 135,
      "second_only": 134,
      "difference": 0.00045495905368516835,
      "low": -0.014170319822780468,
      "high": 0.015080237930150805,
      "p": 1.0,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 322,
      "second": 326,
      "questions": 2198,
      "first_only": 127,
      "second_only": 119,
      "difference": 0.003639672429481347,
      "low": -0.010345577532321246,
      "high": 0.01762492239128394,
      "p": 0.6554659776114778,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 326,
      "second": 374,
      "questions": 2198,
      "first_only": 211,
      "second_only": 205,
      "difference": 0.00272975432211101,
      "low": -0.015457473164688826,
      "high": 0.020916981808910846,
      "p": 0.8063798988033636,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 374,
      "second": 217,
      "questions": 2198,
      "first_only": 205,
      "second_only": 204,
      "difference": 0.00045495905368516835,
      "low": -0.017578946701232553,
      "high": 0.01848886480860289,
      "p": 1.0,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 217,
      "second": 145,
      "questions": 2198,
      "first_only": 124,
      "second_only": 120,
      "difference": 0.0018198362147406734,
      "low": -0.012109063709778005,
      "high": 0.015748736139259353,
      "p": 0.8477486582929077,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 145,
      "second": 120,
      "questions": 2198,
      "first_only": 279,
      "second_only": 258,
      "difference": 0.009554140127388535,
      "low": -0.011106053108983297,
      "high": 0.030214333363760365,
      "p": 0.38812142056379956,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 145,
      "second": 44,
      "questions": 2198,
      "first_only": 271,
      "second_only": 128,
      "difference": 0.06505914467697907,
      "low": 0.047455943209440694,
      "high": 0.08266234614451745,
      "p": 6.670084191087266e-13,
      "p_holm": 4.802460617582831e-11,
      "p_holm_all": 3.7819377363464795e-10
    },
    {
      "first": 120,
      "second": 177,
      "questions": 2198,
      "first_only": 293,
      "second_only": 278,
      "difference": 0.006824385805277525,
      "low": -0.014481888054698789,
      "high": 0.028130659665253837,
      "p": 0.5579928586439417,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 177,
      "second": 346,
      "questions": 2198,
      "first_only": 181,
      "second_only": 151,
      "difference": 0.01364877161055505,
      "low": -0.0025891128617931813,
      "high": 0.02988665608290328,
      "p": 0.11134099001135897,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 346,
      "second": 143,
      "questions": 2198,
      "first_only": 318,
      "second_only": 312,
      "difference": 0.00272975432211101,
      "low": -0.019651942707314244,
      "high": 0.025111451351536262,
      "p": 0.842122409786274,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 143,
      "second": 254,
      "questions": 2198,
      "first_only": 324,
      "second_only": 278,
      "difference": 0.020928116469517744,
      "low": -0.0009333402847602826,
      "high": 0.04278957322379577,
      "p": 0.06655549148604978,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 254,
      "second": 54,
      "questions": 2198,
      "first_only": 312,
      "second_only": 309,
      "difference": 0.001364877161055505,
      "low": -0.020856591122852354,
      "high": 0.023586345444963366,
      "p": 0.9360411996561069,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 54,
      "second": 150,
      "questions": 2198,
      "first_only": 325,
      "second_only": 305,
      "difference": 0.009099181073703366,
      "low": -0.013279574000910723,
      "high": 0.031477936148317454,
      "p": 0.44908872355964985,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 150,
      "second": 44,
      "questions": 2198,
      "first_only": 288,
      "second_only": 286,
      "difference": 0.0009099181073703367,
      "low": -0.020454134613012643,
      "high": 0.022273970827753318,
      "p": 0.9667114435315829,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 44,
      "second": 264,
      "questions": 2198,
      "first_only": 145,
      "second_only": 128,
      "difference": 0.0077343039126478615,
      "low": -0.006995775846243002,
      "high": 0.022464383671538725,
      "p": 0.33287249537889074,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 44,
      "second": 194,
      "questions": 2198,
      "first_only": 268,
      "second_only": 242,
      "difference": 0.011828935395814377,
      "low": -0.0083028588234107,
      "high": 0.03196072961503946,
      "p": 0.2682684298922979,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 44,
      "second": 49,
      "questions": 2198,
      "first_only": 300,
      "second_only": 237,
      "difference": 0.028662420382165606,
      "low": 0.008033138518606254,
      "high": 0.04929170224572496,
      "p": 0.007404857447081556,
      "p_holm": 0.44429144682489335,
      "p_holm_all": 1.0
    },
    {
      "first": 264,
      "second": 194,
      "questions": 2198,
      "first_only": 269,
      "second_only": 260,
      "difference": 0.004094631483166515,
      "low": -0.016414208266830196,
      "high": 0.024603471233163227,
      "p": 0.7280070834123119,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 194,
      "second": 204,
      "questions": 2198,
      "first_only": 278,
      "second_only": 258,
      "difference": 0.009099181073703366,
      "low": -0.011542118616762924,
      "high": 0.029740480764169658,
      "p": 0.41185534013203284,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 204,
      "second": 159,
      "questions": 2198,
      "first_only": 322,
      "second_only": 311,
      "difference": 0.005004549590536852,
      "low": -0.01742968995585384,
      "high": 0.027438789136927544,
      "p": 0.6910576680898435,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 159,
      "second": 49,
      "questions": 2198,
      "first_only": 304,
      "second_only": 298,
      "difference": 0.00272975432211101,
      "low": -0.019148905760362295,
      "high": 0.024608414404584313,
      "p": 0.8385428643521317,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 49,
      "second": 125,
      "questions": 2198,
      "first_only": 268,
      "second_only": 263,
      "difference": 0.0022747952684258415,
      "low": -0.0182732726470048,
      "high": 0.02282286318385648,
      "p": 0.8622122760478431,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 125,
      "second": 52,
      "questions": 2198,
      "first_only": 272,
      "second_only": 266,
      "difference": 0.00272975432211101,
      "low": -0.0179532160725664,
      "high": 0.023412724716788416,
      "p": 0.829351490507984,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 52,
      "second": 334,
      "questions": 2198,
      "first_only": 216,
      "second_only": 188,
      "difference": 0.012738853503184714,
      "low": -0.0051765776278643386,
      "high": 0.030654284634233767,
      "p": 0.17910280027021638,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 334,
      "second": 45,
      "questions": 2198,
      "first_only": 268,
      "second_only": 267,
      "difference": 0.00045495905368516835,
      "low": -0.020170569639752747,
      "high": 0.021080487747123084,
      "p": 1.0,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 45,
      "second": 127,
      "questions": 2198,
      "first_only": 286,
      "second_only": 286,
      "difference": 0.0,
      "low": -0.02132683444629549,
      "high": 0.02132683444629549,
      "p": 1.0,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 45,
      "second": 50,
      "questions": 2198,
      "first_only": 290,
      "second_only": 207,
      "difference": 0.03776160145586897,
      "low": 0.01794483214073702,
      "high": 0.05757837077100093,
      "p": 0.00022780042456489667,
      "p_holm": 0.014807027596718285,
      "p_holm_all": 0.0681123269449041
    },
    {
      "first": 127,
      "second": 128,
      "questions": 2198,
      "first_only": 174,
      "second_only": 169,
      "difference": 0.0022747952684258415,
      "low": -0.014239811705441445,
      "high": 0.018789402242293127,
      "p": 0.8290427375754543,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 128,
      "second": 83,
      "questions": 2198,
      "first_only": 309,
      "second_only": 284,
      "difference": 0.011373976342129208,
      "low": -0.010335611614380794,
      "high": 0.033083564298639206,
      "p": 0.32435025440149456,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 83,
      "second": 53,
      "questions": 2198,
      "first_only": 100,
      "second_only": 97,
      "difference": 0.001364877161055505,
      "low": -0.011150875816214918,
      "high": 0.013880630138325927,
      "p": 0.8867367256095349,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 53,
      "second": 373,
      "questions": 2198,
      "first_only": 319,
      "second_only": 279,
      "difference": 0.018198362147406732,
      "low": -0.0035945106974240755,
      "high": 0.03999123499223754,
      "p": 0.1106738364414517,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 373,
      "second": 50,
      "questions": 2198,
      "first_only": 311,
      "second_only": 301,
      "difference": 0.004549590536851683,
      "low": -0.01750951767154343,
      "high": 0.0266086987452468,
      "p": 0.7160369812792178,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 50,
      "second": 223,
      "questions": 2198,
      "first_only": 408,
      "second_only": 317,
      "difference": 0.041401273885350316,
      "low": 0.017453451813373117,
      "high": 0.06534909595732752,
      "p": 0.0008185461107379673,
      "p_holm": 0.05156840497649194,
      "p_holm_all": 0.22523572165497108
    },
    {
      "first": 223,
      "second": 205,
      "questions": 2198,
      "first_only": 329,
      "second_only": 300,
      "difference": 0.013193812556869881,
      "low": -0.00916360176793051,
      "high": 0.03555122688167027,
      "p": 0.26422130944979083,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 205,
      "second": 131,
      "questions": 2198,
      "first_only": 334,
      "second_only": 318,
      "difference": 0.007279344858962694,
      "low": -0.015488052520659101,
      "high": 0.030046742238584487,
      "p": 0.5569377816972405,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 131,
      "second": 139,
      "questions": 2198,
      "first_only": 317,
      "second_only": 317,
      "difference": 0.0,
      "low": -0.02245292947365585,
      "high": 0.02245292947365585,
      "p": 1.0,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 139,
      "second": 132,
      "questions": 2198,
      "first_only": 346,
      "second_only": 330,
      "difference": 0.007279344858962694,
      "low": -0.015903371158322684,
      "high": 0.03046206087624807,
      "p": 0.5640235791919807,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 132,
      "second": 126,
      "questions": 2198,
      "first_only": 421,
      "second_only": 328,
      "difference": 0.042311191992720654,
      "low": 0.017970916928709595,
      "high": 0.06665146705673171,
      "p": 0.0007641053000487032,
      "p_holm": 0.048902739203117,
      "p_holm_all": 0.21242127341353947
    },
    {
      "first": 126,
      "second": 340,
      "questions": 2198,
      "first_only": 536,
      "second_only": 286,
      "difference": 0.11373976342129208,
      "low": 0.08861976244126205,
      "high": 0.1388597644013221,
      "p": 2.0835598433676e-18,
      "p_holm": 1.541834284092024e-16,
      "p_holm_all": 1.4376562919236439e-15
    },
    {
      "first": 340,
      "second": 141,
      "questions": 2198,
      "first_only": 388,
      "second_only": 346,
      "difference": 0.01910828025477707,
      "low": -0.005037367286177086,
      "high": 0.04325392779573123,
      "p": 0.130137658399862,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 141,
      "second": 243,
      "questions": 2198,
      "first_only": 355,
      "second_only": 347,
      "difference": 0.003639672429481347,
      "low": -0.019986204229966092,
      "high": 0.027265549088928785,
      "p": 0.7916498999128606,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 243,
      "second": 135,
      "questions": 2198,
      "first_only": 447,
      "second_only": 295,
      "difference": 0.06915377616014559,
      "low": 0.045036282552532676,
      "high": 0.09327126976775851,
      "p": 2.6643193119385933e-08,
      "p_holm": 1.8650235183570154e-06,
      "p_holm_all": 1.172300497252981e-05
    },
    {
      "first": 135,
      "second": 51,
      "questions": 2198,
      "first_only": 332,
      "second_only": 306,
      "difference": 0.011828935395814377,
      "low": -0.010689282646469553,
      "high": 0.034347153438098306,
      "p": 0.32229311322473936,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 51,
      "second": 46,
      "questions": 2198,
      "first_only": 295,
      "second_only": 270,
      "difference": 0.011373976342129208,
      "low": -0.009816625686337836,
      "high": 0.03256457837059625,
      "p": 0.31264376433157115,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    },
    {
      "first": 46,
      "second": 122,
      "questions": 2198,
      "first_only": 217,
      "second_only": 208,
      "difference": 0.004094631483166515,
      "low": -0.014287844984190067,
      "high": 0.022477107950523097,
      "p": 0.6980218732848629,
      "p_holm": 1.0,
      "p_holm_all": 1.0
    }
  ],
  "cascades": [
    {
      "decision": 199,
      "frontier": 140,
      "frontier_accuracy": 0.9808917197452229,
      "in_sample": {
        "threshold": 0.920415759950263,
        "answered": 0.3252957233848954,
        "accuracy": 0.9808917197452229
      },
      "curve": [
        [
          0.0,
          0.9809
        ],
        [
          0.01,
          0.9809
        ],
        [
          0.02,
          0.9809
        ],
        [
          0.03,
          0.9809
        ],
        [
          0.04,
          0.9809
        ],
        [
          0.05,
          0.9809
        ],
        [
          0.0601,
          0.9809
        ],
        [
          0.0705,
          0.9809
        ],
        [
          0.0805,
          0.9809
        ],
        [
          0.0905,
          0.9809
        ],
        [
          0.1005,
          0.9809
        ],
        [
          0.1106,
          0.9809
        ],
        [
          0.121,
          0.9809
        ],
        [
          0.131,
          0.9809
        ],
        [
          0.141,
          0.9809
        ],
        [
          0.151,
          0.9809
        ],
        [
          0.1611,
          0.9809
        ],
        [
          0.1715,
          0.9809
        ],
        [
          0.1815,
          0.9809
        ],
        [
          0.1915,
          0.9809
        ],
        [
          0.2015,
          0.9809
        ],
        [
          0.2116,
          0.9809
        ],
        [
          0.222,
          0.9809
        ],
        [
          0.232,
          0.9809
        ],
        [
          0.242,
          0.9809
        ],
        [
          0.252,
          0.9809
        ],
        [
          0.2621,
          0.9813
        ],
        [
          0.2725,
          0.9813
        ],
        [
          0.2825,
          0.9813
        ],
        [
          0.2925,
          0.9813
        ],
        [
          0.3025,
          0.9813
        ],
        [
          0.3126,
          0.9813
        ],
        [
          0.323,
          0.9809
        ],
        [
          0.333,
          0.9804
        ],
        [
          0.343,
          0.98
        ],
        [
          0.353,
          0.9795
        ],
        [
          0.3631,
          0.9795
        ],
        [
          0.3735,
          0.9786
        ],
        [
          0.3835,
          0.9773
        ],
        [
          0.3935,
          0.9768
        ],
        [
          0.4035,
          0.9759
        ],
        [
          0.4136,
          0.9754
        ],
        [
          0.424,
          0.9741
        ],
        [
          0.434,
          0.9722
        ],
        [
          0.444,
          0.9713
        ],
        [
          0.454,
          0.9704
        ],
        [
          0.4641,
          0.97
        ],
        [
          0.4745,
          0.9682
        ],
        [
          0.4845,
          0.965
        ],
        [
          0.4945,
          0.9636
        ],
        [
          0.5045,
          0.9631
        ],
        [
          0.5146,
          0.9618
        ],
        [
          0.525,
          0.9604
        ],
        [
          0.535,
          0.96
        ],
        [
          0.545,
          0.9577
        ],
        [
          0.5551,
          0.955
        ],
        [
          0.5655,
          0.954
        ],
        [
          0.5755,
          0.9522
        ],
        [
          0.5855,
          0.949
        ],
        [
          0.5955,
          0.9454
        ],
        [
          0.6056,
          0.9431
        ],
        [
          0.616,
          0.9418
        ],
        [
          0.626,
          0.9404
        ],
        [
          0.636,
          0.9381
        ],
        [
          0.646,
          0.9354
        ],
        [
          0.6561,
          0.934
        ],
        [
          0.6665,
          0.9295
        ],
        [
          0.6765,
          0.9277
        ],
        [
          0.6865,
          0.924
        ],
        [
          0.6965,
          0.9208
        ],
        [
          0.7066,
          0.9181
        ],
        [
          0.717,
          0.9158
        ],
        [
          0.727,
          0.9126
        ],
        [
          0.737,
          0.9099
        ],
        [
          0.747,
          0.9045
        ],
        [
          0.7571,
          0.8995
        ],
        [
          0.7675,
          0.8958
        ],
        [
          0.7775,
          0.8917
        ],
        [
          0.7875,
          0.8854
        ],
        [
          0.7975,
          0.8822
        ],
        [
          0.8076,
          0.8772
        ],
        [
          0.818,
          0.8735
        ],
        [
          0.828,
          0.8681
        ],
        [
          0.838,
          0.8644
        ],
        [
          0.848,
          0.8585
        ],
        [
          0.8581,
          0.854
        ],
        [
          0.8685,
          0.8471
        ],
        [
          0.8785,
          0.8412
        ],
        [
          0.8885,
          0.8353
        ],
        [
          0.8985,
          0.8303
        ],
        [
          0.9086,
          0.8258
        ],
        [
          0.919,
          0.8203
        ],
        [
          0.929,
          0.8153
        ],
        [
          0.939,
          0.8089
        ],
        [
          0.949,
          0.8025
        ],
        [
          0.9591,
          0.7966
        ],
        [
          0.9695,
          0.7884
        ],
        [
          0.9795,
          0.7834
        ],
        [
          0.9895,
          0.7775
        ],
        [
          0.9995,
          0.7712
        ],
        [
          1.0,
          0.7707
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.33966260789581193,
        "difference": -0.0009489105106740179,
        "worse": 0.55,
        "kept_right": 0.33796193307108247,
        "kept_wrong": 0.0017006748247294751,
        "handed_off": 0.6603373921041883
      },
      "usd_per_1000": 1.6117627063242614
    },
    {
      "decision": 199,
      "frontier": 277,
      "frontier_accuracy": 0.9795268425841674,
      "in_sample": {
        "threshold": 0.9132683223310315,
        "answered": 0.3375796178343949,
        "accuracy": 0.9795268425841674
      },
      "curve": [
        [
          0.0,
          0.9795
        ],
        [
          0.01,
          0.9795
        ],
        [
          0.02,
          0.9795
        ],
        [
          0.03,
          0.9795
        ],
        [
          0.04,
          0.9795
        ],
        [
          0.05,
          0.9795
        ],
        [
          0.0601,
          0.9795
        ],
        [
          0.0705,
          0.9795
        ],
        [
          0.0805,
          0.9795
        ],
        [
          0.0905,
          0.9795
        ],
        [
          0.1005,
          0.9795
        ],
        [
          0.1106,
          0.9795
        ],
        [
          0.121,
          0.9795
        ],
        [
          0.131,
          0.9795
        ],
        [
          0.141,
          0.9795
        ],
        [
          0.151,
          0.9795
        ],
        [
          0.1611,
          0.9795
        ],
        [
          0.1715,
          0.9795
        ],
        [
          0.1815,
          0.9795
        ],
        [
          0.1915,
          0.9795
        ],
        [
          0.2015,
          0.9795
        ],
        [
          0.2116,
          0.9795
        ],
        [
          0.222,
          0.9795
        ],
        [
          0.232,
          0.9795
        ],
        [
          0.242,
          0.9795
        ],
        [
          0.252,
          0.9795
        ],
        [
          0.2621,
          0.98
        ],
        [
          0.2725,
          0.98
        ],
        [
          0.2825,
          0.98
        ],
        [
          0.2925,
          0.9804
        ],
        [
          0.3025,
          0.9804
        ],
        [
          0.3126,
          0.9804
        ],
        [
          0.323,
          0.98
        ],
        [
          0.333,
          0.9795
        ],
        [
          0.343,
          0.9791
        ],
        [
          0.353,
          0.9782
        ],
        [
          0.3631,
          0.9782
        ],
        [
          0.3735,
          0.9773
        ],
        [
          0.3835,
          0.9759
        ],
        [
          0.3935,
          0.9754
        ],
        [
          0.4035,
          0.9745
        ],
        [
          0.4136,
          0.9741
        ],
        [
          0.424,
          0.9727
        ],
        [
          0.434,
          0.9713
        ],
        [
          0.444,
          0.9704
        ],
        [
          0.454,
          0.9695
        ],
        [
          0.4641,
          0.9686
        ],
        [
          0.4745,
          0.9668
        ],
        [
          0.4845,
          0.9636
        ],
        [
          0.4945,
          0.9622
        ],
        [
          0.5045,
          0.9613
        ],
        [
          0.5146,
          0.9604
        ],
        [
          0.525,
          0.9595
        ],
        [
          0.535,
          0.9581
        ],
        [
          0.545,
          0.9559
        ],
        [
          0.5551,
          0.9531
        ],
        [
          0.5655,
          0.9522
        ],
        [
          0.5755,
          0.9504
        ],
        [
          0.5855,
          0.9477
        ],
        [
          0.5955,
          0.9445
        ],
        [
          0.6056,
          0.9418
        ],
        [
          0.616,
          0.9404
        ],
        [
          0.626,
          0.9386
        ],
        [
          0.636,
          0.9363
        ],
        [
          0.646,
          0.934
        ],
        [
          0.6561,
          0.9327
        ],
        [
          0.6665,
          0.9281
        ],
        [
          0.6765,
          0.9263
        ],
        [
          0.6865,
          0.9227
        ],
        [
          0.6965,
          0.9195
        ],
        [
          0.7066,
          0.9167
        ],
        [
          0.717,
          0.914
        ],
        [
          0.727,
          0.9113
        ],
        [
          0.737,
          0.9086
        ],
        [
          0.747,
          0.9031
        ],
        [
          0.7571,
          0.8985
        ],
        [
          0.7675,
          0.8949
        ],
        [
          0.7775,
          0.8908
        ],
        [
          0.7875,
          0.8849
        ],
        [
          0.7975,
          0.8817
        ],
        [
          0.8076,
          0.8767
        ],
        [
          0.818,
          0.8726
        ],
        [
          0.828,
          0.8667
        ],
        [
          0.838,
          0.8626
        ],
        [
          0.848,
          0.8567
        ],
        [
          0.8581,
          0.8521
        ],
        [
          0.8685,
          0.8453
        ],
        [
          0.8785,
          0.8394
        ],
        [
          0.8885,
          0.8339
        ],
        [
          0.8985,
          0.8289
        ],
        [
          0.9086,
          0.8239
        ],
        [
          0.919,
          0.8189
        ],
        [
          0.929,
          0.8139
        ],
        [
          0.939,
          0.8071
        ],
        [
          0.949,
          0.8012
        ],
        [
          0.9591,
          0.7953
        ],
        [
          0.9695,
          0.7875
        ],
        [
          0.9795,
          0.783
        ],
        [
          0.9895,
          0.7775
        ],
        [
          0.9995,
          0.7712
        ],
        [
          1.0,
          0.7707
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.3415731367879182,
        "difference": -0.000639739287961652,
        "worse": 0.46,
        "kept_right": 0.339845109157734,
        "kept_wrong": 0.0017280276301840254,
        "handed_off": 0.6584268632120823
      },
      "usd_per_1000": 4.008062818362677
    },
    {
      "decision": 199,
      "frontier": 276,
      "frontier_accuracy": 0.9777070063694268,
      "in_sample": {
        "threshold": 0.9132683223310315,
        "answered": 0.3375796178343949,
        "accuracy": 0.9777070063694268
      },
      "curve": [
        [
          0.0,
          0.9777
        ],
        [
          0.01,
          0.9777
        ],
        [
          0.02,
          0.9777
        ],
        [
          0.03,
          0.9777
        ],
        [
          0.04,
          0.9777
        ],
        [
          0.05,
          0.9777
        ],
        [
          0.0601,
          0.9777
        ],
        [
          0.0705,
          0.9777
        ],
        [
          0.0805,
          0.9777
        ],
        [
          0.0905,
          0.9777
        ],
        [
          0.1005,
          0.9777
        ],
        [
          0.1106,
          0.9777
        ],
        [
          0.121,
          0.9777
        ],
        [
          0.131,
          0.9777
        ],
        [
          0.141,
          0.9777
        ],
        [
          0.151,
          0.9777
        ],
        [
          0.1611,
          0.9777
        ],
        [
          0.1715,
          0.9777
        ],
        [
          0.1815,
          0.9777
        ],
        [
          0.1915,
          0.9777
        ],
        [
          0.2015,
          0.9777
        ],
        [
          0.2116,
          0.9777
        ],
        [
          0.222,
          0.9777
        ],
        [
          0.232,
          0.9777
        ],
        [
          0.242,
          0.9777
        ],
        [
          0.252,
          0.9777
        ],
        [
          0.2621,
          0.9782
        ],
        [
          0.2725,
          0.9782
        ],
        [
          0.2825,
          0.9782
        ],
        [
          0.2925,
          0.9786
        ],
        [
          0.3025,
          0.9786
        ],
        [
          0.3126,
          0.9786
        ],
        [
          0.323,
          0.9782
        ],
        [
          0.333,
          0.9777
        ],
        [
          0.343,
          0.9773
        ],
        [
          0.353,
          0.9763
        ],
        [
          0.3631,
          0.9763
        ],
        [
          0.3735,
          0.9754
        ],
        [
          0.3835,
          0.9745
        ],
        [
          0.3935,
          0.9741
        ],
        [
          0.4035,
          0.9736
        ],
        [
          0.4136,
          0.9727
        ],
        [
          0.424,
          0.9713
        ],
        [
          0.434,
          0.97
        ],
        [
          0.444,
          0.9691
        ],
        [
          0.454,
          0.9682
        ],
        [
          0.4641,
          0.9672
        ],
        [
          0.4745,
          0.9654
        ],
        [
          0.4845,
          0.9622
        ],
        [
          0.4945,
          0.9609
        ],
        [
          0.5045,
          0.96
        ],
        [
          0.5146,
          0.9591
        ],
        [
          0.525,
          0.9581
        ],
        [
          0.535,
          0.9568
        ],
        [
          0.545,
          0.9545
        ],
        [
          0.5551,
          0.9522
        ],
        [
          0.5655,
          0.9518
        ],
        [
          0.5755,
          0.95
        ],
        [
          0.5855,
          0.9472
        ],
        [
          0.5955,
          0.9445
        ],
        [
          0.6056,
          0.9418
        ],
        [
          0.616,
          0.9404
        ],
        [
          0.626,
          0.9386
        ],
        [
          0.636,
          0.9363
        ],
        [
          0.646,
          0.934
        ],
        [
          0.6561,
          0.9327
        ],
        [
          0.6665,
          0.9281
        ],
        [
          0.6765,
          0.9263
        ],
        [
          0.6865,
          0.9227
        ],
        [
          0.6965,
          0.9195
        ],
        [
          0.7066,
          0.9167
        ],
        [
          0.717,
          0.9145
        ],
        [
          0.727,
          0.9113
        ],
        [
          0.737,
          0.9086
        ],
        [
          0.747,
          0.9035
        ],
        [
          0.7571,
          0.899
        ],
        [
          0.7675,
          0.8954
        ],
        [
          0.7775,
          0.8913
        ],
        [
          0.7875,
          0.8854
        ],
        [
          0.7975,
          0.8826
        ],
        [
          0.8076,
          0.8781
        ],
        [
          0.818,
          0.874
        ],
        [
          0.828,
          0.8681
        ],
        [
          0.838,
          0.864
        ],
        [
          0.848,
          0.8581
        ],
        [
          0.8581,
          0.8535
        ],
        [
          0.8685,
          0.8467
        ],
        [
          0.8785,
          0.8408
        ],
        [
          0.8885,
          0.8348
        ],
        [
          0.8985,
          0.8298
        ],
        [
          0.9086,
          0.8248
        ],
        [
          0.919,
          0.8194
        ],
        [
          0.929,
          0.8144
        ],
        [
          0.939,
          0.8076
        ],
        [
          0.949,
          0.8016
        ],
        [
          0.9591,
          0.7957
        ],
        [
          0.9695,
          0.788
        ],
        [
          0.9795,
          0.7834
        ],
        [
          0.9895,
          0.7775
        ],
        [
          0.9995,
          0.7712
        ],
        [
          1.0,
          0.7707
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.342152153891208,
        "difference": -0.0006548935554456311,
        "worse": 0.46,
        "kept_right": 0.34037562484347467,
        "kept_wrong": 0.0017765290477330998,
        "handed_off": 0.6578478461087925
      },
      "usd_per_1000": 1.081492744331897
    },
    {
      "decision": 199,
      "frontier": 48,
      "frontier_accuracy": 0.9758871701546861,
      "in_sample": {
        "threshold": 0.9132683223310315,
        "answered": 0.3375796178343949,
        "accuracy": 0.9758871701546861
      },
      "curve": [
        [
          0.0,
          0.9759
        ],
        [
          0.01,
          0.9759
        ],
        [
          0.02,
          0.9759
        ],
        [
          0.03,
          0.9759
        ],
        [
          0.04,
          0.9759
        ],
        [
          0.05,
          0.9759
        ],
        [
          0.0601,
          0.9759
        ],
        [
          0.0705,
          0.9759
        ],
        [
          0.0805,
          0.9759
        ],
        [
          0.0905,
          0.9759
        ],
        [
          0.1005,
          0.9759
        ],
        [
          0.1106,
          0.9759
        ],
        [
          0.121,
          0.9759
        ],
        [
          0.131,
          0.9759
        ],
        [
          0.141,
          0.9759
        ],
        [
          0.151,
          0.9759
        ],
        [
          0.1611,
          0.9759
        ],
        [
          0.1715,
          0.9759
        ],
        [
          0.1815,
          0.9759
        ],
        [
          0.1915,
          0.9759
        ],
        [
          0.2015,
          0.9759
        ],
        [
          0.2116,
          0.9759
        ],
        [
          0.222,
          0.9759
        ],
        [
          0.232,
          0.9759
        ],
        [
          0.242,
          0.9759
        ],
        [
          0.252,
          0.9759
        ],
        [
          0.2621,
          0.9763
        ],
        [
          0.2725,
          0.9763
        ],
        [
          0.2825,
          0.9763
        ],
        [
          0.2925,
          0.9768
        ],
        [
          0.3025,
          0.9768
        ],
        [
          0.3126,
          0.9768
        ],
        [
          0.323,
          0.9763
        ],
        [
          0.333,
          0.9759
        ],
        [
          0.343,
          0.9754
        ],
        [
          0.353,
          0.9745
        ],
        [
          0.3631,
          0.9745
        ],
        [
          0.3735,
          0.9736
        ],
        [
          0.3835,
          0.9722
        ],
        [
          0.3935,
          0.9718
        ],
        [
          0.4035,
          0.9713
        ],
        [
          0.4136,
          0.9704
        ],
        [
          0.424,
          0.9695
        ],
        [
          0.434,
          0.9682
        ],
        [
          0.444,
          0.9672
        ],
        [
          0.454,
          0.9663
        ],
        [
          0.4641,
          0.9654
        ],
        [
          0.4745,
          0.9636
        ],
        [
          0.4845,
          0.9604
        ],
        [
          0.4945,
          0.9591
        ],
        [
          0.5045,
          0.9581
        ],
        [
          0.5146,
          0.9572
        ],
        [
          0.525,
          0.9563
        ],
        [
          0.535,
          0.955
        ],
        [
          0.545,
          0.9527
        ],
        [
          0.5551,
          0.95
        ],
        [
          0.5655,
          0.949
        ],
        [
          0.5755,
          0.9472
        ],
        [
          0.5855,
          0.9445
        ],
        [
          0.5955,
          0.9413
        ],
        [
          0.6056,
          0.9386
        ],
        [
          0.616,
          0.9372
        ],
        [
          0.626,
          0.9354
        ],
        [
          0.636,
          0.9336
        ],
        [
          0.646,
          0.9308
        ],
        [
          0.6561,
          0.9295
        ],
        [
          0.6665,
          0.9249
        ],
        [
          0.6765,
          0.9231
        ],
        [
          0.6865,
          0.9195
        ],
        [
          0.6965,
          0.9163
        ],
        [
          0.7066,
          0.914
        ],
        [
          0.717,
          0.9117
        ],
        [
          0.727,
          0.9086
        ],
        [
          0.737,
          0.9058
        ],
        [
          0.747,
          0.9008
        ],
        [
          0.7571,
          0.8963
        ],
        [
          0.7675,
          0.8926
        ],
        [
          0.7775,
          0.8885
        ],
        [
          0.7875,
          0.8822
        ],
        [
          0.7975,
          0.8799
        ],
        [
          0.8076,
          0.8749
        ],
        [
          0.818,
          0.8712
        ],
        [
          0.828,
          0.8658
        ],
        [
          0.838,
          0.8617
        ],
        [
          0.848,
          0.8553
        ],
        [
          0.8581,
          0.8508
        ],
        [
          0.8685,
          0.8439
        ],
        [
          0.8785,
          0.838
        ],
        [
          0.8885,
          0.8326
        ],
        [
          0.8985,
          0.828
        ],
        [
          0.9086,
          0.8239
        ],
        [
          0.919,
          0.8185
        ],
        [
          0.929,
          0.8135
        ],
        [
          0.939,
          0.8066
        ],
        [
          0.949,
          0.8007
        ],
        [
          0.9591,
          0.7948
        ],
        [
          0.9695,
          0.7871
        ],
        [
          0.9795,
          0.783
        ],
        [
          0.9895,
          0.7775
        ],
        [
          0.9995,
          0.7712
        ],
        [
          1.0,
          0.7707
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.34162469881400254,
        "difference": -0.0006427723483195531,
        "worse": 0.46,
        "kept_right": 0.3398936381234604,
        "kept_wrong": 0.0017310606905419264,
        "handed_off": 0.658375301185998
      },
      "usd_per_1000": 3.555229756587364
    },
    {
      "decision": 199,
      "frontier": 191,
      "frontier_accuracy": 0.9740673339399454,
      "in_sample": {
        "threshold": 0.920415759950263,
        "answered": 0.3252957233848954,
        "accuracy": 0.9740673339399454
      },
      "curve": [
        [
          0.0,
          0.9741
        ],
        [
          0.01,
          0.9741
        ],
        [
          0.02,
          0.9741
        ],
        [
          0.03,
          0.9741
        ],
        [
          0.04,
          0.9741
        ],
        [
          0.05,
          0.9741
        ],
        [
          0.0601,
          0.9741
        ],
        [
          0.0705,
          0.9741
        ],
        [
          0.0805,
          0.9741
        ],
        [
          0.0905,
          0.9741
        ],
        [
          0.1005,
          0.9741
        ],
        [
          0.1106,
          0.9741
        ],
        [
          0.121,
          0.9741
        ],
        [
          0.131,
          0.9741
        ],
        [
          0.141,
          0.9741
        ],
        [
          0.151,
          0.9741
        ],
        [
          0.1611,
          0.9741
        ],
        [
          0.1715,
          0.9741
        ],
        [
          0.1815,
          0.9741
        ],
        [
          0.1915,
          0.9741
        ],
        [
          0.2015,
          0.9741
        ],
        [
          0.2116,
          0.9741
        ],
        [
          0.222,
          0.9741
        ],
        [
          0.232,
          0.9741
        ],
        [
          0.242,
          0.9741
        ],
        [
          0.252,
          0.9741
        ],
        [
          0.2621,
          0.9745
        ],
        [
          0.2725,
          0.9745
        ],
        [
          0.2825,
          0.9745
        ],
        [
          0.2925,
          0.9745
        ],
        [
          0.3025,
          0.9745
        ],
        [
          0.3126,
          0.9745
        ],
        [
          0.323,
          0.9741
        ],
        [
          0.333,
          0.9736
        ],
        [
          0.343,
          0.9732
        ],
        [
          0.353,
          0.9722
        ],
        [
          0.3631,
          0.9722
        ],
        [
          0.3735,
          0.9713
        ],
        [
          0.3835,
          0.97
        ],
        [
          0.3935,
          0.9695
        ],
        [
          0.4035,
          0.9691
        ],
        [
          0.4136,
          0.9682
        ],
        [
          0.424,
          0.9668
        ],
        [
          0.434,
          0.9654
        ],
        [
          0.444,
          0.9645
        ],
        [
          0.454,
          0.9641
        ],
        [
          0.4641,
          0.9631
        ],
        [
          0.4745,
          0.9618
        ],
        [
          0.4845,
          0.9591
        ],
        [
          0.4945,
          0.9577
        ],
        [
          0.5045,
          0.9568
        ],
        [
          0.5146,
          0.9559
        ],
        [
          0.525,
          0.9554
        ],
        [
          0.535,
          0.954
        ],
        [
          0.545,
          0.9518
        ],
        [
          0.5551,
          0.9495
        ],
        [
          0.5655,
          0.9495
        ],
        [
          0.5755,
          0.9477
        ],
        [
          0.5855,
          0.9449
        ],
        [
          0.5955,
          0.9413
        ],
        [
          0.6056,
          0.939
        ],
        [
          0.616,
          0.9372
        ],
        [
          0.626,
          0.9354
        ],
        [
          0.636,
          0.9336
        ],
        [
          0.646,
          0.9308
        ],
        [
          0.6561,
          0.9295
        ],
        [
          0.6665,
          0.9249
        ],
        [
          0.6765,
          0.9231
        ],
        [
          0.6865,
          0.9195
        ],
        [
          0.6965,
          0.9163
        ],
        [
          0.7066,
          0.914
        ],
        [
          0.717,
          0.9117
        ],
        [
          0.727,
          0.9086
        ],
        [
          0.737,
          0.9058
        ],
        [
          0.747,
          0.9008
        ],
        [
          0.7571,
          0.8967
        ],
        [
          0.7675,
          0.8931
        ],
        [
          0.7775,
          0.889
        ],
        [
          0.7875,
          0.8831
        ],
        [
          0.7975,
          0.8803
        ],
        [
          0.8076,
          0.8758
        ],
        [
          0.818,
          0.8722
        ],
        [
          0.828,
          0.8662
        ],
        [
          0.838,
          0.8621
        ],
        [
          0.848,
          0.8558
        ],
        [
          0.8581,
          0.8512
        ],
        [
          0.8685,
          0.8444
        ],
        [
          0.8785,
          0.8385
        ],
        [
          0.8885,
          0.833
        ],
        [
          0.8985,
          0.828
        ],
        [
          0.9086,
          0.823
        ],
        [
          0.919,
          0.8176
        ],
        [
          0.929,
          0.8126
        ],
        [
          0.939,
          0.8057
        ],
        [
          0.949,
          0.7998
        ],
        [
          0.9591,
          0.7939
        ],
        [
          0.9695,
          0.7862
        ],
        [
          0.9795,
          0.7821
        ],
        [
          0.9895,
          0.7771
        ],
        [
          0.9995,
          0.7712
        ],
        [
          1.0,
          0.7707
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.33372839169554563,
        "difference": -0.0007396537954783796,
        "worse": 0.4666666666666667,
        "kept_right": 0.3324521833109617,
        "kept_wrong": 0.001276208384583893,
        "handed_off": 0.6662716083044546
      },
      "usd_per_1000": 17.828906207596845
    },
    {
      "decision": 199,
      "frontier": 188,
      "frontier_accuracy": 0.9713375796178344,
      "in_sample": {
        "threshold": 0.920415759950263,
        "answered": 0.3252957233848954,
        "accuracy": 0.9713375796178344
      },
      "curve": [
        [
          0.0,
          0.9713
        ],
        [
          0.01,
          0.9713
        ],
        [
          0.02,
          0.9713
        ],
        [
          0.03,
          0.9713
        ],
        [
          0.04,
          0.9713
        ],
        [
          0.05,
          0.9713
        ],
        [
          0.0601,
          0.9713
        ],
        [
          0.0705,
          0.9713
        ],
        [
          0.0805,
          0.9713
        ],
        [
          0.0905,
          0.9713
        ],
        [
          0.1005,
          0.9713
        ],
        [
          0.1106,
          0.9713
        ],
        [
          0.121,
          0.9713
        ],
        [
          0.131,
          0.9713
        ],
        [
          0.141,
          0.9713
        ],
        [
          0.151,
          0.9713
        ],
        [
          0.1611,
          0.9713
        ],
        [
          0.1715,
          0.9713
        ],
        [
          0.1815,
          0.9713
        ],
        [
          0.1915,
          0.9713
        ],
        [
          0.2015,
          0.9713
        ],
        [
          0.2116,
          0.9713
        ],
        [
          0.222,
          0.9713
        ],
        [
          0.232,
          0.9713
        ],
        [
          0.242,
          0.9713
        ],
        [
          0.252,
          0.9713
        ],
        [
          0.2621,
          0.9718
        ],
        [
          0.2725,
          0.9718
        ],
        [
          0.2825,
          0.9718
        ],
        [
          0.2925,
          0.9718
        ],
        [
          0.3025,
          0.9718
        ],
        [
          0.3126,
          0.9718
        ],
        [
          0.323,
          0.9713
        ],
        [
          0.333,
          0.9709
        ],
        [
          0.343,
          0.9704
        ],
        [
          0.353,
          0.97
        ],
        [
          0.3631,
          0.97
        ],
        [
          0.3735,
          0.9695
        ],
        [
          0.3835,
          0.9682
        ],
        [
          0.3935,
          0.9682
        ],
        [
          0.4035,
          0.9677
        ],
        [
          0.4136,
          0.9672
        ],
        [
          0.424,
          0.9659
        ],
        [
          0.434,
          0.9645
        ],
        [
          0.444,
          0.9636
        ],
        [
          0.454,
          0.9627
        ],
        [
          0.4641,
          0.9622
        ],
        [
          0.4745,
          0.9604
        ],
        [
          0.4845,
          0.9572
        ],
        [
          0.4945,
          0.9559
        ],
        [
          0.5045,
          0.955
        ],
        [
          0.5146,
          0.954
        ],
        [
          0.525,
          0.9531
        ],
        [
          0.535,
          0.9522
        ],
        [
          0.545,
          0.95
        ],
        [
          0.5551,
          0.9472
        ],
        [
          0.5655,
          0.9468
        ],
        [
          0.5755,
          0.9449
        ],
        [
          0.5855,
          0.9418
        ],
        [
          0.5955,
          0.9381
        ],
        [
          0.6056,
          0.9359
        ],
        [
          0.616,
          0.934
        ],
        [
          0.626,
          0.9327
        ],
        [
          0.636,
          0.9304
        ],
        [
          0.646,
          0.9281
        ],
        [
          0.6561,
          0.9272
        ],
        [
          0.6665,
          0.9227
        ],
        [
          0.6765,
          0.9217
        ],
        [
          0.6865,
          0.9181
        ],
        [
          0.6965,
          0.9154
        ],
        [
          0.7066,
          0.9126
        ],
        [
          0.717,
          0.9104
        ],
        [
          0.727,
          0.9072
        ],
        [
          0.737,
          0.9045
        ],
        [
          0.747,
          0.899
        ],
        [
          0.7571,
          0.8944
        ],
        [
          0.7675,
          0.8908
        ],
        [
          0.7775,
          0.8867
        ],
        [
          0.7875,
          0.8808
        ],
        [
          0.7975,
          0.8776
        ],
        [
          0.8076,
          0.8731
        ],
        [
          0.818,
          0.8699
        ],
        [
          0.828,
          0.8644
        ],
        [
          0.838,
          0.8608
        ],
        [
          0.848,
          0.8549
        ],
        [
          0.8581,
          0.8517
        ],
        [
          0.8685,
          0.8449
        ],
        [
          0.8785,
          0.8389
        ],
        [
          0.8885,
          0.8326
        ],
        [
          0.8985,
          0.8276
        ],
        [
          0.9086,
          0.823
        ],
        [
          0.919,
          0.8171
        ],
        [
          0.929,
          0.8126
        ],
        [
          0.939,
          0.8062
        ],
        [
          0.949,
          0.7998
        ],
        [
          0.9591,
          0.7939
        ],
        [
          0.9695,
          0.7862
        ],
        [
          0.9795,
          0.7821
        ],
        [
          0.9895,
          0.7771
        ],
        [
          0.9995,
          0.7712
        ],
        [
          1.0,
          0.7707
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.3429098174371895,
        "difference": -0.001173296626920138,
        "worse": 0.57,
        "kept_right": 0.3408453348231537,
        "kept_wrong": 0.002064482614035918,
        "handed_off": 0.6570901825628104
      },
      "usd_per_1000": 0.647405287998156
    },
    {
      "decision": 199,
      "frontier": 190,
      "frontier_accuracy": 0.9672429481346679,
      "in_sample": {
        "threshold": 0.9071930569698246,
        "answered": 0.34986351228389445,
        "accuracy": 0.9672429481346679
      },
      "curve": [
        [
          0.0,
          0.9672
        ],
        [
          0.01,
          0.9672
        ],
        [
          0.02,
          0.9672
        ],
        [
          0.03,
          0.9672
        ],
        [
          0.04,
          0.9672
        ],
        [
          0.05,
          0.9672
        ],
        [
          0.0601,
          0.9672
        ],
        [
          0.0705,
          0.9672
        ],
        [
          0.0805,
          0.9672
        ],
        [
          0.0905,
          0.9672
        ],
        [
          0.1005,
          0.9672
        ],
        [
          0.1106,
          0.9672
        ],
        [
          0.121,
          0.9672
        ],
        [
          0.131,
          0.9672
        ],
        [
          0.141,
          0.9672
        ],
        [
          0.151,
          0.9672
        ],
        [
          0.1611,
          0.9672
        ],
        [
          0.1715,
          0.9672
        ],
        [
          0.1815,
          0.9672
        ],
        [
          0.1915,
          0.9672
        ],
        [
          0.2015,
          0.9672
        ],
        [
          0.2116,
          0.9672
        ],
        [
          0.222,
          0.9672
        ],
        [
          0.232,
          0.9672
        ],
        [
          0.242,
          0.9672
        ],
        [
          0.252,
          0.9672
        ],
        [
          0.2621,
          0.9677
        ],
        [
          0.2725,
          0.9677
        ],
        [
          0.2825,
          0.9677
        ],
        [
          0.2925,
          0.9682
        ],
        [
          0.3025,
          0.9682
        ],
        [
          0.3126,
          0.9682
        ],
        [
          0.323,
          0.9682
        ],
        [
          0.333,
          0.9677
        ],
        [
          0.343,
          0.9672
        ],
        [
          0.353,
          0.9663
        ],
        [
          0.3631,
          0.9663
        ],
        [
          0.3735,
          0.9654
        ],
        [
          0.3835,
          0.965
        ],
        [
          0.3935,
          0.9645
        ],
        [
          0.4035,
          0.9636
        ],
        [
          0.4136,
          0.9627
        ],
        [
          0.424,
          0.9613
        ],
        [
          0.434,
          0.96
        ],
        [
          0.444,
          0.9591
        ],
        [
          0.454,
          0.9586
        ],
        [
          0.4641,
          0.9577
        ],
        [
          0.4745,
          0.9563
        ],
        [
          0.4845,
          0.9531
        ],
        [
          0.4945,
          0.9518
        ],
        [
          0.5045,
          0.9509
        ],
        [
          0.5146,
          0.95
        ],
        [
          0.525,
          0.95
        ],
        [
          0.535,
          0.9486
        ],
        [
          0.545,
          0.9463
        ],
        [
          0.5551,
          0.9436
        ],
        [
          0.5655,
          0.9431
        ],
        [
          0.5755,
          0.9413
        ],
        [
          0.5855,
          0.9386
        ],
        [
          0.5955,
          0.9349
        ],
        [
          0.6056,
          0.9322
        ],
        [
          0.616,
          0.9308
        ],
        [
          0.626,
          0.929
        ],
        [
          0.636,
          0.9281
        ],
        [
          0.646,
          0.9263
        ],
        [
          0.6561,
          0.9249
        ],
        [
          0.6665,
          0.9204
        ],
        [
          0.6765,
          0.9186
        ],
        [
          0.6865,
          0.9149
        ],
        [
          0.6965,
          0.9117
        ],
        [
          0.7066,
          0.9095
        ],
        [
          0.717,
          0.9067
        ],
        [
          0.727,
          0.904
        ],
        [
          0.737,
          0.9013
        ],
        [
          0.747,
          0.8963
        ],
        [
          0.7571,
          0.8917
        ],
        [
          0.7675,
          0.8881
        ],
        [
          0.7775,
          0.884
        ],
        [
          0.7875,
          0.8781
        ],
        [
          0.7975,
          0.8758
        ],
        [
          0.8076,
          0.8712
        ],
        [
          0.818,
          0.8681
        ],
        [
          0.828,
          0.8621
        ],
        [
          0.838,
          0.859
        ],
        [
          0.848,
          0.853
        ],
        [
          0.8581,
          0.849
        ],
        [
          0.8685,
          0.8421
        ],
        [
          0.8785,
          0.8362
        ],
        [
          0.8885,
          0.8308
        ],
        [
          0.8985,
          0.8262
        ],
        [
          0.9086,
          0.8212
        ],
        [
          0.919,
          0.8162
        ],
        [
          0.929,
          0.8107
        ],
        [
          0.939,
          0.8044
        ],
        [
          0.949,
          0.7985
        ],
        [
          0.9591,
          0.793
        ],
        [
          0.9695,
          0.7848
        ],
        [
          0.9795,
          0.7821
        ],
        [
          0.9895,
          0.7771
        ],
        [
          0.9995,
          0.7712
        ],
        [
          1.0,
          0.7707
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.3530133436548086,
        "difference": -0.0007700232223720493,
        "worse": 0.45,
        "kept_right": 0.3503698324865172,
        "kept_wrong": 0.002643511168291161,
        "handed_off": 0.6469866563451918
      },
      "usd_per_1000": 3.617095932911081
    },
    {
      "decision": 199,
      "frontier": 124,
      "frontier_accuracy": 0.9667879890809827,
      "in_sample": {
        "threshold": 0.9132683223310315,
        "answered": 0.3375796178343949,
        "accuracy": 0.9667879890809827
      },
      "curve": [
        [
          0.0,
          0.9668
        ],
        [
          0.01,
          0.9668
        ],
        [
          0.02,
          0.9668
        ],
        [
          0.03,
          0.9668
        ],
        [
          0.04,
          0.9668
        ],
        [
          0.05,
          0.9668
        ],
        [
          0.0601,
          0.9668
        ],
        [
          0.0705,
          0.9668
        ],
        [
          0.0805,
          0.9668
        ],
        [
          0.0905,
          0.9668
        ],
        [
          0.1005,
          0.9668
        ],
        [
          0.1106,
          0.9668
        ],
        [
          0.121,
          0.9668
        ],
        [
          0.131,
          0.9668
        ],
        [
          0.141,
          0.9668
        ],
        [
          0.151,
          0.9668
        ],
        [
          0.1611,
          0.9668
        ],
        [
          0.1715,
          0.9668
        ],
        [
          0.1815,
          0.9668
        ],
        [
          0.1915,
          0.9668
        ],
        [
          0.2015,
          0.9668
        ],
        [
          0.2116,
          0.9668
        ],
        [
          0.222,
          0.9668
        ],
        [
          0.232,
          0.9668
        ],
        [
          0.242,
          0.9668
        ],
        [
          0.252,
          0.9668
        ],
        [
          0.2621,
          0.9672
        ],
        [
          0.2725,
          0.9672
        ],
        [
          0.2825,
          0.9672
        ],
        [
          0.2925,
          0.9677
        ],
        [
          0.3025,
          0.9677
        ],
        [
          0.3126,
          0.9677
        ],
        [
          0.323,
          0.9672
        ],
        [
          0.333,
          0.9668
        ],
        [
          0.343,
          0.9663
        ],
        [
          0.353,
          0.9654
        ],
        [
          0.3631,
          0.9654
        ],
        [
          0.3735,
          0.9645
        ],
        [
          0.3835,
          0.9636
        ],
        [
          0.3935,
          0.9631
        ],
        [
          0.4035,
          0.9627
        ],
        [
          0.4136,
          0.9622
        ],
        [
          0.424,
          0.9613
        ],
        [
          0.434,
          0.96
        ],
        [
          0.444,
          0.9591
        ],
        [
          0.454,
          0.9586
        ],
        [
          0.4641,
          0.9577
        ],
        [
          0.4745,
          0.9559
        ],
        [
          0.4845,
          0.9527
        ],
        [
          0.4945,
          0.9518
        ],
        [
          0.5045,
          0.9509
        ],
        [
          0.5146,
          0.95
        ],
        [
          0.525,
          0.949
        ],
        [
          0.535,
          0.9477
        ],
        [
          0.545,
          0.9454
        ],
        [
          0.5551,
          0.9431
        ],
        [
          0.5655,
          0.9422
        ],
        [
          0.5755,
          0.9404
        ],
        [
          0.5855,
          0.9377
        ],
        [
          0.5955,
          0.9345
        ],
        [
          0.6056,
          0.9322
        ],
        [
          0.616,
          0.9308
        ],
        [
          0.626,
          0.929
        ],
        [
          0.636,
          0.9272
        ],
        [
          0.646,
          0.9249
        ],
        [
          0.6561,
          0.9236
        ],
        [
          0.6665,
          0.919
        ],
        [
          0.6765,
          0.9172
        ],
        [
          0.6865,
          0.914
        ],
        [
          0.6965,
          0.9108
        ],
        [
          0.7066,
          0.9086
        ],
        [
          0.717,
          0.9067
        ],
        [
          0.727,
          0.904
        ],
        [
          0.737,
          0.9013
        ],
        [
          0.747,
          0.8967
        ],
        [
          0.7571,
          0.8926
        ],
        [
          0.7675,
          0.889
        ],
        [
          0.7775,
          0.8854
        ],
        [
          0.7875,
          0.8794
        ],
        [
          0.7975,
          0.8772
        ],
        [
          0.8076,
          0.8722
        ],
        [
          0.818,
          0.869
        ],
        [
          0.828,
          0.8635
        ],
        [
          0.838,
          0.8599
        ],
        [
          0.848,
          0.854
        ],
        [
          0.8581,
          0.8494
        ],
        [
          0.8685,
          0.8426
        ],
        [
          0.8785,
          0.8367
        ],
        [
          0.8885,
          0.8312
        ],
        [
          0.8985,
          0.8262
        ],
        [
          0.9086,
          0.8217
        ],
        [
          0.919,
          0.8162
        ],
        [
          0.929,
          0.8112
        ],
        [
          0.939,
          0.8048
        ],
        [
          0.949,
          0.7989
        ],
        [
          0.9591,
          0.7934
        ],
        [
          0.9695,
          0.7862
        ],
        [
          0.9795,
          0.7821
        ],
        [
          0.9895,
          0.7766
        ],
        [
          0.9995,
          0.7712
        ],
        [
          1.0,
          0.7707
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.3428618047286546,
        "difference": -0.0007307090551725854,
        "worse": 0.46,
        "kept_right": 0.34097610202186773,
        "kept_wrong": 0.001885702706786679,
        "handed_off": 0.657138195271346
      },
      "usd_per_1000": 0.7571497499742376
    },
    {
      "decision": 199,
      "frontier": 278,
      "frontier_accuracy": 0.9499545040946314,
      "in_sample": {
        "threshold": 0.9000736319729711,
        "answered": 0.364877161055505,
        "accuracy": 0.9499545040946314
      },
      "curve": [
        [
          0.0,
          0.95
        ],
        [
          0.01,
          0.95
        ],
        [
          0.02,
          0.95
        ],
        [
          0.03,
          0.95
        ],
        [
          0.04,
          0.95
        ],
        [
          0.05,
          0.95
        ],
        [
          0.0601,
          0.95
        ],
        [
          0.0705,
          0.95
        ],
        [
          0.0805,
          0.95
        ],
        [
          0.0905,
          0.95
        ],
        [
          0.1005,
          0.95
        ],
        [
          0.1106,
          0.95
        ],
        [
          0.121,
          0.95
        ],
        [
          0.131,
          0.95
        ],
        [
          0.141,
          0.95
        ],
        [
          0.151,
          0.95
        ],
        [
          0.1611,
          0.95
        ],
        [
          0.1715,
          0.95
        ],
        [
          0.1815,
          0.95
        ],
        [
          0.1915,
          0.95
        ],
        [
          0.2015,
          0.95
        ],
        [
          0.2116,
          0.9504
        ],
        [
          0.222,
          0.9504
        ],
        [
          0.232,
          0.9504
        ],
        [
          0.242,
          0.9504
        ],
        [
          0.252,
          0.9504
        ],
        [
          0.2621,
          0.9509
        ],
        [
          0.2725,
          0.9513
        ],
        [
          0.2825,
          0.9513
        ],
        [
          0.2925,
          0.9513
        ],
        [
          0.3025,
          0.9518
        ],
        [
          0.3126,
          0.9518
        ],
        [
          0.323,
          0.9518
        ],
        [
          0.333,
          0.9513
        ],
        [
          0.343,
          0.9509
        ],
        [
          0.353,
          0.95
        ],
        [
          0.3631,
          0.95
        ],
        [
          0.3735,
          0.9495
        ],
        [
          0.3835,
          0.9481
        ],
        [
          0.3935,
          0.9477
        ],
        [
          0.4035,
          0.9472
        ],
        [
          0.4136,
          0.9468
        ],
        [
          0.424,
          0.9463
        ],
        [
          0.434,
          0.9449
        ],
        [
          0.444,
          0.944
        ],
        [
          0.454,
          0.944
        ],
        [
          0.4641,
          0.944
        ],
        [
          0.4745,
          0.9427
        ],
        [
          0.4845,
          0.9399
        ],
        [
          0.4945,
          0.939
        ],
        [
          0.5045,
          0.9386
        ],
        [
          0.5146,
          0.9381
        ],
        [
          0.525,
          0.9372
        ],
        [
          0.535,
          0.9359
        ],
        [
          0.545,
          0.934
        ],
        [
          0.5551,
          0.9318
        ],
        [
          0.5655,
          0.9313
        ],
        [
          0.5755,
          0.9295
        ],
        [
          0.5855,
          0.9272
        ],
        [
          0.5955,
          0.924
        ],
        [
          0.6056,
          0.9217
        ],
        [
          0.616,
          0.9208
        ],
        [
          0.626,
          0.919
        ],
        [
          0.636,
          0.9186
        ],
        [
          0.646,
          0.9167
        ],
        [
          0.6561,
          0.9154
        ],
        [
          0.6665,
          0.9108
        ],
        [
          0.6765,
          0.909
        ],
        [
          0.6865,
          0.9049
        ],
        [
          0.6965,
          0.9017
        ],
        [
          0.7066,
          0.8999
        ],
        [
          0.717,
          0.8981
        ],
        [
          0.727,
          0.8958
        ],
        [
          0.737,
          0.8926
        ],
        [
          0.747,
          0.8881
        ],
        [
          0.7571,
          0.884
        ],
        [
          0.7675,
          0.8808
        ],
        [
          0.7775,
          0.8767
        ],
        [
          0.7875,
          0.8708
        ],
        [
          0.7975,
          0.8681
        ],
        [
          0.8076,
          0.864
        ],
        [
          0.818,
          0.8612
        ],
        [
          0.828,
          0.8558
        ],
        [
          0.838,
          0.8526
        ],
        [
          0.848,
          0.8467
        ],
        [
          0.8581,
          0.843
        ],
        [
          0.8685,
          0.8371
        ],
        [
          0.8785,
          0.8326
        ],
        [
          0.8885,
          0.8271
        ],
        [
          0.8985,
          0.8226
        ],
        [
          0.9086,
          0.8185
        ],
        [
          0.919,
          0.813
        ],
        [
          0.929,
          0.8085
        ],
        [
          0.939,
          0.8025
        ],
        [
          0.949,
          0.7962
        ],
        [
          0.9591,
          0.7907
        ],
        [
          0.9695,
          0.7843
        ],
        [
          0.9795,
          0.7812
        ],
        [
          0.9895,
          0.7762
        ],
        [
          0.9995,
          0.7712
        ],
        [
          1.0,
          0.7707
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.3680696052244301,
        "difference": -0.0006064443914873734,
        "worse": 0.43333333333333335,
        "kept_right": 0.36414658799245797,
        "kept_wrong": 0.003923017231972113,
        "handed_off": 0.63193039477557
      },
      "usd_per_1000": 4.1274636042781125
    },
    {
      "decision": 199,
      "frontier": 275,
      "frontier_accuracy": 0.9426751592356688,
      "in_sample": {
        "threshold": 0.9071930569698246,
        "answered": 0.34986351228389445,
        "accuracy": 0.9426751592356688
      },
      "curve": [
        [
          0.0,
          0.9427
        ],
        [
          0.01,
          0.9427
        ],
        [
          0.02,
          0.9427
        ],
        [
          0.03,
          0.9427
        ],
        [
          0.04,
          0.9427
        ],
        [
          0.05,
          0.9427
        ],
        [
          0.0601,
          0.9427
        ],
        [
          0.0705,
          0.9427
        ],
        [
          0.0805,
          0.9427
        ],
        [
          0.0905,
          0.9427
        ],
        [
          0.1005,
          0.9427
        ],
        [
          0.1106,
          0.9427
        ],
        [
          0.121,
          0.9427
        ],
        [
          0.131,
          0.9427
        ],
        [
          0.141,
          0.9427
        ],
        [
          0.151,
          0.9427
        ],
        [
          0.1611,
          0.9427
        ],
        [
          0.1715,
          0.9427
        ],
        [
          0.1815,
          0.9427
        ],
        [
          0.1915,
          0.9427
        ],
        [
          0.2015,
          0.9427
        ],
        [
          0.2116,
          0.9427
        ],
        [
          0.222,
          0.9427
        ],
        [
          0.232,
          0.9427
        ],
        [
          0.242,
          0.9427
        ],
        [
          0.252,
          0.9427
        ],
        [
          0.2621,
          0.9431
        ],
        [
          0.2725,
          0.9431
        ],
        [
          0.2825,
          0.9431
        ],
        [
          0.2925,
          0.9436
        ],
        [
          0.3025,
          0.9436
        ],
        [
          0.3126,
          0.9436
        ],
        [
          0.323,
          0.9431
        ],
        [
          0.333,
          0.9427
        ],
        [
          0.343,
          0.9427
        ],
        [
          0.353,
          0.9418
        ],
        [
          0.3631,
          0.9418
        ],
        [
          0.3735,
          0.9413
        ],
        [
          0.3835,
          0.9399
        ],
        [
          0.3935,
          0.9395
        ],
        [
          0.4035,
          0.939
        ],
        [
          0.4136,
          0.939
        ],
        [
          0.424,
          0.9381
        ],
        [
          0.434,
          0.9372
        ],
        [
          0.444,
          0.9363
        ],
        [
          0.454,
          0.9359
        ],
        [
          0.4641,
          0.9359
        ],
        [
          0.4745,
          0.934
        ],
        [
          0.4845,
          0.9313
        ],
        [
          0.4945,
          0.9304
        ],
        [
          0.5045,
          0.9299
        ],
        [
          0.5146,
          0.929
        ],
        [
          0.525,
          0.9286
        ],
        [
          0.535,
          0.9268
        ],
        [
          0.545,
          0.9245
        ],
        [
          0.5551,
          0.9231
        ],
        [
          0.5655,
          0.9227
        ],
        [
          0.5755,
          0.9208
        ],
        [
          0.5855,
          0.9195
        ],
        [
          0.5955,
          0.9158
        ],
        [
          0.6056,
          0.914
        ],
        [
          0.616,
          0.9122
        ],
        [
          0.626,
          0.9108
        ],
        [
          0.636,
          0.9095
        ],
        [
          0.646,
          0.9081
        ],
        [
          0.6561,
          0.9072
        ],
        [
          0.6665,
          0.9026
        ],
        [
          0.6765,
          0.9008
        ],
        [
          0.6865,
          0.8976
        ],
        [
          0.6965,
          0.8944
        ],
        [
          0.7066,
          0.8917
        ],
        [
          0.717,
          0.8899
        ],
        [
          0.727,
          0.8876
        ],
        [
          0.737,
          0.8854
        ],
        [
          0.747,
          0.8808
        ],
        [
          0.7571,
          0.8781
        ],
        [
          0.7675,
          0.8744
        ],
        [
          0.7775,
          0.8717
        ],
        [
          0.7875,
          0.8662
        ],
        [
          0.7975,
          0.8649
        ],
        [
          0.8076,
          0.8608
        ],
        [
          0.818,
          0.8581
        ],
        [
          0.828,
          0.853
        ],
        [
          0.838,
          0.8508
        ],
        [
          0.848,
          0.8449
        ],
        [
          0.8581,
          0.8412
        ],
        [
          0.8685,
          0.8362
        ],
        [
          0.8785,
          0.8308
        ],
        [
          0.8885,
          0.8258
        ],
        [
          0.8985,
          0.8221
        ],
        [
          0.9086,
          0.818
        ],
        [
          0.919,
          0.8126
        ],
        [
          0.929,
          0.8076
        ],
        [
          0.939,
          0.8012
        ],
        [
          0.949,
          0.7957
        ],
        [
          0.9591,
          0.7907
        ],
        [
          0.9695,
          0.783
        ],
        [
          0.9795,
          0.7803
        ],
        [
          0.9895,
          0.7753
        ],
        [
          0.9995,
          0.7712
        ],
        [
          1.0,
          0.7707
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.3474914156850193,
        "difference": -0.0006155354162042707,
        "worse": 0.45,
        "kept_right": 0.3452964727570106,
        "kept_wrong": 0.002194942928008527,
        "handed_off": 0.6525085843149814
      },
      "usd_per_1000": 0.3500627088270299
    },
    {
      "decision": 199,
      "frontier": 192,
      "frontier_accuracy": 0.9331210191082803,
      "in_sample": {
        "threshold": 0.8949701699809105,
        "answered": 0.37443130118289353,
        "accuracy": 0.9331210191082803
      },
      "curve": [
        [
          0.0,
          0.9331
        ],
        [
          0.01,
          0.9331
        ],
        [
          0.02,
          0.9331
        ],
        [
          0.03,
          0.9331
        ],
        [
          0.04,
          0.9331
        ],
        [
          0.05,
          0.9331
        ],
        [
          0.0601,
          0.9331
        ],
        [
          0.0705,
          0.9331
        ],
        [
          0.0805,
          0.9331
        ],
        [
          0.0905,
          0.9331
        ],
        [
          0.1005,
          0.9331
        ],
        [
          0.1106,
          0.9331
        ],
        [
          0.121,
          0.9331
        ],
        [
          0.131,
          0.9331
        ],
        [
          0.141,
          0.9331
        ],
        [
          0.151,
          0.9331
        ],
        [
          0.1611,
          0.9331
        ],
        [
          0.1715,
          0.9331
        ],
        [
          0.1815,
          0.9331
        ],
        [
          0.1915,
          0.9331
        ],
        [
          0.2015,
          0.9331
        ],
        [
          0.2116,
          0.9336
        ],
        [
          0.222,
          0.9336
        ],
        [
          0.232,
          0.9336
        ],
        [
          0.242,
          0.934
        ],
        [
          0.252,
          0.934
        ],
        [
          0.2621,
          0.9345
        ],
        [
          0.2725,
          0.9345
        ],
        [
          0.2825,
          0.9345
        ],
        [
          0.2925,
          0.9349
        ],
        [
          0.3025,
          0.9349
        ],
        [
          0.3126,
          0.9349
        ],
        [
          0.323,
          0.9349
        ],
        [
          0.333,
          0.9349
        ],
        [
          0.343,
          0.9345
        ],
        [
          0.353,
          0.9336
        ],
        [
          0.3631,
          0.9336
        ],
        [
          0.3735,
          0.9331
        ],
        [
          0.3835,
          0.9318
        ],
        [
          0.3935,
          0.9313
        ],
        [
          0.4035,
          0.9304
        ],
        [
          0.4136,
          0.9299
        ],
        [
          0.424,
          0.9299
        ],
        [
          0.434,
          0.9286
        ],
        [
          0.444,
          0.9281
        ],
        [
          0.454,
          0.9281
        ],
        [
          0.4641,
          0.9272
        ],
        [
          0.4745,
          0.9263
        ],
        [
          0.4845,
          0.9236
        ],
        [
          0.4945,
          0.9227
        ],
        [
          0.5045,
          0.9213
        ],
        [
          0.5146,
          0.9208
        ],
        [
          0.525,
          0.9208
        ],
        [
          0.535,
          0.9199
        ],
        [
          0.545,
          0.9186
        ],
        [
          0.5551,
          0.9167
        ],
        [
          0.5655,
          0.9163
        ],
        [
          0.5755,
          0.9145
        ],
        [
          0.5855,
          0.9126
        ],
        [
          0.5955,
          0.9095
        ],
        [
          0.6056,
          0.9081
        ],
        [
          0.616,
          0.9067
        ],
        [
          0.626,
          0.9058
        ],
        [
          0.636,
          0.9058
        ],
        [
          0.646,
          0.904
        ],
        [
          0.6561,
          0.9026
        ],
        [
          0.6665,
          0.8985
        ],
        [
          0.6765,
          0.8967
        ],
        [
          0.6865,
          0.8935
        ],
        [
          0.6965,
          0.8908
        ],
        [
          0.7066,
          0.889
        ],
        [
          0.717,
          0.8876
        ],
        [
          0.727,
          0.8858
        ],
        [
          0.737,
          0.884
        ],
        [
          0.747,
          0.8799
        ],
        [
          0.7571,
          0.8758
        ],
        [
          0.7675,
          0.8731
        ],
        [
          0.7775,
          0.8699
        ],
        [
          0.7875,
          0.8635
        ],
        [
          0.7975,
          0.8608
        ],
        [
          0.8076,
          0.8571
        ],
        [
          0.818,
          0.854
        ],
        [
          0.828,
          0.849
        ],
        [
          0.838,
          0.8462
        ],
        [
          0.848,
          0.8412
        ],
        [
          0.8581,
          0.8371
        ],
        [
          0.8685,
          0.8312
        ],
        [
          0.8785,
          0.8267
        ],
        [
          0.8885,
          0.8217
        ],
        [
          0.8985,
          0.8171
        ],
        [
          0.9086,
          0.8135
        ],
        [
          0.919,
          0.8094
        ],
        [
          0.929,
          0.8048
        ],
        [
          0.939,
          0.7994
        ],
        [
          0.949,
          0.7939
        ],
        [
          0.9591,
          0.7889
        ],
        [
          0.9695,
          0.783
        ],
        [
          0.9795,
          0.7798
        ],
        [
          0.9895,
          0.7762
        ],
        [
          0.9995,
          0.7712
        ],
        [
          1.0,
          0.7707
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.3781776447400145,
        "difference": -0.0010097758155391662,
        "worse": 0.45,
        "kept_right": 0.37314794232071513,
        "kept_wrong": 0.005029702419299302,
        "handed_off": 0.6218223552599856
      },
      "usd_per_1000": 7.233425691484622
    },
    {
      "decision": 199,
      "frontier": 123,
      "frontier_accuracy": 0.9326660600545951,
      "in_sample": {
        "threshold": 0.9237098977857325,
        "answered": 0.3175614194722475,
        "accuracy": 0.9326660600545951
      },
      "curve": [
        [
          0.0,
          0.9327
        ],
        [
          0.01,
          0.9327
        ],
        [
          0.02,
          0.9327
        ],
        [
          0.03,
          0.9327
        ],
        [
          0.04,
          0.9327
        ],
        [
          0.05,
          0.9327
        ],
        [
          0.0601,
          0.9327
        ],
        [
          0.0705,
          0.9327
        ],
        [
          0.0805,
          0.9327
        ],
        [
          0.0905,
          0.9327
        ],
        [
          0.1005,
          0.9327
        ],
        [
          0.1106,
          0.9327
        ],
        [
          0.121,
          0.9327
        ],
        [
          0.131,
          0.9327
        ],
        [
          0.141,
          0.9327
        ],
        [
          0.151,
          0.9327
        ],
        [
          0.1611,
          0.9327
        ],
        [
          0.1715,
          0.9327
        ],
        [
          0.1815,
          0.9327
        ],
        [
          0.1915,
          0.9327
        ],
        [
          0.2015,
          0.9327
        ],
        [
          0.2116,
          0.9327
        ],
        [
          0.222,
          0.9327
        ],
        [
          0.232,
          0.9327
        ],
        [
          0.242,
          0.9327
        ],
        [
          0.252,
          0.9327
        ],
        [
          0.2621,
          0.9327
        ],
        [
          0.2725,
          0.9327
        ],
        [
          0.2825,
          0.9327
        ],
        [
          0.2925,
          0.9327
        ],
        [
          0.3025,
          0.9327
        ],
        [
          0.3126,
          0.9327
        ],
        [
          0.323,
          0.9322
        ],
        [
          0.333,
          0.9318
        ],
        [
          0.343,
          0.9313
        ],
        [
          0.353,
          0.9313
        ],
        [
          0.3631,
          0.9313
        ],
        [
          0.3735,
          0.9304
        ],
        [
          0.3835,
          0.929
        ],
        [
          0.3935,
          0.9286
        ],
        [
          0.4035,
          0.9281
        ],
        [
          0.4136,
          0.9277
        ],
        [
          0.424,
          0.9268
        ],
        [
          0.434,
          0.9254
        ],
        [
          0.444,
          0.9249
        ],
        [
          0.454,
          0.924
        ],
        [
          0.4641,
          0.9236
        ],
        [
          0.4745,
          0.9222
        ],
        [
          0.4845,
          0.9195
        ],
        [
          0.4945,
          0.9186
        ],
        [
          0.5045,
          0.9186
        ],
        [
          0.5146,
          0.9172
        ],
        [
          0.525,
          0.9167
        ],
        [
          0.535,
          0.9158
        ],
        [
          0.545,
          0.9136
        ],
        [
          0.5551,
          0.9113
        ],
        [
          0.5655,
          0.9108
        ],
        [
          0.5755,
          0.9095
        ],
        [
          0.5855,
          0.9063
        ],
        [
          0.5955,
          0.9035
        ],
        [
          0.6056,
          0.9022
        ],
        [
          0.616,
          0.9008
        ],
        [
          0.626,
          0.9004
        ],
        [
          0.636,
          0.8999
        ],
        [
          0.646,
          0.8981
        ],
        [
          0.6561,
          0.8981
        ],
        [
          0.6665,
          0.8935
        ],
        [
          0.6765,
          0.8917
        ],
        [
          0.6865,
          0.8885
        ],
        [
          0.6965,
          0.8863
        ],
        [
          0.7066,
          0.8835
        ],
        [
          0.717,
          0.8822
        ],
        [
          0.727,
          0.8794
        ],
        [
          0.737,
          0.8785
        ],
        [
          0.747,
          0.8744
        ],
        [
          0.7571,
          0.8708
        ],
        [
          0.7675,
          0.8676
        ],
        [
          0.7775,
          0.8635
        ],
        [
          0.7875,
          0.8599
        ],
        [
          0.7975,
          0.8571
        ],
        [
          0.8076,
          0.853
        ],
        [
          0.818,
          0.8503
        ],
        [
          0.828,
          0.8458
        ],
        [
          0.838,
          0.8435
        ],
        [
          0.848,
          0.8376
        ],
        [
          0.8581,
          0.8353
        ],
        [
          0.8685,
          0.8294
        ],
        [
          0.8785,
          0.8248
        ],
        [
          0.8885,
          0.8194
        ],
        [
          0.8985,
          0.8153
        ],
        [
          0.9086,
          0.8121
        ],
        [
          0.919,
          0.8071
        ],
        [
          0.929,
          0.8044
        ],
        [
          0.939,
          0.7998
        ],
        [
          0.949,
          0.7944
        ],
        [
          0.9591,
          0.7894
        ],
        [
          0.9695,
          0.7825
        ],
        [
          0.9795,
          0.7798
        ],
        [
          0.9895,
          0.7762
        ],
        [
          0.9995,
          0.7712
        ],
        [
          1.0,
          0.7707
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.33140122896980106,
        "difference": -0.0010701669246398686,
        "worse": 0.5266666666666666,
        "kept_right": 0.33008251067467415,
        "kept_wrong": 0.0013187182951270825,
        "handed_off": 0.6685987710301989
      },
      "usd_per_1000": 0.19391332528696562
    },
    {
      "decision": 199,
      "frontier": 274,
      "frontier_accuracy": 0.9085532302092811,
      "in_sample": {
        "threshold": 0.9071930569698246,
        "answered": 0.34986351228389445,
        "accuracy": 0.9085532302092811
      },
      "curve": [
        [
          0.0,
          0.9086
        ],
        [
          0.01,
          0.9086
        ],
        [
          0.02,
          0.9086
        ],
        [
          0.03,
          0.9086
        ],
        [
          0.04,
          0.9086
        ],
        [
          0.05,
          0.9086
        ],
        [
          0.0601,
          0.9086
        ],
        [
          0.0705,
          0.9086
        ],
        [
          0.0805,
          0.9086
        ],
        [
          0.0905,
          0.9086
        ],
        [
          0.1005,
          0.9086
        ],
        [
          0.1106,
          0.9086
        ],
        [
          0.121,
          0.9086
        ],
        [
          0.131,
          0.9086
        ],
        [
          0.141,
          0.9086
        ],
        [
          0.151,
          0.9086
        ],
        [
          0.1611,
          0.9086
        ],
        [
          0.1715,
          0.9086
        ],
        [
          0.1815,
          0.9086
        ],
        [
          0.1915,
          0.9086
        ],
        [
          0.2015,
          0.9086
        ],
        [
          0.2116,
          0.9086
        ],
        [
          0.222,
          0.9086
        ],
        [
          0.232,
          0.9086
        ],
        [
          0.242,
          0.9086
        ],
        [
          0.252,
          0.9086
        ],
        [
          0.2621,
          0.909
        ],
        [
          0.2725,
          0.909
        ],
        [
          0.2825,
          0.909
        ],
        [
          0.2925,
          0.9095
        ],
        [
          0.3025,
          0.9095
        ],
        [
          0.3126,
          0.9095
        ],
        [
          0.323,
          0.909
        ],
        [
          0.333,
          0.9086
        ],
        [
          0.343,
          0.9086
        ],
        [
          0.353,
          0.9081
        ],
        [
          0.3631,
          0.9081
        ],
        [
          0.3735,
          0.9081
        ],
        [
          0.3835,
          0.9072
        ],
        [
          0.3935,
          0.9067
        ],
        [
          0.4035,
          0.9058
        ],
        [
          0.4136,
          0.9058
        ],
        [
          0.424,
          0.9049
        ],
        [
          0.434,
          0.9035
        ],
        [
          0.444,
          0.9026
        ],
        [
          0.454,
          0.9031
        ],
        [
          0.4641,
          0.9031
        ],
        [
          0.4745,
          0.9017
        ],
        [
          0.4845,
          0.899
        ],
        [
          0.4945,
          0.8981
        ],
        [
          0.5045,
          0.8985
        ],
        [
          0.5146,
          0.8981
        ],
        [
          0.525,
          0.8976
        ],
        [
          0.535,
          0.8963
        ],
        [
          0.545,
          0.894
        ],
        [
          0.5551,
          0.8922
        ],
        [
          0.5655,
          0.8926
        ],
        [
          0.5755,
          0.8908
        ],
        [
          0.5855,
          0.889
        ],
        [
          0.5955,
          0.8867
        ],
        [
          0.6056,
          0.8849
        ],
        [
          0.616,
          0.884
        ],
        [
          0.626,
          0.8831
        ],
        [
          0.636,
          0.8813
        ],
        [
          0.646,
          0.8799
        ],
        [
          0.6561,
          0.8803
        ],
        [
          0.6665,
          0.8772
        ],
        [
          0.6765,
          0.8763
        ],
        [
          0.6865,
          0.8731
        ],
        [
          0.6965,
          0.8712
        ],
        [
          0.7066,
          0.869
        ],
        [
          0.717,
          0.8681
        ],
        [
          0.727,
          0.8658
        ],
        [
          0.737,
          0.8635
        ],
        [
          0.747,
          0.859
        ],
        [
          0.7571,
          0.8558
        ],
        [
          0.7675,
          0.8521
        ],
        [
          0.7775,
          0.8494
        ],
        [
          0.7875,
          0.8449
        ],
        [
          0.7975,
          0.8449
        ],
        [
          0.8076,
          0.8417
        ],
        [
          0.818,
          0.8399
        ],
        [
          0.828,
          0.8348
        ],
        [
          0.838,
          0.8326
        ],
        [
          0.848,
          0.8276
        ],
        [
          0.8581,
          0.8258
        ],
        [
          0.8685,
          0.8203
        ],
        [
          0.8785,
          0.8157
        ],
        [
          0.8885,
          0.8116
        ],
        [
          0.8985,
          0.8076
        ],
        [
          0.9086,
          0.8039
        ],
        [
          0.919,
          0.8003
        ],
        [
          0.929,
          0.7966
        ],
        [
          0.939,
          0.7921
        ],
        [
          0.949,
          0.7889
        ],
        [
          0.9591,
          0.7843
        ],
        [
          0.9695,
          0.7789
        ],
        [
          0.9795,
          0.7771
        ],
        [
          0.9895,
          0.7739
        ],
        [
          0.9995,
          0.7712
        ],
        [
          1.0,
          0.7707
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.3618113955858659,
        "difference": -0.0008490453489093967,
        "worse": 0.49,
        "kept_right": 0.35823382520408825,
        "kept_wrong": 0.0035775703817775505,
        "handed_off": 0.6381886044141345
      },
      "usd_per_1000": 0.28636356743055014
    },
    {
      "decision": 199,
      "frontier": 47,
      "frontier_accuracy": 0.9044585987261147,
      "in_sample": {
        "threshold": 0.8334889618528571,
        "answered": 0.4599636032757052,
        "accuracy": 0.9044585987261147
      },
      "curve": [
        [
          0.0,
          0.9045
        ],
        [
          0.01,
          0.9045
        ],
        [
          0.02,
          0.9045
        ],
        [
          0.03,
          0.9045
        ],
        [
          0.04,
          0.9045
        ],
        [
          0.05,
          0.9045
        ],
        [
          0.0601,
          0.9045
        ],
        [
          0.0705,
          0.9045
        ],
        [
          0.0805,
          0.9045
        ],
        [
          0.0905,
          0.9045
        ],
        [
          0.1005,
          0.9045
        ],
        [
          0.1106,
          0.9045
        ],
        [
          0.121,
          0.9045
        ],
        [
          0.131,
          0.9045
        ],
        [
          0.141,
          0.9045
        ],
        [
          0.151,
          0.9045
        ],
        [
          0.1611,
          0.9045
        ],
        [
          0.1715,
          0.9045
        ],
        [
          0.1815,
          0.9045
        ],
        [
          0.1915,
          0.9045
        ],
        [
          0.2015,
          0.9045
        ],
        [
          0.2116,
          0.9049
        ],
        [
          0.222,
          0.9049
        ],
        [
          0.232,
          0.9049
        ],
        [
          0.242,
          0.9049
        ],
        [
          0.252,
          0.9049
        ],
        [
          0.2621,
          0.9049
        ],
        [
          0.2725,
          0.9054
        ],
        [
          0.2825,
          0.9054
        ],
        [
          0.2925,
          0.9058
        ],
        [
          0.3025,
          0.9063
        ],
        [
          0.3126,
          0.9067
        ],
        [
          0.323,
          0.9067
        ],
        [
          0.333,
          0.9072
        ],
        [
          0.343,
          0.9067
        ],
        [
          0.353,
          0.9058
        ],
        [
          0.3631,
          0.9067
        ],
        [
          0.3735,
          0.9063
        ],
        [
          0.3835,
          0.9049
        ],
        [
          0.3935,
          0.9049
        ],
        [
          0.4035,
          0.9049
        ],
        [
          0.4136,
          0.9054
        ],
        [
          0.424,
          0.9058
        ],
        [
          0.434,
          0.9045
        ],
        [
          0.444,
          0.904
        ],
        [
          0.454,
          0.9049
        ],
        [
          0.4641,
          0.904
        ],
        [
          0.4745,
          0.9031
        ],
        [
          0.4845,
          0.9004
        ],
        [
          0.4945,
          0.8995
        ],
        [
          0.5045,
          0.8995
        ],
        [
          0.5146,
          0.899
        ],
        [
          0.525,
          0.899
        ],
        [
          0.535,
          0.8985
        ],
        [
          0.545,
          0.8976
        ],
        [
          0.5551,
          0.8963
        ],
        [
          0.5655,
          0.8958
        ],
        [
          0.5755,
          0.894
        ],
        [
          0.5855,
          0.8922
        ],
        [
          0.5955,
          0.8885
        ],
        [
          0.6056,
          0.8876
        ],
        [
          0.616,
          0.8872
        ],
        [
          0.626,
          0.8867
        ],
        [
          0.636,
          0.8867
        ],
        [
          0.646,
          0.8849
        ],
        [
          0.6561,
          0.8849
        ],
        [
          0.6665,
          0.8813
        ],
        [
          0.6765,
          0.8794
        ],
        [
          0.6865,
          0.8767
        ],
        [
          0.6965,
          0.874
        ],
        [
          0.7066,
          0.8726
        ],
        [
          0.717,
          0.8708
        ],
        [
          0.727,
          0.8685
        ],
        [
          0.737,
          0.8672
        ],
        [
          0.747,
          0.8626
        ],
        [
          0.7571,
          0.859
        ],
        [
          0.7675,
          0.8562
        ],
        [
          0.7775,
          0.8526
        ],
        [
          0.7875,
          0.848
        ],
        [
          0.7975,
          0.8471
        ],
        [
          0.8076,
          0.8444
        ],
        [
          0.818,
          0.843
        ],
        [
          0.828,
          0.838
        ],
        [
          0.838,
          0.8362
        ],
        [
          0.848,
          0.8303
        ],
        [
          0.8581,
          0.828
        ],
        [
          0.8685,
          0.8235
        ],
        [
          0.8785,
          0.8194
        ],
        [
          0.8885,
          0.8144
        ],
        [
          0.8985,
          0.8112
        ],
        [
          0.9086,
          0.8085
        ],
        [
          0.919,
          0.8044
        ],
        [
          0.929,
          0.7998
        ],
        [
          0.939,
          0.7944
        ],
        [
          0.949,
          0.7903
        ],
        [
          0.9591,
          0.7875
        ],
        [
          0.9695,
          0.7807
        ],
        [
          0.9795,
          0.7793
        ],
        [
          0.9895,
          0.7757
        ],
        [
          0.9995,
          0.7712
        ],
        [
          1.0,
          0.7707
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.4510413530227064,
        "difference": -0.001651651622911792,
        "worse": 0.49666666666666665,
        "kept_right": 0.4368025050199066,
        "kept_wrong": 0.01423884800279977,
        "handed_off": 0.5489586469772938
      },
      "usd_per_1000": 2.72102623758235
    },
    {
      "decision": 199,
      "frontier": 193,
      "frontier_accuracy": 0.89171974522293,
      "in_sample": {
        "threshold": 0.8274329093228885,
        "answered": 0.467697907188353,
        "accuracy": 0.89171974522293
      },
      "curve": [
        [
          0.0,
          0.8917
        ],
        [
          0.01,
          0.8917
        ],
        [
          0.02,
          0.8917
        ],
        [
          0.03,
          0.8917
        ],
        [
          0.04,
          0.8917
        ],
        [
          0.05,
          0.8917
        ],
        [
          0.0601,
          0.8917
        ],
        [
          0.0705,
          0.8917
        ],
        [
          0.0805,
          0.8917
        ],
        [
          0.0905,
          0.8917
        ],
        [
          0.1005,
          0.8917
        ],
        [
          0.1106,
          0.8917
        ],
        [
          0.121,
          0.8917
        ],
        [
          0.131,
          0.8917
        ],
        [
          0.141,
          0.8917
        ],
        [
          0.151,
          0.8922
        ],
        [
          0.1611,
          0.8922
        ],
        [
          0.1715,
          0.8922
        ],
        [
          0.1815,
          0.8922
        ],
        [
          0.1915,
          0.8922
        ],
        [
          0.2015,
          0.8922
        ],
        [
          0.2116,
          0.8922
        ],
        [
          0.222,
          0.8922
        ],
        [
          0.232,
          0.8922
        ],
        [
          0.242,
          0.8922
        ],
        [
          0.252,
          0.8922
        ],
        [
          0.2621,
          0.8922
        ],
        [
          0.2725,
          0.8922
        ],
        [
          0.2825,
          0.8922
        ],
        [
          0.2925,
          0.8926
        ],
        [
          0.3025,
          0.8931
        ],
        [
          0.3126,
          0.8931
        ],
        [
          0.323,
          0.8931
        ],
        [
          0.333,
          0.8935
        ],
        [
          0.343,
          0.8931
        ],
        [
          0.353,
          0.8922
        ],
        [
          0.3631,
          0.8926
        ],
        [
          0.3735,
          0.8926
        ],
        [
          0.3835,
          0.8917
        ],
        [
          0.3935,
          0.8913
        ],
        [
          0.4035,
          0.8913
        ],
        [
          0.4136,
          0.8913
        ],
        [
          0.424,
          0.8922
        ],
        [
          0.434,
          0.8908
        ],
        [
          0.444,
          0.8908
        ],
        [
          0.454,
          0.8917
        ],
        [
          0.4641,
          0.8917
        ],
        [
          0.4745,
          0.8908
        ],
        [
          0.4845,
          0.8881
        ],
        [
          0.4945,
          0.8881
        ],
        [
          0.5045,
          0.8881
        ],
        [
          0.5146,
          0.8885
        ],
        [
          0.525,
          0.8881
        ],
        [
          0.535,
          0.8876
        ],
        [
          0.545,
          0.8858
        ],
        [
          0.5551,
          0.8854
        ],
        [
          0.5655,
          0.8854
        ],
        [
          0.5755,
          0.8835
        ],
        [
          0.5855,
          0.8831
        ],
        [
          0.5955,
          0.8808
        ],
        [
          0.6056,
          0.8794
        ],
        [
          0.616,
          0.8781
        ],
        [
          0.626,
          0.8781
        ],
        [
          0.636,
          0.8785
        ],
        [
          0.646,
          0.8776
        ],
        [
          0.6561,
          0.8785
        ],
        [
          0.6665,
          0.8758
        ],
        [
          0.6765,
          0.8749
        ],
        [
          0.6865,
          0.8726
        ],
        [
          0.6965,
          0.8699
        ],
        [
          0.7066,
          0.8681
        ],
        [
          0.717,
          0.8667
        ],
        [
          0.727,
          0.864
        ],
        [
          0.737,
          0.8626
        ],
        [
          0.747,
          0.8594
        ],
        [
          0.7571,
          0.8567
        ],
        [
          0.7675,
          0.8544
        ],
        [
          0.7775,
          0.8526
        ],
        [
          0.7875,
          0.849
        ],
        [
          0.7975,
          0.8467
        ],
        [
          0.8076,
          0.843
        ],
        [
          0.818,
          0.8394
        ],
        [
          0.828,
          0.8344
        ],
        [
          0.838,
          0.8326
        ],
        [
          0.848,
          0.8289
        ],
        [
          0.8581,
          0.8258
        ],
        [
          0.8685,
          0.8217
        ],
        [
          0.8785,
          0.8171
        ],
        [
          0.8885,
          0.813
        ],
        [
          0.8985,
          0.8094
        ],
        [
          0.9086,
          0.8066
        ],
        [
          0.919,
          0.8021
        ],
        [
          0.929,
          0.798
        ],
        [
          0.939,
          0.793
        ],
        [
          0.949,
          0.7889
        ],
        [
          0.9591,
          0.7853
        ],
        [
          0.9695,
          0.7798
        ],
        [
          0.9795,
          0.7775
        ],
        [
          0.9895,
          0.7748
        ],
        [
          0.9995,
          0.7712
        ],
        [
          1.0,
          0.7707
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.4628424301030797,
        "difference": -0.0031534541815321834,
        "worse": 0.5333333333333333,
        "kept_right": 0.44613134854254466,
        "kept_wrong": 0.016711081560535572,
        "handed_off": 0.5371575698969199
      },
      "usd_per_1000": 4.176015045191006
    },
    {
      "decision": 199,
      "frontier": 55,
      "frontier_accuracy": 0.8503184713375797,
      "in_sample": {
        "threshold": 0.5204495914012839,
        "answered": 0.759326660600546,
        "accuracy": 0.8503184713375797
      },
      "curve": [
        [
          0.0,
          0.8503
        ],
        [
          0.01,
          0.8503
        ],
        [
          0.02,
          0.8503
        ],
        [
          0.03,
          0.8503
        ],
        [
          0.04,
          0.8508
        ],
        [
          0.05,
          0.8508
        ],
        [
          0.0601,
          0.8517
        ],
        [
          0.0705,
          0.8517
        ],
        [
          0.0805,
          0.8517
        ],
        [
          0.0905,
          0.8521
        ],
        [
          0.1005,
          0.8521
        ],
        [
          0.1106,
          0.8521
        ],
        [
          0.121,
          0.8521
        ],
        [
          0.131,
          0.8526
        ],
        [
          0.141,
          0.8535
        ],
        [
          0.151,
          0.8544
        ],
        [
          0.1611,
          0.8544
        ],
        [
          0.1715,
          0.8544
        ],
        [
          0.1815,
          0.8544
        ],
        [
          0.1915,
          0.8553
        ],
        [
          0.2015,
          0.8558
        ],
        [
          0.2116,
          0.8558
        ],
        [
          0.222,
          0.8562
        ],
        [
          0.232,
          0.8567
        ],
        [
          0.242,
          0.8571
        ],
        [
          0.252,
          0.8576
        ],
        [
          0.2621,
          0.8599
        ],
        [
          0.2725,
          0.8612
        ],
        [
          0.2825,
          0.8617
        ],
        [
          0.2925,
          0.8621
        ],
        [
          0.3025,
          0.8626
        ],
        [
          0.3126,
          0.8635
        ],
        [
          0.323,
          0.8644
        ],
        [
          0.333,
          0.8649
        ],
        [
          0.343,
          0.8649
        ],
        [
          0.353,
          0.8653
        ],
        [
          0.3631,
          0.8653
        ],
        [
          0.3735,
          0.8653
        ],
        [
          0.3835,
          0.8653
        ],
        [
          0.3935,
          0.8653
        ],
        [
          0.4035,
          0.8653
        ],
        [
          0.4136,
          0.8658
        ],
        [
          0.424,
          0.8667
        ],
        [
          0.434,
          0.8662
        ],
        [
          0.444,
          0.8672
        ],
        [
          0.454,
          0.8676
        ],
        [
          0.4641,
          0.8685
        ],
        [
          0.4745,
          0.8681
        ],
        [
          0.4845,
          0.8667
        ],
        [
          0.4945,
          0.8667
        ],
        [
          0.5045,
          0.8676
        ],
        [
          0.5146,
          0.8685
        ],
        [
          0.525,
          0.8685
        ],
        [
          0.535,
          0.8694
        ],
        [
          0.545,
          0.8685
        ],
        [
          0.5551,
          0.8685
        ],
        [
          0.5655,
          0.8681
        ],
        [
          0.5755,
          0.8672
        ],
        [
          0.5855,
          0.8681
        ],
        [
          0.5955,
          0.8662
        ],
        [
          0.6056,
          0.8649
        ],
        [
          0.616,
          0.8649
        ],
        [
          0.626,
          0.8649
        ],
        [
          0.636,
          0.8658
        ],
        [
          0.646,
          0.8653
        ],
        [
          0.6561,
          0.8667
        ],
        [
          0.6665,
          0.864
        ],
        [
          0.6765,
          0.8626
        ],
        [
          0.6865,
          0.8594
        ],
        [
          0.6965,
          0.8581
        ],
        [
          0.7066,
          0.8571
        ],
        [
          0.717,
          0.8567
        ],
        [
          0.727,
          0.8558
        ],
        [
          0.737,
          0.8549
        ],
        [
          0.747,
          0.8517
        ],
        [
          0.7571,
          0.8503
        ],
        [
          0.7675,
          0.8476
        ],
        [
          0.7775,
          0.8462
        ],
        [
          0.7875,
          0.843
        ],
        [
          0.7975,
          0.8417
        ],
        [
          0.8076,
          0.8376
        ],
        [
          0.818,
          0.8358
        ],
        [
          0.828,
          0.8303
        ],
        [
          0.838,
          0.8289
        ],
        [
          0.848,
          0.8239
        ],
        [
          0.8581,
          0.8212
        ],
        [
          0.8685,
          0.8171
        ],
        [
          0.8785,
          0.8139
        ],
        [
          0.8885,
          0.8094
        ],
        [
          0.8985,
          0.8053
        ],
        [
          0.9086,
          0.803
        ],
        [
          0.919,
          0.8003
        ],
        [
          0.929,
          0.7966
        ],
        [
          0.939,
          0.7921
        ],
        [
          0.949,
          0.7889
        ],
        [
          0.9591,
          0.7853
        ],
        [
          0.9695,
          0.7807
        ],
        [
          0.9795,
          0.778
        ],
        [
          0.9895,
          0.7743
        ],
        [
          0.9995,
          0.7712
        ],
        [
          1.0,
          0.7707
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.7540109493784212,
        "difference": -0.0007490366953526507,
        "worse": 0.45,
        "kept_right": 0.6613869814873974,
        "kept_wrong": 0.09262396789102392,
        "handed_off": 0.24598905062157916
      },
      "usd_per_1000": 0.9132644909370751
    },
    {
      "decision": 119,
      "frontier": 140,
      "frontier_accuracy": 0.9808917197452229,
      "in_sample": {
        "threshold": 0.9999999924404349,
        "answered": 0.1305732484076433,
        "accuracy": 0.9808917197452229
      },
      "curve": [
        [
          0.0,
          0.9809
        ],
        [
          0.01,
          0.9809
        ],
        [
          0.02,
          0.9809
        ],
        [
          0.03,
          0.9809
        ],
        [
          0.04,
          0.9809
        ],
        [
          0.05,
          0.9809
        ],
        [
          0.0601,
          0.9809
        ],
        [
          0.0705,
          0.9809
        ],
        [
          0.0805,
          0.9809
        ],
        [
          0.0905,
          0.9809
        ],
        [
          0.1005,
          0.9809
        ],
        [
          0.1106,
          0.9809
        ],
        [
          0.121,
          0.9809
        ],
        [
          0.131,
          0.9804
        ],
        [
          0.141,
          0.9804
        ],
        [
          0.151,
          0.98
        ],
        [
          0.1611,
          0.98
        ],
        [
          0.1715,
          0.98
        ],
        [
          0.1815,
          0.9786
        ],
        [
          0.1915,
          0.9786
        ],
        [
          0.2015,
          0.9786
        ],
        [
          0.2116,
          0.9786
        ],
        [
          0.222,
          0.9786
        ],
        [
          0.232,
          0.9786
        ],
        [
          0.242,
          0.9786
        ],
        [
          0.252,
          0.9782
        ],
        [
          0.2621,
          0.9782
        ],
        [
          0.2725,
          0.9782
        ],
        [
          0.2825,
          0.9773
        ],
        [
          0.2925,
          0.9763
        ],
        [
          0.3025,
          0.9768
        ],
        [
          0.3126,
          0.9763
        ],
        [
          0.323,
          0.9759
        ],
        [
          0.333,
          0.975
        ],
        [
          0.343,
          0.9732
        ],
        [
          0.353,
          0.9718
        ],
        [
          0.3631,
          0.9709
        ],
        [
          0.3735,
          0.97
        ],
        [
          0.3835,
          0.9695
        ],
        [
          0.3935,
          0.9691
        ],
        [
          0.4035,
          0.9691
        ],
        [
          0.4136,
          0.9686
        ],
        [
          0.424,
          0.9682
        ],
        [
          0.434,
          0.9663
        ],
        [
          0.444,
          0.965
        ],
        [
          0.454,
          0.9636
        ],
        [
          0.4641,
          0.9622
        ],
        [
          0.4745,
          0.9609
        ],
        [
          0.4845,
          0.9595
        ],
        [
          0.4945,
          0.9572
        ],
        [
          0.5045,
          0.9536
        ],
        [
          0.5146,
          0.9522
        ],
        [
          0.525,
          0.9495
        ],
        [
          0.535,
          0.949
        ],
        [
          0.545,
          0.9449
        ],
        [
          0.5551,
          0.9427
        ],
        [
          0.5655,
          0.939
        ],
        [
          0.5755,
          0.9363
        ],
        [
          0.5855,
          0.9331
        ],
        [
          0.5955,
          0.9313
        ],
        [
          0.6056,
          0.929
        ],
        [
          0.616,
          0.9245
        ],
        [
          0.626,
          0.9222
        ],
        [
          0.636,
          0.9181
        ],
        [
          0.646,
          0.914
        ],
        [
          0.6561,
          0.9104
        ],
        [
          0.6665,
          0.9067
        ],
        [
          0.6765,
          0.9026
        ],
        [
          0.6865,
          0.8999
        ],
        [
          0.6965,
          0.8949
        ],
        [
          0.7066,
          0.8922
        ],
        [
          0.717,
          0.8872
        ],
        [
          0.727,
          0.884
        ],
        [
          0.737,
          0.879
        ],
        [
          0.747,
          0.8758
        ],
        [
          0.7571,
          0.8717
        ],
        [
          0.7675,
          0.8681
        ],
        [
          0.7775,
          0.8631
        ],
        [
          0.7875,
          0.8567
        ],
        [
          0.7975,
          0.8512
        ],
        [
          0.8076,
          0.8458
        ],
        [
          0.818,
          0.8408
        ],
        [
          0.828,
          0.8344
        ],
        [
          0.838,
          0.8289
        ],
        [
          0.848,
          0.8217
        ],
        [
          0.8581,
          0.8148
        ],
        [
          0.8685,
          0.8085
        ],
        [
          0.8785,
          0.8057
        ],
        [
          0.8885,
          0.7998
        ],
        [
          0.8985,
          0.7953
        ],
        [
          0.9086,
          0.788
        ],
        [
          0.919,
          0.7816
        ],
        [
          0.929,
          0.7739
        ],
        [
          0.939,
          0.7662
        ],
        [
          0.949,
          0.7621
        ],
        [
          0.9591,
          0.7557
        ],
        [
          0.9695,
          0.7498
        ],
        [
          0.9795,
          0.7443
        ],
        [
          0.9895,
          0.7366
        ],
        [
          0.9995,
          0.7316
        ],
        [
          1.0,
          0.7311
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.14776905546275246,
        "difference": -0.000870067474754946,
        "worse": 0.5,
        "kept_right": 0.14688989155922894,
        "kept_wrong": 0.0008791639035235061,
        "handed_off": 0.8522309445372479
      },
      "usd_per_1000": 2.0616345726548344
    },
    {
      "decision": 119,
      "frontier": 277,
      "frontier_accuracy": 0.9795268425841674,
      "in_sample": {
        "threshold": 0.9999999924404349,
        "answered": 0.1305732484076433,
        "accuracy": 0.9795268425841674
      },
      "curve": [
        [
          0.0,
          0.9795
        ],
        [
          0.01,
          0.9795
        ],
        [
          0.02,
          0.9795
        ],
        [
          0.03,
          0.9795
        ],
        [
          0.04,
          0.9795
        ],
        [
          0.05,
          0.9795
        ],
        [
          0.0601,
          0.9795
        ],
        [
          0.0705,
          0.9795
        ],
        [
          0.0805,
          0.9795
        ],
        [
          0.0905,
          0.9795
        ],
        [
          0.1005,
          0.9795
        ],
        [
          0.1106,
          0.9795
        ],
        [
          0.121,
          0.9795
        ],
        [
          0.131,
          0.9791
        ],
        [
          0.141,
          0.9791
        ],
        [
          0.151,
          0.9786
        ],
        [
          0.1611,
          0.9786
        ],
        [
          0.1715,
          0.9786
        ],
        [
          0.1815,
          0.9773
        ],
        [
          0.1915,
          0.9768
        ],
        [
          0.2015,
          0.9768
        ],
        [
          0.2116,
          0.9768
        ],
        [
          0.222,
          0.9768
        ],
        [
          0.232,
          0.9768
        ],
        [
          0.242,
          0.9768
        ],
        [
          0.252,
          0.9763
        ],
        [
          0.2621,
          0.9768
        ],
        [
          0.2725,
          0.9768
        ],
        [
          0.2825,
          0.9759
        ],
        [
          0.2925,
          0.975
        ],
        [
          0.3025,
          0.975
        ],
        [
          0.3126,
          0.9741
        ],
        [
          0.323,
          0.9736
        ],
        [
          0.333,
          0.9727
        ],
        [
          0.343,
          0.9709
        ],
        [
          0.353,
          0.9695
        ],
        [
          0.3631,
          0.9686
        ],
        [
          0.3735,
          0.9682
        ],
        [
          0.3835,
          0.9677
        ],
        [
          0.3935,
          0.9672
        ],
        [
          0.4035,
          0.9677
        ],
        [
          0.4136,
          0.9672
        ],
        [
          0.424,
          0.9668
        ],
        [
          0.434,
          0.965
        ],
        [
          0.444,
          0.9636
        ],
        [
          0.454,
          0.9622
        ],
        [
          0.4641,
          0.9604
        ],
        [
          0.4745,
          0.9591
        ],
        [
          0.4845,
          0.9577
        ],
        [
          0.4945,
          0.9554
        ],
        [
          0.5045,
          0.9518
        ],
        [
          0.5146,
          0.9495
        ],
        [
          0.525,
          0.9468
        ],
        [
          0.535,
          0.9459
        ],
        [
          0.545,
          0.9422
        ],
        [
          0.5551,
          0.9399
        ],
        [
          0.5655,
          0.9368
        ],
        [
          0.5755,
          0.934
        ],
        [
          0.5855,
          0.9308
        ],
        [
          0.5955,
          0.929
        ],
        [
          0.6056,
          0.9268
        ],
        [
          0.616,
          0.9227
        ],
        [
          0.626,
          0.9208
        ],
        [
          0.636,
          0.9167
        ],
        [
          0.646,
          0.9126
        ],
        [
          0.6561,
          0.909
        ],
        [
          0.6665,
          0.9045
        ],
        [
          0.6765,
          0.9008
        ],
        [
          0.6865,
          0.8981
        ],
        [
          0.6965,
          0.894
        ],
        [
          0.7066,
          0.8913
        ],
        [
          0.717,
          0.8858
        ],
        [
          0.727,
          0.8826
        ],
        [
          0.737,
          0.8776
        ],
        [
          0.747,
          0.8744
        ],
        [
          0.7571,
          0.8708
        ],
        [
          0.7675,
          0.8676
        ],
        [
          0.7775,
          0.8626
        ],
        [
          0.7875,
          0.8558
        ],
        [
          0.7975,
          0.8503
        ],
        [
          0.8076,
          0.8449
        ],
        [
          0.818,
          0.8399
        ],
        [
          0.828,
          0.8335
        ],
        [
          0.838,
          0.828
        ],
        [
          0.848,
          0.8207
        ],
        [
          0.8581,
          0.8139
        ],
        [
          0.8685,
          0.8076
        ],
        [
          0.8785,
          0.8048
        ],
        [
          0.8885,
          0.7989
        ],
        [
          0.8985,
          0.7944
        ],
        [
          0.9086,
          0.7871
        ],
        [
          0.919,
          0.7807
        ],
        [
          0.929,
          0.773
        ],
        [
          0.939,
          0.7652
        ],
        [
          0.949,
          0.7611
        ],
        [
          0.9591,
          0.7548
        ],
        [
          0.9695,
          0.7489
        ],
        [
          0.9795,
          0.7443
        ],
        [
          0.9895,
          0.7366
        ],
        [
          0.9995,
          0.7316
        ],
        [
          1.0,
          0.7311
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.14885582398780722,
        "difference": -0.0009398334278664147,
        "worse": 0.5,
        "kept_right": 0.14791599055994084,
        "kept_wrong": 0.0009398334278664147,
        "handed_off": 0.8511441760121932
      },
      "usd_per_1000": 5.162661855127568
    },
    {
      "decision": 119,
      "frontier": 276,
      "frontier_accuracy": 0.9777070063694268,
      "in_sample": {
        "threshold": 0.9999999924404349,
        "answered": 0.1305732484076433,
        "accuracy": 0.9777070063694268
      },
      "curve": [
        [
          0.0,
          0.9777
        ],
        [
          0.01,
          0.9777
        ],
        [
          0.02,
          0.9777
        ],
        [
          0.03,
          0.9777
        ],
        [
          0.04,
          0.9777
        ],
        [
          0.05,
          0.9777
        ],
        [
          0.0601,
          0.9777
        ],
        [
          0.0705,
          0.9777
        ],
        [
          0.0805,
          0.9777
        ],
        [
          0.0905,
          0.9777
        ],
        [
          0.1005,
          0.9777
        ],
        [
          0.1106,
          0.9777
        ],
        [
          0.121,
          0.9777
        ],
        [
          0.131,
          0.9773
        ],
        [
          0.141,
          0.9773
        ],
        [
          0.151,
          0.9768
        ],
        [
          0.1611,
          0.9768
        ],
        [
          0.1715,
          0.9768
        ],
        [
          0.1815,
          0.9754
        ],
        [
          0.1915,
          0.975
        ],
        [
          0.2015,
          0.975
        ],
        [
          0.2116,
          0.975
        ],
        [
          0.222,
          0.975
        ],
        [
          0.232,
          0.975
        ],
        [
          0.242,
          0.975
        ],
        [
          0.252,
          0.9745
        ],
        [
          0.2621,
          0.975
        ],
        [
          0.2725,
          0.975
        ],
        [
          0.2825,
          0.9741
        ],
        [
          0.2925,
          0.9732
        ],
        [
          0.3025,
          0.9732
        ],
        [
          0.3126,
          0.9722
        ],
        [
          0.323,
          0.9718
        ],
        [
          0.333,
          0.9713
        ],
        [
          0.343,
          0.9695
        ],
        [
          0.353,
          0.9682
        ],
        [
          0.3631,
          0.9672
        ],
        [
          0.3735,
          0.9668
        ],
        [
          0.3835,
          0.9663
        ],
        [
          0.3935,
          0.9659
        ],
        [
          0.4035,
          0.9663
        ],
        [
          0.4136,
          0.9659
        ],
        [
          0.424,
          0.9654
        ],
        [
          0.434,
          0.9641
        ],
        [
          0.444,
          0.9627
        ],
        [
          0.454,
          0.9613
        ],
        [
          0.4641,
          0.9595
        ],
        [
          0.4745,
          0.9581
        ],
        [
          0.4845,
          0.9568
        ],
        [
          0.4945,
          0.9545
        ],
        [
          0.5045,
          0.9509
        ],
        [
          0.5146,
          0.9486
        ],
        [
          0.525,
          0.9454
        ],
        [
          0.535,
          0.9445
        ],
        [
          0.545,
          0.9409
        ],
        [
          0.5551,
          0.9386
        ],
        [
          0.5655,
          0.9354
        ],
        [
          0.5755,
          0.9327
        ],
        [
          0.5855,
          0.9295
        ],
        [
          0.5955,
          0.9277
        ],
        [
          0.6056,
          0.9254
        ],
        [
          0.616,
          0.9217
        ],
        [
          0.626,
          0.9199
        ],
        [
          0.636,
          0.9158
        ],
        [
          0.646,
          0.9117
        ],
        [
          0.6561,
          0.9086
        ],
        [
          0.6665,
          0.904
        ],
        [
          0.6765,
          0.9008
        ],
        [
          0.6865,
          0.8981
        ],
        [
          0.6965,
          0.8935
        ],
        [
          0.7066,
          0.8908
        ],
        [
          0.717,
          0.8854
        ],
        [
          0.727,
          0.8822
        ],
        [
          0.737,
          0.8776
        ],
        [
          0.747,
          0.8744
        ],
        [
          0.7571,
          0.8708
        ],
        [
          0.7675,
          0.8672
        ],
        [
          0.7775,
          0.8621
        ],
        [
          0.7875,
          0.8558
        ],
        [
          0.7975,
          0.8503
        ],
        [
          0.8076,
          0.8449
        ],
        [
          0.818,
          0.8399
        ],
        [
          0.828,
          0.8335
        ],
        [
          0.838,
          0.828
        ],
        [
          0.848,
          0.8207
        ],
        [
          0.8581,
          0.8139
        ],
        [
          0.8685,
          0.8076
        ],
        [
          0.8785,
          0.8048
        ],
        [
          0.8885,
          0.7989
        ],
        [
          0.8985,
          0.7944
        ],
        [
          0.9086,
          0.7871
        ],
        [
          0.919,
          0.7807
        ],
        [
          0.929,
          0.7734
        ],
        [
          0.939,
          0.7652
        ],
        [
          0.949,
          0.7607
        ],
        [
          0.9591,
          0.7543
        ],
        [
          0.9695,
          0.7484
        ],
        [
          0.9795,
          0.7443
        ],
        [
          0.9895,
          0.7366
        ],
        [
          0.9995,
          0.7316
        ],
        [
          1.0,
          0.7311
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.14885582398780722,
        "difference": -0.0009398334278664147,
        "worse": 0.5,
        "kept_right": 0.14791599055994084,
        "kept_wrong": 0.0009398334278664147,
        "handed_off": 0.8511441760121932
      },
      "usd_per_1000": 1.3807177180348031
    },
    {
      "decision": 119,
      "frontier": 48,
      "frontier_accuracy": 0.9758871701546861,
      "in_sample": {
        "threshold": 0.9999999924404349,
        "answered": 0.1305732484076433,
        "accuracy": 0.9758871701546861
      },
      "curve": [
        [
          0.0,
          0.9759
        ],
        [
          0.01,
          0.9759
        ],
        [
          0.02,
          0.9759
        ],
        [
          0.03,
          0.9759
        ],
        [
          0.04,
          0.9759
        ],
        [
          0.05,
          0.9759
        ],
        [
          0.0601,
          0.9759
        ],
        [
          0.0705,
          0.9759
        ],
        [
          0.0805,
          0.9759
        ],
        [
          0.0905,
          0.9759
        ],
        [
          0.1005,
          0.9759
        ],
        [
          0.1106,
          0.9759
        ],
        [
          0.121,
          0.9759
        ],
        [
          0.131,
          0.9754
        ],
        [
          0.141,
          0.9754
        ],
        [
          0.151,
          0.975
        ],
        [
          0.1611,
          0.975
        ],
        [
          0.1715,
          0.975
        ],
        [
          0.1815,
          0.9736
        ],
        [
          0.1915,
          0.9732
        ],
        [
          0.2015,
          0.9732
        ],
        [
          0.2116,
          0.9732
        ],
        [
          0.222,
          0.9732
        ],
        [
          0.232,
          0.9732
        ],
        [
          0.242,
          0.9732
        ],
        [
          0.252,
          0.9727
        ],
        [
          0.2621,
          0.9732
        ],
        [
          0.2725,
          0.9732
        ],
        [
          0.2825,
          0.9722
        ],
        [
          0.2925,
          0.9713
        ],
        [
          0.3025,
          0.9713
        ],
        [
          0.3126,
          0.9704
        ],
        [
          0.323,
          0.97
        ],
        [
          0.333,
          0.9691
        ],
        [
          0.343,
          0.9672
        ],
        [
          0.353,
          0.9659
        ],
        [
          0.3631,
          0.965
        ],
        [
          0.3735,
          0.9645
        ],
        [
          0.3835,
          0.9641
        ],
        [
          0.3935,
          0.9636
        ],
        [
          0.4035,
          0.9641
        ],
        [
          0.4136,
          0.9636
        ],
        [
          0.424,
          0.9631
        ],
        [
          0.434,
          0.9613
        ],
        [
          0.444,
          0.96
        ],
        [
          0.454,
          0.9586
        ],
        [
          0.4641,
          0.9568
        ],
        [
          0.4745,
          0.9554
        ],
        [
          0.4845,
          0.954
        ],
        [
          0.4945,
          0.9518
        ],
        [
          0.5045,
          0.9481
        ],
        [
          0.5146,
          0.9459
        ],
        [
          0.525,
          0.9431
        ],
        [
          0.535,
          0.9422
        ],
        [
          0.545,
          0.9386
        ],
        [
          0.5551,
          0.9363
        ],
        [
          0.5655,
          0.9331
        ],
        [
          0.5755,
          0.9304
        ],
        [
          0.5855,
          0.9272
        ],
        [
          0.5955,
          0.9254
        ],
        [
          0.6056,
          0.9231
        ],
        [
          0.616,
          0.9195
        ],
        [
          0.626,
          0.9177
        ],
        [
          0.636,
          0.9136
        ],
        [
          0.646,
          0.9099
        ],
        [
          0.6561,
          0.9063
        ],
        [
          0.6665,
          0.9017
        ],
        [
          0.6765,
          0.8976
        ],
        [
          0.6865,
          0.8949
        ],
        [
          0.6965,
          0.8908
        ],
        [
          0.7066,
          0.8881
        ],
        [
          0.717,
          0.8835
        ],
        [
          0.727,
          0.8803
        ],
        [
          0.737,
          0.8758
        ],
        [
          0.747,
          0.8731
        ],
        [
          0.7571,
          0.869
        ],
        [
          0.7675,
          0.8658
        ],
        [
          0.7775,
          0.8612
        ],
        [
          0.7875,
          0.8553
        ],
        [
          0.7975,
          0.8499
        ],
        [
          0.8076,
          0.8444
        ],
        [
          0.818,
          0.8394
        ],
        [
          0.828,
          0.833
        ],
        [
          0.838,
          0.8276
        ],
        [
          0.848,
          0.8203
        ],
        [
          0.8581,
          0.8135
        ],
        [
          0.8685,
          0.8071
        ],
        [
          0.8785,
          0.8044
        ],
        [
          0.8885,
          0.7985
        ],
        [
          0.8985,
          0.7939
        ],
        [
          0.9086,
          0.7866
        ],
        [
          0.919,
          0.7803
        ],
        [
          0.929,
          0.7725
        ],
        [
          0.939,
          0.7643
        ],
        [
          0.949,
          0.7598
        ],
        [
          0.9591,
          0.7539
        ],
        [
          0.9695,
          0.748
        ],
        [
          0.9795,
          0.7439
        ],
        [
          0.9895,
          0.7361
        ],
        [
          0.9995,
          0.7316
        ],
        [
          1.0,
          0.7311
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.14885582398780722,
        "difference": -0.0009398334278664147,
        "worse": 0.5,
        "kept_right": 0.14791599055994084,
        "kept_wrong": 0.0009398334278664147,
        "handed_off": 0.8511441760121932
      },
      "usd_per_1000": 4.577645945149231
    },
    {
      "decision": 119,
      "frontier": 191,
      "frontier_accuracy": 0.9740673339399454,
      "in_sample": {
        "threshold": 0.9999999924404349,
        "answered": 0.1305732484076433,
        "accuracy": 0.9740673339399454
      },
      "curve": [
        [
          0.0,
          0.9741
        ],
        [
          0.01,
          0.9741
        ],
        [
          0.02,
          0.9741
        ],
        [
          0.03,
          0.9741
        ],
        [
          0.04,
          0.9741
        ],
        [
          0.05,
          0.9741
        ],
        [
          0.0601,
          0.9741
        ],
        [
          0.0705,
          0.9741
        ],
        [
          0.0805,
          0.9741
        ],
        [
          0.0905,
          0.9741
        ],
        [
          0.1005,
          0.9741
        ],
        [
          0.1106,
          0.9741
        ],
        [
          0.121,
          0.9741
        ],
        [
          0.131,
          0.9736
        ],
        [
          0.141,
          0.9736
        ],
        [
          0.151,
          0.9732
        ],
        [
          0.1611,
          0.9732
        ],
        [
          0.1715,
          0.9732
        ],
        [
          0.1815,
          0.9718
        ],
        [
          0.1915,
          0.9713
        ],
        [
          0.2015,
          0.9713
        ],
        [
          0.2116,
          0.9713
        ],
        [
          0.222,
          0.9713
        ],
        [
          0.232,
          0.9713
        ],
        [
          0.242,
          0.9713
        ],
        [
          0.252,
          0.9709
        ],
        [
          0.2621,
          0.9709
        ],
        [
          0.2725,
          0.9709
        ],
        [
          0.2825,
          0.97
        ],
        [
          0.2925,
          0.9691
        ],
        [
          0.3025,
          0.9691
        ],
        [
          0.3126,
          0.9682
        ],
        [
          0.323,
          0.9677
        ],
        [
          0.333,
          0.9672
        ],
        [
          0.343,
          0.9654
        ],
        [
          0.353,
          0.9641
        ],
        [
          0.3631,
          0.9631
        ],
        [
          0.3735,
          0.9627
        ],
        [
          0.3835,
          0.9627
        ],
        [
          0.3935,
          0.9622
        ],
        [
          0.4035,
          0.9627
        ],
        [
          0.4136,
          0.9622
        ],
        [
          0.424,
          0.9618
        ],
        [
          0.434,
          0.96
        ],
        [
          0.444,
          0.9586
        ],
        [
          0.454,
          0.9572
        ],
        [
          0.4641,
          0.9554
        ],
        [
          0.4745,
          0.954
        ],
        [
          0.4845,
          0.9527
        ],
        [
          0.4945,
          0.9504
        ],
        [
          0.5045,
          0.9468
        ],
        [
          0.5146,
          0.9445
        ],
        [
          0.525,
          0.9413
        ],
        [
          0.535,
          0.9404
        ],
        [
          0.545,
          0.9368
        ],
        [
          0.5551,
          0.9349
        ],
        [
          0.5655,
          0.9322
        ],
        [
          0.5755,
          0.9295
        ],
        [
          0.5855,
          0.9268
        ],
        [
          0.5955,
          0.9249
        ],
        [
          0.6056,
          0.9227
        ],
        [
          0.616,
          0.9186
        ],
        [
          0.626,
          0.9167
        ],
        [
          0.636,
          0.9131
        ],
        [
          0.646,
          0.9095
        ],
        [
          0.6561,
          0.9063
        ],
        [
          0.6665,
          0.9017
        ],
        [
          0.6765,
          0.8976
        ],
        [
          0.6865,
          0.8949
        ],
        [
          0.6965,
          0.8908
        ],
        [
          0.7066,
          0.8881
        ],
        [
          0.717,
          0.8831
        ],
        [
          0.727,
          0.8799
        ],
        [
          0.737,
          0.8749
        ],
        [
          0.747,
          0.8717
        ],
        [
          0.7571,
          0.8685
        ],
        [
          0.7675,
          0.8649
        ],
        [
          0.7775,
          0.8599
        ],
        [
          0.7875,
          0.854
        ],
        [
          0.7975,
          0.8485
        ],
        [
          0.8076,
          0.843
        ],
        [
          0.818,
          0.838
        ],
        [
          0.828,
          0.8317
        ],
        [
          0.838,
          0.8262
        ],
        [
          0.848,
          0.8189
        ],
        [
          0.8581,
          0.8121
        ],
        [
          0.8685,
          0.8053
        ],
        [
          0.8785,
          0.8025
        ],
        [
          0.8885,
          0.7966
        ],
        [
          0.8985,
          0.7925
        ],
        [
          0.9086,
          0.7853
        ],
        [
          0.919,
          0.7789
        ],
        [
          0.929,
          0.7716
        ],
        [
          0.939,
          0.7639
        ],
        [
          0.949,
          0.7598
        ],
        [
          0.9591,
          0.7534
        ],
        [
          0.9695,
          0.7475
        ],
        [
          0.9795,
          0.7439
        ],
        [
          0.9895,
          0.7361
        ],
        [
          0.9995,
          0.7316
        ],
        [
          1.0,
          0.7311
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.14615379414273794,
        "difference": -0.0008670399140395683,
        "worse": 0.5,
        "kept_right": 0.14528675422869833,
        "kept_wrong": 0.0008670399140395683,
        "handed_off": 0.8538462058572625
      },
      "usd_per_1000": 22.82988035755018
    },
    {
      "decision": 119,
      "frontier": 188,
      "frontier_accuracy": 0.9713375796178344,
      "in_sample": {
        "threshold": 0.9999999924404349,
        "answered": 0.1305732484076433,
        "accuracy": 0.9713375796178344
      },
      "curve": [
        [
          0.0,
          0.9713
        ],
        [
          0.01,
          0.9713
        ],
        [
          0.02,
          0.9713
        ],
        [
          0.03,
          0.9713
        ],
        [
          0.04,
          0.9713
        ],
        [
          0.05,
          0.9713
        ],
        [
          0.0601,
          0.9713
        ],
        [
          0.0705,
          0.9713
        ],
        [
          0.0805,
          0.9713
        ],
        [
          0.0905,
          0.9713
        ],
        [
          0.1005,
          0.9713
        ],
        [
          0.1106,
          0.9713
        ],
        [
          0.121,
          0.9713
        ],
        [
          0.131,
          0.9709
        ],
        [
          0.141,
          0.9709
        ],
        [
          0.151,
          0.9704
        ],
        [
          0.1611,
          0.9704
        ],
        [
          0.1715,
          0.9704
        ],
        [
          0.1815,
          0.9691
        ],
        [
          0.1915,
          0.9691
        ],
        [
          0.2015,
          0.9691
        ],
        [
          0.2116,
          0.9691
        ],
        [
          0.222,
          0.9691
        ],
        [
          0.232,
          0.9691
        ],
        [
          0.242,
          0.9691
        ],
        [
          0.252,
          0.9691
        ],
        [
          0.2621,
          0.9691
        ],
        [
          0.2725,
          0.9691
        ],
        [
          0.2825,
          0.9682
        ],
        [
          0.2925,
          0.9677
        ],
        [
          0.3025,
          0.9682
        ],
        [
          0.3126,
          0.9677
        ],
        [
          0.323,
          0.9672
        ],
        [
          0.333,
          0.9668
        ],
        [
          0.343,
          0.965
        ],
        [
          0.353,
          0.9636
        ],
        [
          0.3631,
          0.9627
        ],
        [
          0.3735,
          0.9622
        ],
        [
          0.3835,
          0.9618
        ],
        [
          0.3935,
          0.9613
        ],
        [
          0.4035,
          0.9613
        ],
        [
          0.4136,
          0.9609
        ],
        [
          0.424,
          0.9604
        ],
        [
          0.434,
          0.9586
        ],
        [
          0.444,
          0.9572
        ],
        [
          0.454,
          0.9559
        ],
        [
          0.4641,
          0.9545
        ],
        [
          0.4745,
          0.9531
        ],
        [
          0.4845,
          0.9518
        ],
        [
          0.4945,
          0.9495
        ],
        [
          0.5045,
          0.9459
        ],
        [
          0.5146,
          0.9445
        ],
        [
          0.525,
          0.9418
        ],
        [
          0.535,
          0.9413
        ],
        [
          0.545,
          0.9372
        ],
        [
          0.5551,
          0.9349
        ],
        [
          0.5655,
          0.9322
        ],
        [
          0.5755,
          0.9295
        ],
        [
          0.5855,
          0.9263
        ],
        [
          0.5955,
          0.9245
        ],
        [
          0.6056,
          0.9222
        ],
        [
          0.616,
          0.9186
        ],
        [
          0.626,
          0.9167
        ],
        [
          0.636,
          0.9126
        ],
        [
          0.646,
          0.909
        ],
        [
          0.6561,
          0.9058
        ],
        [
          0.6665,
          0.9013
        ],
        [
          0.6765,
          0.8976
        ],
        [
          0.6865,
          0.8949
        ],
        [
          0.6965,
          0.8899
        ],
        [
          0.7066,
          0.8876
        ],
        [
          0.717,
          0.8826
        ],
        [
          0.727,
          0.8794
        ],
        [
          0.737,
          0.8744
        ],
        [
          0.747,
          0.8712
        ],
        [
          0.7571,
          0.8672
        ],
        [
          0.7675,
          0.8635
        ],
        [
          0.7775,
          0.8585
        ],
        [
          0.7875,
          0.8526
        ],
        [
          0.7975,
          0.8471
        ],
        [
          0.8076,
          0.8417
        ],
        [
          0.818,
          0.8367
        ],
        [
          0.828,
          0.8303
        ],
        [
          0.838,
          0.8248
        ],
        [
          0.848,
          0.818
        ],
        [
          0.8581,
          0.8116
        ],
        [
          0.8685,
          0.8053
        ],
        [
          0.8785,
          0.8025
        ],
        [
          0.8885,
          0.7971
        ],
        [
          0.8985,
          0.7916
        ],
        [
          0.9086,
          0.7848
        ],
        [
          0.919,
          0.7789
        ],
        [
          0.929,
          0.7712
        ],
        [
          0.939,
          0.7639
        ],
        [
          0.949,
          0.7607
        ],
        [
          0.9591,
          0.7548
        ],
        [
          0.9695,
          0.7489
        ],
        [
          0.9795,
          0.7434
        ],
        [
          0.9895,
          0.7361
        ],
        [
          0.9995,
          0.7316
        ],
        [
          1.0,
          0.7311
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1519482066945511,
        "difference": -0.0010095829675147018,
        "worse": 0.5,
        "kept_right": 0.15086586877073052,
        "kept_wrong": 0.0010823379238206641,
        "handed_off": 0.8480517933054492
      },
      "usd_per_1000": 0.8170468769855304
    },
    {
      "decision": 119,
      "frontier": 190,
      "frontier_accuracy": 0.9672429481346679,
      "in_sample": {
        "threshold": 0.9999999907788629,
        "answered": 0.1464968152866242,
        "accuracy": 0.9672429481346679
      },
      "curve": [
        [
          0.0,
          0.9672
        ],
        [
          0.01,
          0.9672
        ],
        [
          0.02,
          0.9672
        ],
        [
          0.03,
          0.9672
        ],
        [
          0.04,
          0.9672
        ],
        [
          0.05,
          0.9672
        ],
        [
          0.0601,
          0.9672
        ],
        [
          0.0705,
          0.9672
        ],
        [
          0.0805,
          0.9672
        ],
        [
          0.0905,
          0.9672
        ],
        [
          0.1005,
          0.9672
        ],
        [
          0.1106,
          0.9672
        ],
        [
          0.121,
          0.9672
        ],
        [
          0.131,
          0.9668
        ],
        [
          0.141,
          0.9672
        ],
        [
          0.151,
          0.9668
        ],
        [
          0.1611,
          0.9668
        ],
        [
          0.1715,
          0.9668
        ],
        [
          0.1815,
          0.9654
        ],
        [
          0.1915,
          0.965
        ],
        [
          0.2015,
          0.965
        ],
        [
          0.2116,
          0.965
        ],
        [
          0.222,
          0.965
        ],
        [
          0.232,
          0.965
        ],
        [
          0.242,
          0.965
        ],
        [
          0.252,
          0.9645
        ],
        [
          0.2621,
          0.965
        ],
        [
          0.2725,
          0.965
        ],
        [
          0.2825,
          0.9641
        ],
        [
          0.2925,
          0.9631
        ],
        [
          0.3025,
          0.9631
        ],
        [
          0.3126,
          0.9622
        ],
        [
          0.323,
          0.9618
        ],
        [
          0.333,
          0.9613
        ],
        [
          0.343,
          0.9595
        ],
        [
          0.353,
          0.9581
        ],
        [
          0.3631,
          0.9572
        ],
        [
          0.3735,
          0.9568
        ],
        [
          0.3835,
          0.9563
        ],
        [
          0.3935,
          0.9563
        ],
        [
          0.4035,
          0.9568
        ],
        [
          0.4136,
          0.9563
        ],
        [
          0.424,
          0.9559
        ],
        [
          0.434,
          0.954
        ],
        [
          0.444,
          0.9527
        ],
        [
          0.454,
          0.9513
        ],
        [
          0.4641,
          0.9495
        ],
        [
          0.4745,
          0.9481
        ],
        [
          0.4845,
          0.9468
        ],
        [
          0.4945,
          0.9445
        ],
        [
          0.5045,
          0.9409
        ],
        [
          0.5146,
          0.9386
        ],
        [
          0.525,
          0.9354
        ],
        [
          0.535,
          0.9349
        ],
        [
          0.545,
          0.9313
        ],
        [
          0.5551,
          0.929
        ],
        [
          0.5655,
          0.9263
        ],
        [
          0.5755,
          0.9236
        ],
        [
          0.5855,
          0.9208
        ],
        [
          0.5955,
          0.9195
        ],
        [
          0.6056,
          0.9172
        ],
        [
          0.616,
          0.9126
        ],
        [
          0.626,
          0.9108
        ],
        [
          0.636,
          0.9067
        ],
        [
          0.646,
          0.9026
        ],
        [
          0.6561,
          0.899
        ],
        [
          0.6665,
          0.8949
        ],
        [
          0.6765,
          0.8913
        ],
        [
          0.6865,
          0.8885
        ],
        [
          0.6965,
          0.8849
        ],
        [
          0.7066,
          0.8822
        ],
        [
          0.717,
          0.8772
        ],
        [
          0.727,
          0.8744
        ],
        [
          0.737,
          0.8703
        ],
        [
          0.747,
          0.8676
        ],
        [
          0.7571,
          0.8644
        ],
        [
          0.7675,
          0.8612
        ],
        [
          0.7775,
          0.8562
        ],
        [
          0.7875,
          0.8508
        ],
        [
          0.7975,
          0.8453
        ],
        [
          0.8076,
          0.8399
        ],
        [
          0.818,
          0.8348
        ],
        [
          0.828,
          0.8285
        ],
        [
          0.838,
          0.8235
        ],
        [
          0.848,
          0.8167
        ],
        [
          0.8581,
          0.8098
        ],
        [
          0.8685,
          0.8035
        ],
        [
          0.8785,
          0.8007
        ],
        [
          0.8885,
          0.7948
        ],
        [
          0.8985,
          0.7907
        ],
        [
          0.9086,
          0.7834
        ],
        [
          0.919,
          0.7771
        ],
        [
          0.929,
          0.7702
        ],
        [
          0.939,
          0.763
        ],
        [
          0.949,
          0.7593
        ],
        [
          0.9591,
          0.7534
        ],
        [
          0.9695,
          0.748
        ],
        [
          0.9795,
          0.7439
        ],
        [
          0.9895,
          0.7361
        ],
        [
          0.9995,
          0.7316
        ],
        [
          1.0,
          0.7311
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.16908697456346072,
        "difference": -0.001261266546977777,
        "worse": 0.49666666666666665,
        "kept_right": 0.1675650078606535,
        "kept_wrong": 0.001521966702807245,
        "handed_off": 0.8309130254365388
      },
      "usd_per_1000": 4.626954983856047
    },
    {
      "decision": 119,
      "frontier": 124,
      "frontier_accuracy": 0.9667879890809827,
      "in_sample": {
        "threshold": 0.9999999924404349,
        "answered": 0.1305732484076433,
        "accuracy": 0.9667879890809827
      },
      "curve": [
        [
          0.0,
          0.9668
        ],
        [
          0.01,
          0.9668
        ],
        [
          0.02,
          0.9668
        ],
        [
          0.03,
          0.9668
        ],
        [
          0.04,
          0.9668
        ],
        [
          0.05,
          0.9668
        ],
        [
          0.0601,
          0.9668
        ],
        [
          0.0705,
          0.9668
        ],
        [
          0.0805,
          0.9668
        ],
        [
          0.0905,
          0.9668
        ],
        [
          0.1005,
          0.9668
        ],
        [
          0.1106,
          0.9668
        ],
        [
          0.121,
          0.9668
        ],
        [
          0.131,
          0.9663
        ],
        [
          0.141,
          0.9663
        ],
        [
          0.151,
          0.9659
        ],
        [
          0.1611,
          0.9659
        ],
        [
          0.1715,
          0.9659
        ],
        [
          0.1815,
          0.9645
        ],
        [
          0.1915,
          0.9641
        ],
        [
          0.2015,
          0.9641
        ],
        [
          0.2116,
          0.9641
        ],
        [
          0.222,
          0.9641
        ],
        [
          0.232,
          0.9641
        ],
        [
          0.242,
          0.9641
        ],
        [
          0.252,
          0.9636
        ],
        [
          0.2621,
          0.9641
        ],
        [
          0.2725,
          0.9641
        ],
        [
          0.2825,
          0.9631
        ],
        [
          0.2925,
          0.9622
        ],
        [
          0.3025,
          0.9622
        ],
        [
          0.3126,
          0.9613
        ],
        [
          0.323,
          0.9609
        ],
        [
          0.333,
          0.96
        ],
        [
          0.343,
          0.9581
        ],
        [
          0.353,
          0.9568
        ],
        [
          0.3631,
          0.9559
        ],
        [
          0.3735,
          0.9554
        ],
        [
          0.3835,
          0.955
        ],
        [
          0.3935,
          0.9545
        ],
        [
          0.4035,
          0.955
        ],
        [
          0.4136,
          0.9545
        ],
        [
          0.424,
          0.954
        ],
        [
          0.434,
          0.9522
        ],
        [
          0.444,
          0.9509
        ],
        [
          0.454,
          0.9495
        ],
        [
          0.4641,
          0.9477
        ],
        [
          0.4745,
          0.9463
        ],
        [
          0.4845,
          0.9449
        ],
        [
          0.4945,
          0.9427
        ],
        [
          0.5045,
          0.939
        ],
        [
          0.5146,
          0.9368
        ],
        [
          0.525,
          0.9345
        ],
        [
          0.535,
          0.9336
        ],
        [
          0.545,
          0.9299
        ],
        [
          0.5551,
          0.9281
        ],
        [
          0.5655,
          0.9249
        ],
        [
          0.5755,
          0.9222
        ],
        [
          0.5855,
          0.9195
        ],
        [
          0.5955,
          0.9177
        ],
        [
          0.6056,
          0.9154
        ],
        [
          0.616,
          0.9117
        ],
        [
          0.626,
          0.9099
        ],
        [
          0.636,
          0.9058
        ],
        [
          0.646,
          0.9017
        ],
        [
          0.6561,
          0.8985
        ],
        [
          0.6665,
          0.8954
        ],
        [
          0.6765,
          0.8926
        ],
        [
          0.6865,
          0.8899
        ],
        [
          0.6965,
          0.8858
        ],
        [
          0.7066,
          0.8835
        ],
        [
          0.717,
          0.8785
        ],
        [
          0.727,
          0.8753
        ],
        [
          0.737,
          0.8708
        ],
        [
          0.747,
          0.8676
        ],
        [
          0.7571,
          0.864
        ],
        [
          0.7675,
          0.8608
        ],
        [
          0.7775,
          0.8558
        ],
        [
          0.7875,
          0.8499
        ],
        [
          0.7975,
          0.8453
        ],
        [
          0.8076,
          0.8399
        ],
        [
          0.818,
          0.8348
        ],
        [
          0.828,
          0.8285
        ],
        [
          0.838,
          0.823
        ],
        [
          0.848,
          0.8162
        ],
        [
          0.8581,
          0.8098
        ],
        [
          0.8685,
          0.8039
        ],
        [
          0.8785,
          0.8012
        ],
        [
          0.8885,
          0.7957
        ],
        [
          0.8985,
          0.7912
        ],
        [
          0.9086,
          0.7839
        ],
        [
          0.919,
          0.778
        ],
        [
          0.929,
          0.7707
        ],
        [
          0.939,
          0.7634
        ],
        [
          0.949,
          0.7598
        ],
        [
          0.9591,
          0.7543
        ],
        [
          0.9695,
          0.7484
        ],
        [
          0.9795,
          0.7439
        ],
        [
          0.9895,
          0.7361
        ],
        [
          0.9995,
          0.7316
        ],
        [
          1.0,
          0.7311
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.14885582398780722,
        "difference": -0.0009398334278664147,
        "worse": 0.5,
        "kept_right": 0.14791599055994084,
        "kept_wrong": 0.0009398334278664147,
        "handed_off": 0.8511441760121932
      },
      "usd_per_1000": 0.9621104522078497
    },
    {
      "decision": 119,
      "frontier": 278,
      "frontier_accuracy": 0.9499545040946314,
      "in_sample": {
        "threshold": 0.9999999924404349,
        "answered": 0.1305732484076433,
        "accuracy": 0.9499545040946314
      },
      "curve": [
        [
          0.0,
          0.95
        ],
        [
          0.01,
          0.95
        ],
        [
          0.02,
          0.95
        ],
        [
          0.03,
          0.95
        ],
        [
          0.04,
          0.95
        ],
        [
          0.05,
          0.95
        ],
        [
          0.0601,
          0.95
        ],
        [
          0.0705,
          0.95
        ],
        [
          0.0805,
          0.95
        ],
        [
          0.0905,
          0.95
        ],
        [
          0.1005,
          0.95
        ],
        [
          0.1106,
          0.95
        ],
        [
          0.121,
          0.95
        ],
        [
          0.131,
          0.9495
        ],
        [
          0.141,
          0.9495
        ],
        [
          0.151,
          0.949
        ],
        [
          0.1611,
          0.949
        ],
        [
          0.1715,
          0.949
        ],
        [
          0.1815,
          0.9477
        ],
        [
          0.1915,
          0.9472
        ],
        [
          0.2015,
          0.9472
        ],
        [
          0.2116,
          0.9472
        ],
        [
          0.222,
          0.9472
        ],
        [
          0.232,
          0.9472
        ],
        [
          0.242,
          0.9472
        ],
        [
          0.252,
          0.9468
        ],
        [
          0.2621,
          0.9472
        ],
        [
          0.2725,
          0.9472
        ],
        [
          0.2825,
          0.9468
        ],
        [
          0.2925,
          0.9459
        ],
        [
          0.3025,
          0.9459
        ],
        [
          0.3126,
          0.9449
        ],
        [
          0.323,
          0.9445
        ],
        [
          0.333,
          0.9445
        ],
        [
          0.343,
          0.9431
        ],
        [
          0.353,
          0.9418
        ],
        [
          0.3631,
          0.9409
        ],
        [
          0.3735,
          0.9404
        ],
        [
          0.3835,
          0.9409
        ],
        [
          0.3935,
          0.9404
        ],
        [
          0.4035,
          0.9413
        ],
        [
          0.4136,
          0.9409
        ],
        [
          0.424,
          0.9404
        ],
        [
          0.434,
          0.9386
        ],
        [
          0.444,
          0.9372
        ],
        [
          0.454,
          0.9363
        ],
        [
          0.4641,
          0.9354
        ],
        [
          0.4745,
          0.934
        ],
        [
          0.4845,
          0.9331
        ],
        [
          0.4945,
          0.9318
        ],
        [
          0.5045,
          0.9281
        ],
        [
          0.5146,
          0.9268
        ],
        [
          0.525,
          0.9249
        ],
        [
          0.535,
          0.924
        ],
        [
          0.545,
          0.9204
        ],
        [
          0.5551,
          0.9181
        ],
        [
          0.5655,
          0.9158
        ],
        [
          0.5755,
          0.9131
        ],
        [
          0.5855,
          0.9108
        ],
        [
          0.5955,
          0.909
        ],
        [
          0.6056,
          0.9067
        ],
        [
          0.616,
          0.9026
        ],
        [
          0.626,
          0.9013
        ],
        [
          0.636,
          0.8976
        ],
        [
          0.646,
          0.8949
        ],
        [
          0.6561,
          0.8917
        ],
        [
          0.6665,
          0.8881
        ],
        [
          0.6765,
          0.8849
        ],
        [
          0.6865,
          0.8822
        ],
        [
          0.6965,
          0.8781
        ],
        [
          0.7066,
          0.8758
        ],
        [
          0.717,
          0.8717
        ],
        [
          0.727,
          0.8685
        ],
        [
          0.737,
          0.8635
        ],
        [
          0.747,
          0.8608
        ],
        [
          0.7571,
          0.8576
        ],
        [
          0.7675,
          0.854
        ],
        [
          0.7775,
          0.849
        ],
        [
          0.7875,
          0.8435
        ],
        [
          0.7975,
          0.8385
        ],
        [
          0.8076,
          0.833
        ],
        [
          0.818,
          0.8285
        ],
        [
          0.828,
          0.8226
        ],
        [
          0.838,
          0.8167
        ],
        [
          0.848,
          0.8098
        ],
        [
          0.8581,
          0.8035
        ],
        [
          0.8685,
          0.7975
        ],
        [
          0.8785,
          0.7948
        ],
        [
          0.8885,
          0.7903
        ],
        [
          0.8985,
          0.7857
        ],
        [
          0.9086,
          0.7798
        ],
        [
          0.919,
          0.7739
        ],
        [
          0.929,
          0.7666
        ],
        [
          0.939,
          0.7598
        ],
        [
          0.949,
          0.7571
        ],
        [
          0.9591,
          0.7525
        ],
        [
          0.9695,
          0.7466
        ],
        [
          0.9795,
          0.7425
        ],
        [
          0.9895,
          0.7352
        ],
        [
          0.9995,
          0.7316
        ],
        [
          1.0,
          0.7311
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1506392678626765,
        "difference": -0.0010246132047559975,
        "worse": 0.5033333333333333,
        "kept_right": 0.1496055802477572,
        "kept_wrong": 0.001033687614919337,
        "handed_off": 0.849360732137324
      },
      "usd_per_1000": 5.528341468663125
    },
    {
      "decision": 119,
      "frontier": 275,
      "frontier_accuracy": 0.9426751592356688,
      "in_sample": {
        "threshold": 0.9999999907788629,
        "answered": 0.1464968152866242,
        "accuracy": 0.9426751592356688
      },
      "curve": [
        [
          0.0,
          0.9427
        ],
        [
          0.01,
          0.9427
        ],
        [
          0.02,
          0.9427
        ],
        [
          0.03,
          0.9427
        ],
        [
          0.04,
          0.9427
        ],
        [
          0.05,
          0.9427
        ],
        [
          0.0601,
          0.9427
        ],
        [
          0.0705,
          0.9427
        ],
        [
          0.0805,
          0.9427
        ],
        [
          0.0905,
          0.9427
        ],
        [
          0.1005,
          0.9431
        ],
        [
          0.1106,
          0.9431
        ],
        [
          0.121,
          0.9431
        ],
        [
          0.131,
          0.9427
        ],
        [
          0.141,
          0.9427
        ],
        [
          0.151,
          0.9422
        ],
        [
          0.1611,
          0.9422
        ],
        [
          0.1715,
          0.9422
        ],
        [
          0.1815,
          0.9409
        ],
        [
          0.1915,
          0.9404
        ],
        [
          0.2015,
          0.9404
        ],
        [
          0.2116,
          0.9404
        ],
        [
          0.222,
          0.9404
        ],
        [
          0.232,
          0.9404
        ],
        [
          0.242,
          0.9404
        ],
        [
          0.252,
          0.9399
        ],
        [
          0.2621,
          0.9409
        ],
        [
          0.2725,
          0.9409
        ],
        [
          0.2825,
          0.9399
        ],
        [
          0.2925,
          0.939
        ],
        [
          0.3025,
          0.939
        ],
        [
          0.3126,
          0.9381
        ],
        [
          0.323,
          0.9377
        ],
        [
          0.333,
          0.9368
        ],
        [
          0.343,
          0.9349
        ],
        [
          0.353,
          0.9336
        ],
        [
          0.3631,
          0.9327
        ],
        [
          0.3735,
          0.9322
        ],
        [
          0.3835,
          0.9322
        ],
        [
          0.3935,
          0.9318
        ],
        [
          0.4035,
          0.9322
        ],
        [
          0.4136,
          0.9322
        ],
        [
          0.424,
          0.9318
        ],
        [
          0.434,
          0.9299
        ],
        [
          0.444,
          0.9286
        ],
        [
          0.454,
          0.9277
        ],
        [
          0.4641,
          0.9263
        ],
        [
          0.4745,
          0.9254
        ],
        [
          0.4845,
          0.9245
        ],
        [
          0.4945,
          0.9231
        ],
        [
          0.5045,
          0.9199
        ],
        [
          0.5146,
          0.9186
        ],
        [
          0.525,
          0.9167
        ],
        [
          0.535,
          0.9167
        ],
        [
          0.545,
          0.9149
        ],
        [
          0.5551,
          0.9136
        ],
        [
          0.5655,
          0.9113
        ],
        [
          0.5755,
          0.9086
        ],
        [
          0.5855,
          0.9063
        ],
        [
          0.5955,
          0.9045
        ],
        [
          0.6056,
          0.9022
        ],
        [
          0.616,
          0.8985
        ],
        [
          0.626,
          0.8963
        ],
        [
          0.636,
          0.8926
        ],
        [
          0.646,
          0.8899
        ],
        [
          0.6561,
          0.8872
        ],
        [
          0.6665,
          0.8831
        ],
        [
          0.6765,
          0.8794
        ],
        [
          0.6865,
          0.8772
        ],
        [
          0.6965,
          0.8735
        ],
        [
          0.7066,
          0.8708
        ],
        [
          0.717,
          0.8658
        ],
        [
          0.727,
          0.8626
        ],
        [
          0.737,
          0.8594
        ],
        [
          0.747,
          0.8571
        ],
        [
          0.7571,
          0.854
        ],
        [
          0.7675,
          0.8503
        ],
        [
          0.7775,
          0.8462
        ],
        [
          0.7875,
          0.8412
        ],
        [
          0.7975,
          0.8371
        ],
        [
          0.8076,
          0.8317
        ],
        [
          0.818,
          0.8271
        ],
        [
          0.828,
          0.8221
        ],
        [
          0.838,
          0.8167
        ],
        [
          0.848,
          0.8098
        ],
        [
          0.8581,
          0.8039
        ],
        [
          0.8685,
          0.7975
        ],
        [
          0.8785,
          0.7948
        ],
        [
          0.8885,
          0.7916
        ],
        [
          0.8985,
          0.7871
        ],
        [
          0.9086,
          0.7807
        ],
        [
          0.919,
          0.7748
        ],
        [
          0.929,
          0.768
        ],
        [
          0.939,
          0.7607
        ],
        [
          0.949,
          0.7571
        ],
        [
          0.9591,
          0.7516
        ],
        [
          0.9695,
          0.7461
        ],
        [
          0.9795,
          0.7425
        ],
        [
          0.9895,
          0.7348
        ],
        [
          0.9995,
          0.7316
        ],
        [
          1.0,
          0.7311
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.17586312705738538,
        "difference": -0.0011884508889155661,
        "worse": 0.51,
        "kept_right": 0.17414414917254353,
        "kept_wrong": 0.001718977884841822,
        "handed_off": 0.8241368729426147
      },
      "usd_per_1000": 0.42402929463639805
    },
    {
      "decision": 119,
      "frontier": 192,
      "frontier_accuracy": 0.9331210191082803,
      "in_sample": {
        "threshold": 0.9999999924404349,
        "answered": 0.1305732484076433,
        "accuracy": 0.9331210191082803
      },
      "curve": [
        [
          0.0,
          0.9331
        ],
        [
          0.01,
          0.9331
        ],
        [
          0.02,
          0.9331
        ],
        [
          0.03,
          0.9331
        ],
        [
          0.04,
          0.9331
        ],
        [
          0.05,
          0.9331
        ],
        [
          0.0601,
          0.9331
        ],
        [
          0.0705,
          0.9331
        ],
        [
          0.0805,
          0.9331
        ],
        [
          0.0905,
          0.9331
        ],
        [
          0.1005,
          0.9331
        ],
        [
          0.1106,
          0.9331
        ],
        [
          0.121,
          0.9331
        ],
        [
          0.131,
          0.9327
        ],
        [
          0.141,
          0.9327
        ],
        [
          0.151,
          0.9322
        ],
        [
          0.1611,
          0.9322
        ],
        [
          0.1715,
          0.9322
        ],
        [
          0.1815,
          0.9308
        ],
        [
          0.1915,
          0.9304
        ],
        [
          0.2015,
          0.9304
        ],
        [
          0.2116,
          0.9304
        ],
        [
          0.222,
          0.9304
        ],
        [
          0.232,
          0.9304
        ],
        [
          0.242,
          0.9304
        ],
        [
          0.252,
          0.9299
        ],
        [
          0.2621,
          0.9308
        ],
        [
          0.2725,
          0.9308
        ],
        [
          0.2825,
          0.9299
        ],
        [
          0.2925,
          0.929
        ],
        [
          0.3025,
          0.9295
        ],
        [
          0.3126,
          0.9286
        ],
        [
          0.323,
          0.9286
        ],
        [
          0.333,
          0.9286
        ],
        [
          0.343,
          0.9272
        ],
        [
          0.353,
          0.9263
        ],
        [
          0.3631,
          0.9254
        ],
        [
          0.3735,
          0.9254
        ],
        [
          0.3835,
          0.9263
        ],
        [
          0.3935,
          0.9268
        ],
        [
          0.4035,
          0.9272
        ],
        [
          0.4136,
          0.9268
        ],
        [
          0.424,
          0.9268
        ],
        [
          0.434,
          0.9249
        ],
        [
          0.444,
          0.9236
        ],
        [
          0.454,
          0.9227
        ],
        [
          0.4641,
          0.9213
        ],
        [
          0.4745,
          0.9204
        ],
        [
          0.4845,
          0.919
        ],
        [
          0.4945,
          0.9177
        ],
        [
          0.5045,
          0.914
        ],
        [
          0.5146,
          0.9126
        ],
        [
          0.525,
          0.9104
        ],
        [
          0.535,
          0.9099
        ],
        [
          0.545,
          0.9067
        ],
        [
          0.5551,
          0.9049
        ],
        [
          0.5655,
          0.9026
        ],
        [
          0.5755,
          0.8999
        ],
        [
          0.5855,
          0.8981
        ],
        [
          0.5955,
          0.8981
        ],
        [
          0.6056,
          0.8963
        ],
        [
          0.616,
          0.8922
        ],
        [
          0.626,
          0.8913
        ],
        [
          0.636,
          0.8881
        ],
        [
          0.646,
          0.8854
        ],
        [
          0.6561,
          0.8826
        ],
        [
          0.6665,
          0.8785
        ],
        [
          0.6765,
          0.8749
        ],
        [
          0.6865,
          0.8726
        ],
        [
          0.6965,
          0.869
        ],
        [
          0.7066,
          0.8667
        ],
        [
          0.717,
          0.8617
        ],
        [
          0.727,
          0.8594
        ],
        [
          0.737,
          0.8553
        ],
        [
          0.747,
          0.8526
        ],
        [
          0.7571,
          0.8494
        ],
        [
          0.7675,
          0.8467
        ],
        [
          0.7775,
          0.8417
        ],
        [
          0.7875,
          0.8362
        ],
        [
          0.7975,
          0.8312
        ],
        [
          0.8076,
          0.8262
        ],
        [
          0.818,
          0.8221
        ],
        [
          0.828,
          0.8157
        ],
        [
          0.838,
          0.8112
        ],
        [
          0.848,
          0.8071
        ],
        [
          0.8581,
          0.8007
        ],
        [
          0.8685,
          0.7953
        ],
        [
          0.8785,
          0.793
        ],
        [
          0.8885,
          0.7884
        ],
        [
          0.8985,
          0.7839
        ],
        [
          0.9086,
          0.778
        ],
        [
          0.919,
          0.7725
        ],
        [
          0.929,
          0.7652
        ],
        [
          0.939,
          0.7584
        ],
        [
          0.949,
          0.7575
        ],
        [
          0.9591,
          0.753
        ],
        [
          0.9695,
          0.748
        ],
        [
          0.9795,
          0.7439
        ],
        [
          0.9895,
          0.7361
        ],
        [
          0.9995,
          0.7316
        ],
        [
          1.0,
          0.7311
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.16520497245080948,
        "difference": -0.0015156910895967564,
        "worse": 0.5266666666666666,
        "kept_right": 0.1634225481727083,
        "kept_wrong": 0.0017824242781011679,
        "handed_off": 0.8347950275491909
      },
      "usd_per_1000": 9.691607411261334
    },
    {
      "decision": 119,
      "frontier": 123,
      "frontier_accuracy": 0.9326660600545951,
      "in_sample": {
        "threshold": 0.9999999907788629,
        "answered": 0.1464968152866242,
        "accuracy": 0.9326660600545951
      },
      "curve": [
        [
          0.0,
          0.9327
        ],
        [
          0.01,
          0.9327
        ],
        [
          0.02,
          0.9327
        ],
        [
          0.03,
          0.9331
        ],
        [
          0.04,
          0.9331
        ],
        [
          0.05,
          0.9331
        ],
        [
          0.0601,
          0.9331
        ],
        [
          0.0705,
          0.9331
        ],
        [
          0.0805,
          0.9331
        ],
        [
          0.0905,
          0.9331
        ],
        [
          0.1005,
          0.9331
        ],
        [
          0.1106,
          0.9331
        ],
        [
          0.121,
          0.9331
        ],
        [
          0.131,
          0.9327
        ],
        [
          0.141,
          0.9327
        ],
        [
          0.151,
          0.9322
        ],
        [
          0.1611,
          0.9322
        ],
        [
          0.1715,
          0.9322
        ],
        [
          0.1815,
          0.9308
        ],
        [
          0.1915,
          0.9308
        ],
        [
          0.2015,
          0.9308
        ],
        [
          0.2116,
          0.9308
        ],
        [
          0.222,
          0.9308
        ],
        [
          0.232,
          0.9308
        ],
        [
          0.242,
          0.9308
        ],
        [
          0.252,
          0.9308
        ],
        [
          0.2621,
          0.9308
        ],
        [
          0.2725,
          0.9308
        ],
        [
          0.2825,
          0.9299
        ],
        [
          0.2925,
          0.9295
        ],
        [
          0.3025,
          0.9299
        ],
        [
          0.3126,
          0.9295
        ],
        [
          0.323,
          0.929
        ],
        [
          0.333,
          0.9277
        ],
        [
          0.343,
          0.9258
        ],
        [
          0.353,
          0.9245
        ],
        [
          0.3631,
          0.924
        ],
        [
          0.3735,
          0.9236
        ],
        [
          0.3835,
          0.9236
        ],
        [
          0.3935,
          0.9236
        ],
        [
          0.4035,
          0.9236
        ],
        [
          0.4136,
          0.9231
        ],
        [
          0.424,
          0.9227
        ],
        [
          0.434,
          0.9208
        ],
        [
          0.444,
          0.9199
        ],
        [
          0.454,
          0.9186
        ],
        [
          0.4641,
          0.9172
        ],
        [
          0.4745,
          0.9163
        ],
        [
          0.4845,
          0.9149
        ],
        [
          0.4945,
          0.9126
        ],
        [
          0.5045,
          0.9104
        ],
        [
          0.5146,
          0.9099
        ],
        [
          0.525,
          0.9072
        ],
        [
          0.535,
          0.9072
        ],
        [
          0.545,
          0.9031
        ],
        [
          0.5551,
          0.9017
        ],
        [
          0.5655,
          0.8995
        ],
        [
          0.5755,
          0.8967
        ],
        [
          0.5855,
          0.894
        ],
        [
          0.5955,
          0.8931
        ],
        [
          0.6056,
          0.8908
        ],
        [
          0.616,
          0.8863
        ],
        [
          0.626,
          0.8844
        ],
        [
          0.636,
          0.8813
        ],
        [
          0.646,
          0.879
        ],
        [
          0.6561,
          0.8763
        ],
        [
          0.6665,
          0.8731
        ],
        [
          0.6765,
          0.8712
        ],
        [
          0.6865,
          0.869
        ],
        [
          0.6965,
          0.8644
        ],
        [
          0.7066,
          0.8621
        ],
        [
          0.717,
          0.8581
        ],
        [
          0.727,
          0.8553
        ],
        [
          0.737,
          0.8508
        ],
        [
          0.747,
          0.8485
        ],
        [
          0.7571,
          0.8453
        ],
        [
          0.7675,
          0.843
        ],
        [
          0.7775,
          0.8389
        ],
        [
          0.7875,
          0.8353
        ],
        [
          0.7975,
          0.8317
        ],
        [
          0.8076,
          0.8267
        ],
        [
          0.818,
          0.8226
        ],
        [
          0.828,
          0.8171
        ],
        [
          0.838,
          0.8121
        ],
        [
          0.848,
          0.8053
        ],
        [
          0.8581,
          0.7989
        ],
        [
          0.8685,
          0.7948
        ],
        [
          0.8785,
          0.793
        ],
        [
          0.8885,
          0.7903
        ],
        [
          0.8985,
          0.7866
        ],
        [
          0.9086,
          0.7812
        ],
        [
          0.919,
          0.7762
        ],
        [
          0.929,
          0.7693
        ],
        [
          0.939,
          0.7621
        ],
        [
          0.949,
          0.7589
        ],
        [
          0.9591,
          0.7539
        ],
        [
          0.9695,
          0.7484
        ],
        [
          0.9795,
          0.7434
        ],
        [
          0.9895,
          0.7361
        ],
        [
          0.9995,
          0.7316
        ],
        [
          1.0,
          0.7311
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.16929363702647776,
        "difference": -0.00090056358897558,
        "worse": 0.48333333333333334,
        "kept_right": 0.16780495970492812,
        "kept_wrong": 0.0014886773215496052,
        "handed_off": 0.8307063629735222
      },
      "usd_per_1000": 0.2231143879051463
    },
    {
      "decision": 119,
      "frontier": 274,
      "frontier_accuracy": 0.9085532302092811,
      "in_sample": {
        "threshold": 0.9999999924404349,
        "answered": 0.1305732484076433,
        "accuracy": 0.9085532302092811
      },
      "curve": [
        [
          0.0,
          0.9086
        ],
        [
          0.01,
          0.9086
        ],
        [
          0.02,
          0.9086
        ],
        [
          0.03,
          0.9086
        ],
        [
          0.04,
          0.9086
        ],
        [
          0.05,
          0.9086
        ],
        [
          0.0601,
          0.9086
        ],
        [
          0.0705,
          0.9086
        ],
        [
          0.0805,
          0.9086
        ],
        [
          0.0905,
          0.9086
        ],
        [
          0.1005,
          0.9086
        ],
        [
          0.1106,
          0.9086
        ],
        [
          0.121,
          0.9086
        ],
        [
          0.131,
          0.9081
        ],
        [
          0.141,
          0.9081
        ],
        [
          0.151,
          0.9076
        ],
        [
          0.1611,
          0.9076
        ],
        [
          0.1715,
          0.9076
        ],
        [
          0.1815,
          0.9063
        ],
        [
          0.1915,
          0.9058
        ],
        [
          0.2015,
          0.9058
        ],
        [
          0.2116,
          0.9058
        ],
        [
          0.222,
          0.9058
        ],
        [
          0.232,
          0.9058
        ],
        [
          0.242,
          0.9058
        ],
        [
          0.252,
          0.9054
        ],
        [
          0.2621,
          0.9058
        ],
        [
          0.2725,
          0.9058
        ],
        [
          0.2825,
          0.9058
        ],
        [
          0.2925,
          0.9054
        ],
        [
          0.3025,
          0.9058
        ],
        [
          0.3126,
          0.9049
        ],
        [
          0.323,
          0.9049
        ],
        [
          0.333,
          0.9045
        ],
        [
          0.343,
          0.9031
        ],
        [
          0.353,
          0.9017
        ],
        [
          0.3631,
          0.9008
        ],
        [
          0.3735,
          0.9004
        ],
        [
          0.3835,
          0.9004
        ],
        [
          0.3935,
          0.9004
        ],
        [
          0.4035,
          0.9004
        ],
        [
          0.4136,
          0.9004
        ],
        [
          0.424,
          0.8999
        ],
        [
          0.434,
          0.899
        ],
        [
          0.444,
          0.8976
        ],
        [
          0.454,
          0.8972
        ],
        [
          0.4641,
          0.8958
        ],
        [
          0.4745,
          0.8949
        ],
        [
          0.4845,
          0.894
        ],
        [
          0.4945,
          0.8926
        ],
        [
          0.5045,
          0.8899
        ],
        [
          0.5146,
          0.8885
        ],
        [
          0.525,
          0.8863
        ],
        [
          0.535,
          0.8863
        ],
        [
          0.545,
          0.8844
        ],
        [
          0.5551,
          0.8835
        ],
        [
          0.5655,
          0.8831
        ],
        [
          0.5755,
          0.8803
        ],
        [
          0.5855,
          0.8785
        ],
        [
          0.5955,
          0.8767
        ],
        [
          0.6056,
          0.8744
        ],
        [
          0.616,
          0.8703
        ],
        [
          0.626,
          0.8685
        ],
        [
          0.636,
          0.8662
        ],
        [
          0.646,
          0.864
        ],
        [
          0.6561,
          0.8608
        ],
        [
          0.6665,
          0.8571
        ],
        [
          0.6765,
          0.854
        ],
        [
          0.6865,
          0.8517
        ],
        [
          0.6965,
          0.8485
        ],
        [
          0.7066,
          0.8467
        ],
        [
          0.717,
          0.8426
        ],
        [
          0.727,
          0.8399
        ],
        [
          0.737,
          0.8367
        ],
        [
          0.747,
          0.8358
        ],
        [
          0.7571,
          0.8326
        ],
        [
          0.7675,
          0.8303
        ],
        [
          0.7775,
          0.8258
        ],
        [
          0.7875,
          0.8235
        ],
        [
          0.7975,
          0.8212
        ],
        [
          0.8076,
          0.8162
        ],
        [
          0.818,
          0.8121
        ],
        [
          0.828,
          0.8089
        ],
        [
          0.838,
          0.8039
        ],
        [
          0.848,
          0.7998
        ],
        [
          0.8581,
          0.7934
        ],
        [
          0.8685,
          0.7884
        ],
        [
          0.8785,
          0.7862
        ],
        [
          0.8885,
          0.7825
        ],
        [
          0.8985,
          0.7789
        ],
        [
          0.9086,
          0.7739
        ],
        [
          0.919,
          0.7689
        ],
        [
          0.929,
          0.763
        ],
        [
          0.939,
          0.7566
        ],
        [
          0.949,
          0.7539
        ],
        [
          0.9591,
          0.7502
        ],
        [
          0.9695,
          0.7448
        ],
        [
          0.9795,
          0.7411
        ],
        [
          0.9895,
          0.7348
        ],
        [
          0.9995,
          0.7316
        ],
        [
          1.0,
          0.7311
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.15445730632730015,
        "difference": -0.0011065500221806902,
        "worse": 0.5033333333333333,
        "kept_right": 0.15320527679345317,
        "kept_wrong": 0.0012520295338470018,
        "handed_off": 0.8455426936727001
      },
      "usd_per_1000": 0.3604089186486025
    },
    {
      "decision": 119,
      "frontier": 47,
      "frontier_accuracy": 0.9044585987261147,
      "in_sample": {
        "threshold": 0.9999999924404349,
        "answered": 0.1305732484076433,
        "accuracy": 0.9044585987261147
      },
      "curve": [
        [
          0.0,
          0.9045
        ],
        [
          0.01,
          0.9045
        ],
        [
          0.02,
          0.9045
        ],
        [
          0.03,
          0.9045
        ],
        [
          0.04,
          0.9045
        ],
        [
          0.05,
          0.9045
        ],
        [
          0.0601,
          0.9045
        ],
        [
          0.0705,
          0.9045
        ],
        [
          0.0805,
          0.9045
        ],
        [
          0.0905,
          0.9045
        ],
        [
          0.1005,
          0.9045
        ],
        [
          0.1106,
          0.9045
        ],
        [
          0.121,
          0.9045
        ],
        [
          0.131,
          0.904
        ],
        [
          0.141,
          0.904
        ],
        [
          0.151,
          0.9035
        ],
        [
          0.1611,
          0.9035
        ],
        [
          0.1715,
          0.9035
        ],
        [
          0.1815,
          0.9022
        ],
        [
          0.1915,
          0.9017
        ],
        [
          0.2015,
          0.9017
        ],
        [
          0.2116,
          0.9017
        ],
        [
          0.222,
          0.9017
        ],
        [
          0.232,
          0.9017
        ],
        [
          0.242,
          0.9017
        ],
        [
          0.252,
          0.9013
        ],
        [
          0.2621,
          0.9017
        ],
        [
          0.2725,
          0.9017
        ],
        [
          0.2825,
          0.9008
        ],
        [
          0.2925,
          0.8999
        ],
        [
          0.3025,
          0.9004
        ],
        [
          0.3126,
          0.8999
        ],
        [
          0.323,
          0.8995
        ],
        [
          0.333,
          0.8985
        ],
        [
          0.343,
          0.8976
        ],
        [
          0.353,
          0.8967
        ],
        [
          0.3631,
          0.8958
        ],
        [
          0.3735,
          0.8967
        ],
        [
          0.3835,
          0.8972
        ],
        [
          0.3935,
          0.8967
        ],
        [
          0.4035,
          0.8976
        ],
        [
          0.4136,
          0.8972
        ],
        [
          0.424,
          0.8972
        ],
        [
          0.434,
          0.8954
        ],
        [
          0.444,
          0.8944
        ],
        [
          0.454,
          0.8935
        ],
        [
          0.4641,
          0.8926
        ],
        [
          0.4745,
          0.8917
        ],
        [
          0.4845,
          0.8908
        ],
        [
          0.4945,
          0.8894
        ],
        [
          0.5045,
          0.8863
        ],
        [
          0.5146,
          0.8849
        ],
        [
          0.525,
          0.8831
        ],
        [
          0.535,
          0.8822
        ],
        [
          0.545,
          0.8799
        ],
        [
          0.5551,
          0.8785
        ],
        [
          0.5655,
          0.8763
        ],
        [
          0.5755,
          0.8744
        ],
        [
          0.5855,
          0.8726
        ],
        [
          0.5955,
          0.8726
        ],
        [
          0.6056,
          0.8708
        ],
        [
          0.616,
          0.8667
        ],
        [
          0.626,
          0.8658
        ],
        [
          0.636,
          0.8631
        ],
        [
          0.646,
          0.8621
        ],
        [
          0.6561,
          0.8599
        ],
        [
          0.6665,
          0.8562
        ],
        [
          0.6765,
          0.853
        ],
        [
          0.6865,
          0.8503
        ],
        [
          0.6965,
          0.8467
        ],
        [
          0.7066,
          0.8462
        ],
        [
          0.717,
          0.8421
        ],
        [
          0.727,
          0.8394
        ],
        [
          0.737,
          0.8348
        ],
        [
          0.747,
          0.8326
        ],
        [
          0.7571,
          0.8294
        ],
        [
          0.7675,
          0.8267
        ],
        [
          0.7775,
          0.823
        ],
        [
          0.7875,
          0.818
        ],
        [
          0.7975,
          0.8135
        ],
        [
          0.8076,
          0.808
        ],
        [
          0.818,
          0.8035
        ],
        [
          0.828,
          0.7998
        ],
        [
          0.838,
          0.7953
        ],
        [
          0.848,
          0.7903
        ],
        [
          0.8581,
          0.7862
        ],
        [
          0.8685,
          0.783
        ],
        [
          0.8785,
          0.7825
        ],
        [
          0.8885,
          0.7798
        ],
        [
          0.8985,
          0.7757
        ],
        [
          0.9086,
          0.7712
        ],
        [
          0.919,
          0.7662
        ],
        [
          0.929,
          0.7598
        ],
        [
          0.939,
          0.7543
        ],
        [
          0.949,
          0.753
        ],
        [
          0.9591,
          0.7498
        ],
        [
          0.9695,
          0.7443
        ],
        [
          0.9795,
          0.7407
        ],
        [
          0.9895,
          0.7352
        ],
        [
          0.9995,
          0.7316
        ],
        [
          1.0,
          0.7311
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1499556319957371,
        "difference": -0.0010186020954371447,
        "worse": 0.5,
        "kept_right": 0.1489158260210453,
        "kept_wrong": 0.0010398059746918175,
        "handed_off": 0.8500443680042632
      },
      "usd_per_1000": 4.191216837182538
    },
    {
      "decision": 119,
      "frontier": 193,
      "frontier_accuracy": 0.89171974522293,
      "in_sample": {
        "threshold": 0.9999999924404349,
        "answered": 0.1305732484076433,
        "accuracy": 0.89171974522293
      },
      "curve": [
        [
          0.0,
          0.8917
        ],
        [
          0.01,
          0.8917
        ],
        [
          0.02,
          0.8917
        ],
        [
          0.03,
          0.8917
        ],
        [
          0.04,
          0.8917
        ],
        [
          0.05,
          0.8917
        ],
        [
          0.0601,
          0.8917
        ],
        [
          0.0705,
          0.8917
        ],
        [
          0.0805,
          0.8917
        ],
        [
          0.0905,
          0.8917
        ],
        [
          0.1005,
          0.8917
        ],
        [
          0.1106,
          0.8917
        ],
        [
          0.121,
          0.8917
        ],
        [
          0.131,
          0.8913
        ],
        [
          0.141,
          0.8913
        ],
        [
          0.151,
          0.8908
        ],
        [
          0.1611,
          0.8908
        ],
        [
          0.1715,
          0.8908
        ],
        [
          0.1815,
          0.8894
        ],
        [
          0.1915,
          0.8894
        ],
        [
          0.2015,
          0.8894
        ],
        [
          0.2116,
          0.8894
        ],
        [
          0.222,
          0.8899
        ],
        [
          0.232,
          0.8899
        ],
        [
          0.242,
          0.8899
        ],
        [
          0.252,
          0.8894
        ],
        [
          0.2621,
          0.8904
        ],
        [
          0.2725,
          0.8904
        ],
        [
          0.2825,
          0.8894
        ],
        [
          0.2925,
          0.8885
        ],
        [
          0.3025,
          0.889
        ],
        [
          0.3126,
          0.8881
        ],
        [
          0.323,
          0.8876
        ],
        [
          0.333,
          0.8872
        ],
        [
          0.343,
          0.8863
        ],
        [
          0.353,
          0.8858
        ],
        [
          0.3631,
          0.8867
        ],
        [
          0.3735,
          0.8867
        ],
        [
          0.3835,
          0.8872
        ],
        [
          0.3935,
          0.8876
        ],
        [
          0.4035,
          0.889
        ],
        [
          0.4136,
          0.8885
        ],
        [
          0.424,
          0.889
        ],
        [
          0.434,
          0.8881
        ],
        [
          0.444,
          0.8881
        ],
        [
          0.454,
          0.8876
        ],
        [
          0.4641,
          0.8867
        ],
        [
          0.4745,
          0.8863
        ],
        [
          0.4845,
          0.8863
        ],
        [
          0.4945,
          0.8858
        ],
        [
          0.5045,
          0.8835
        ],
        [
          0.5146,
          0.884
        ],
        [
          0.525,
          0.8813
        ],
        [
          0.535,
          0.8808
        ],
        [
          0.545,
          0.8776
        ],
        [
          0.5551,
          0.8758
        ],
        [
          0.5655,
          0.8744
        ],
        [
          0.5755,
          0.8722
        ],
        [
          0.5855,
          0.8703
        ],
        [
          0.5955,
          0.8699
        ],
        [
          0.6056,
          0.8676
        ],
        [
          0.616,
          0.8658
        ],
        [
          0.626,
          0.8653
        ],
        [
          0.636,
          0.8631
        ],
        [
          0.646,
          0.8608
        ],
        [
          0.6561,
          0.8581
        ],
        [
          0.6665,
          0.8558
        ],
        [
          0.6765,
          0.8526
        ],
        [
          0.6865,
          0.8499
        ],
        [
          0.6965,
          0.8462
        ],
        [
          0.7066,
          0.8439
        ],
        [
          0.717,
          0.8394
        ],
        [
          0.727,
          0.8367
        ],
        [
          0.737,
          0.8335
        ],
        [
          0.747,
          0.8326
        ],
        [
          0.7571,
          0.8308
        ],
        [
          0.7675,
          0.8276
        ],
        [
          0.7775,
          0.8235
        ],
        [
          0.7875,
          0.8185
        ],
        [
          0.7975,
          0.8153
        ],
        [
          0.8076,
          0.8107
        ],
        [
          0.818,
          0.8071
        ],
        [
          0.828,
          0.8035
        ],
        [
          0.838,
          0.7994
        ],
        [
          0.848,
          0.7939
        ],
        [
          0.8581,
          0.7889
        ],
        [
          0.8685,
          0.7862
        ],
        [
          0.8785,
          0.7834
        ],
        [
          0.8885,
          0.7803
        ],
        [
          0.8985,
          0.7757
        ],
        [
          0.9086,
          0.7707
        ],
        [
          0.919,
          0.7662
        ],
        [
          0.929,
          0.7607
        ],
        [
          0.939,
          0.7539
        ],
        [
          0.949,
          0.7516
        ],
        [
          0.9591,
          0.7493
        ],
        [
          0.9695,
          0.7443
        ],
        [
          0.9795,
          0.7411
        ],
        [
          0.9895,
          0.7348
        ],
        [
          0.9995,
          0.7316
        ],
        [
          1.0,
          0.7311
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.2621786882371094,
        "difference": -0.004016884698254913,
        "worse": 0.6533333333333333,
        "kept_right": 0.25277149469958937,
        "kept_wrong": 0.009407193537520292,
        "handed_off": 0.7378213117628902
      },
      "usd_per_1000": 5.716337010087484
    },
    {
      "decision": 119,
      "frontier": 55,
      "frontier_accuracy": 0.8503184713375797,
      "in_sample": {
        "threshold": 0.9999037271941633,
        "answered": 0.60828025477707,
        "accuracy": 0.8503184713375797
      },
      "curve": [
        [
          0.0,
          0.8503
        ],
        [
          0.01,
          0.8503
        ],
        [
          0.02,
          0.8503
        ],
        [
          0.03,
          0.8508
        ],
        [
          0.04,
          0.8508
        ],
        [
          0.05,
          0.8512
        ],
        [
          0.0601,
          0.8512
        ],
        [
          0.0705,
          0.8512
        ],
        [
          0.0805,
          0.853
        ],
        [
          0.0905,
          0.853
        ],
        [
          0.1005,
          0.8535
        ],
        [
          0.1106,
          0.854
        ],
        [
          0.121,
          0.854
        ],
        [
          0.131,
          0.8544
        ],
        [
          0.141,
          0.8544
        ],
        [
          0.151,
          0.854
        ],
        [
          0.1611,
          0.8544
        ],
        [
          0.1715,
          0.8549
        ],
        [
          0.1815,
          0.8544
        ],
        [
          0.1915,
          0.8549
        ],
        [
          0.2015,
          0.8553
        ],
        [
          0.2116,
          0.8558
        ],
        [
          0.222,
          0.8558
        ],
        [
          0.232,
          0.8558
        ],
        [
          0.242,
          0.8562
        ],
        [
          0.252,
          0.8562
        ],
        [
          0.2621,
          0.8576
        ],
        [
          0.2725,
          0.8576
        ],
        [
          0.2825,
          0.8571
        ],
        [
          0.2925,
          0.8571
        ],
        [
          0.3025,
          0.8585
        ],
        [
          0.3126,
          0.8594
        ],
        [
          0.323,
          0.8594
        ],
        [
          0.333,
          0.8594
        ],
        [
          0.343,
          0.8594
        ],
        [
          0.353,
          0.8585
        ],
        [
          0.3631,
          0.8594
        ],
        [
          0.3735,
          0.8608
        ],
        [
          0.3835,
          0.8612
        ],
        [
          0.3935,
          0.8631
        ],
        [
          0.4035,
          0.8649
        ],
        [
          0.4136,
          0.8649
        ],
        [
          0.424,
          0.8653
        ],
        [
          0.434,
          0.8649
        ],
        [
          0.444,
          0.864
        ],
        [
          0.454,
          0.864
        ],
        [
          0.4641,
          0.8631
        ],
        [
          0.4745,
          0.8635
        ],
        [
          0.4845,
          0.8644
        ],
        [
          0.4945,
          0.8631
        ],
        [
          0.5045,
          0.8631
        ],
        [
          0.5146,
          0.8635
        ],
        [
          0.525,
          0.8617
        ],
        [
          0.535,
          0.8603
        ],
        [
          0.545,
          0.8581
        ],
        [
          0.5551,
          0.8571
        ],
        [
          0.5655,
          0.8549
        ],
        [
          0.5755,
          0.853
        ],
        [
          0.5855,
          0.8521
        ],
        [
          0.5955,
          0.8512
        ],
        [
          0.6056,
          0.8503
        ],
        [
          0.616,
          0.8471
        ],
        [
          0.626,
          0.8485
        ],
        [
          0.636,
          0.8471
        ],
        [
          0.646,
          0.8467
        ],
        [
          0.6561,
          0.8444
        ],
        [
          0.6665,
          0.8408
        ],
        [
          0.6765,
          0.8385
        ],
        [
          0.6865,
          0.8358
        ],
        [
          0.6965,
          0.8317
        ],
        [
          0.7066,
          0.8298
        ],
        [
          0.717,
          0.8267
        ],
        [
          0.727,
          0.8235
        ],
        [
          0.737,
          0.8207
        ],
        [
          0.747,
          0.8207
        ],
        [
          0.7571,
          0.8185
        ],
        [
          0.7675,
          0.8157
        ],
        [
          0.7775,
          0.8112
        ],
        [
          0.7875,
          0.8066
        ],
        [
          0.7975,
          0.8035
        ],
        [
          0.8076,
          0.8003
        ],
        [
          0.818,
          0.7985
        ],
        [
          0.828,
          0.7939
        ],
        [
          0.838,
          0.7912
        ],
        [
          0.848,
          0.7875
        ],
        [
          0.8581,
          0.7834
        ],
        [
          0.8685,
          0.7793
        ],
        [
          0.8785,
          0.778
        ],
        [
          0.8885,
          0.7757
        ],
        [
          0.8985,
          0.7721
        ],
        [
          0.9086,
          0.7689
        ],
        [
          0.919,
          0.7648
        ],
        [
          0.929,
          0.7584
        ],
        [
          0.939,
          0.7516
        ],
        [
          0.949,
          0.7498
        ],
        [
          0.9591,
          0.7484
        ],
        [
          0.9695,
          0.7434
        ],
        [
          0.9795,
          0.7402
        ],
        [
          0.9895,
          0.7338
        ],
        [
          0.9995,
          0.7316
        ],
        [
          1.0,
          0.7311
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.6085168661771586,
        "difference": -0.0012330249110587166,
        "worse": 0.45666666666666667,
        "kept_right": 0.5441068047326055,
        "kept_wrong": 0.0644100614445532,
        "handed_off": 0.3914831338228413
      },
      "usd_per_1000": 1.4306099775563401
    },
    {
      "decision": 248,
      "frontier": 140,
      "frontier_accuracy": 0.9808917197452229,
      "in_sample": {
        "threshold": 0.9549926987304493,
        "answered": 0.21155595996360327,
        "accuracy": 0.9808917197452229
      },
      "curve": [
        [
          0.0,
          0.9809
        ],
        [
          0.01,
          0.9809
        ],
        [
          0.02,
          0.9809
        ],
        [
          0.03,
          0.9809
        ],
        [
          0.04,
          0.9809
        ],
        [
          0.05,
          0.9809
        ],
        [
          0.0601,
          0.9809
        ],
        [
          0.0705,
          0.9809
        ],
        [
          0.0805,
          0.9809
        ],
        [
          0.0905,
          0.9809
        ],
        [
          0.1005,
          0.9809
        ],
        [
          0.1106,
          0.9809
        ],
        [
          0.121,
          0.9809
        ],
        [
          0.131,
          0.9809
        ],
        [
          0.141,
          0.9809
        ],
        [
          0.151,
          0.9809
        ],
        [
          0.1611,
          0.9809
        ],
        [
          0.1715,
          0.9809
        ],
        [
          0.1815,
          0.9809
        ],
        [
          0.1915,
          0.9809
        ],
        [
          0.2015,
          0.9809
        ],
        [
          0.2116,
          0.9809
        ],
        [
          0.222,
          0.9804
        ],
        [
          0.232,
          0.9804
        ],
        [
          0.242,
          0.9795
        ],
        [
          0.252,
          0.9786
        ],
        [
          0.2621,
          0.9782
        ],
        [
          0.2725,
          0.9782
        ],
        [
          0.2825,
          0.9777
        ],
        [
          0.2925,
          0.9777
        ],
        [
          0.3025,
          0.9773
        ],
        [
          0.3126,
          0.9768
        ],
        [
          0.323,
          0.9763
        ],
        [
          0.333,
          0.9759
        ],
        [
          0.343,
          0.975
        ],
        [
          0.353,
          0.9732
        ],
        [
          0.3631,
          0.9727
        ],
        [
          0.3735,
          0.9709
        ],
        [
          0.3835,
          0.9691
        ],
        [
          0.3935,
          0.9682
        ],
        [
          0.4035,
          0.9654
        ],
        [
          0.4136,
          0.9641
        ],
        [
          0.424,
          0.9622
        ],
        [
          0.434,
          0.9609
        ],
        [
          0.444,
          0.9572
        ],
        [
          0.454,
          0.9559
        ],
        [
          0.4641,
          0.9522
        ],
        [
          0.4745,
          0.9495
        ],
        [
          0.4845,
          0.9468
        ],
        [
          0.4945,
          0.9449
        ],
        [
          0.5045,
          0.9427
        ],
        [
          0.5146,
          0.9413
        ],
        [
          0.525,
          0.9386
        ],
        [
          0.535,
          0.934
        ],
        [
          0.545,
          0.9318
        ],
        [
          0.5551,
          0.9281
        ],
        [
          0.5655,
          0.9222
        ],
        [
          0.5755,
          0.9177
        ],
        [
          0.5855,
          0.9122
        ],
        [
          0.5955,
          0.9086
        ],
        [
          0.6056,
          0.9035
        ],
        [
          0.616,
          0.8985
        ],
        [
          0.626,
          0.8958
        ],
        [
          0.636,
          0.894
        ],
        [
          0.646,
          0.8926
        ],
        [
          0.6561,
          0.8876
        ],
        [
          0.6665,
          0.8831
        ],
        [
          0.6765,
          0.8799
        ],
        [
          0.6865,
          0.8753
        ],
        [
          0.6965,
          0.8712
        ],
        [
          0.7066,
          0.8672
        ],
        [
          0.717,
          0.8631
        ],
        [
          0.727,
          0.8571
        ],
        [
          0.737,
          0.8508
        ],
        [
          0.747,
          0.8462
        ],
        [
          0.7571,
          0.8412
        ],
        [
          0.7675,
          0.8358
        ],
        [
          0.7775,
          0.8298
        ],
        [
          0.7875,
          0.823
        ],
        [
          0.7975,
          0.8157
        ],
        [
          0.8076,
          0.8112
        ],
        [
          0.818,
          0.8035
        ],
        [
          0.828,
          0.7957
        ],
        [
          0.838,
          0.7894
        ],
        [
          0.848,
          0.783
        ],
        [
          0.8581,
          0.7775
        ],
        [
          0.8685,
          0.773
        ],
        [
          0.8785,
          0.7657
        ],
        [
          0.8885,
          0.7589
        ],
        [
          0.8985,
          0.7525
        ],
        [
          0.9086,
          0.7461
        ],
        [
          0.919,
          0.7389
        ],
        [
          0.929,
          0.732
        ],
        [
          0.939,
          0.7261
        ],
        [
          0.949,
          0.7193
        ],
        [
          0.9591,
          0.7138
        ],
        [
          0.9695,
          0.7079
        ],
        [
          0.9795,
          0.7011
        ],
        [
          0.9895,
          0.6929
        ],
        [
          0.9995,
          0.6852
        ],
        [
          1.0,
          0.6847
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.22513282891278408,
        "difference": -0.0008215687044501233,
        "worse": 0.5033333333333333,
        "kept_right": 0.22407781626069118,
        "kept_wrong": 0.0010550126520929273,
        "handed_off": 0.7748671710872156
      },
      "usd_per_1000": 1.8888531232253272
    },
    {
      "decision": 248,
      "frontier": 277,
      "frontier_accuracy": 0.9795268425841674,
      "in_sample": {
        "threshold": 0.9438777175486791,
        "answered": 0.23339399454049137,
        "accuracy": 0.9795268425841674
      },
      "curve": [
        [
          0.0,
          0.9795
        ],
        [
          0.01,
          0.9795
        ],
        [
          0.02,
          0.9795
        ],
        [
          0.03,
          0.9795
        ],
        [
          0.04,
          0.9795
        ],
        [
          0.05,
          0.9795
        ],
        [
          0.0601,
          0.9795
        ],
        [
          0.0705,
          0.9795
        ],
        [
          0.0805,
          0.9795
        ],
        [
          0.0905,
          0.9795
        ],
        [
          0.1005,
          0.9795
        ],
        [
          0.1106,
          0.9795
        ],
        [
          0.121,
          0.9795
        ],
        [
          0.131,
          0.9795
        ],
        [
          0.141,
          0.9795
        ],
        [
          0.151,
          0.9795
        ],
        [
          0.1611,
          0.9795
        ],
        [
          0.1715,
          0.9795
        ],
        [
          0.1815,
          0.9795
        ],
        [
          0.1915,
          0.9795
        ],
        [
          0.2015,
          0.9795
        ],
        [
          0.2116,
          0.98
        ],
        [
          0.222,
          0.9795
        ],
        [
          0.232,
          0.9795
        ],
        [
          0.242,
          0.9786
        ],
        [
          0.252,
          0.9777
        ],
        [
          0.2621,
          0.9773
        ],
        [
          0.2725,
          0.9773
        ],
        [
          0.2825,
          0.9768
        ],
        [
          0.2925,
          0.9768
        ],
        [
          0.3025,
          0.9763
        ],
        [
          0.3126,
          0.9759
        ],
        [
          0.323,
          0.9754
        ],
        [
          0.333,
          0.975
        ],
        [
          0.343,
          0.9736
        ],
        [
          0.353,
          0.9718
        ],
        [
          0.3631,
          0.9713
        ],
        [
          0.3735,
          0.9695
        ],
        [
          0.3835,
          0.9682
        ],
        [
          0.3935,
          0.9672
        ],
        [
          0.4035,
          0.9645
        ],
        [
          0.4136,
          0.9627
        ],
        [
          0.424,
          0.9609
        ],
        [
          0.434,
          0.9604
        ],
        [
          0.444,
          0.9563
        ],
        [
          0.454,
          0.9545
        ],
        [
          0.4641,
          0.9513
        ],
        [
          0.4745,
          0.9486
        ],
        [
          0.4845,
          0.9459
        ],
        [
          0.4945,
          0.944
        ],
        [
          0.5045,
          0.9418
        ],
        [
          0.5146,
          0.9404
        ],
        [
          0.525,
          0.9372
        ],
        [
          0.535,
          0.9331
        ],
        [
          0.545,
          0.9308
        ],
        [
          0.5551,
          0.9272
        ],
        [
          0.5655,
          0.9217
        ],
        [
          0.5755,
          0.9172
        ],
        [
          0.5855,
          0.9117
        ],
        [
          0.5955,
          0.9081
        ],
        [
          0.6056,
          0.9031
        ],
        [
          0.616,
          0.8981
        ],
        [
          0.626,
          0.8954
        ],
        [
          0.636,
          0.8931
        ],
        [
          0.646,
          0.8913
        ],
        [
          0.6561,
          0.8863
        ],
        [
          0.6665,
          0.8813
        ],
        [
          0.6765,
          0.8785
        ],
        [
          0.6865,
          0.8744
        ],
        [
          0.6965,
          0.8703
        ],
        [
          0.7066,
          0.8667
        ],
        [
          0.717,
          0.8626
        ],
        [
          0.727,
          0.8562
        ],
        [
          0.737,
          0.8499
        ],
        [
          0.747,
          0.8458
        ],
        [
          0.7571,
          0.8408
        ],
        [
          0.7675,
          0.8353
        ],
        [
          0.7775,
          0.8294
        ],
        [
          0.7875,
          0.8226
        ],
        [
          0.7975,
          0.8157
        ],
        [
          0.8076,
          0.8112
        ],
        [
          0.818,
          0.8039
        ],
        [
          0.828,
          0.7971
        ],
        [
          0.838,
          0.7907
        ],
        [
          0.848,
          0.7843
        ],
        [
          0.8581,
          0.778
        ],
        [
          0.8685,
          0.7734
        ],
        [
          0.8785,
          0.7662
        ],
        [
          0.8885,
          0.7593
        ],
        [
          0.8985,
          0.7525
        ],
        [
          0.9086,
          0.7461
        ],
        [
          0.919,
          0.7389
        ],
        [
          0.929,
          0.732
        ],
        [
          0.939,
          0.7261
        ],
        [
          0.949,
          0.7193
        ],
        [
          0.9591,
          0.7143
        ],
        [
          0.9695,
          0.7075
        ],
        [
          0.9795,
          0.7006
        ],
        [
          0.9895,
          0.6929
        ],
        [
          0.9995,
          0.6852
        ],
        [
          1.0,
          0.6847
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.2364342302336929,
        "difference": -0.0009490456800917263,
        "worse": 0.48,
        "kept_right": 0.23479095203490558,
        "kept_wrong": 0.0016432781987874296,
        "handed_off": 0.763565769766307
      },
      "usd_per_1000": 4.645819710392941
    },
    {
      "decision": 248,
      "frontier": 276,
      "frontier_accuracy": 0.9777070063694268,
      "in_sample": {
        "threshold": 0.9401461695392861,
        "answered": 0.2411282984531392,
        "accuracy": 0.9777070063694268
      },
      "curve": [
        [
          0.0,
          0.9777
        ],
        [
          0.01,
          0.9777
        ],
        [
          0.02,
          0.9777
        ],
        [
          0.03,
          0.9777
        ],
        [
          0.04,
          0.9777
        ],
        [
          0.05,
          0.9777
        ],
        [
          0.0601,
          0.9777
        ],
        [
          0.0705,
          0.9777
        ],
        [
          0.0805,
          0.9777
        ],
        [
          0.0905,
          0.9777
        ],
        [
          0.1005,
          0.9777
        ],
        [
          0.1106,
          0.9777
        ],
        [
          0.121,
          0.9777
        ],
        [
          0.131,
          0.9777
        ],
        [
          0.141,
          0.9777
        ],
        [
          0.151,
          0.9777
        ],
        [
          0.1611,
          0.9777
        ],
        [
          0.1715,
          0.9782
        ],
        [
          0.1815,
          0.9782
        ],
        [
          0.1915,
          0.9782
        ],
        [
          0.2015,
          0.9782
        ],
        [
          0.2116,
          0.9786
        ],
        [
          0.222,
          0.9782
        ],
        [
          0.232,
          0.9782
        ],
        [
          0.242,
          0.9773
        ],
        [
          0.252,
          0.9763
        ],
        [
          0.2621,
          0.9759
        ],
        [
          0.2725,
          0.9759
        ],
        [
          0.2825,
          0.9754
        ],
        [
          0.2925,
          0.9754
        ],
        [
          0.3025,
          0.975
        ],
        [
          0.3126,
          0.9745
        ],
        [
          0.323,
          0.9741
        ],
        [
          0.333,
          0.9736
        ],
        [
          0.343,
          0.9722
        ],
        [
          0.353,
          0.9704
        ],
        [
          0.3631,
          0.97
        ],
        [
          0.3735,
          0.9682
        ],
        [
          0.3835,
          0.9668
        ],
        [
          0.3935,
          0.9663
        ],
        [
          0.4035,
          0.9641
        ],
        [
          0.4136,
          0.9622
        ],
        [
          0.424,
          0.9604
        ],
        [
          0.434,
          0.96
        ],
        [
          0.444,
          0.9559
        ],
        [
          0.454,
          0.954
        ],
        [
          0.4641,
          0.9509
        ],
        [
          0.4745,
          0.9481
        ],
        [
          0.4845,
          0.9454
        ],
        [
          0.4945,
          0.9436
        ],
        [
          0.5045,
          0.9413
        ],
        [
          0.5146,
          0.9399
        ],
        [
          0.525,
          0.9372
        ],
        [
          0.535,
          0.9336
        ],
        [
          0.545,
          0.9313
        ],
        [
          0.5551,
          0.9277
        ],
        [
          0.5655,
          0.9222
        ],
        [
          0.5755,
          0.9177
        ],
        [
          0.5855,
          0.9126
        ],
        [
          0.5955,
          0.909
        ],
        [
          0.6056,
          0.904
        ],
        [
          0.616,
          0.899
        ],
        [
          0.626,
          0.8963
        ],
        [
          0.636,
          0.894
        ],
        [
          0.646,
          0.8917
        ],
        [
          0.6561,
          0.8867
        ],
        [
          0.6665,
          0.8817
        ],
        [
          0.6765,
          0.8785
        ],
        [
          0.6865,
          0.8744
        ],
        [
          0.6965,
          0.8703
        ],
        [
          0.7066,
          0.8667
        ],
        [
          0.717,
          0.8626
        ],
        [
          0.727,
          0.8562
        ],
        [
          0.737,
          0.8499
        ],
        [
          0.747,
          0.8462
        ],
        [
          0.7571,
          0.8412
        ],
        [
          0.7675,
          0.8358
        ],
        [
          0.7775,
          0.8298
        ],
        [
          0.7875,
          0.823
        ],
        [
          0.7975,
          0.8157
        ],
        [
          0.8076,
          0.8116
        ],
        [
          0.818,
          0.8044
        ],
        [
          0.828,
          0.7966
        ],
        [
          0.838,
          0.7903
        ],
        [
          0.848,
          0.7839
        ],
        [
          0.8581,
          0.7775
        ],
        [
          0.8685,
          0.773
        ],
        [
          0.8785,
          0.7657
        ],
        [
          0.8885,
          0.7589
        ],
        [
          0.8985,
          0.752
        ],
        [
          0.9086,
          0.7461
        ],
        [
          0.919,
          0.7389
        ],
        [
          0.929,
          0.732
        ],
        [
          0.939,
          0.7261
        ],
        [
          0.949,
          0.7193
        ],
        [
          0.9591,
          0.7143
        ],
        [
          0.9695,
          0.7075
        ],
        [
          0.9795,
          0.7006
        ],
        [
          0.9895,
          0.6929
        ],
        [
          0.9995,
          0.6852
        ],
        [
          1.0,
          0.6847
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.2477085373738853,
        "difference": -0.0010066599702269314,
        "worse": 0.49,
        "kept_right": 0.24537101578261067,
        "kept_wrong": 0.0023375215912746377,
        "handed_off": 0.7522914626261143
      },
      "usd_per_1000": 1.2347293173619862
    },
    {
      "decision": 248,
      "frontier": 48,
      "frontier_accuracy": 0.9758871701546861,
      "in_sample": {
        "threshold": 0.9438777175486791,
        "answered": 0.23339399454049137,
        "accuracy": 0.9758871701546861
      },
      "curve": [
        [
          0.0,
          0.9759
        ],
        [
          0.01,
          0.9759
        ],
        [
          0.02,
          0.9759
        ],
        [
          0.03,
          0.9759
        ],
        [
          0.04,
          0.9759
        ],
        [
          0.05,
          0.9759
        ],
        [
          0.0601,
          0.9759
        ],
        [
          0.0705,
          0.9759
        ],
        [
          0.0805,
          0.9759
        ],
        [
          0.0905,
          0.9759
        ],
        [
          0.1005,
          0.9759
        ],
        [
          0.1106,
          0.9759
        ],
        [
          0.121,
          0.9759
        ],
        [
          0.131,
          0.9759
        ],
        [
          0.141,
          0.9759
        ],
        [
          0.151,
          0.9759
        ],
        [
          0.1611,
          0.9759
        ],
        [
          0.1715,
          0.9759
        ],
        [
          0.1815,
          0.9759
        ],
        [
          0.1915,
          0.9759
        ],
        [
          0.2015,
          0.9759
        ],
        [
          0.2116,
          0.9763
        ],
        [
          0.222,
          0.9759
        ],
        [
          0.232,
          0.9759
        ],
        [
          0.242,
          0.975
        ],
        [
          0.252,
          0.9741
        ],
        [
          0.2621,
          0.9736
        ],
        [
          0.2725,
          0.9736
        ],
        [
          0.2825,
          0.9732
        ],
        [
          0.2925,
          0.9732
        ],
        [
          0.3025,
          0.9727
        ],
        [
          0.3126,
          0.9722
        ],
        [
          0.323,
          0.9718
        ],
        [
          0.333,
          0.9713
        ],
        [
          0.343,
          0.97
        ],
        [
          0.353,
          0.9682
        ],
        [
          0.3631,
          0.9677
        ],
        [
          0.3735,
          0.9659
        ],
        [
          0.3835,
          0.9645
        ],
        [
          0.3935,
          0.9636
        ],
        [
          0.4035,
          0.9613
        ],
        [
          0.4136,
          0.9595
        ],
        [
          0.424,
          0.9581
        ],
        [
          0.434,
          0.9577
        ],
        [
          0.444,
          0.9536
        ],
        [
          0.454,
          0.9518
        ],
        [
          0.4641,
          0.9486
        ],
        [
          0.4745,
          0.9459
        ],
        [
          0.4845,
          0.9431
        ],
        [
          0.4945,
          0.9413
        ],
        [
          0.5045,
          0.9395
        ],
        [
          0.5146,
          0.9381
        ],
        [
          0.525,
          0.9354
        ],
        [
          0.535,
          0.9313
        ],
        [
          0.545,
          0.929
        ],
        [
          0.5551,
          0.9254
        ],
        [
          0.5655,
          0.9199
        ],
        [
          0.5755,
          0.9154
        ],
        [
          0.5855,
          0.9099
        ],
        [
          0.5955,
          0.9063
        ],
        [
          0.6056,
          0.9017
        ],
        [
          0.616,
          0.8967
        ],
        [
          0.626,
          0.894
        ],
        [
          0.636,
          0.8917
        ],
        [
          0.646,
          0.8894
        ],
        [
          0.6561,
          0.8844
        ],
        [
          0.6665,
          0.8794
        ],
        [
          0.6765,
          0.8767
        ],
        [
          0.6865,
          0.8722
        ],
        [
          0.6965,
          0.8681
        ],
        [
          0.7066,
          0.8644
        ],
        [
          0.717,
          0.8608
        ],
        [
          0.727,
          0.8549
        ],
        [
          0.737,
          0.8485
        ],
        [
          0.747,
          0.8444
        ],
        [
          0.7571,
          0.8394
        ],
        [
          0.7675,
          0.8335
        ],
        [
          0.7775,
          0.8276
        ],
        [
          0.7875,
          0.8207
        ],
        [
          0.7975,
          0.8139
        ],
        [
          0.8076,
          0.8098
        ],
        [
          0.818,
          0.803
        ],
        [
          0.828,
          0.7962
        ],
        [
          0.838,
          0.7898
        ],
        [
          0.848,
          0.7834
        ],
        [
          0.8581,
          0.7775
        ],
        [
          0.8685,
          0.773
        ],
        [
          0.8785,
          0.7657
        ],
        [
          0.8885,
          0.7589
        ],
        [
          0.8985,
          0.752
        ],
        [
          0.9086,
          0.7461
        ],
        [
          0.919,
          0.7389
        ],
        [
          0.929,
          0.732
        ],
        [
          0.939,
          0.7261
        ],
        [
          0.949,
          0.7193
        ],
        [
          0.9591,
          0.7138
        ],
        [
          0.9695,
          0.7075
        ],
        [
          0.9795,
          0.7006
        ],
        [
          0.9895,
          0.6929
        ],
        [
          0.9995,
          0.6852
        ],
        [
          1.0,
          0.6847
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.2364342302336929,
        "difference": -0.0009490456800917263,
        "worse": 0.48,
        "kept_right": 0.23479095203490558,
        "kept_wrong": 0.0016432781987874296,
        "handed_off": 0.763565769766307
      },
      "usd_per_1000": 4.1209989617848395
    },
    {
      "decision": 248,
      "frontier": 191,
      "frontier_accuracy": 0.9740673339399454,
      "in_sample": {
        "threshold": 0.9549926987304493,
        "answered": 0.21155595996360327,
        "accuracy": 0.9740673339399454
      },
      "curve": [
        [
          0.0,
          0.9741
        ],
        [
          0.01,
          0.9741
        ],
        [
          0.02,
          0.9741
        ],
        [
          0.03,
          0.9741
        ],
        [
          0.04,
          0.9741
        ],
        [
          0.05,
          0.9741
        ],
        [
          0.0601,
          0.9741
        ],
        [
          0.0705,
          0.9741
        ],
        [
          0.0805,
          0.9741
        ],
        [
          0.0905,
          0.9741
        ],
        [
          0.1005,
          0.9741
        ],
        [
          0.1106,
          0.9741
        ],
        [
          0.121,
          0.9741
        ],
        [
          0.131,
          0.9741
        ],
        [
          0.141,
          0.9741
        ],
        [
          0.151,
          0.9741
        ],
        [
          0.1611,
          0.9741
        ],
        [
          0.1715,
          0.9741
        ],
        [
          0.1815,
          0.9741
        ],
        [
          0.1915,
          0.9741
        ],
        [
          0.2015,
          0.9741
        ],
        [
          0.2116,
          0.9741
        ],
        [
          0.222,
          0.9736
        ],
        [
          0.232,
          0.9736
        ],
        [
          0.242,
          0.9727
        ],
        [
          0.252,
          0.9718
        ],
        [
          0.2621,
          0.9713
        ],
        [
          0.2725,
          0.9713
        ],
        [
          0.2825,
          0.9709
        ],
        [
          0.2925,
          0.9709
        ],
        [
          0.3025,
          0.9704
        ],
        [
          0.3126,
          0.97
        ],
        [
          0.323,
          0.9695
        ],
        [
          0.333,
          0.9691
        ],
        [
          0.343,
          0.9677
        ],
        [
          0.353,
          0.9668
        ],
        [
          0.3631,
          0.9663
        ],
        [
          0.3735,
          0.9645
        ],
        [
          0.3835,
          0.9631
        ],
        [
          0.3935,
          0.9622
        ],
        [
          0.4035,
          0.96
        ],
        [
          0.4136,
          0.9581
        ],
        [
          0.424,
          0.9568
        ],
        [
          0.434,
          0.9563
        ],
        [
          0.444,
          0.9522
        ],
        [
          0.454,
          0.9504
        ],
        [
          0.4641,
          0.9468
        ],
        [
          0.4745,
          0.944
        ],
        [
          0.4845,
          0.9413
        ],
        [
          0.4945,
          0.9395
        ],
        [
          0.5045,
          0.9372
        ],
        [
          0.5146,
          0.9359
        ],
        [
          0.525,
          0.9331
        ],
        [
          0.535,
          0.929
        ],
        [
          0.545,
          0.9268
        ],
        [
          0.5551,
          0.9231
        ],
        [
          0.5655,
          0.9177
        ],
        [
          0.5755,
          0.9126
        ],
        [
          0.5855,
          0.9076
        ],
        [
          0.5955,
          0.904
        ],
        [
          0.6056,
          0.8995
        ],
        [
          0.616,
          0.8944
        ],
        [
          0.626,
          0.8917
        ],
        [
          0.636,
          0.8899
        ],
        [
          0.646,
          0.8876
        ],
        [
          0.6561,
          0.8826
        ],
        [
          0.6665,
          0.8776
        ],
        [
          0.6765,
          0.8749
        ],
        [
          0.6865,
          0.8703
        ],
        [
          0.6965,
          0.8667
        ],
        [
          0.7066,
          0.8631
        ],
        [
          0.717,
          0.859
        ],
        [
          0.727,
          0.853
        ],
        [
          0.737,
          0.8467
        ],
        [
          0.747,
          0.8426
        ],
        [
          0.7571,
          0.838
        ],
        [
          0.7675,
          0.8321
        ],
        [
          0.7775,
          0.8267
        ],
        [
          0.7875,
          0.8198
        ],
        [
          0.7975,
          0.813
        ],
        [
          0.8076,
          0.8089
        ],
        [
          0.818,
          0.8021
        ],
        [
          0.828,
          0.7948
        ],
        [
          0.838,
          0.7889
        ],
        [
          0.848,
          0.783
        ],
        [
          0.8581,
          0.7775
        ],
        [
          0.8685,
          0.773
        ],
        [
          0.8785,
          0.7657
        ],
        [
          0.8885,
          0.7589
        ],
        [
          0.8985,
          0.752
        ],
        [
          0.9086,
          0.7452
        ],
        [
          0.919,
          0.7379
        ],
        [
          0.929,
          0.7311
        ],
        [
          0.939,
          0.7252
        ],
        [
          0.949,
          0.7184
        ],
        [
          0.9591,
          0.7134
        ],
        [
          0.9695,
          0.707
        ],
        [
          0.9795,
          0.7002
        ],
        [
          0.9895,
          0.6929
        ],
        [
          0.9995,
          0.6852
        ],
        [
          1.0,
          0.6847
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.22513282891278408,
        "difference": -0.0008215687044501233,
        "worse": 0.5033333333333333,
        "kept_right": 0.22407781626069118,
        "kept_wrong": 0.0010550126520929273,
        "handed_off": 0.7748671710872156
      },
      "usd_per_1000": 20.73253242814131
    },
    {
      "decision": 248,
      "frontier": 188,
      "frontier_accuracy": 0.9713375796178344,
      "in_sample": {
        "threshold": 0.9401461695392861,
        "answered": 0.2411282984531392,
        "accuracy": 0.9713375796178344
      },
      "curve": [
        [
          0.0,
          0.9713
        ],
        [
          0.01,
          0.9713
        ],
        [
          0.02,
          0.9713
        ],
        [
          0.03,
          0.9713
        ],
        [
          0.04,
          0.9713
        ],
        [
          0.05,
          0.9713
        ],
        [
          0.0601,
          0.9713
        ],
        [
          0.0705,
          0.9713
        ],
        [
          0.0805,
          0.9713
        ],
        [
          0.0905,
          0.9713
        ],
        [
          0.1005,
          0.9713
        ],
        [
          0.1106,
          0.9713
        ],
        [
          0.121,
          0.9713
        ],
        [
          0.131,
          0.9713
        ],
        [
          0.141,
          0.9713
        ],
        [
          0.151,
          0.9713
        ],
        [
          0.1611,
          0.9713
        ],
        [
          0.1715,
          0.9713
        ],
        [
          0.1815,
          0.9713
        ],
        [
          0.1915,
          0.9718
        ],
        [
          0.2015,
          0.9718
        ],
        [
          0.2116,
          0.9718
        ],
        [
          0.222,
          0.9718
        ],
        [
          0.232,
          0.9718
        ],
        [
          0.242,
          0.9709
        ],
        [
          0.252,
          0.97
        ],
        [
          0.2621,
          0.9695
        ],
        [
          0.2725,
          0.9695
        ],
        [
          0.2825,
          0.9691
        ],
        [
          0.2925,
          0.9695
        ],
        [
          0.3025,
          0.9691
        ],
        [
          0.3126,
          0.9686
        ],
        [
          0.323,
          0.9682
        ],
        [
          0.333,
          0.9677
        ],
        [
          0.343,
          0.9672
        ],
        [
          0.353,
          0.9654
        ],
        [
          0.3631,
          0.965
        ],
        [
          0.3735,
          0.9631
        ],
        [
          0.3835,
          0.9618
        ],
        [
          0.3935,
          0.9609
        ],
        [
          0.4035,
          0.9586
        ],
        [
          0.4136,
          0.9572
        ],
        [
          0.424,
          0.9559
        ],
        [
          0.434,
          0.955
        ],
        [
          0.444,
          0.9513
        ],
        [
          0.454,
          0.9495
        ],
        [
          0.4641,
          0.9468
        ],
        [
          0.4745,
          0.944
        ],
        [
          0.4845,
          0.9413
        ],
        [
          0.4945,
          0.9395
        ],
        [
          0.5045,
          0.9372
        ],
        [
          0.5146,
          0.9359
        ],
        [
          0.525,
          0.9336
        ],
        [
          0.535,
          0.929
        ],
        [
          0.545,
          0.9268
        ],
        [
          0.5551,
          0.9231
        ],
        [
          0.5655,
          0.9172
        ],
        [
          0.5755,
          0.9122
        ],
        [
          0.5855,
          0.9063
        ],
        [
          0.5955,
          0.9026
        ],
        [
          0.6056,
          0.8976
        ],
        [
          0.616,
          0.8926
        ],
        [
          0.626,
          0.8899
        ],
        [
          0.636,
          0.8885
        ],
        [
          0.646,
          0.8872
        ],
        [
          0.6561,
          0.8822
        ],
        [
          0.6665,
          0.8776
        ],
        [
          0.6765,
          0.8744
        ],
        [
          0.6865,
          0.8703
        ],
        [
          0.6965,
          0.8662
        ],
        [
          0.7066,
          0.8621
        ],
        [
          0.717,
          0.8581
        ],
        [
          0.727,
          0.8521
        ],
        [
          0.737,
          0.8458
        ],
        [
          0.747,
          0.8421
        ],
        [
          0.7571,
          0.8371
        ],
        [
          0.7675,
          0.8326
        ],
        [
          0.7775,
          0.8267
        ],
        [
          0.7875,
          0.8198
        ],
        [
          0.7975,
          0.813
        ],
        [
          0.8076,
          0.8085
        ],
        [
          0.818,
          0.8007
        ],
        [
          0.828,
          0.7934
        ],
        [
          0.838,
          0.7875
        ],
        [
          0.848,
          0.7812
        ],
        [
          0.8581,
          0.7757
        ],
        [
          0.8685,
          0.7712
        ],
        [
          0.8785,
          0.7639
        ],
        [
          0.8885,
          0.7566
        ],
        [
          0.8985,
          0.7507
        ],
        [
          0.9086,
          0.7448
        ],
        [
          0.919,
          0.7379
        ],
        [
          0.929,
          0.7311
        ],
        [
          0.939,
          0.7257
        ],
        [
          0.949,
          0.7193
        ],
        [
          0.9591,
          0.7138
        ],
        [
          0.9695,
          0.7075
        ],
        [
          0.9795,
          0.7006
        ],
        [
          0.9895,
          0.6929
        ],
        [
          0.9995,
          0.6852
        ],
        [
          1.0,
          0.6847
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.2518582391635817,
        "difference": -0.0011917156164963038,
        "worse": 0.49,
        "kept_right": 0.2491870418630423,
        "kept_wrong": 0.002671197300539426,
        "handed_off": 0.748141760836418
      },
      "usd_per_1000": 0.7351590716669488
    },
    {
      "decision": 248,
      "frontier": 190,
      "frontier_accuracy": 0.9672429481346679,
      "in_sample": {
        "threshold": 0.9359281587918308,
        "answered": 0.2470427661510464,
        "accuracy": 0.9672429481346679
      },
      "curve": [
        [
          0.0,
          0.9672
        ],
        [
          0.01,
          0.9672
        ],
        [
          0.02,
          0.9672
        ],
        [
          0.03,
          0.9672
        ],
        [
          0.04,
          0.9672
        ],
        [
          0.05,
          0.9672
        ],
        [
          0.0601,
          0.9672
        ],
        [
          0.0705,
          0.9672
        ],
        [
          0.0805,
          0.9672
        ],
        [
          0.0905,
          0.9672
        ],
        [
          0.1005,
          0.9672
        ],
        [
          0.1106,
          0.9672
        ],
        [
          0.121,
          0.9672
        ],
        [
          0.131,
          0.9672
        ],
        [
          0.141,
          0.9672
        ],
        [
          0.151,
          0.9672
        ],
        [
          0.1611,
          0.9672
        ],
        [
          0.1715,
          0.9672
        ],
        [
          0.1815,
          0.9672
        ],
        [
          0.1915,
          0.9672
        ],
        [
          0.2015,
          0.9672
        ],
        [
          0.2116,
          0.9677
        ],
        [
          0.222,
          0.9677
        ],
        [
          0.232,
          0.9677
        ],
        [
          0.242,
          0.9672
        ],
        [
          0.252,
          0.9663
        ],
        [
          0.2621,
          0.9659
        ],
        [
          0.2725,
          0.9659
        ],
        [
          0.2825,
          0.9654
        ],
        [
          0.2925,
          0.9659
        ],
        [
          0.3025,
          0.9654
        ],
        [
          0.3126,
          0.965
        ],
        [
          0.323,
          0.9645
        ],
        [
          0.333,
          0.9641
        ],
        [
          0.343,
          0.9631
        ],
        [
          0.353,
          0.9618
        ],
        [
          0.3631,
          0.9613
        ],
        [
          0.3735,
          0.9595
        ],
        [
          0.3835,
          0.9581
        ],
        [
          0.3935,
          0.9577
        ],
        [
          0.4035,
          0.955
        ],
        [
          0.4136,
          0.9531
        ],
        [
          0.424,
          0.9513
        ],
        [
          0.434,
          0.9509
        ],
        [
          0.444,
          0.9468
        ],
        [
          0.454,
          0.9449
        ],
        [
          0.4641,
          0.9413
        ],
        [
          0.4745,
          0.9386
        ],
        [
          0.4845,
          0.9359
        ],
        [
          0.4945,
          0.934
        ],
        [
          0.5045,
          0.9318
        ],
        [
          0.5146,
          0.9308
        ],
        [
          0.525,
          0.9286
        ],
        [
          0.535,
          0.9245
        ],
        [
          0.545,
          0.9222
        ],
        [
          0.5551,
          0.9186
        ],
        [
          0.5655,
          0.9131
        ],
        [
          0.5755,
          0.9086
        ],
        [
          0.5855,
          0.9031
        ],
        [
          0.5955,
          0.8995
        ],
        [
          0.6056,
          0.8949
        ],
        [
          0.616,
          0.8899
        ],
        [
          0.626,
          0.8872
        ],
        [
          0.636,
          0.8854
        ],
        [
          0.646,
          0.8831
        ],
        [
          0.6561,
          0.8781
        ],
        [
          0.6665,
          0.8731
        ],
        [
          0.6765,
          0.8703
        ],
        [
          0.6865,
          0.8662
        ],
        [
          0.6965,
          0.8621
        ],
        [
          0.7066,
          0.8585
        ],
        [
          0.717,
          0.8544
        ],
        [
          0.727,
          0.849
        ],
        [
          0.737,
          0.8426
        ],
        [
          0.747,
          0.8389
        ],
        [
          0.7571,
          0.8344
        ],
        [
          0.7675,
          0.8289
        ],
        [
          0.7775,
          0.8239
        ],
        [
          0.7875,
          0.8171
        ],
        [
          0.7975,
          0.8107
        ],
        [
          0.8076,
          0.8066
        ],
        [
          0.818,
          0.7994
        ],
        [
          0.828,
          0.793
        ],
        [
          0.838,
          0.7871
        ],
        [
          0.848,
          0.7812
        ],
        [
          0.8581,
          0.7753
        ],
        [
          0.8685,
          0.7707
        ],
        [
          0.8785,
          0.7634
        ],
        [
          0.8885,
          0.7566
        ],
        [
          0.8985,
          0.7498
        ],
        [
          0.9086,
          0.7434
        ],
        [
          0.919,
          0.7366
        ],
        [
          0.929,
          0.7298
        ],
        [
          0.939,
          0.7243
        ],
        [
          0.949,
          0.7175
        ],
        [
          0.9591,
          0.7125
        ],
        [
          0.9695,
          0.7066
        ],
        [
          0.9795,
          0.6997
        ],
        [
          0.9895,
          0.6924
        ],
        [
          0.9995,
          0.6852
        ],
        [
          1.0,
          0.6847
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.2632807133807822,
        "difference": -0.0014917573423651536,
        "worse": 0.51,
        "kept_right": 0.25998200813723976,
        "kept_wrong": 0.003298705243542378,
        "handed_off": 0.7367192866192177
      },
      "usd_per_1000": 4.116804857487332
    },
    {
      "decision": 248,
      "frontier": 124,
      "frontier_accuracy": 0.9667879890809827,
      "in_sample": {
        "threshold": 0.9438777175486791,
        "answered": 0.23339399454049137,
        "accuracy": 0.9667879890809827
      },
      "curve": [
        [
          0.0,
          0.9668
        ],
        [
          0.01,
          0.9668
        ],
        [
          0.02,
          0.9668
        ],
        [
          0.03,
          0.9668
        ],
        [
          0.04,
          0.9668
        ],
        [
          0.05,
          0.9668
        ],
        [
          0.0601,
          0.9668
        ],
        [
          0.0705,
          0.9668
        ],
        [
          0.0805,
          0.9668
        ],
        [
          0.0905,
          0.9668
        ],
        [
          0.1005,
          0.9668
        ],
        [
          0.1106,
          0.9668
        ],
        [
          0.121,
          0.9668
        ],
        [
          0.131,
          0.9668
        ],
        [
          0.141,
          0.9668
        ],
        [
          0.151,
          0.9668
        ],
        [
          0.1611,
          0.9668
        ],
        [
          0.1715,
          0.9668
        ],
        [
          0.1815,
          0.9668
        ],
        [
          0.1915,
          0.9668
        ],
        [
          0.2015,
          0.9668
        ],
        [
          0.2116,
          0.9672
        ],
        [
          0.222,
          0.9668
        ],
        [
          0.232,
          0.9668
        ],
        [
          0.242,
          0.9659
        ],
        [
          0.252,
          0.965
        ],
        [
          0.2621,
          0.9645
        ],
        [
          0.2725,
          0.9645
        ],
        [
          0.2825,
          0.9641
        ],
        [
          0.2925,
          0.9641
        ],
        [
          0.3025,
          0.9636
        ],
        [
          0.3126,
          0.9631
        ],
        [
          0.323,
          0.9627
        ],
        [
          0.333,
          0.9622
        ],
        [
          0.343,
          0.9609
        ],
        [
          0.353,
          0.9591
        ],
        [
          0.3631,
          0.9586
        ],
        [
          0.3735,
          0.9568
        ],
        [
          0.3835,
          0.9554
        ],
        [
          0.3935,
          0.955
        ],
        [
          0.4035,
          0.9527
        ],
        [
          0.4136,
          0.9509
        ],
        [
          0.424,
          0.949
        ],
        [
          0.434,
          0.9486
        ],
        [
          0.444,
          0.9449
        ],
        [
          0.454,
          0.9436
        ],
        [
          0.4641,
          0.9404
        ],
        [
          0.4745,
          0.9377
        ],
        [
          0.4845,
          0.9349
        ],
        [
          0.4945,
          0.9331
        ],
        [
          0.5045,
          0.9313
        ],
        [
          0.5146,
          0.9304
        ],
        [
          0.525,
          0.9272
        ],
        [
          0.535,
          0.9236
        ],
        [
          0.545,
          0.9213
        ],
        [
          0.5551,
          0.9177
        ],
        [
          0.5655,
          0.9122
        ],
        [
          0.5755,
          0.9076
        ],
        [
          0.5855,
          0.9026
        ],
        [
          0.5955,
          0.899
        ],
        [
          0.6056,
          0.8944
        ],
        [
          0.616,
          0.8899
        ],
        [
          0.626,
          0.8872
        ],
        [
          0.636,
          0.8854
        ],
        [
          0.646,
          0.8835
        ],
        [
          0.6561,
          0.8785
        ],
        [
          0.6665,
          0.8735
        ],
        [
          0.6765,
          0.8708
        ],
        [
          0.6865,
          0.8667
        ],
        [
          0.6965,
          0.8631
        ],
        [
          0.7066,
          0.8594
        ],
        [
          0.717,
          0.8558
        ],
        [
          0.727,
          0.8499
        ],
        [
          0.737,
          0.8435
        ],
        [
          0.747,
          0.8389
        ],
        [
          0.7571,
          0.8339
        ],
        [
          0.7675,
          0.8285
        ],
        [
          0.7775,
          0.8226
        ],
        [
          0.7875,
          0.8162
        ],
        [
          0.7975,
          0.8098
        ],
        [
          0.8076,
          0.8062
        ],
        [
          0.818,
          0.7989
        ],
        [
          0.828,
          0.7916
        ],
        [
          0.838,
          0.7857
        ],
        [
          0.848,
          0.7793
        ],
        [
          0.8581,
          0.7739
        ],
        [
          0.8685,
          0.7693
        ],
        [
          0.8785,
          0.7621
        ],
        [
          0.8885,
          0.7548
        ],
        [
          0.8985,
          0.748
        ],
        [
          0.9086,
          0.742
        ],
        [
          0.919,
          0.7352
        ],
        [
          0.929,
          0.7284
        ],
        [
          0.939,
          0.7229
        ],
        [
          0.949,
          0.7166
        ],
        [
          0.9591,
          0.7125
        ],
        [
          0.9695,
          0.7066
        ],
        [
          0.9795,
          0.6997
        ],
        [
          0.9895,
          0.6924
        ],
        [
          0.9995,
          0.6852
        ],
        [
          1.0,
          0.6847
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.2364342302336929,
        "difference": -0.0009490456800917263,
        "worse": 0.48,
        "kept_right": 0.23479095203490558,
        "kept_wrong": 0.0016432781987874296,
        "handed_off": 0.763565769766307
      },
      "usd_per_1000": 0.8774836801322002
    },
    {
      "decision": 248,
      "frontier": 278,
      "frontier_accuracy": 0.9499545040946314,
      "in_sample": {
        "threshold": 0.9549926987304493,
        "answered": 0.21155595996360327,
        "accuracy": 0.9499545040946314
      },
      "curve": [
        [
          0.0,
          0.95
        ],
        [
          0.01,
          0.95
        ],
        [
          0.02,
          0.95
        ],
        [
          0.03,
          0.95
        ],
        [
          0.04,
          0.95
        ],
        [
          0.05,
          0.95
        ],
        [
          0.0601,
          0.95
        ],
        [
          0.0705,
          0.95
        ],
        [
          0.0805,
          0.95
        ],
        [
          0.0905,
          0.95
        ],
        [
          0.1005,
          0.95
        ],
        [
          0.1106,
          0.95
        ],
        [
          0.121,
          0.95
        ],
        [
          0.131,
          0.95
        ],
        [
          0.141,
          0.95
        ],
        [
          0.151,
          0.95
        ],
        [
          0.1611,
          0.95
        ],
        [
          0.1715,
          0.95
        ],
        [
          0.1815,
          0.95
        ],
        [
          0.1915,
          0.95
        ],
        [
          0.2015,
          0.95
        ],
        [
          0.2116,
          0.95
        ],
        [
          0.222,
          0.9495
        ],
        [
          0.232,
          0.9495
        ],
        [
          0.242,
          0.949
        ],
        [
          0.252,
          0.9481
        ],
        [
          0.2621,
          0.9481
        ],
        [
          0.2725,
          0.9481
        ],
        [
          0.2825,
          0.9477
        ],
        [
          0.2925,
          0.9477
        ],
        [
          0.3025,
          0.9477
        ],
        [
          0.3126,
          0.9472
        ],
        [
          0.323,
          0.9468
        ],
        [
          0.333,
          0.9463
        ],
        [
          0.343,
          0.9459
        ],
        [
          0.353,
          0.944
        ],
        [
          0.3631,
          0.9436
        ],
        [
          0.3735,
          0.9422
        ],
        [
          0.3835,
          0.9409
        ],
        [
          0.3935,
          0.9404
        ],
        [
          0.4035,
          0.9381
        ],
        [
          0.4136,
          0.9363
        ],
        [
          0.424,
          0.9349
        ],
        [
          0.434,
          0.9345
        ],
        [
          0.444,
          0.9313
        ],
        [
          0.454,
          0.9299
        ],
        [
          0.4641,
          0.9268
        ],
        [
          0.4745,
          0.9245
        ],
        [
          0.4845,
          0.9217
        ],
        [
          0.4945,
          0.9208
        ],
        [
          0.5045,
          0.9195
        ],
        [
          0.5146,
          0.9181
        ],
        [
          0.525,
          0.9158
        ],
        [
          0.535,
          0.9122
        ],
        [
          0.545,
          0.9099
        ],
        [
          0.5551,
          0.9063
        ],
        [
          0.5655,
          0.9013
        ],
        [
          0.5755,
          0.8967
        ],
        [
          0.5855,
          0.8922
        ],
        [
          0.5955,
          0.8885
        ],
        [
          0.6056,
          0.8835
        ],
        [
          0.616,
          0.8785
        ],
        [
          0.626,
          0.8758
        ],
        [
          0.636,
          0.8744
        ],
        [
          0.646,
          0.8726
        ],
        [
          0.6561,
          0.8676
        ],
        [
          0.6665,
          0.864
        ],
        [
          0.6765,
          0.8612
        ],
        [
          0.6865,
          0.8576
        ],
        [
          0.6965,
          0.8535
        ],
        [
          0.7066,
          0.8494
        ],
        [
          0.717,
          0.8458
        ],
        [
          0.727,
          0.8399
        ],
        [
          0.737,
          0.8348
        ],
        [
          0.747,
          0.8317
        ],
        [
          0.7571,
          0.8267
        ],
        [
          0.7675,
          0.8212
        ],
        [
          0.7775,
          0.8157
        ],
        [
          0.7875,
          0.8103
        ],
        [
          0.7975,
          0.8039
        ],
        [
          0.8076,
          0.7998
        ],
        [
          0.818,
          0.7934
        ],
        [
          0.828,
          0.7871
        ],
        [
          0.838,
          0.7816
        ],
        [
          0.848,
          0.7757
        ],
        [
          0.8581,
          0.7707
        ],
        [
          0.8685,
          0.7662
        ],
        [
          0.8785,
          0.7598
        ],
        [
          0.8885,
          0.753
        ],
        [
          0.8985,
          0.7461
        ],
        [
          0.9086,
          0.7402
        ],
        [
          0.919,
          0.7334
        ],
        [
          0.929,
          0.7275
        ],
        [
          0.939,
          0.7229
        ],
        [
          0.949,
          0.7161
        ],
        [
          0.9591,
          0.7116
        ],
        [
          0.9695,
          0.7061
        ],
        [
          0.9795,
          0.6993
        ],
        [
          0.9895,
          0.692
        ],
        [
          0.9995,
          0.6847
        ],
        [
          1.0,
          0.6847
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.2363287384299998,
        "difference": -0.001197486782396448,
        "worse": 0.5266666666666666,
        "kept_right": 0.2345097493246493,
        "kept_wrong": 0.001818989105350616,
        "handed_off": 0.7636712615700001
      },
      "usd_per_1000": 4.9849730888349875
    },
    {
      "decision": 248,
      "frontier": 275,
      "frontier_accuracy": 0.9426751592356688,
      "in_sample": {
        "threshold": 0.9401461695392861,
        "answered": 0.2411282984531392,
        "accuracy": 0.9426751592356688
      },
      "curve": [
        [
          0.0,
          0.9427
        ],
        [
          0.01,
          0.9427
        ],
        [
          0.02,
          0.9427
        ],
        [
          0.03,
          0.9427
        ],
        [
          0.04,
          0.9427
        ],
        [
          0.05,
          0.9427
        ],
        [
          0.0601,
          0.9427
        ],
        [
          0.0705,
          0.9427
        ],
        [
          0.0805,
          0.9427
        ],
        [
          0.0905,
          0.9427
        ],
        [
          0.1005,
          0.9427
        ],
        [
          0.1106,
          0.9427
        ],
        [
          0.121,
          0.9427
        ],
        [
          0.131,
          0.9427
        ],
        [
          0.141,
          0.9427
        ],
        [
          0.151,
          0.9427
        ],
        [
          0.1611,
          0.9427
        ],
        [
          0.1715,
          0.9427
        ],
        [
          0.1815,
          0.9427
        ],
        [
          0.1915,
          0.9427
        ],
        [
          0.2015,
          0.9427
        ],
        [
          0.2116,
          0.9431
        ],
        [
          0.222,
          0.9427
        ],
        [
          0.232,
          0.9427
        ],
        [
          0.242,
          0.9422
        ],
        [
          0.252,
          0.9413
        ],
        [
          0.2621,
          0.9409
        ],
        [
          0.2725,
          0.9409
        ],
        [
          0.2825,
          0.9404
        ],
        [
          0.2925,
          0.9404
        ],
        [
          0.3025,
          0.9399
        ],
        [
          0.3126,
          0.9395
        ],
        [
          0.323,
          0.939
        ],
        [
          0.333,
          0.9386
        ],
        [
          0.343,
          0.9381
        ],
        [
          0.353,
          0.9363
        ],
        [
          0.3631,
          0.9359
        ],
        [
          0.3735,
          0.9345
        ],
        [
          0.3835,
          0.9331
        ],
        [
          0.3935,
          0.9327
        ],
        [
          0.4035,
          0.9308
        ],
        [
          0.4136,
          0.9295
        ],
        [
          0.424,
          0.9281
        ],
        [
          0.434,
          0.9277
        ],
        [
          0.444,
          0.9245
        ],
        [
          0.454,
          0.9227
        ],
        [
          0.4641,
          0.9195
        ],
        [
          0.4745,
          0.9177
        ],
        [
          0.4845,
          0.9154
        ],
        [
          0.4945,
          0.9136
        ],
        [
          0.5045,
          0.9117
        ],
        [
          0.5146,
          0.9113
        ],
        [
          0.525,
          0.909
        ],
        [
          0.535,
          0.9054
        ],
        [
          0.545,
          0.9031
        ],
        [
          0.5551,
          0.8999
        ],
        [
          0.5655,
          0.8963
        ],
        [
          0.5755,
          0.8913
        ],
        [
          0.5855,
          0.8872
        ],
        [
          0.5955,
          0.8844
        ],
        [
          0.6056,
          0.8803
        ],
        [
          0.616,
          0.8758
        ],
        [
          0.626,
          0.8731
        ],
        [
          0.636,
          0.8712
        ],
        [
          0.646,
          0.869
        ],
        [
          0.6561,
          0.864
        ],
        [
          0.6665,
          0.8603
        ],
        [
          0.6765,
          0.8581
        ],
        [
          0.6865,
          0.854
        ],
        [
          0.6965,
          0.8503
        ],
        [
          0.7066,
          0.8471
        ],
        [
          0.717,
          0.843
        ],
        [
          0.727,
          0.8371
        ],
        [
          0.737,
          0.8308
        ],
        [
          0.747,
          0.828
        ],
        [
          0.7571,
          0.8235
        ],
        [
          0.7675,
          0.8185
        ],
        [
          0.7775,
          0.813
        ],
        [
          0.7875,
          0.8062
        ],
        [
          0.7975,
          0.7998
        ],
        [
          0.8076,
          0.7957
        ],
        [
          0.818,
          0.7898
        ],
        [
          0.828,
          0.7834
        ],
        [
          0.838,
          0.7789
        ],
        [
          0.848,
          0.773
        ],
        [
          0.8581,
          0.768
        ],
        [
          0.8685,
          0.7639
        ],
        [
          0.8785,
          0.7575
        ],
        [
          0.8885,
          0.7511
        ],
        [
          0.8985,
          0.7448
        ],
        [
          0.9086,
          0.7393
        ],
        [
          0.919,
          0.732
        ],
        [
          0.929,
          0.7257
        ],
        [
          0.939,
          0.7207
        ],
        [
          0.949,
          0.7143
        ],
        [
          0.9591,
          0.7111
        ],
        [
          0.9695,
          0.7056
        ],
        [
          0.9795,
          0.6993
        ],
        [
          0.9895,
          0.6929
        ],
        [
          0.9995,
          0.6852
        ],
        [
          1.0,
          0.6847
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.24311645724652087,
        "difference": -0.0010339497082953075,
        "worse": 0.4666666666666667,
        "kept_right": 0.24100627444776332,
        "kept_wrong": 0.0021101827987577235,
        "handed_off": 0.7568835427534789
      },
      "usd_per_1000": 0.40379606847129645
    },
    {
      "decision": 248,
      "frontier": 192,
      "frontier_accuracy": 0.9331210191082803,
      "in_sample": {
        "threshold": 0.9401461695392861,
        "answered": 0.2411282984531392,
        "accuracy": 0.9331210191082803
      },
      "curve": [
        [
          0.0,
          0.9331
        ],
        [
          0.01,
          0.9331
        ],
        [
          0.02,
          0.9331
        ],
        [
          0.03,
          0.9331
        ],
        [
          0.04,
          0.9331
        ],
        [
          0.05,
          0.9331
        ],
        [
          0.0601,
          0.9331
        ],
        [
          0.0705,
          0.9331
        ],
        [
          0.0805,
          0.9331
        ],
        [
          0.0905,
          0.9331
        ],
        [
          0.1005,
          0.9331
        ],
        [
          0.1106,
          0.9331
        ],
        [
          0.121,
          0.9331
        ],
        [
          0.131,
          0.9331
        ],
        [
          0.141,
          0.9331
        ],
        [
          0.151,
          0.9331
        ],
        [
          0.1611,
          0.9331
        ],
        [
          0.1715,
          0.9331
        ],
        [
          0.1815,
          0.9331
        ],
        [
          0.1915,
          0.9331
        ],
        [
          0.2015,
          0.9331
        ],
        [
          0.2116,
          0.9336
        ],
        [
          0.222,
          0.9336
        ],
        [
          0.232,
          0.9336
        ],
        [
          0.242,
          0.9327
        ],
        [
          0.252,
          0.9318
        ],
        [
          0.2621,
          0.9313
        ],
        [
          0.2725,
          0.9313
        ],
        [
          0.2825,
          0.9308
        ],
        [
          0.2925,
          0.9308
        ],
        [
          0.3025,
          0.9308
        ],
        [
          0.3126,
          0.9304
        ],
        [
          0.323,
          0.9299
        ],
        [
          0.333,
          0.9295
        ],
        [
          0.343,
          0.9286
        ],
        [
          0.353,
          0.9272
        ],
        [
          0.3631,
          0.9268
        ],
        [
          0.3735,
          0.9258
        ],
        [
          0.3835,
          0.9245
        ],
        [
          0.3935,
          0.9236
        ],
        [
          0.4035,
          0.9213
        ],
        [
          0.4136,
          0.9208
        ],
        [
          0.424,
          0.9199
        ],
        [
          0.434,
          0.919
        ],
        [
          0.444,
          0.9158
        ],
        [
          0.454,
          0.9145
        ],
        [
          0.4641,
          0.9122
        ],
        [
          0.4745,
          0.9095
        ],
        [
          0.4845,
          0.9072
        ],
        [
          0.4945,
          0.9063
        ],
        [
          0.5045,
          0.9054
        ],
        [
          0.5146,
          0.9045
        ],
        [
          0.525,
          0.9022
        ],
        [
          0.535,
          0.8985
        ],
        [
          0.545,
          0.8963
        ],
        [
          0.5551,
          0.8935
        ],
        [
          0.5655,
          0.8894
        ],
        [
          0.5755,
          0.8844
        ],
        [
          0.5855,
          0.8799
        ],
        [
          0.5955,
          0.8767
        ],
        [
          0.6056,
          0.8726
        ],
        [
          0.616,
          0.8685
        ],
        [
          0.626,
          0.8658
        ],
        [
          0.636,
          0.8644
        ],
        [
          0.646,
          0.8621
        ],
        [
          0.6561,
          0.8571
        ],
        [
          0.6665,
          0.853
        ],
        [
          0.6765,
          0.8508
        ],
        [
          0.6865,
          0.8471
        ],
        [
          0.6965,
          0.843
        ],
        [
          0.7066,
          0.8399
        ],
        [
          0.717,
          0.8367
        ],
        [
          0.727,
          0.8321
        ],
        [
          0.737,
          0.8276
        ],
        [
          0.747,
          0.8244
        ],
        [
          0.7571,
          0.8198
        ],
        [
          0.7675,
          0.8148
        ],
        [
          0.7775,
          0.8112
        ],
        [
          0.7875,
          0.8053
        ],
        [
          0.7975,
          0.7989
        ],
        [
          0.8076,
          0.7953
        ],
        [
          0.818,
          0.788
        ],
        [
          0.828,
          0.7821
        ],
        [
          0.838,
          0.7771
        ],
        [
          0.848,
          0.7712
        ],
        [
          0.8581,
          0.7657
        ],
        [
          0.8685,
          0.7611
        ],
        [
          0.8785,
          0.7543
        ],
        [
          0.8885,
          0.7484
        ],
        [
          0.8985,
          0.742
        ],
        [
          0.9086,
          0.7357
        ],
        [
          0.919,
          0.7302
        ],
        [
          0.929,
          0.7238
        ],
        [
          0.939,
          0.7193
        ],
        [
          0.949,
          0.7129
        ],
        [
          0.9591,
          0.7088
        ],
        [
          0.9695,
          0.7038
        ],
        [
          0.9795,
          0.6979
        ],
        [
          0.9895,
          0.692
        ],
        [
          0.9995,
          0.6847
        ],
        [
          1.0,
          0.6847
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.24907306913910895,
        "difference": -0.0011430321140264816,
        "worse": 0.52,
        "kept_right": 0.2466293544243215,
        "kept_wrong": 0.002443714714787468,
        "handed_off": 0.7509269308608908
      },
      "usd_per_1000": 8.732304763022995
    },
    {
      "decision": 248,
      "frontier": 123,
      "frontier_accuracy": 0.9326660600545951,
      "in_sample": {
        "threshold": 0.9549926987304493,
        "answered": 0.21155595996360327,
        "accuracy": 0.9326660600545951
      },
      "curve": [
        [
          0.0,
          0.9327
        ],
        [
          0.01,
          0.9327
        ],
        [
          0.02,
          0.9327
        ],
        [
          0.03,
          0.9327
        ],
        [
          0.04,
          0.9327
        ],
        [
          0.05,
          0.9327
        ],
        [
          0.0601,
          0.9327
        ],
        [
          0.0705,
          0.9327
        ],
        [
          0.0805,
          0.9327
        ],
        [
          0.0905,
          0.9327
        ],
        [
          0.1005,
          0.9327
        ],
        [
          0.1106,
          0.9327
        ],
        [
          0.121,
          0.9327
        ],
        [
          0.131,
          0.9327
        ],
        [
          0.141,
          0.9327
        ],
        [
          0.151,
          0.9327
        ],
        [
          0.1611,
          0.9327
        ],
        [
          0.1715,
          0.9327
        ],
        [
          0.1815,
          0.9327
        ],
        [
          0.1915,
          0.9327
        ],
        [
          0.2015,
          0.9327
        ],
        [
          0.2116,
          0.9327
        ],
        [
          0.222,
          0.9322
        ],
        [
          0.232,
          0.9322
        ],
        [
          0.242,
          0.9313
        ],
        [
          0.252,
          0.9304
        ],
        [
          0.2621,
          0.9299
        ],
        [
          0.2725,
          0.9299
        ],
        [
          0.2825,
          0.9295
        ],
        [
          0.2925,
          0.9295
        ],
        [
          0.3025,
          0.929
        ],
        [
          0.3126,
          0.9286
        ],
        [
          0.323,
          0.9281
        ],
        [
          0.333,
          0.9277
        ],
        [
          0.343,
          0.9272
        ],
        [
          0.353,
          0.9263
        ],
        [
          0.3631,
          0.9258
        ],
        [
          0.3735,
          0.924
        ],
        [
          0.3835,
          0.9227
        ],
        [
          0.3935,
          0.9222
        ],
        [
          0.4035,
          0.9199
        ],
        [
          0.4136,
          0.919
        ],
        [
          0.424,
          0.9186
        ],
        [
          0.434,
          0.9186
        ],
        [
          0.444,
          0.9158
        ],
        [
          0.454,
          0.9145
        ],
        [
          0.4641,
          0.9122
        ],
        [
          0.4745,
          0.9095
        ],
        [
          0.4845,
          0.9072
        ],
        [
          0.4945,
          0.9054
        ],
        [
          0.5045,
          0.904
        ],
        [
          0.5146,
          0.9035
        ],
        [
          0.525,
          0.9026
        ],
        [
          0.535,
          0.8981
        ],
        [
          0.545,
          0.8963
        ],
        [
          0.5551,
          0.8926
        ],
        [
          0.5655,
          0.8867
        ],
        [
          0.5755,
          0.8817
        ],
        [
          0.5855,
          0.8785
        ],
        [
          0.5955,
          0.8753
        ],
        [
          0.6056,
          0.8708
        ],
        [
          0.616,
          0.8658
        ],
        [
          0.626,
          0.8631
        ],
        [
          0.636,
          0.8621
        ],
        [
          0.646,
          0.8603
        ],
        [
          0.6561,
          0.8562
        ],
        [
          0.6665,
          0.8521
        ],
        [
          0.6765,
          0.8503
        ],
        [
          0.6865,
          0.8471
        ],
        [
          0.6965,
          0.8435
        ],
        [
          0.7066,
          0.8399
        ],
        [
          0.717,
          0.8362
        ],
        [
          0.727,
          0.8312
        ],
        [
          0.737,
          0.8248
        ],
        [
          0.747,
          0.8207
        ],
        [
          0.7571,
          0.8176
        ],
        [
          0.7675,
          0.8126
        ],
        [
          0.7775,
          0.8066
        ],
        [
          0.7875,
          0.8003
        ],
        [
          0.7975,
          0.7934
        ],
        [
          0.8076,
          0.7907
        ],
        [
          0.818,
          0.7839
        ],
        [
          0.828,
          0.7771
        ],
        [
          0.838,
          0.7716
        ],
        [
          0.848,
          0.7666
        ],
        [
          0.8581,
          0.763
        ],
        [
          0.8685,
          0.7589
        ],
        [
          0.8785,
          0.752
        ],
        [
          0.8885,
          0.7457
        ],
        [
          0.8985,
          0.7393
        ],
        [
          0.9086,
          0.7338
        ],
        [
          0.919,
          0.7284
        ],
        [
          0.929,
          0.7234
        ],
        [
          0.939,
          0.7179
        ],
        [
          0.949,
          0.712
        ],
        [
          0.9591,
          0.7088
        ],
        [
          0.9695,
          0.7038
        ],
        [
          0.9795,
          0.6975
        ],
        [
          0.9895,
          0.6906
        ],
        [
          0.9995,
          0.6852
        ],
        [
          1.0,
          0.6847
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.22616103637411256,
        "difference": -0.0008973952133976517,
        "worse": 0.5033333333333333,
        "kept_right": 0.22500896579056676,
        "kept_wrong": 0.001152070583545764,
        "handed_off": 0.7738389636258871
      },
      "usd_per_1000": 0.2222102172789507
    },
    {
      "decision": 248,
      "frontier": 274,
      "frontier_accuracy": 0.9085532302092811,
      "in_sample": {
        "threshold": 0.9168286892583957,
        "answered": 0.2752502274795268,
        "accuracy": 0.9085532302092811
      },
      "curve": [
        [
          0.0,
          0.9086
        ],
        [
          0.01,
          0.9086
        ],
        [
          0.02,
          0.9086
        ],
        [
          0.03,
          0.9086
        ],
        [
          0.04,
          0.9086
        ],
        [
          0.05,
          0.9086
        ],
        [
          0.0601,
          0.9086
        ],
        [
          0.0705,
          0.9086
        ],
        [
          0.0805,
          0.9086
        ],
        [
          0.0905,
          0.9086
        ],
        [
          0.1005,
          0.9086
        ],
        [
          0.1106,
          0.9086
        ],
        [
          0.121,
          0.9086
        ],
        [
          0.131,
          0.9086
        ],
        [
          0.141,
          0.9086
        ],
        [
          0.151,
          0.9086
        ],
        [
          0.1611,
          0.9086
        ],
        [
          0.1715,
          0.909
        ],
        [
          0.1815,
          0.909
        ],
        [
          0.1915,
          0.9095
        ],
        [
          0.2015,
          0.9095
        ],
        [
          0.2116,
          0.9099
        ],
        [
          0.222,
          0.9099
        ],
        [
          0.232,
          0.9099
        ],
        [
          0.242,
          0.9095
        ],
        [
          0.252,
          0.909
        ],
        [
          0.2621,
          0.9086
        ],
        [
          0.2725,
          0.9086
        ],
        [
          0.2825,
          0.9081
        ],
        [
          0.2925,
          0.9081
        ],
        [
          0.3025,
          0.9081
        ],
        [
          0.3126,
          0.9081
        ],
        [
          0.323,
          0.9076
        ],
        [
          0.333,
          0.9072
        ],
        [
          0.343,
          0.9072
        ],
        [
          0.353,
          0.9058
        ],
        [
          0.3631,
          0.9054
        ],
        [
          0.3735,
          0.904
        ],
        [
          0.3835,
          0.9031
        ],
        [
          0.3935,
          0.9026
        ],
        [
          0.4035,
          0.9004
        ],
        [
          0.4136,
          0.8995
        ],
        [
          0.424,
          0.8985
        ],
        [
          0.434,
          0.8976
        ],
        [
          0.444,
          0.894
        ],
        [
          0.454,
          0.8922
        ],
        [
          0.4641,
          0.8894
        ],
        [
          0.4745,
          0.8872
        ],
        [
          0.4845,
          0.8849
        ],
        [
          0.4945,
          0.8835
        ],
        [
          0.5045,
          0.8813
        ],
        [
          0.5146,
          0.8803
        ],
        [
          0.525,
          0.8785
        ],
        [
          0.535,
          0.8744
        ],
        [
          0.545,
          0.8726
        ],
        [
          0.5551,
          0.869
        ],
        [
          0.5655,
          0.8658
        ],
        [
          0.5755,
          0.8617
        ],
        [
          0.5855,
          0.8576
        ],
        [
          0.5955,
          0.8549
        ],
        [
          0.6056,
          0.8512
        ],
        [
          0.616,
          0.8476
        ],
        [
          0.626,
          0.8449
        ],
        [
          0.636,
          0.8439
        ],
        [
          0.646,
          0.8421
        ],
        [
          0.6561,
          0.8376
        ],
        [
          0.6665,
          0.8344
        ],
        [
          0.6765,
          0.833
        ],
        [
          0.6865,
          0.8294
        ],
        [
          0.6965,
          0.8267
        ],
        [
          0.7066,
          0.8235
        ],
        [
          0.717,
          0.8203
        ],
        [
          0.727,
          0.8144
        ],
        [
          0.737,
          0.8089
        ],
        [
          0.747,
          0.8066
        ],
        [
          0.7571,
          0.803
        ],
        [
          0.7675,
          0.798
        ],
        [
          0.7775,
          0.793
        ],
        [
          0.7875,
          0.7875
        ],
        [
          0.7975,
          0.7816
        ],
        [
          0.8076,
          0.7793
        ],
        [
          0.818,
          0.7739
        ],
        [
          0.828,
          0.768
        ],
        [
          0.838,
          0.7643
        ],
        [
          0.848,
          0.7589
        ],
        [
          0.8581,
          0.7543
        ],
        [
          0.8685,
          0.7502
        ],
        [
          0.8785,
          0.7452
        ],
        [
          0.8885,
          0.7389
        ],
        [
          0.8985,
          0.7338
        ],
        [
          0.9086,
          0.7284
        ],
        [
          0.919,
          0.7225
        ],
        [
          0.929,
          0.7166
        ],
        [
          0.939,
          0.7116
        ],
        [
          0.949,
          0.7079
        ],
        [
          0.9591,
          0.7052
        ],
        [
          0.9695,
          0.7015
        ],
        [
          0.9795,
          0.6961
        ],
        [
          0.9895,
          0.6906
        ],
        [
          0.9995,
          0.6847
        ],
        [
          1.0,
          0.6847
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.3038610791532604,
        "difference": -0.001491862597498103,
        "worse": 0.56,
        "kept_right": 0.2975185582665776,
        "kept_wrong": 0.006342520886683036,
        "handed_off": 0.6961389208467399
      },
      "usd_per_1000": 0.31109570576626944
    },
    {
      "decision": 248,
      "frontier": 47,
      "frontier_accuracy": 0.9044585987261147,
      "in_sample": {
        "threshold": 0.8892715854938671,
        "answered": 0.31528662420382164,
        "accuracy": 0.9044585987261147
      },
      "curve": [
        [
          0.0,
          0.9045
        ],
        [
          0.01,
          0.9045
        ],
        [
          0.02,
          0.9045
        ],
        [
          0.03,
          0.9045
        ],
        [
          0.04,
          0.9045
        ],
        [
          0.05,
          0.9045
        ],
        [
          0.0601,
          0.9045
        ],
        [
          0.0705,
          0.9045
        ],
        [
          0.0805,
          0.9045
        ],
        [
          0.0905,
          0.9045
        ],
        [
          0.1005,
          0.9045
        ],
        [
          0.1106,
          0.9045
        ],
        [
          0.121,
          0.9045
        ],
        [
          0.131,
          0.9045
        ],
        [
          0.141,
          0.9045
        ],
        [
          0.151,
          0.9045
        ],
        [
          0.1611,
          0.9045
        ],
        [
          0.1715,
          0.9045
        ],
        [
          0.1815,
          0.9045
        ],
        [
          0.1915,
          0.9049
        ],
        [
          0.2015,
          0.9049
        ],
        [
          0.2116,
          0.9054
        ],
        [
          0.222,
          0.9049
        ],
        [
          0.232,
          0.9054
        ],
        [
          0.242,
          0.9049
        ],
        [
          0.252,
          0.9045
        ],
        [
          0.2621,
          0.9049
        ],
        [
          0.2725,
          0.9049
        ],
        [
          0.2825,
          0.9045
        ],
        [
          0.2925,
          0.9045
        ],
        [
          0.3025,
          0.9045
        ],
        [
          0.3126,
          0.9045
        ],
        [
          0.323,
          0.904
        ],
        [
          0.333,
          0.9031
        ],
        [
          0.343,
          0.9017
        ],
        [
          0.353,
          0.9004
        ],
        [
          0.3631,
          0.9004
        ],
        [
          0.3735,
          0.899
        ],
        [
          0.3835,
          0.8981
        ],
        [
          0.3935,
          0.8972
        ],
        [
          0.4035,
          0.8963
        ],
        [
          0.4136,
          0.8949
        ],
        [
          0.424,
          0.894
        ],
        [
          0.434,
          0.894
        ],
        [
          0.444,
          0.8926
        ],
        [
          0.454,
          0.8908
        ],
        [
          0.4641,
          0.8885
        ],
        [
          0.4745,
          0.8858
        ],
        [
          0.4845,
          0.8831
        ],
        [
          0.4945,
          0.8826
        ],
        [
          0.5045,
          0.8813
        ],
        [
          0.5146,
          0.8799
        ],
        [
          0.525,
          0.8785
        ],
        [
          0.535,
          0.8753
        ],
        [
          0.545,
          0.8735
        ],
        [
          0.5551,
          0.8708
        ],
        [
          0.5655,
          0.8667
        ],
        [
          0.5755,
          0.8631
        ],
        [
          0.5855,
          0.8594
        ],
        [
          0.5955,
          0.8562
        ],
        [
          0.6056,
          0.8517
        ],
        [
          0.616,
          0.8476
        ],
        [
          0.626,
          0.8449
        ],
        [
          0.636,
          0.843
        ],
        [
          0.646,
          0.8417
        ],
        [
          0.6561,
          0.838
        ],
        [
          0.6665,
          0.8344
        ],
        [
          0.6765,
          0.8326
        ],
        [
          0.6865,
          0.8289
        ],
        [
          0.6965,
          0.8262
        ],
        [
          0.7066,
          0.823
        ],
        [
          0.717,
          0.8194
        ],
        [
          0.727,
          0.8144
        ],
        [
          0.737,
          0.8103
        ],
        [
          0.747,
          0.8066
        ],
        [
          0.7571,
          0.803
        ],
        [
          0.7675,
          0.7971
        ],
        [
          0.7775,
          0.7925
        ],
        [
          0.7875,
          0.7862
        ],
        [
          0.7975,
          0.7803
        ],
        [
          0.8076,
          0.7766
        ],
        [
          0.818,
          0.7702
        ],
        [
          0.828,
          0.7643
        ],
        [
          0.838,
          0.7593
        ],
        [
          0.848,
          0.7534
        ],
        [
          0.8581,
          0.7489
        ],
        [
          0.8685,
          0.7457
        ],
        [
          0.8785,
          0.7398
        ],
        [
          0.8885,
          0.7352
        ],
        [
          0.8985,
          0.7302
        ],
        [
          0.9086,
          0.7261
        ],
        [
          0.919,
          0.7207
        ],
        [
          0.929,
          0.7161
        ],
        [
          0.939,
          0.712
        ],
        [
          0.949,
          0.7061
        ],
        [
          0.9591,
          0.7029
        ],
        [
          0.9695,
          0.6997
        ],
        [
          0.9795,
          0.6934
        ],
        [
          0.9895,
          0.6884
        ],
        [
          0.9995,
          0.6847
        ],
        [
          1.0,
          0.6847
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.30013310684291633,
        "difference": -0.0012974040517823476,
        "worse": 0.44333333333333336,
        "kept_right": 0.29456713362900744,
        "kept_wrong": 0.005565973213908954,
        "handed_off": 0.6998668931570837
      },
      "usd_per_1000": 3.4651233858079418
    },
    {
      "decision": 248,
      "frontier": 193,
      "frontier_accuracy": 0.89171974522293,
      "in_sample": {
        "threshold": 0.9025956556359601,
        "answered": 0.29754322111010006,
        "accuracy": 0.89171974522293
      },
      "curve": [
        [
          0.0,
          0.8917
        ],
        [
          0.01,
          0.8917
        ],
        [
          0.02,
          0.8917
        ],
        [
          0.03,
          0.8917
        ],
        [
          0.04,
          0.8917
        ],
        [
          0.05,
          0.8917
        ],
        [
          0.0601,
          0.8917
        ],
        [
          0.0705,
          0.8917
        ],
        [
          0.0805,
          0.8917
        ],
        [
          0.0905,
          0.8917
        ],
        [
          0.1005,
          0.8917
        ],
        [
          0.1106,
          0.8917
        ],
        [
          0.121,
          0.8917
        ],
        [
          0.131,
          0.8917
        ],
        [
          0.141,
          0.8917
        ],
        [
          0.151,
          0.8917
        ],
        [
          0.1611,
          0.8917
        ],
        [
          0.1715,
          0.8917
        ],
        [
          0.1815,
          0.8922
        ],
        [
          0.1915,
          0.8922
        ],
        [
          0.2015,
          0.8922
        ],
        [
          0.2116,
          0.8926
        ],
        [
          0.222,
          0.8926
        ],
        [
          0.232,
          0.8926
        ],
        [
          0.242,
          0.8922
        ],
        [
          0.252,
          0.8917
        ],
        [
          0.2621,
          0.8917
        ],
        [
          0.2725,
          0.8917
        ],
        [
          0.2825,
          0.8917
        ],
        [
          0.2925,
          0.8917
        ],
        [
          0.3025,
          0.8913
        ],
        [
          0.3126,
          0.8913
        ],
        [
          0.323,
          0.8908
        ],
        [
          0.333,
          0.8904
        ],
        [
          0.343,
          0.8899
        ],
        [
          0.353,
          0.889
        ],
        [
          0.3631,
          0.889
        ],
        [
          0.3735,
          0.8881
        ],
        [
          0.3835,
          0.8863
        ],
        [
          0.3935,
          0.8858
        ],
        [
          0.4035,
          0.884
        ],
        [
          0.4136,
          0.8822
        ],
        [
          0.424,
          0.8826
        ],
        [
          0.434,
          0.8822
        ],
        [
          0.444,
          0.8799
        ],
        [
          0.454,
          0.879
        ],
        [
          0.4641,
          0.8785
        ],
        [
          0.4745,
          0.8758
        ],
        [
          0.4845,
          0.8735
        ],
        [
          0.4945,
          0.8731
        ],
        [
          0.5045,
          0.8731
        ],
        [
          0.5146,
          0.8722
        ],
        [
          0.525,
          0.8703
        ],
        [
          0.535,
          0.8667
        ],
        [
          0.545,
          0.8649
        ],
        [
          0.5551,
          0.8617
        ],
        [
          0.5655,
          0.8581
        ],
        [
          0.5755,
          0.854
        ],
        [
          0.5855,
          0.8508
        ],
        [
          0.5955,
          0.8476
        ],
        [
          0.6056,
          0.8439
        ],
        [
          0.616,
          0.8403
        ],
        [
          0.626,
          0.8389
        ],
        [
          0.636,
          0.838
        ],
        [
          0.646,
          0.8362
        ],
        [
          0.6561,
          0.8317
        ],
        [
          0.6665,
          0.8294
        ],
        [
          0.6765,
          0.8285
        ],
        [
          0.6865,
          0.8248
        ],
        [
          0.6965,
          0.8212
        ],
        [
          0.7066,
          0.818
        ],
        [
          0.717,
          0.8144
        ],
        [
          0.727,
          0.8094
        ],
        [
          0.737,
          0.8053
        ],
        [
          0.747,
          0.8021
        ],
        [
          0.7571,
          0.798
        ],
        [
          0.7675,
          0.7934
        ],
        [
          0.7775,
          0.7889
        ],
        [
          0.7875,
          0.7843
        ],
        [
          0.7975,
          0.7784
        ],
        [
          0.8076,
          0.7757
        ],
        [
          0.818,
          0.7712
        ],
        [
          0.828,
          0.7652
        ],
        [
          0.838,
          0.7607
        ],
        [
          0.848,
          0.7557
        ],
        [
          0.8581,
          0.7516
        ],
        [
          0.8685,
          0.7489
        ],
        [
          0.8785,
          0.7429
        ],
        [
          0.8885,
          0.7366
        ],
        [
          0.8985,
          0.7311
        ],
        [
          0.9086,
          0.7257
        ],
        [
          0.919,
          0.7211
        ],
        [
          0.929,
          0.7161
        ],
        [
          0.939,
          0.7129
        ],
        [
          0.949,
          0.7084
        ],
        [
          0.9591,
          0.7061
        ],
        [
          0.9695,
          0.7011
        ],
        [
          0.9795,
          0.6961
        ],
        [
          0.9895,
          0.6902
        ],
        [
          0.9995,
          0.6847
        ],
        [
          1.0,
          0.6847
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.30187236710970766,
        "difference": -0.001264346331573234,
        "worse": 0.5733333333333334,
        "kept_right": 0.2960666331965821,
        "kept_wrong": 0.005805733913125618,
        "handed_off": 0.6981276328902922
      },
      "usd_per_1000": 5.423176160497151
    },
    {
      "decision": 248,
      "frontier": 55,
      "frontier_accuracy": 0.8503184713375797,
      "in_sample": {
        "threshold": 0.6940946862959242,
        "answered": 0.532302092811647,
        "accuracy": 0.8503184713375797
      },
      "curve": [
        [
          0.0,
          0.8503
        ],
        [
          0.01,
          0.8503
        ],
        [
          0.02,
          0.8503
        ],
        [
          0.03,
          0.8503
        ],
        [
          0.04,
          0.8508
        ],
        [
          0.05,
          0.8508
        ],
        [
          0.0601,
          0.8508
        ],
        [
          0.0705,
          0.8512
        ],
        [
          0.0805,
          0.8517
        ],
        [
          0.0905,
          0.8517
        ],
        [
          0.1005,
          0.8517
        ],
        [
          0.1106,
          0.8521
        ],
        [
          0.121,
          0.8521
        ],
        [
          0.131,
          0.8526
        ],
        [
          0.141,
          0.853
        ],
        [
          0.151,
          0.853
        ],
        [
          0.1611,
          0.854
        ],
        [
          0.1715,
          0.8544
        ],
        [
          0.1815,
          0.8549
        ],
        [
          0.1915,
          0.8553
        ],
        [
          0.2015,
          0.8558
        ],
        [
          0.2116,
          0.8558
        ],
        [
          0.222,
          0.8558
        ],
        [
          0.232,
          0.8567
        ],
        [
          0.242,
          0.8562
        ],
        [
          0.252,
          0.8562
        ],
        [
          0.2621,
          0.8567
        ],
        [
          0.2725,
          0.8581
        ],
        [
          0.2825,
          0.8581
        ],
        [
          0.2925,
          0.859
        ],
        [
          0.3025,
          0.8594
        ],
        [
          0.3126,
          0.8603
        ],
        [
          0.323,
          0.8617
        ],
        [
          0.333,
          0.8626
        ],
        [
          0.343,
          0.8626
        ],
        [
          0.353,
          0.8617
        ],
        [
          0.3631,
          0.8626
        ],
        [
          0.3735,
          0.8626
        ],
        [
          0.3835,
          0.8612
        ],
        [
          0.3935,
          0.8608
        ],
        [
          0.4035,
          0.8594
        ],
        [
          0.4136,
          0.859
        ],
        [
          0.424,
          0.859
        ],
        [
          0.434,
          0.8594
        ],
        [
          0.444,
          0.8571
        ],
        [
          0.454,
          0.8558
        ],
        [
          0.4641,
          0.8567
        ],
        [
          0.4745,
          0.8553
        ],
        [
          0.4845,
          0.8535
        ],
        [
          0.4945,
          0.8535
        ],
        [
          0.5045,
          0.8535
        ],
        [
          0.5146,
          0.853
        ],
        [
          0.525,
          0.8517
        ],
        [
          0.535,
          0.8494
        ],
        [
          0.545,
          0.848
        ],
        [
          0.5551,
          0.8453
        ],
        [
          0.5655,
          0.8417
        ],
        [
          0.5755,
          0.838
        ],
        [
          0.5855,
          0.8344
        ],
        [
          0.5955,
          0.8321
        ],
        [
          0.6056,
          0.8276
        ],
        [
          0.616,
          0.8253
        ],
        [
          0.626,
          0.8235
        ],
        [
          0.636,
          0.8221
        ],
        [
          0.646,
          0.8212
        ],
        [
          0.6561,
          0.8171
        ],
        [
          0.6665,
          0.8139
        ],
        [
          0.6765,
          0.8116
        ],
        [
          0.6865,
          0.8085
        ],
        [
          0.6965,
          0.8076
        ],
        [
          0.7066,
          0.8048
        ],
        [
          0.717,
          0.8021
        ],
        [
          0.727,
          0.7975
        ],
        [
          0.737,
          0.7934
        ],
        [
          0.747,
          0.7907
        ],
        [
          0.7571,
          0.788
        ],
        [
          0.7675,
          0.7834
        ],
        [
          0.7775,
          0.7807
        ],
        [
          0.7875,
          0.7757
        ],
        [
          0.7975,
          0.7702
        ],
        [
          0.8076,
          0.7675
        ],
        [
          0.818,
          0.7625
        ],
        [
          0.828,
          0.7584
        ],
        [
          0.838,
          0.7539
        ],
        [
          0.848,
          0.7498
        ],
        [
          0.8581,
          0.7466
        ],
        [
          0.8685,
          0.7448
        ],
        [
          0.8785,
          0.7384
        ],
        [
          0.8885,
          0.7348
        ],
        [
          0.8985,
          0.7293
        ],
        [
          0.9086,
          0.7252
        ],
        [
          0.919,
          0.7216
        ],
        [
          0.929,
          0.717
        ],
        [
          0.939,
          0.7138
        ],
        [
          0.949,
          0.7097
        ],
        [
          0.9591,
          0.707
        ],
        [
          0.9695,
          0.7025
        ],
        [
          0.9795,
          0.6965
        ],
        [
          0.9895,
          0.6906
        ],
        [
          0.9995,
          0.6847
        ],
        [
          1.0,
          0.6847
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.5246571432162458,
        "difference": -0.0008168977365752996,
        "worse": 0.49,
        "kept_right": 0.4742760954184199,
        "kept_wrong": 0.05038104779782619,
        "handed_off": 0.47534285678375404
      },
      "usd_per_1000": 1.7514308820737416
    },
    {
      "decision": 129,
      "frontier": 140,
      "frontier_accuracy": 0.9808917197452229,
      "in_sample": {
        "threshold": 0.8914050026003353,
        "answered": 0.19472247497725204,
        "accuracy": 0.9808917197452229
      },
      "curve": [
        [
          0.0,
          0.9809
        ],
        [
          0.01,
          0.9809
        ],
        [
          0.02,
          0.9809
        ],
        [
          0.03,
          0.9809
        ],
        [
          0.04,
          0.9809
        ],
        [
          0.05,
          0.9809
        ],
        [
          0.0601,
          0.9809
        ],
        [
          0.0705,
          0.9809
        ],
        [
          0.0805,
          0.9809
        ],
        [
          0.0905,
          0.9809
        ],
        [
          0.1005,
          0.9809
        ],
        [
          0.1106,
          0.9809
        ],
        [
          0.121,
          0.9809
        ],
        [
          0.131,
          0.9809
        ],
        [
          0.141,
          0.9809
        ],
        [
          0.151,
          0.9809
        ],
        [
          0.1611,
          0.9809
        ],
        [
          0.1715,
          0.9809
        ],
        [
          0.1815,
          0.9809
        ],
        [
          0.1915,
          0.9809
        ],
        [
          0.2015,
          0.9804
        ],
        [
          0.2116,
          0.98
        ],
        [
          0.222,
          0.9791
        ],
        [
          0.232,
          0.9777
        ],
        [
          0.242,
          0.9768
        ],
        [
          0.252,
          0.9754
        ],
        [
          0.2621,
          0.9741
        ],
        [
          0.2725,
          0.9736
        ],
        [
          0.2825,
          0.9736
        ],
        [
          0.2925,
          0.9736
        ],
        [
          0.3025,
          0.9727
        ],
        [
          0.3126,
          0.9718
        ],
        [
          0.323,
          0.9682
        ],
        [
          0.333,
          0.9668
        ],
        [
          0.343,
          0.965
        ],
        [
          0.353,
          0.9622
        ],
        [
          0.3631,
          0.9613
        ],
        [
          0.3735,
          0.9609
        ],
        [
          0.3835,
          0.9604
        ],
        [
          0.3935,
          0.96
        ],
        [
          0.4035,
          0.9572
        ],
        [
          0.4136,
          0.955
        ],
        [
          0.424,
          0.9536
        ],
        [
          0.434,
          0.9518
        ],
        [
          0.444,
          0.949
        ],
        [
          0.454,
          0.9449
        ],
        [
          0.4641,
          0.9422
        ],
        [
          0.4745,
          0.9404
        ],
        [
          0.4845,
          0.9377
        ],
        [
          0.4945,
          0.9336
        ],
        [
          0.5045,
          0.9308
        ],
        [
          0.5146,
          0.9277
        ],
        [
          0.525,
          0.9249
        ],
        [
          0.535,
          0.9227
        ],
        [
          0.545,
          0.9181
        ],
        [
          0.5551,
          0.9131
        ],
        [
          0.5655,
          0.9104
        ],
        [
          0.5755,
          0.9072
        ],
        [
          0.5855,
          0.9049
        ],
        [
          0.5955,
          0.8995
        ],
        [
          0.6056,
          0.8949
        ],
        [
          0.616,
          0.8908
        ],
        [
          0.626,
          0.8872
        ],
        [
          0.636,
          0.8826
        ],
        [
          0.646,
          0.8794
        ],
        [
          0.6561,
          0.8749
        ],
        [
          0.6665,
          0.8694
        ],
        [
          0.6765,
          0.8644
        ],
        [
          0.6865,
          0.8612
        ],
        [
          0.6965,
          0.8562
        ],
        [
          0.7066,
          0.8517
        ],
        [
          0.717,
          0.8471
        ],
        [
          0.727,
          0.8439
        ],
        [
          0.737,
          0.8389
        ],
        [
          0.747,
          0.8344
        ],
        [
          0.7571,
          0.8276
        ],
        [
          0.7675,
          0.8212
        ],
        [
          0.7775,
          0.8157
        ],
        [
          0.7875,
          0.8116
        ],
        [
          0.7975,
          0.8053
        ],
        [
          0.8076,
          0.7985
        ],
        [
          0.818,
          0.7925
        ],
        [
          0.828,
          0.7875
        ],
        [
          0.838,
          0.7816
        ],
        [
          0.848,
          0.7748
        ],
        [
          0.8581,
          0.768
        ],
        [
          0.8685,
          0.7621
        ],
        [
          0.8785,
          0.7543
        ],
        [
          0.8885,
          0.7498
        ],
        [
          0.8985,
          0.742
        ],
        [
          0.9086,
          0.7361
        ],
        [
          0.919,
          0.7298
        ],
        [
          0.929,
          0.7225
        ],
        [
          0.939,
          0.7179
        ],
        [
          0.949,
          0.7106
        ],
        [
          0.9591,
          0.7052
        ],
        [
          0.9695,
          0.697
        ],
        [
          0.9795,
          0.6902
        ],
        [
          0.9895,
          0.6843
        ],
        [
          0.9995,
          0.6793
        ],
        [
          1.0,
          0.6788
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.20357802170792177,
        "difference": -0.0009579960350227274,
        "worse": 0.52,
        "kept_right": 0.2020956254040043,
        "kept_wrong": 0.0014823963039174062,
        "handed_off": 0.7964219782920782
      },
      "usd_per_1000": 2.0867060660725123
    },
    {
      "decision": 129,
      "frontier": 277,
      "frontier_accuracy": 0.9795268425841674,
      "in_sample": {
        "threshold": 0.8914050026003353,
        "answered": 0.19472247497725204,
        "accuracy": 0.9795268425841674
      },
      "curve": [
        [
          0.0,
          0.9795
        ],
        [
          0.01,
          0.9795
        ],
        [
          0.02,
          0.9795
        ],
        [
          0.03,
          0.9795
        ],
        [
          0.04,
          0.9795
        ],
        [
          0.05,
          0.9795
        ],
        [
          0.0601,
          0.9795
        ],
        [
          0.0705,
          0.9795
        ],
        [
          0.0805,
          0.9795
        ],
        [
          0.0905,
          0.9795
        ],
        [
          0.1005,
          0.9795
        ],
        [
          0.1106,
          0.9795
        ],
        [
          0.121,
          0.9795
        ],
        [
          0.131,
          0.9795
        ],
        [
          0.141,
          0.9795
        ],
        [
          0.151,
          0.9795
        ],
        [
          0.1611,
          0.9795
        ],
        [
          0.1715,
          0.9795
        ],
        [
          0.1815,
          0.9795
        ],
        [
          0.1915,
          0.9795
        ],
        [
          0.2015,
          0.9791
        ],
        [
          0.2116,
          0.9782
        ],
        [
          0.222,
          0.9773
        ],
        [
          0.232,
          0.9759
        ],
        [
          0.242,
          0.975
        ],
        [
          0.252,
          0.9736
        ],
        [
          0.2621,
          0.9722
        ],
        [
          0.2725,
          0.9718
        ],
        [
          0.2825,
          0.9718
        ],
        [
          0.2925,
          0.9718
        ],
        [
          0.3025,
          0.9709
        ],
        [
          0.3126,
          0.97
        ],
        [
          0.323,
          0.9663
        ],
        [
          0.333,
          0.9654
        ],
        [
          0.343,
          0.9636
        ],
        [
          0.353,
          0.9609
        ],
        [
          0.3631,
          0.96
        ],
        [
          0.3735,
          0.9595
        ],
        [
          0.3835,
          0.9591
        ],
        [
          0.3935,
          0.9586
        ],
        [
          0.4035,
          0.9559
        ],
        [
          0.4136,
          0.9536
        ],
        [
          0.424,
          0.9522
        ],
        [
          0.434,
          0.95
        ],
        [
          0.444,
          0.9472
        ],
        [
          0.454,
          0.9431
        ],
        [
          0.4641,
          0.9404
        ],
        [
          0.4745,
          0.9386
        ],
        [
          0.4845,
          0.9359
        ],
        [
          0.4945,
          0.9313
        ],
        [
          0.5045,
          0.9286
        ],
        [
          0.5146,
          0.9254
        ],
        [
          0.525,
          0.9222
        ],
        [
          0.535,
          0.9199
        ],
        [
          0.545,
          0.9154
        ],
        [
          0.5551,
          0.9104
        ],
        [
          0.5655,
          0.9076
        ],
        [
          0.5755,
          0.9049
        ],
        [
          0.5855,
          0.9026
        ],
        [
          0.5955,
          0.8972
        ],
        [
          0.6056,
          0.8926
        ],
        [
          0.616,
          0.8885
        ],
        [
          0.626,
          0.8849
        ],
        [
          0.636,
          0.8803
        ],
        [
          0.646,
          0.8776
        ],
        [
          0.6561,
          0.8735
        ],
        [
          0.6665,
          0.8681
        ],
        [
          0.6765,
          0.8631
        ],
        [
          0.6865,
          0.8599
        ],
        [
          0.6965,
          0.854
        ],
        [
          0.7066,
          0.8494
        ],
        [
          0.717,
          0.8449
        ],
        [
          0.727,
          0.8412
        ],
        [
          0.737,
          0.8362
        ],
        [
          0.747,
          0.8317
        ],
        [
          0.7571,
          0.8253
        ],
        [
          0.7675,
          0.8189
        ],
        [
          0.7775,
          0.8135
        ],
        [
          0.7875,
          0.8098
        ],
        [
          0.7975,
          0.8035
        ],
        [
          0.8076,
          0.7971
        ],
        [
          0.818,
          0.7912
        ],
        [
          0.828,
          0.7857
        ],
        [
          0.838,
          0.7798
        ],
        [
          0.848,
          0.773
        ],
        [
          0.8581,
          0.7671
        ],
        [
          0.8685,
          0.7611
        ],
        [
          0.8785,
          0.7543
        ],
        [
          0.8885,
          0.7498
        ],
        [
          0.8985,
          0.742
        ],
        [
          0.9086,
          0.7361
        ],
        [
          0.919,
          0.7298
        ],
        [
          0.929,
          0.7229
        ],
        [
          0.939,
          0.7184
        ],
        [
          0.949,
          0.7111
        ],
        [
          0.9591,
          0.7056
        ],
        [
          0.9695,
          0.6965
        ],
        [
          0.9795,
          0.6897
        ],
        [
          0.9895,
          0.6843
        ],
        [
          0.9995,
          0.6793
        ],
        [
          1.0,
          0.6788
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.20115218258219725,
        "difference": -0.0009457369812094933,
        "worse": 0.52,
        "kept_right": 0.19994268755865646,
        "kept_wrong": 0.0012094950235406914,
        "handed_off": 0.7988478174178029
      },
      "usd_per_1000": 5.005534553283764
    },
    {
      "decision": 129,
      "frontier": 276,
      "frontier_accuracy": 0.9777070063694268,
      "in_sample": {
        "threshold": 0.8812402594760806,
        "answered": 0.20336669699727025,
        "accuracy": 0.9777070063694268
      },
      "curve": [
        [
          0.0,
          0.9777
        ],
        [
          0.01,
          0.9777
        ],
        [
          0.02,
          0.9777
        ],
        [
          0.03,
          0.9777
        ],
        [
          0.04,
          0.9777
        ],
        [
          0.05,
          0.9777
        ],
        [
          0.0601,
          0.9777
        ],
        [
          0.0705,
          0.9777
        ],
        [
          0.0805,
          0.9777
        ],
        [
          0.0905,
          0.9777
        ],
        [
          0.1005,
          0.9777
        ],
        [
          0.1106,
          0.9777
        ],
        [
          0.121,
          0.9777
        ],
        [
          0.131,
          0.9777
        ],
        [
          0.141,
          0.9777
        ],
        [
          0.151,
          0.9782
        ],
        [
          0.1611,
          0.9782
        ],
        [
          0.1715,
          0.9782
        ],
        [
          0.1815,
          0.9782
        ],
        [
          0.1915,
          0.9782
        ],
        [
          0.2015,
          0.9777
        ],
        [
          0.2116,
          0.9768
        ],
        [
          0.222,
          0.9763
        ],
        [
          0.232,
          0.975
        ],
        [
          0.242,
          0.9736
        ],
        [
          0.252,
          0.9722
        ],
        [
          0.2621,
          0.9709
        ],
        [
          0.2725,
          0.9704
        ],
        [
          0.2825,
          0.9704
        ],
        [
          0.2925,
          0.9704
        ],
        [
          0.3025,
          0.9695
        ],
        [
          0.3126,
          0.9686
        ],
        [
          0.323,
          0.965
        ],
        [
          0.333,
          0.9641
        ],
        [
          0.343,
          0.9622
        ],
        [
          0.353,
          0.9595
        ],
        [
          0.3631,
          0.9591
        ],
        [
          0.3735,
          0.9586
        ],
        [
          0.3835,
          0.9586
        ],
        [
          0.3935,
          0.9581
        ],
        [
          0.4035,
          0.9554
        ],
        [
          0.4136,
          0.9531
        ],
        [
          0.424,
          0.9518
        ],
        [
          0.434,
          0.9495
        ],
        [
          0.444,
          0.9468
        ],
        [
          0.454,
          0.9427
        ],
        [
          0.4641,
          0.9399
        ],
        [
          0.4745,
          0.9381
        ],
        [
          0.4845,
          0.9354
        ],
        [
          0.4945,
          0.9308
        ],
        [
          0.5045,
          0.9281
        ],
        [
          0.5146,
          0.9258
        ],
        [
          0.525,
          0.9227
        ],
        [
          0.535,
          0.9204
        ],
        [
          0.545,
          0.9154
        ],
        [
          0.5551,
          0.9104
        ],
        [
          0.5655,
          0.9076
        ],
        [
          0.5755,
          0.9049
        ],
        [
          0.5855,
          0.9026
        ],
        [
          0.5955,
          0.8972
        ],
        [
          0.6056,
          0.8931
        ],
        [
          0.616,
          0.889
        ],
        [
          0.626,
          0.8854
        ],
        [
          0.636,
          0.8808
        ],
        [
          0.646,
          0.8781
        ],
        [
          0.6561,
          0.8744
        ],
        [
          0.6665,
          0.869
        ],
        [
          0.6765,
          0.8644
        ],
        [
          0.6865,
          0.8612
        ],
        [
          0.6965,
          0.8553
        ],
        [
          0.7066,
          0.8508
        ],
        [
          0.717,
          0.8462
        ],
        [
          0.727,
          0.8426
        ],
        [
          0.737,
          0.8376
        ],
        [
          0.747,
          0.833
        ],
        [
          0.7571,
          0.8267
        ],
        [
          0.7675,
          0.8203
        ],
        [
          0.7775,
          0.8148
        ],
        [
          0.7875,
          0.8112
        ],
        [
          0.7975,
          0.8048
        ],
        [
          0.8076,
          0.7985
        ],
        [
          0.818,
          0.7925
        ],
        [
          0.828,
          0.7871
        ],
        [
          0.838,
          0.7812
        ],
        [
          0.848,
          0.7743
        ],
        [
          0.8581,
          0.768
        ],
        [
          0.8685,
          0.7621
        ],
        [
          0.8785,
          0.7548
        ],
        [
          0.8885,
          0.7502
        ],
        [
          0.8985,
          0.7425
        ],
        [
          0.9086,
          0.7366
        ],
        [
          0.919,
          0.7302
        ],
        [
          0.929,
          0.7229
        ],
        [
          0.939,
          0.7184
        ],
        [
          0.949,
          0.7111
        ],
        [
          0.9591,
          0.7056
        ],
        [
          0.9695,
          0.6965
        ],
        [
          0.9795,
          0.6897
        ],
        [
          0.9895,
          0.6843
        ],
        [
          0.9995,
          0.6793
        ],
        [
          1.0,
          0.6788
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.2086005432051061,
        "difference": -0.0010124505727664684,
        "worse": 0.5466666666666666,
        "kept_right": 0.2064573718762424,
        "kept_wrong": 0.0021431713288636777,
        "handed_off": 0.7913994567948941
      },
      "usd_per_1000": 1.4438795617661495
    },
    {
      "decision": 129,
      "frontier": 48,
      "frontier_accuracy": 0.9758871701546861,
      "in_sample": {
        "threshold": 0.8914050026003353,
        "answered": 0.19472247497725204,
        "accuracy": 0.9758871701546861
      },
      "curve": [
        [
          0.0,
          0.9759
        ],
        [
          0.01,
          0.9759
        ],
        [
          0.02,
          0.9759
        ],
        [
          0.03,
          0.9759
        ],
        [
          0.04,
          0.9759
        ],
        [
          0.05,
          0.9759
        ],
        [
          0.0601,
          0.9759
        ],
        [
          0.0705,
          0.9759
        ],
        [
          0.0805,
          0.9759
        ],
        [
          0.0905,
          0.9759
        ],
        [
          0.1005,
          0.9759
        ],
        [
          0.1106,
          0.9759
        ],
        [
          0.121,
          0.9759
        ],
        [
          0.131,
          0.9759
        ],
        [
          0.141,
          0.9759
        ],
        [
          0.151,
          0.9759
        ],
        [
          0.1611,
          0.9759
        ],
        [
          0.1715,
          0.9759
        ],
        [
          0.1815,
          0.9759
        ],
        [
          0.1915,
          0.9759
        ],
        [
          0.2015,
          0.9754
        ],
        [
          0.2116,
          0.9745
        ],
        [
          0.222,
          0.9736
        ],
        [
          0.232,
          0.9722
        ],
        [
          0.242,
          0.9709
        ],
        [
          0.252,
          0.9695
        ],
        [
          0.2621,
          0.9682
        ],
        [
          0.2725,
          0.9677
        ],
        [
          0.2825,
          0.9677
        ],
        [
          0.2925,
          0.9677
        ],
        [
          0.3025,
          0.9668
        ],
        [
          0.3126,
          0.9659
        ],
        [
          0.323,
          0.9622
        ],
        [
          0.333,
          0.9613
        ],
        [
          0.343,
          0.9595
        ],
        [
          0.353,
          0.9568
        ],
        [
          0.3631,
          0.9559
        ],
        [
          0.3735,
          0.9554
        ],
        [
          0.3835,
          0.9554
        ],
        [
          0.3935,
          0.955
        ],
        [
          0.4035,
          0.9522
        ],
        [
          0.4136,
          0.95
        ],
        [
          0.424,
          0.9486
        ],
        [
          0.434,
          0.9463
        ],
        [
          0.444,
          0.9436
        ],
        [
          0.454,
          0.9395
        ],
        [
          0.4641,
          0.9368
        ],
        [
          0.4745,
          0.9349
        ],
        [
          0.4845,
          0.9322
        ],
        [
          0.4945,
          0.9277
        ],
        [
          0.5045,
          0.9249
        ],
        [
          0.5146,
          0.9222
        ],
        [
          0.525,
          0.919
        ],
        [
          0.535,
          0.9167
        ],
        [
          0.545,
          0.9122
        ],
        [
          0.5551,
          0.9072
        ],
        [
          0.5655,
          0.9045
        ],
        [
          0.5755,
          0.9017
        ],
        [
          0.5855,
          0.8995
        ],
        [
          0.5955,
          0.894
        ],
        [
          0.6056,
          0.8904
        ],
        [
          0.616,
          0.8863
        ],
        [
          0.626,
          0.8826
        ],
        [
          0.636,
          0.8781
        ],
        [
          0.646,
          0.8753
        ],
        [
          0.6561,
          0.8717
        ],
        [
          0.6665,
          0.8662
        ],
        [
          0.6765,
          0.8617
        ],
        [
          0.6865,
          0.8585
        ],
        [
          0.6965,
          0.8526
        ],
        [
          0.7066,
          0.8476
        ],
        [
          0.717,
          0.843
        ],
        [
          0.727,
          0.8394
        ],
        [
          0.737,
          0.8344
        ],
        [
          0.747,
          0.8298
        ],
        [
          0.7571,
          0.8235
        ],
        [
          0.7675,
          0.8171
        ],
        [
          0.7775,
          0.8112
        ],
        [
          0.7875,
          0.8076
        ],
        [
          0.7975,
          0.8012
        ],
        [
          0.8076,
          0.7948
        ],
        [
          0.818,
          0.7894
        ],
        [
          0.828,
          0.7843
        ],
        [
          0.838,
          0.7784
        ],
        [
          0.848,
          0.7716
        ],
        [
          0.8581,
          0.7652
        ],
        [
          0.8685,
          0.7589
        ],
        [
          0.8785,
          0.752
        ],
        [
          0.8885,
          0.7475
        ],
        [
          0.8985,
          0.7407
        ],
        [
          0.9086,
          0.7348
        ],
        [
          0.919,
          0.7288
        ],
        [
          0.929,
          0.7225
        ],
        [
          0.939,
          0.7179
        ],
        [
          0.949,
          0.7106
        ],
        [
          0.9591,
          0.7052
        ],
        [
          0.9695,
          0.6965
        ],
        [
          0.9795,
          0.6897
        ],
        [
          0.9895,
          0.6843
        ],
        [
          0.9995,
          0.6793
        ],
        [
          1.0,
          0.6788
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.20115218258219725,
        "difference": -0.0009457369812094933,
        "worse": 0.52,
        "kept_right": 0.19994268755865646,
        "kept_wrong": 0.0012094950235406914,
        "handed_off": 0.7988478174178029
      },
      "usd_per_1000": 4.456463437000653
    },
    {
      "decision": 129,
      "frontier": 191,
      "frontier_accuracy": 0.9740673339399454,
      "in_sample": {
        "threshold": 0.8914050026003353,
        "answered": 0.19472247497725204,
        "accuracy": 0.9740673339399454
      },
      "curve": [
        [
          0.0,
          0.9741
        ],
        [
          0.01,
          0.9741
        ],
        [
          0.02,
          0.9741
        ],
        [
          0.03,
          0.9741
        ],
        [
          0.04,
          0.9741
        ],
        [
          0.05,
          0.9741
        ],
        [
          0.0601,
          0.9741
        ],
        [
          0.0705,
          0.9741
        ],
        [
          0.0805,
          0.9741
        ],
        [
          0.0905,
          0.9741
        ],
        [
          0.1005,
          0.9741
        ],
        [
          0.1106,
          0.9741
        ],
        [
          0.121,
          0.9741
        ],
        [
          0.131,
          0.9741
        ],
        [
          0.141,
          0.9741
        ],
        [
          0.151,
          0.9741
        ],
        [
          0.1611,
          0.9741
        ],
        [
          0.1715,
          0.9741
        ],
        [
          0.1815,
          0.9741
        ],
        [
          0.1915,
          0.9741
        ],
        [
          0.2015,
          0.9736
        ],
        [
          0.2116,
          0.9727
        ],
        [
          0.222,
          0.9718
        ],
        [
          0.232,
          0.9704
        ],
        [
          0.242,
          0.9691
        ],
        [
          0.252,
          0.9677
        ],
        [
          0.2621,
          0.9663
        ],
        [
          0.2725,
          0.9659
        ],
        [
          0.2825,
          0.9659
        ],
        [
          0.2925,
          0.9659
        ],
        [
          0.3025,
          0.965
        ],
        [
          0.3126,
          0.9641
        ],
        [
          0.323,
          0.9604
        ],
        [
          0.333,
          0.9595
        ],
        [
          0.343,
          0.9577
        ],
        [
          0.353,
          0.9554
        ],
        [
          0.3631,
          0.955
        ],
        [
          0.3735,
          0.9545
        ],
        [
          0.3835,
          0.9545
        ],
        [
          0.3935,
          0.954
        ],
        [
          0.4035,
          0.9513
        ],
        [
          0.4136,
          0.9495
        ],
        [
          0.424,
          0.9481
        ],
        [
          0.434,
          0.9454
        ],
        [
          0.444,
          0.9427
        ],
        [
          0.454,
          0.9386
        ],
        [
          0.4641,
          0.9363
        ],
        [
          0.4745,
          0.9345
        ],
        [
          0.4845,
          0.9318
        ],
        [
          0.4945,
          0.9272
        ],
        [
          0.5045,
          0.9245
        ],
        [
          0.5146,
          0.9217
        ],
        [
          0.525,
          0.9186
        ],
        [
          0.535,
          0.9163
        ],
        [
          0.545,
          0.9113
        ],
        [
          0.5551,
          0.9063
        ],
        [
          0.5655,
          0.9035
        ],
        [
          0.5755,
          0.9008
        ],
        [
          0.5855,
          0.8985
        ],
        [
          0.5955,
          0.8931
        ],
        [
          0.6056,
          0.889
        ],
        [
          0.616,
          0.8849
        ],
        [
          0.626,
          0.8813
        ],
        [
          0.636,
          0.8772
        ],
        [
          0.646,
          0.874
        ],
        [
          0.6561,
          0.8708
        ],
        [
          0.6665,
          0.8653
        ],
        [
          0.6765,
          0.8603
        ],
        [
          0.6865,
          0.8571
        ],
        [
          0.6965,
          0.8512
        ],
        [
          0.7066,
          0.8471
        ],
        [
          0.717,
          0.8426
        ],
        [
          0.727,
          0.8389
        ],
        [
          0.737,
          0.8339
        ],
        [
          0.747,
          0.8294
        ],
        [
          0.7571,
          0.823
        ],
        [
          0.7675,
          0.8167
        ],
        [
          0.7775,
          0.8107
        ],
        [
          0.7875,
          0.8071
        ],
        [
          0.7975,
          0.8007
        ],
        [
          0.8076,
          0.7948
        ],
        [
          0.818,
          0.7889
        ],
        [
          0.828,
          0.7839
        ],
        [
          0.838,
          0.778
        ],
        [
          0.848,
          0.7712
        ],
        [
          0.8581,
          0.7648
        ],
        [
          0.8685,
          0.7584
        ],
        [
          0.8785,
          0.752
        ],
        [
          0.8885,
          0.7475
        ],
        [
          0.8985,
          0.7402
        ],
        [
          0.9086,
          0.7343
        ],
        [
          0.919,
          0.7284
        ],
        [
          0.929,
          0.7216
        ],
        [
          0.939,
          0.7179
        ],
        [
          0.949,
          0.7106
        ],
        [
          0.9591,
          0.7052
        ],
        [
          0.9695,
          0.6965
        ],
        [
          0.9795,
          0.6897
        ],
        [
          0.9895,
          0.6843
        ],
        [
          0.9995,
          0.6793
        ],
        [
          1.0,
          0.6788
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.20115218258219725,
        "difference": -0.0009457369812094933,
        "worse": 0.52,
        "kept_right": 0.19994268755865646,
        "kept_wrong": 0.0012094950235406914,
        "handed_off": 0.7988478174178029
      },
      "usd_per_1000": 21.51942930228917
    },
    {
      "decision": 129,
      "frontier": 188,
      "frontier_accuracy": 0.9713375796178344,
      "in_sample": {
        "threshold": 0.8762650624733633,
        "answered": 0.20655141037306643,
        "accuracy": 0.9713375796178344
      },
      "curve": [
        [
          0.0,
          0.9713
        ],
        [
          0.01,
          0.9713
        ],
        [
          0.02,
          0.9713
        ],
        [
          0.03,
          0.9713
        ],
        [
          0.04,
          0.9713
        ],
        [
          0.05,
          0.9713
        ],
        [
          0.0601,
          0.9713
        ],
        [
          0.0705,
          0.9713
        ],
        [
          0.0805,
          0.9713
        ],
        [
          0.0905,
          0.9713
        ],
        [
          0.1005,
          0.9713
        ],
        [
          0.1106,
          0.9713
        ],
        [
          0.121,
          0.9713
        ],
        [
          0.131,
          0.9713
        ],
        [
          0.141,
          0.9713
        ],
        [
          0.151,
          0.9713
        ],
        [
          0.1611,
          0.9713
        ],
        [
          0.1715,
          0.9713
        ],
        [
          0.1815,
          0.9713
        ],
        [
          0.1915,
          0.9718
        ],
        [
          0.2015,
          0.9713
        ],
        [
          0.2116,
          0.9709
        ],
        [
          0.222,
          0.9709
        ],
        [
          0.232,
          0.97
        ],
        [
          0.242,
          0.9695
        ],
        [
          0.252,
          0.9682
        ],
        [
          0.2621,
          0.9668
        ],
        [
          0.2725,
          0.9663
        ],
        [
          0.2825,
          0.9663
        ],
        [
          0.2925,
          0.9663
        ],
        [
          0.3025,
          0.9654
        ],
        [
          0.3126,
          0.9645
        ],
        [
          0.323,
          0.9609
        ],
        [
          0.333,
          0.9595
        ],
        [
          0.343,
          0.9577
        ],
        [
          0.353,
          0.955
        ],
        [
          0.3631,
          0.9545
        ],
        [
          0.3735,
          0.954
        ],
        [
          0.3835,
          0.9536
        ],
        [
          0.3935,
          0.9531
        ],
        [
          0.4035,
          0.9504
        ],
        [
          0.4136,
          0.9477
        ],
        [
          0.424,
          0.9463
        ],
        [
          0.434,
          0.9445
        ],
        [
          0.444,
          0.9418
        ],
        [
          0.454,
          0.9377
        ],
        [
          0.4641,
          0.9349
        ],
        [
          0.4745,
          0.9336
        ],
        [
          0.4845,
          0.9308
        ],
        [
          0.4945,
          0.9268
        ],
        [
          0.5045,
          0.924
        ],
        [
          0.5146,
          0.9208
        ],
        [
          0.525,
          0.9177
        ],
        [
          0.535,
          0.9154
        ],
        [
          0.545,
          0.9108
        ],
        [
          0.5551,
          0.9058
        ],
        [
          0.5655,
          0.9031
        ],
        [
          0.5755,
          0.8999
        ],
        [
          0.5855,
          0.8972
        ],
        [
          0.5955,
          0.8917
        ],
        [
          0.6056,
          0.8876
        ],
        [
          0.616,
          0.8835
        ],
        [
          0.626,
          0.8799
        ],
        [
          0.636,
          0.8753
        ],
        [
          0.646,
          0.8722
        ],
        [
          0.6561,
          0.8685
        ],
        [
          0.6665,
          0.8631
        ],
        [
          0.6765,
          0.8581
        ],
        [
          0.6865,
          0.8549
        ],
        [
          0.6965,
          0.8499
        ],
        [
          0.7066,
          0.8453
        ],
        [
          0.717,
          0.8408
        ],
        [
          0.727,
          0.8376
        ],
        [
          0.737,
          0.8326
        ],
        [
          0.747,
          0.828
        ],
        [
          0.7571,
          0.8212
        ],
        [
          0.7675,
          0.8148
        ],
        [
          0.7775,
          0.8098
        ],
        [
          0.7875,
          0.8057
        ],
        [
          0.7975,
          0.7994
        ],
        [
          0.8076,
          0.7925
        ],
        [
          0.818,
          0.7871
        ],
        [
          0.828,
          0.7821
        ],
        [
          0.838,
          0.7762
        ],
        [
          0.848,
          0.7693
        ],
        [
          0.8581,
          0.7634
        ],
        [
          0.8685,
          0.7575
        ],
        [
          0.8785,
          0.7502
        ],
        [
          0.8885,
          0.7457
        ],
        [
          0.8985,
          0.7389
        ],
        [
          0.9086,
          0.7329
        ],
        [
          0.919,
          0.7266
        ],
        [
          0.929,
          0.7197
        ],
        [
          0.939,
          0.7152
        ],
        [
          0.949,
          0.7084
        ],
        [
          0.9591,
          0.7034
        ],
        [
          0.9695,
          0.6961
        ],
        [
          0.9795,
          0.6902
        ],
        [
          0.9895,
          0.6843
        ],
        [
          0.9995,
          0.6793
        ],
        [
          1.0,
          0.6788
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.22216439822531334,
        "difference": -0.0018189730676810964,
        "worse": 0.6633333333333333,
        "kept_right": 0.21815983768782365,
        "kept_wrong": 0.004004560537489606,
        "handed_off": 0.7778356017746866
      },
      "usd_per_1000": 0.9094769413643534
    },
    {
      "decision": 129,
      "frontier": 190,
      "frontier_accuracy": 0.9672429481346679,
      "in_sample": {
        "threshold": 0.8609069687541565,
        "answered": 0.22020018198362148,
        "accuracy": 0.9672429481346679
      },
      "curve": [
        [
          0.0,
          0.9672
        ],
        [
          0.01,
          0.9672
        ],
        [
          0.02,
          0.9672
        ],
        [
          0.03,
          0.9672
        ],
        [
          0.04,
          0.9672
        ],
        [
          0.05,
          0.9672
        ],
        [
          0.0601,
          0.9672
        ],
        [
          0.0705,
          0.9672
        ],
        [
          0.0805,
          0.9672
        ],
        [
          0.0905,
          0.9672
        ],
        [
          0.1005,
          0.9672
        ],
        [
          0.1106,
          0.9672
        ],
        [
          0.121,
          0.9672
        ],
        [
          0.131,
          0.9672
        ],
        [
          0.141,
          0.9672
        ],
        [
          0.151,
          0.9672
        ],
        [
          0.1611,
          0.9672
        ],
        [
          0.1715,
          0.9677
        ],
        [
          0.1815,
          0.9682
        ],
        [
          0.1915,
          0.9682
        ],
        [
          0.2015,
          0.9677
        ],
        [
          0.2116,
          0.9668
        ],
        [
          0.222,
          0.9668
        ],
        [
          0.232,
          0.9654
        ],
        [
          0.242,
          0.9641
        ],
        [
          0.252,
          0.9627
        ],
        [
          0.2621,
          0.9613
        ],
        [
          0.2725,
          0.9609
        ],
        [
          0.2825,
          0.9609
        ],
        [
          0.2925,
          0.9609
        ],
        [
          0.3025,
          0.96
        ],
        [
          0.3126,
          0.9591
        ],
        [
          0.323,
          0.9554
        ],
        [
          0.333,
          0.9545
        ],
        [
          0.343,
          0.9527
        ],
        [
          0.353,
          0.9509
        ],
        [
          0.3631,
          0.9504
        ],
        [
          0.3735,
          0.95
        ],
        [
          0.3835,
          0.9495
        ],
        [
          0.3935,
          0.949
        ],
        [
          0.4035,
          0.9463
        ],
        [
          0.4136,
          0.944
        ],
        [
          0.424,
          0.9427
        ],
        [
          0.434,
          0.9399
        ],
        [
          0.444,
          0.9377
        ],
        [
          0.454,
          0.934
        ],
        [
          0.4641,
          0.9313
        ],
        [
          0.4745,
          0.9295
        ],
        [
          0.4845,
          0.9268
        ],
        [
          0.4945,
          0.9222
        ],
        [
          0.5045,
          0.9195
        ],
        [
          0.5146,
          0.9167
        ],
        [
          0.525,
          0.9136
        ],
        [
          0.535,
          0.9113
        ],
        [
          0.545,
          0.9067
        ],
        [
          0.5551,
          0.9017
        ],
        [
          0.5655,
          0.899
        ],
        [
          0.5755,
          0.8967
        ],
        [
          0.5855,
          0.8944
        ],
        [
          0.5955,
          0.8894
        ],
        [
          0.6056,
          0.8849
        ],
        [
          0.616,
          0.8808
        ],
        [
          0.626,
          0.8772
        ],
        [
          0.636,
          0.8731
        ],
        [
          0.646,
          0.8703
        ],
        [
          0.6561,
          0.8667
        ],
        [
          0.6665,
          0.8612
        ],
        [
          0.6765,
          0.8571
        ],
        [
          0.6865,
          0.854
        ],
        [
          0.6965,
          0.848
        ],
        [
          0.7066,
          0.8435
        ],
        [
          0.717,
          0.8389
        ],
        [
          0.727,
          0.8353
        ],
        [
          0.737,
          0.8303
        ],
        [
          0.747,
          0.8258
        ],
        [
          0.7571,
          0.8194
        ],
        [
          0.7675,
          0.813
        ],
        [
          0.7775,
          0.8076
        ],
        [
          0.7875,
          0.8039
        ],
        [
          0.7975,
          0.7975
        ],
        [
          0.8076,
          0.7916
        ],
        [
          0.818,
          0.7857
        ],
        [
          0.828,
          0.7807
        ],
        [
          0.838,
          0.7748
        ],
        [
          0.848,
          0.768
        ],
        [
          0.8581,
          0.7621
        ],
        [
          0.8685,
          0.7571
        ],
        [
          0.8785,
          0.7507
        ],
        [
          0.8885,
          0.7461
        ],
        [
          0.8985,
          0.7398
        ],
        [
          0.9086,
          0.7338
        ],
        [
          0.919,
          0.7284
        ],
        [
          0.929,
          0.7216
        ],
        [
          0.939,
          0.7175
        ],
        [
          0.949,
          0.7102
        ],
        [
          0.9591,
          0.7047
        ],
        [
          0.9695,
          0.6961
        ],
        [
          0.9795,
          0.6902
        ],
        [
          0.9895,
          0.6843
        ],
        [
          0.9995,
          0.6793
        ],
        [
          1.0,
          0.6788
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.2193366729448072,
        "difference": -0.001360784504866207,
        "worse": 0.5333333333333333,
        "kept_right": 0.21581141945198823,
        "kept_wrong": 0.0035252534928190095,
        "handed_off": 0.7806633270551931
      },
      "usd_per_1000": 4.507217722933898
    },
    {
      "decision": 129,
      "frontier": 124,
      "frontier_accuracy": 0.9667879890809827,
      "in_sample": {
        "threshold": 0.8914050026003353,
        "answered": 0.19472247497725204,
        "accuracy": 0.9667879890809827
      },
      "curve": [
        [
          0.0,
          0.9668
        ],
        [
          0.01,
          0.9668
        ],
        [
          0.02,
          0.9668
        ],
        [
          0.03,
          0.9668
        ],
        [
          0.04,
          0.9668
        ],
        [
          0.05,
          0.9668
        ],
        [
          0.0601,
          0.9668
        ],
        [
          0.0705,
          0.9668
        ],
        [
          0.0805,
          0.9668
        ],
        [
          0.0905,
          0.9668
        ],
        [
          0.1005,
          0.9668
        ],
        [
          0.1106,
          0.9668
        ],
        [
          0.121,
          0.9668
        ],
        [
          0.131,
          0.9668
        ],
        [
          0.141,
          0.9668
        ],
        [
          0.151,
          0.9668
        ],
        [
          0.1611,
          0.9668
        ],
        [
          0.1715,
          0.9668
        ],
        [
          0.1815,
          0.9668
        ],
        [
          0.1915,
          0.9668
        ],
        [
          0.2015,
          0.9663
        ],
        [
          0.2116,
          0.9654
        ],
        [
          0.222,
          0.965
        ],
        [
          0.232,
          0.9636
        ],
        [
          0.242,
          0.9627
        ],
        [
          0.252,
          0.9613
        ],
        [
          0.2621,
          0.96
        ],
        [
          0.2725,
          0.9595
        ],
        [
          0.2825,
          0.9595
        ],
        [
          0.2925,
          0.9595
        ],
        [
          0.3025,
          0.9586
        ],
        [
          0.3126,
          0.9577
        ],
        [
          0.323,
          0.954
        ],
        [
          0.333,
          0.9531
        ],
        [
          0.343,
          0.9513
        ],
        [
          0.353,
          0.9486
        ],
        [
          0.3631,
          0.9477
        ],
        [
          0.3735,
          0.9472
        ],
        [
          0.3835,
          0.9477
        ],
        [
          0.3935,
          0.9472
        ],
        [
          0.4035,
          0.9445
        ],
        [
          0.4136,
          0.9422
        ],
        [
          0.424,
          0.9409
        ],
        [
          0.434,
          0.9386
        ],
        [
          0.444,
          0.9359
        ],
        [
          0.454,
          0.9318
        ],
        [
          0.4641,
          0.929
        ],
        [
          0.4745,
          0.9272
        ],
        [
          0.4845,
          0.9245
        ],
        [
          0.4945,
          0.9199
        ],
        [
          0.5045,
          0.9172
        ],
        [
          0.5146,
          0.9149
        ],
        [
          0.525,
          0.9122
        ],
        [
          0.535,
          0.9099
        ],
        [
          0.545,
          0.9058
        ],
        [
          0.5551,
          0.9008
        ],
        [
          0.5655,
          0.8985
        ],
        [
          0.5755,
          0.8958
        ],
        [
          0.5855,
          0.8935
        ],
        [
          0.5955,
          0.8885
        ],
        [
          0.6056,
          0.8849
        ],
        [
          0.616,
          0.8808
        ],
        [
          0.626,
          0.8772
        ],
        [
          0.636,
          0.8726
        ],
        [
          0.646,
          0.8699
        ],
        [
          0.6561,
          0.8662
        ],
        [
          0.6665,
          0.8608
        ],
        [
          0.6765,
          0.8567
        ],
        [
          0.6865,
          0.8535
        ],
        [
          0.6965,
          0.8476
        ],
        [
          0.7066,
          0.843
        ],
        [
          0.717,
          0.8385
        ],
        [
          0.727,
          0.8348
        ],
        [
          0.737,
          0.8298
        ],
        [
          0.747,
          0.8258
        ],
        [
          0.7571,
          0.8194
        ],
        [
          0.7675,
          0.8135
        ],
        [
          0.7775,
          0.808
        ],
        [
          0.7875,
          0.8044
        ],
        [
          0.7975,
          0.798
        ],
        [
          0.8076,
          0.7925
        ],
        [
          0.818,
          0.7871
        ],
        [
          0.828,
          0.7821
        ],
        [
          0.838,
          0.7762
        ],
        [
          0.848,
          0.7693
        ],
        [
          0.8581,
          0.7643
        ],
        [
          0.8685,
          0.7584
        ],
        [
          0.8785,
          0.752
        ],
        [
          0.8885,
          0.7475
        ],
        [
          0.8985,
          0.7407
        ],
        [
          0.9086,
          0.7352
        ],
        [
          0.919,
          0.7293
        ],
        [
          0.929,
          0.722
        ],
        [
          0.939,
          0.7175
        ],
        [
          0.949,
          0.7102
        ],
        [
          0.9591,
          0.7047
        ],
        [
          0.9695,
          0.6961
        ],
        [
          0.9795,
          0.6902
        ],
        [
          0.9895,
          0.6843
        ],
        [
          0.9995,
          0.6793
        ],
        [
          1.0,
          0.6788
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.2017189412333017,
        "difference": -0.0009517949151601076,
        "worse": 0.52,
        "kept_right": 0.200403363349573,
        "kept_wrong": 0.0013155778837285622,
        "handed_off": 0.7982810587666985
      },
      "usd_per_1000": 1.0624345483937163
    },
    {
      "decision": 129,
      "frontier": 278,
      "frontier_accuracy": 0.9499545040946314,
      "in_sample": {
        "threshold": 0.8914050026003353,
        "answered": 0.19472247497725204,
        "accuracy": 0.9499545040946314
      },
      "curve": [
        [
          0.0,
          0.95
        ],
        [
          0.01,
          0.95
        ],
        [
          0.02,
          0.95
        ],
        [
          0.03,
          0.95
        ],
        [
          0.04,
          0.95
        ],
        [
          0.05,
          0.95
        ],
        [
          0.0601,
          0.95
        ],
        [
          0.0705,
          0.95
        ],
        [
          0.0805,
          0.95
        ],
        [
          0.0905,
          0.95
        ],
        [
          0.1005,
          0.95
        ],
        [
          0.1106,
          0.95
        ],
        [
          0.121,
          0.95
        ],
        [
          0.131,
          0.95
        ],
        [
          0.141,
          0.95
        ],
        [
          0.151,
          0.95
        ],
        [
          0.1611,
          0.95
        ],
        [
          0.1715,
          0.95
        ],
        [
          0.1815,
          0.95
        ],
        [
          0.1915,
          0.95
        ],
        [
          0.2015,
          0.9495
        ],
        [
          0.2116,
          0.9495
        ],
        [
          0.222,
          0.9486
        ],
        [
          0.232,
          0.9472
        ],
        [
          0.242,
          0.9463
        ],
        [
          0.252,
          0.9449
        ],
        [
          0.2621,
          0.944
        ],
        [
          0.2725,
          0.944
        ],
        [
          0.2825,
          0.944
        ],
        [
          0.2925,
          0.944
        ],
        [
          0.3025,
          0.9436
        ],
        [
          0.3126,
          0.9427
        ],
        [
          0.323,
          0.9395
        ],
        [
          0.333,
          0.9386
        ],
        [
          0.343,
          0.9368
        ],
        [
          0.353,
          0.934
        ],
        [
          0.3631,
          0.9336
        ],
        [
          0.3735,
          0.9336
        ],
        [
          0.3835,
          0.934
        ],
        [
          0.3935,
          0.9336
        ],
        [
          0.4035,
          0.9313
        ],
        [
          0.4136,
          0.929
        ],
        [
          0.424,
          0.9277
        ],
        [
          0.434,
          0.9258
        ],
        [
          0.444,
          0.9231
        ],
        [
          0.454,
          0.9195
        ],
        [
          0.4641,
          0.9167
        ],
        [
          0.4745,
          0.9149
        ],
        [
          0.4845,
          0.9131
        ],
        [
          0.4945,
          0.909
        ],
        [
          0.5045,
          0.9063
        ],
        [
          0.5146,
          0.9031
        ],
        [
          0.525,
          0.9008
        ],
        [
          0.535,
          0.8985
        ],
        [
          0.545,
          0.894
        ],
        [
          0.5551,
          0.889
        ],
        [
          0.5655,
          0.8867
        ],
        [
          0.5755,
          0.884
        ],
        [
          0.5855,
          0.8822
        ],
        [
          0.5955,
          0.8767
        ],
        [
          0.6056,
          0.8735
        ],
        [
          0.616,
          0.8694
        ],
        [
          0.626,
          0.8658
        ],
        [
          0.636,
          0.8617
        ],
        [
          0.646,
          0.859
        ],
        [
          0.6561,
          0.8553
        ],
        [
          0.6665,
          0.8503
        ],
        [
          0.6765,
          0.8458
        ],
        [
          0.6865,
          0.8426
        ],
        [
          0.6965,
          0.8367
        ],
        [
          0.7066,
          0.8326
        ],
        [
          0.717,
          0.828
        ],
        [
          0.727,
          0.8244
        ],
        [
          0.737,
          0.8194
        ],
        [
          0.747,
          0.8148
        ],
        [
          0.7571,
          0.8089
        ],
        [
          0.7675,
          0.8035
        ],
        [
          0.7775,
          0.7994
        ],
        [
          0.7875,
          0.7957
        ],
        [
          0.7975,
          0.7903
        ],
        [
          0.8076,
          0.7843
        ],
        [
          0.818,
          0.7793
        ],
        [
          0.828,
          0.7743
        ],
        [
          0.838,
          0.7693
        ],
        [
          0.848,
          0.763
        ],
        [
          0.8581,
          0.7575
        ],
        [
          0.8685,
          0.7525
        ],
        [
          0.8785,
          0.7466
        ],
        [
          0.8885,
          0.7429
        ],
        [
          0.8985,
          0.7375
        ],
        [
          0.9086,
          0.7325
        ],
        [
          0.919,
          0.7266
        ],
        [
          0.929,
          0.7197
        ],
        [
          0.939,
          0.7157
        ],
        [
          0.949,
          0.7088
        ],
        [
          0.9591,
          0.7034
        ],
        [
          0.9695,
          0.6956
        ],
        [
          0.9795,
          0.6893
        ],
        [
          0.9895,
          0.6843
        ],
        [
          0.9995,
          0.6793
        ],
        [
          1.0,
          0.6788
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.20678220896293822,
        "difference": -0.0009367784243091379,
        "worse": 0.52,
        "kept_right": 0.20499666884902049,
        "kept_wrong": 0.0017855401139176981,
        "handed_off": 0.7932177910370619
      },
      "usd_per_1000": 5.322995957386796
    },
    {
      "decision": 129,
      "frontier": 275,
      "frontier_accuracy": 0.9426751592356688,
      "in_sample": {
        "threshold": 0.8914050026003353,
        "answered": 0.19472247497725204,
        "accuracy": 0.9426751592356688
      },
      "curve": [
        [
          0.0,
          0.9427
        ],
        [
          0.01,
          0.9427
        ],
        [
          0.02,
          0.9427
        ],
        [
          0.03,
          0.9427
        ],
        [
          0.04,
          0.9427
        ],
        [
          0.05,
          0.9427
        ],
        [
          0.0601,
          0.9427
        ],
        [
          0.0705,
          0.9427
        ],
        [
          0.0805,
          0.9427
        ],
        [
          0.0905,
          0.9427
        ],
        [
          0.1005,
          0.9427
        ],
        [
          0.1106,
          0.9427
        ],
        [
          0.121,
          0.9427
        ],
        [
          0.131,
          0.9427
        ],
        [
          0.141,
          0.9427
        ],
        [
          0.151,
          0.9427
        ],
        [
          0.1611,
          0.9427
        ],
        [
          0.1715,
          0.9427
        ],
        [
          0.1815,
          0.9427
        ],
        [
          0.1915,
          0.9427
        ],
        [
          0.2015,
          0.9422
        ],
        [
          0.2116,
          0.9418
        ],
        [
          0.222,
          0.9409
        ],
        [
          0.232,
          0.9395
        ],
        [
          0.242,
          0.9386
        ],
        [
          0.252,
          0.9372
        ],
        [
          0.2621,
          0.9359
        ],
        [
          0.2725,
          0.9359
        ],
        [
          0.2825,
          0.9359
        ],
        [
          0.2925,
          0.9359
        ],
        [
          0.3025,
          0.9349
        ],
        [
          0.3126,
          0.9345
        ],
        [
          0.323,
          0.9318
        ],
        [
          0.333,
          0.9308
        ],
        [
          0.343,
          0.929
        ],
        [
          0.353,
          0.9263
        ],
        [
          0.3631,
          0.9258
        ],
        [
          0.3735,
          0.9258
        ],
        [
          0.3835,
          0.9263
        ],
        [
          0.3935,
          0.9263
        ],
        [
          0.4035,
          0.9245
        ],
        [
          0.4136,
          0.9222
        ],
        [
          0.424,
          0.9208
        ],
        [
          0.434,
          0.9181
        ],
        [
          0.444,
          0.9154
        ],
        [
          0.454,
          0.9117
        ],
        [
          0.4641,
          0.9095
        ],
        [
          0.4745,
          0.9086
        ],
        [
          0.4845,
          0.9067
        ],
        [
          0.4945,
          0.9026
        ],
        [
          0.5045,
          0.8999
        ],
        [
          0.5146,
          0.8976
        ],
        [
          0.525,
          0.8949
        ],
        [
          0.535,
          0.8926
        ],
        [
          0.545,
          0.8881
        ],
        [
          0.5551,
          0.8835
        ],
        [
          0.5655,
          0.8822
        ],
        [
          0.5755,
          0.8799
        ],
        [
          0.5855,
          0.8776
        ],
        [
          0.5955,
          0.8735
        ],
        [
          0.6056,
          0.8699
        ],
        [
          0.616,
          0.8662
        ],
        [
          0.626,
          0.8631
        ],
        [
          0.636,
          0.859
        ],
        [
          0.646,
          0.8558
        ],
        [
          0.6561,
          0.853
        ],
        [
          0.6665,
          0.8485
        ],
        [
          0.6765,
          0.8444
        ],
        [
          0.6865,
          0.8412
        ],
        [
          0.6965,
          0.8358
        ],
        [
          0.7066,
          0.8317
        ],
        [
          0.717,
          0.8267
        ],
        [
          0.727,
          0.823
        ],
        [
          0.737,
          0.8189
        ],
        [
          0.747,
          0.8148
        ],
        [
          0.7571,
          0.8085
        ],
        [
          0.7675,
          0.8025
        ],
        [
          0.7775,
          0.798
        ],
        [
          0.7875,
          0.7953
        ],
        [
          0.7975,
          0.7898
        ],
        [
          0.8076,
          0.7848
        ],
        [
          0.818,
          0.7807
        ],
        [
          0.828,
          0.7766
        ],
        [
          0.838,
          0.7712
        ],
        [
          0.848,
          0.7657
        ],
        [
          0.8581,
          0.7602
        ],
        [
          0.8685,
          0.7543
        ],
        [
          0.8785,
          0.7484
        ],
        [
          0.8885,
          0.7439
        ],
        [
          0.8985,
          0.7366
        ],
        [
          0.9086,
          0.7307
        ],
        [
          0.919,
          0.7252
        ],
        [
          0.929,
          0.7193
        ],
        [
          0.939,
          0.7157
        ],
        [
          0.949,
          0.7093
        ],
        [
          0.9591,
          0.7038
        ],
        [
          0.9695,
          0.6952
        ],
        [
          0.9795,
          0.6884
        ],
        [
          0.9895,
          0.6833
        ],
        [
          0.9995,
          0.6793
        ],
        [
          1.0,
          0.6788
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.2037175424843852,
        "difference": -0.0009852935782438375,
        "worse": 0.52,
        "kept_right": 0.20220481557688874,
        "kept_wrong": 0.0015127269074964173,
        "handed_off": 0.7962824575156147
      },
      "usd_per_1000": 0.5697769705206592
    },
    {
      "decision": 129,
      "frontier": 192,
      "frontier_accuracy": 0.9331210191082803,
      "in_sample": {
        "threshold": 0.8650550330769855,
        "answered": 0.21474067333939945,
        "accuracy": 0.9331210191082803
      },
      "curve": [
        [
          0.0,
          0.9331
        ],
        [
          0.01,
          0.9331
        ],
        [
          0.02,
          0.9331
        ],
        [
          0.03,
          0.9331
        ],
        [
          0.04,
          0.9331
        ],
        [
          0.05,
          0.9331
        ],
        [
          0.0601,
          0.9331
        ],
        [
          0.0705,
          0.9331
        ],
        [
          0.0805,
          0.9331
        ],
        [
          0.0905,
          0.9331
        ],
        [
          0.1005,
          0.9331
        ],
        [
          0.1106,
          0.9331
        ],
        [
          0.121,
          0.9331
        ],
        [
          0.131,
          0.9331
        ],
        [
          0.141,
          0.9331
        ],
        [
          0.151,
          0.9331
        ],
        [
          0.1611,
          0.9331
        ],
        [
          0.1715,
          0.9336
        ],
        [
          0.1815,
          0.9336
        ],
        [
          0.1915,
          0.9336
        ],
        [
          0.2015,
          0.9331
        ],
        [
          0.2116,
          0.9327
        ],
        [
          0.222,
          0.9322
        ],
        [
          0.232,
          0.9308
        ],
        [
          0.242,
          0.9299
        ],
        [
          0.252,
          0.9286
        ],
        [
          0.2621,
          0.9277
        ],
        [
          0.2725,
          0.9272
        ],
        [
          0.2825,
          0.9272
        ],
        [
          0.2925,
          0.9277
        ],
        [
          0.3025,
          0.9272
        ],
        [
          0.3126,
          0.9263
        ],
        [
          0.323,
          0.9231
        ],
        [
          0.333,
          0.9227
        ],
        [
          0.343,
          0.9217
        ],
        [
          0.353,
          0.9195
        ],
        [
          0.3631,
          0.919
        ],
        [
          0.3735,
          0.9186
        ],
        [
          0.3835,
          0.919
        ],
        [
          0.3935,
          0.9186
        ],
        [
          0.4035,
          0.9163
        ],
        [
          0.4136,
          0.9145
        ],
        [
          0.424,
          0.9136
        ],
        [
          0.434,
          0.9113
        ],
        [
          0.444,
          0.909
        ],
        [
          0.454,
          0.9054
        ],
        [
          0.4641,
          0.9026
        ],
        [
          0.4745,
          0.9017
        ],
        [
          0.4845,
          0.8995
        ],
        [
          0.4945,
          0.8954
        ],
        [
          0.5045,
          0.8931
        ],
        [
          0.5146,
          0.8908
        ],
        [
          0.525,
          0.8885
        ],
        [
          0.535,
          0.8867
        ],
        [
          0.545,
          0.884
        ],
        [
          0.5551,
          0.8794
        ],
        [
          0.5655,
          0.8781
        ],
        [
          0.5755,
          0.8758
        ],
        [
          0.5855,
          0.874
        ],
        [
          0.5955,
          0.8694
        ],
        [
          0.6056,
          0.8662
        ],
        [
          0.616,
          0.8621
        ],
        [
          0.626,
          0.8585
        ],
        [
          0.636,
          0.8558
        ],
        [
          0.646,
          0.8526
        ],
        [
          0.6561,
          0.849
        ],
        [
          0.6665,
          0.8449
        ],
        [
          0.6765,
          0.8408
        ],
        [
          0.6865,
          0.838
        ],
        [
          0.6965,
          0.8321
        ],
        [
          0.7066,
          0.828
        ],
        [
          0.717,
          0.8244
        ],
        [
          0.727,
          0.8207
        ],
        [
          0.737,
          0.8157
        ],
        [
          0.747,
          0.8112
        ],
        [
          0.7571,
          0.8057
        ],
        [
          0.7675,
          0.7994
        ],
        [
          0.7775,
          0.7948
        ],
        [
          0.7875,
          0.7912
        ],
        [
          0.7975,
          0.7862
        ],
        [
          0.8076,
          0.7816
        ],
        [
          0.818,
          0.7762
        ],
        [
          0.828,
          0.7716
        ],
        [
          0.838,
          0.7666
        ],
        [
          0.848,
          0.7598
        ],
        [
          0.8581,
          0.7543
        ],
        [
          0.8685,
          0.7502
        ],
        [
          0.8785,
          0.7448
        ],
        [
          0.8885,
          0.7416
        ],
        [
          0.8985,
          0.7352
        ],
        [
          0.9086,
          0.7307
        ],
        [
          0.919,
          0.7247
        ],
        [
          0.929,
          0.7184
        ],
        [
          0.939,
          0.7143
        ],
        [
          0.949,
          0.7079
        ],
        [
          0.9591,
          0.7029
        ],
        [
          0.9695,
          0.6943
        ],
        [
          0.9795,
          0.6888
        ],
        [
          0.9895,
          0.6843
        ],
        [
          0.9995,
          0.6793
        ],
        [
          1.0,
          0.6788
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.21642291907899278,
        "difference": -0.0014153934479321866,
        "worse": 0.5533333333333333,
        "kept_right": 0.21338871874068174,
        "kept_wrong": 0.0030342003383109577,
        "handed_off": 0.7835770809210074
      },
      "usd_per_1000": 9.257068453831787
    },
    {
      "decision": 129,
      "frontier": 123,
      "frontier_accuracy": 0.9326660600545951,
      "in_sample": {
        "threshold": 0.8762650624733633,
        "answered": 0.20655141037306643,
        "accuracy": 0.9326660600545951
      },
      "curve": [
        [
          0.0,
          0.9327
        ],
        [
          0.01,
          0.9327
        ],
        [
          0.02,
          0.9327
        ],
        [
          0.03,
          0.9327
        ],
        [
          0.04,
          0.9327
        ],
        [
          0.05,
          0.9327
        ],
        [
          0.0601,
          0.9327
        ],
        [
          0.0705,
          0.9327
        ],
        [
          0.0805,
          0.9327
        ],
        [
          0.0905,
          0.9327
        ],
        [
          0.1005,
          0.9327
        ],
        [
          0.1106,
          0.9327
        ],
        [
          0.121,
          0.9327
        ],
        [
          0.131,
          0.9327
        ],
        [
          0.141,
          0.9331
        ],
        [
          0.151,
          0.9331
        ],
        [
          0.1611,
          0.9331
        ],
        [
          0.1715,
          0.9331
        ],
        [
          0.1815,
          0.9331
        ],
        [
          0.1915,
          0.9331
        ],
        [
          0.2015,
          0.9327
        ],
        [
          0.2116,
          0.9322
        ],
        [
          0.222,
          0.9318
        ],
        [
          0.232,
          0.9304
        ],
        [
          0.242,
          0.9295
        ],
        [
          0.252,
          0.9281
        ],
        [
          0.2621,
          0.9268
        ],
        [
          0.2725,
          0.9263
        ],
        [
          0.2825,
          0.9263
        ],
        [
          0.2925,
          0.9263
        ],
        [
          0.3025,
          0.9254
        ],
        [
          0.3126,
          0.9249
        ],
        [
          0.323,
          0.9213
        ],
        [
          0.333,
          0.9199
        ],
        [
          0.343,
          0.9181
        ],
        [
          0.353,
          0.9154
        ],
        [
          0.3631,
          0.9149
        ],
        [
          0.3735,
          0.9149
        ],
        [
          0.3835,
          0.9154
        ],
        [
          0.3935,
          0.9154
        ],
        [
          0.4035,
          0.9126
        ],
        [
          0.4136,
          0.9108
        ],
        [
          0.424,
          0.9095
        ],
        [
          0.434,
          0.9081
        ],
        [
          0.444,
          0.9054
        ],
        [
          0.454,
          0.9017
        ],
        [
          0.4641,
          0.8995
        ],
        [
          0.4745,
          0.8985
        ],
        [
          0.4845,
          0.8958
        ],
        [
          0.4945,
          0.8917
        ],
        [
          0.5045,
          0.8894
        ],
        [
          0.5146,
          0.8867
        ],
        [
          0.525,
          0.884
        ],
        [
          0.535,
          0.8817
        ],
        [
          0.545,
          0.8776
        ],
        [
          0.5551,
          0.8731
        ],
        [
          0.5655,
          0.8703
        ],
        [
          0.5755,
          0.8681
        ],
        [
          0.5855,
          0.8658
        ],
        [
          0.5955,
          0.8612
        ],
        [
          0.6056,
          0.8571
        ],
        [
          0.616,
          0.8544
        ],
        [
          0.626,
          0.8508
        ],
        [
          0.636,
          0.8467
        ],
        [
          0.646,
          0.8444
        ],
        [
          0.6561,
          0.8417
        ],
        [
          0.6665,
          0.8367
        ],
        [
          0.6765,
          0.8321
        ],
        [
          0.6865,
          0.8289
        ],
        [
          0.6965,
          0.8253
        ],
        [
          0.7066,
          0.8217
        ],
        [
          0.717,
          0.8167
        ],
        [
          0.727,
          0.8139
        ],
        [
          0.737,
          0.8098
        ],
        [
          0.747,
          0.8057
        ],
        [
          0.7571,
          0.8003
        ],
        [
          0.7675,
          0.7939
        ],
        [
          0.7775,
          0.7889
        ],
        [
          0.7875,
          0.7857
        ],
        [
          0.7975,
          0.7803
        ],
        [
          0.8076,
          0.7743
        ],
        [
          0.818,
          0.7698
        ],
        [
          0.828,
          0.7652
        ],
        [
          0.838,
          0.7598
        ],
        [
          0.848,
          0.7539
        ],
        [
          0.8581,
          0.7493
        ],
        [
          0.8685,
          0.7434
        ],
        [
          0.8785,
          0.7379
        ],
        [
          0.8885,
          0.7338
        ],
        [
          0.8985,
          0.7284
        ],
        [
          0.9086,
          0.7238
        ],
        [
          0.919,
          0.7184
        ],
        [
          0.929,
          0.712
        ],
        [
          0.939,
          0.7097
        ],
        [
          0.949,
          0.7038
        ],
        [
          0.9591,
          0.6993
        ],
        [
          0.9695,
          0.6929
        ],
        [
          0.9795,
          0.6865
        ],
        [
          0.9895,
          0.682
        ],
        [
          0.9995,
          0.6793
        ],
        [
          1.0,
          0.6788
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.21340390926026326,
        "difference": -0.001294545064736697,
        "worse": 0.5533333333333333,
        "kept_right": 0.2106270369651545,
        "kept_wrong": 0.002776872295108827,
        "handed_off": 0.7865960907397366
      },
      "usd_per_1000": 0.3713462173239203
    },
    {
      "decision": 129,
      "frontier": 274,
      "frontier_accuracy": 0.9085532302092811,
      "in_sample": {
        "threshold": 0.8552889310598578,
        "answered": 0.22202001819836215,
        "accuracy": 0.9085532302092811
      },
      "curve": [
        [
          0.0,
          0.9086
        ],
        [
          0.01,
          0.9086
        ],
        [
          0.02,
          0.9086
        ],
        [
          0.03,
          0.9086
        ],
        [
          0.04,
          0.9086
        ],
        [
          0.05,
          0.9086
        ],
        [
          0.0601,
          0.9086
        ],
        [
          0.0705,
          0.9086
        ],
        [
          0.0805,
          0.9086
        ],
        [
          0.0905,
          0.9086
        ],
        [
          0.1005,
          0.9086
        ],
        [
          0.1106,
          0.9086
        ],
        [
          0.121,
          0.9086
        ],
        [
          0.131,
          0.9086
        ],
        [
          0.141,
          0.9086
        ],
        [
          0.151,
          0.909
        ],
        [
          0.1611,
          0.909
        ],
        [
          0.1715,
          0.909
        ],
        [
          0.1815,
          0.909
        ],
        [
          0.1915,
          0.909
        ],
        [
          0.2015,
          0.9086
        ],
        [
          0.2116,
          0.9086
        ],
        [
          0.222,
          0.9086
        ],
        [
          0.232,
          0.9081
        ],
        [
          0.242,
          0.9072
        ],
        [
          0.252,
          0.9058
        ],
        [
          0.2621,
          0.9049
        ],
        [
          0.2725,
          0.9049
        ],
        [
          0.2825,
          0.9049
        ],
        [
          0.2925,
          0.9049
        ],
        [
          0.3025,
          0.904
        ],
        [
          0.3126,
          0.9035
        ],
        [
          0.323,
          0.9004
        ],
        [
          0.333,
          0.8999
        ],
        [
          0.343,
          0.8985
        ],
        [
          0.353,
          0.8967
        ],
        [
          0.3631,
          0.8967
        ],
        [
          0.3735,
          0.8967
        ],
        [
          0.3835,
          0.8967
        ],
        [
          0.3935,
          0.8967
        ],
        [
          0.4035,
          0.8944
        ],
        [
          0.4136,
          0.8926
        ],
        [
          0.424,
          0.8913
        ],
        [
          0.434,
          0.8885
        ],
        [
          0.444,
          0.8858
        ],
        [
          0.454,
          0.8822
        ],
        [
          0.4641,
          0.8803
        ],
        [
          0.4745,
          0.8794
        ],
        [
          0.4845,
          0.8772
        ],
        [
          0.4945,
          0.8731
        ],
        [
          0.5045,
          0.8703
        ],
        [
          0.5146,
          0.8681
        ],
        [
          0.525,
          0.8653
        ],
        [
          0.535,
          0.8635
        ],
        [
          0.545,
          0.8603
        ],
        [
          0.5551,
          0.8562
        ],
        [
          0.5655,
          0.8558
        ],
        [
          0.5755,
          0.8544
        ],
        [
          0.5855,
          0.8521
        ],
        [
          0.5955,
          0.849
        ],
        [
          0.6056,
          0.8453
        ],
        [
          0.616,
          0.8417
        ],
        [
          0.626,
          0.8394
        ],
        [
          0.636,
          0.8362
        ],
        [
          0.646,
          0.8335
        ],
        [
          0.6561,
          0.8308
        ],
        [
          0.6665,
          0.8267
        ],
        [
          0.6765,
          0.8221
        ],
        [
          0.6865,
          0.8203
        ],
        [
          0.6965,
          0.8153
        ],
        [
          0.7066,
          0.8116
        ],
        [
          0.717,
          0.808
        ],
        [
          0.727,
          0.8048
        ],
        [
          0.737,
          0.8016
        ],
        [
          0.747,
          0.798
        ],
        [
          0.7571,
          0.7916
        ],
        [
          0.7675,
          0.7857
        ],
        [
          0.7775,
          0.7812
        ],
        [
          0.7875,
          0.7775
        ],
        [
          0.7975,
          0.7721
        ],
        [
          0.8076,
          0.7671
        ],
        [
          0.818,
          0.763
        ],
        [
          0.828,
          0.7607
        ],
        [
          0.838,
          0.7552
        ],
        [
          0.848,
          0.7498
        ],
        [
          0.8581,
          0.7457
        ],
        [
          0.8685,
          0.7411
        ],
        [
          0.8785,
          0.7361
        ],
        [
          0.8885,
          0.732
        ],
        [
          0.8985,
          0.7266
        ],
        [
          0.9086,
          0.722
        ],
        [
          0.919,
          0.717
        ],
        [
          0.929,
          0.7116
        ],
        [
          0.939,
          0.7084
        ],
        [
          0.949,
          0.7029
        ],
        [
          0.9591,
          0.6979
        ],
        [
          0.9695,
          0.6924
        ],
        [
          0.9795,
          0.6874
        ],
        [
          0.9895,
          0.6824
        ],
        [
          0.9995,
          0.6793
        ],
        [
          1.0,
          0.6788
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.23255581272950163,
        "difference": -0.0015433201561850499,
        "worse": 0.6233333333333333,
        "kept_right": 0.22746261346189262,
        "kept_wrong": 0.005093199267609087,
        "handed_off": 0.7674441872704982
      },
      "usd_per_1000": 0.4871989079724671
    },
    {
      "decision": 129,
      "frontier": 47,
      "frontier_accuracy": 0.9044585987261147,
      "in_sample": {
        "threshold": 0.8650550330769855,
        "answered": 0.21474067333939945,
        "accuracy": 0.9044585987261147
      },
      "curve": [
        [
          0.0,
          0.9045
        ],
        [
          0.01,
          0.9045
        ],
        [
          0.02,
          0.9045
        ],
        [
          0.03,
          0.9045
        ],
        [
          0.04,
          0.9045
        ],
        [
          0.05,
          0.9045
        ],
        [
          0.0601,
          0.9045
        ],
        [
          0.0705,
          0.9045
        ],
        [
          0.0805,
          0.9045
        ],
        [
          0.0905,
          0.9045
        ],
        [
          0.1005,
          0.9045
        ],
        [
          0.1106,
          0.9045
        ],
        [
          0.121,
          0.9049
        ],
        [
          0.131,
          0.9049
        ],
        [
          0.141,
          0.9049
        ],
        [
          0.151,
          0.9049
        ],
        [
          0.1611,
          0.9049
        ],
        [
          0.1715,
          0.9049
        ],
        [
          0.1815,
          0.9049
        ],
        [
          0.1915,
          0.9049
        ],
        [
          0.2015,
          0.9049
        ],
        [
          0.2116,
          0.9045
        ],
        [
          0.222,
          0.9035
        ],
        [
          0.232,
          0.9022
        ],
        [
          0.242,
          0.9013
        ],
        [
          0.252,
          0.8999
        ],
        [
          0.2621,
          0.899
        ],
        [
          0.2725,
          0.899
        ],
        [
          0.2825,
          0.899
        ],
        [
          0.2925,
          0.899
        ],
        [
          0.3025,
          0.899
        ],
        [
          0.3126,
          0.8981
        ],
        [
          0.323,
          0.8954
        ],
        [
          0.333,
          0.8954
        ],
        [
          0.343,
          0.8949
        ],
        [
          0.353,
          0.8926
        ],
        [
          0.3631,
          0.8922
        ],
        [
          0.3735,
          0.8922
        ],
        [
          0.3835,
          0.8926
        ],
        [
          0.3935,
          0.8926
        ],
        [
          0.4035,
          0.8908
        ],
        [
          0.4136,
          0.889
        ],
        [
          0.424,
          0.8876
        ],
        [
          0.434,
          0.8858
        ],
        [
          0.444,
          0.8831
        ],
        [
          0.454,
          0.8799
        ],
        [
          0.4641,
          0.8776
        ],
        [
          0.4745,
          0.8763
        ],
        [
          0.4845,
          0.8744
        ],
        [
          0.4945,
          0.8708
        ],
        [
          0.5045,
          0.8685
        ],
        [
          0.5146,
          0.8662
        ],
        [
          0.525,
          0.8644
        ],
        [
          0.535,
          0.8631
        ],
        [
          0.545,
          0.8599
        ],
        [
          0.5551,
          0.8558
        ],
        [
          0.5655,
          0.8535
        ],
        [
          0.5755,
          0.8521
        ],
        [
          0.5855,
          0.8508
        ],
        [
          0.5955,
          0.8458
        ],
        [
          0.6056,
          0.8421
        ],
        [
          0.616,
          0.8389
        ],
        [
          0.626,
          0.8353
        ],
        [
          0.636,
          0.8321
        ],
        [
          0.646,
          0.8303
        ],
        [
          0.6561,
          0.8262
        ],
        [
          0.6665,
          0.8217
        ],
        [
          0.6765,
          0.8189
        ],
        [
          0.6865,
          0.8167
        ],
        [
          0.6965,
          0.8126
        ],
        [
          0.7066,
          0.8085
        ],
        [
          0.717,
          0.8053
        ],
        [
          0.727,
          0.8025
        ],
        [
          0.737,
          0.7994
        ],
        [
          0.747,
          0.7962
        ],
        [
          0.7571,
          0.7907
        ],
        [
          0.7675,
          0.7857
        ],
        [
          0.7775,
          0.7812
        ],
        [
          0.7875,
          0.7793
        ],
        [
          0.7975,
          0.773
        ],
        [
          0.8076,
          0.768
        ],
        [
          0.818,
          0.7634
        ],
        [
          0.828,
          0.7589
        ],
        [
          0.838,
          0.753
        ],
        [
          0.848,
          0.747
        ],
        [
          0.8581,
          0.7425
        ],
        [
          0.8685,
          0.7398
        ],
        [
          0.8785,
          0.7352
        ],
        [
          0.8885,
          0.7307
        ],
        [
          0.8985,
          0.7261
        ],
        [
          0.9086,
          0.7216
        ],
        [
          0.919,
          0.7161
        ],
        [
          0.929,
          0.7106
        ],
        [
          0.939,
          0.7084
        ],
        [
          0.949,
          0.7038
        ],
        [
          0.9591,
          0.6988
        ],
        [
          0.9695,
          0.692
        ],
        [
          0.9795,
          0.6861
        ],
        [
          0.9895,
          0.6824
        ],
        [
          0.9995,
          0.6793
        ],
        [
          1.0,
          0.6788
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.21795782026197813,
        "difference": -0.0010455604463890144,
        "worse": 0.5133333333333333,
        "kept_right": 0.21490543562897846,
        "kept_wrong": 0.003052384632999544,
        "handed_off": 0.7820421797380219
      },
      "usd_per_1000": 4.0160053417212485
    },
    {
      "decision": 129,
      "frontier": 193,
      "frontier_accuracy": 0.89171974522293,
      "in_sample": {
        "threshold": 0.8444715688557025,
        "answered": 0.23339399454049137,
        "accuracy": 0.89171974522293
      },
      "curve": [
        [
          0.0,
          0.8917
        ],
        [
          0.01,
          0.8917
        ],
        [
          0.02,
          0.8917
        ],
        [
          0.03,
          0.8917
        ],
        [
          0.04,
          0.8917
        ],
        [
          0.05,
          0.8917
        ],
        [
          0.0601,
          0.8917
        ],
        [
          0.0705,
          0.8917
        ],
        [
          0.0805,
          0.8917
        ],
        [
          0.0905,
          0.8917
        ],
        [
          0.1005,
          0.8917
        ],
        [
          0.1106,
          0.8917
        ],
        [
          0.121,
          0.8922
        ],
        [
          0.131,
          0.8922
        ],
        [
          0.141,
          0.8922
        ],
        [
          0.151,
          0.8922
        ],
        [
          0.1611,
          0.8922
        ],
        [
          0.1715,
          0.8922
        ],
        [
          0.1815,
          0.8926
        ],
        [
          0.1915,
          0.8926
        ],
        [
          0.2015,
          0.8926
        ],
        [
          0.2116,
          0.8922
        ],
        [
          0.222,
          0.8931
        ],
        [
          0.232,
          0.8917
        ],
        [
          0.242,
          0.8908
        ],
        [
          0.252,
          0.8899
        ],
        [
          0.2621,
          0.889
        ],
        [
          0.2725,
          0.8885
        ],
        [
          0.2825,
          0.8885
        ],
        [
          0.2925,
          0.889
        ],
        [
          0.3025,
          0.8885
        ],
        [
          0.3126,
          0.8881
        ],
        [
          0.323,
          0.8849
        ],
        [
          0.333,
          0.884
        ],
        [
          0.343,
          0.8835
        ],
        [
          0.353,
          0.8822
        ],
        [
          0.3631,
          0.8817
        ],
        [
          0.3735,
          0.8822
        ],
        [
          0.3835,
          0.8826
        ],
        [
          0.3935,
          0.8822
        ],
        [
          0.4035,
          0.8813
        ],
        [
          0.4136,
          0.8794
        ],
        [
          0.424,
          0.8785
        ],
        [
          0.434,
          0.8776
        ],
        [
          0.444,
          0.8749
        ],
        [
          0.454,
          0.8712
        ],
        [
          0.4641,
          0.8685
        ],
        [
          0.4745,
          0.8681
        ],
        [
          0.4845,
          0.8662
        ],
        [
          0.4945,
          0.8617
        ],
        [
          0.5045,
          0.8599
        ],
        [
          0.5146,
          0.8576
        ],
        [
          0.525,
          0.8567
        ],
        [
          0.535,
          0.8553
        ],
        [
          0.545,
          0.8526
        ],
        [
          0.5551,
          0.848
        ],
        [
          0.5655,
          0.8467
        ],
        [
          0.5755,
          0.8449
        ],
        [
          0.5855,
          0.843
        ],
        [
          0.5955,
          0.8385
        ],
        [
          0.6056,
          0.8367
        ],
        [
          0.616,
          0.8335
        ],
        [
          0.626,
          0.8303
        ],
        [
          0.636,
          0.8276
        ],
        [
          0.646,
          0.8253
        ],
        [
          0.6561,
          0.823
        ],
        [
          0.6665,
          0.8198
        ],
        [
          0.6765,
          0.8157
        ],
        [
          0.6865,
          0.8139
        ],
        [
          0.6965,
          0.8094
        ],
        [
          0.7066,
          0.8048
        ],
        [
          0.717,
          0.8025
        ],
        [
          0.727,
          0.7989
        ],
        [
          0.737,
          0.7957
        ],
        [
          0.747,
          0.7912
        ],
        [
          0.7571,
          0.7862
        ],
        [
          0.7675,
          0.7821
        ],
        [
          0.7775,
          0.7771
        ],
        [
          0.7875,
          0.7743
        ],
        [
          0.7975,
          0.7689
        ],
        [
          0.8076,
          0.7652
        ],
        [
          0.818,
          0.7611
        ],
        [
          0.828,
          0.7566
        ],
        [
          0.838,
          0.752
        ],
        [
          0.848,
          0.7466
        ],
        [
          0.8581,
          0.7434
        ],
        [
          0.8685,
          0.7389
        ],
        [
          0.8785,
          0.7334
        ],
        [
          0.8885,
          0.7311
        ],
        [
          0.8985,
          0.7261
        ],
        [
          0.9086,
          0.7216
        ],
        [
          0.919,
          0.7166
        ],
        [
          0.929,
          0.7106
        ],
        [
          0.939,
          0.707
        ],
        [
          0.949,
          0.7015
        ],
        [
          0.9591,
          0.6979
        ],
        [
          0.9695,
          0.6902
        ],
        [
          0.9795,
          0.6852
        ],
        [
          0.9895,
          0.6811
        ],
        [
          0.9995,
          0.6793
        ],
        [
          1.0,
          0.6788
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.2470975578312444,
        "difference": -0.0014761870422039481,
        "worse": 0.54,
        "kept_right": 0.24047689147122733,
        "kept_wrong": 0.006620666360016907,
        "handed_off": 0.7529024421687557
      },
      "usd_per_1000": 5.993258565198822
    },
    {
      "decision": 129,
      "frontier": 55,
      "frontier_accuracy": 0.8503184713375797,
      "in_sample": {
        "threshold": 0.7228668415197546,
        "answered": 0.4454049135577798,
        "accuracy": 0.8503184713375797
      },
      "curve": [
        [
          0.0,
          0.8503
        ],
        [
          0.01,
          0.8503
        ],
        [
          0.02,
          0.8503
        ],
        [
          0.03,
          0.8508
        ],
        [
          0.04,
          0.8508
        ],
        [
          0.05,
          0.8508
        ],
        [
          0.0601,
          0.8508
        ],
        [
          0.0705,
          0.8512
        ],
        [
          0.0805,
          0.8512
        ],
        [
          0.0905,
          0.8512
        ],
        [
          0.1005,
          0.8512
        ],
        [
          0.1106,
          0.8517
        ],
        [
          0.121,
          0.8517
        ],
        [
          0.131,
          0.8517
        ],
        [
          0.141,
          0.8517
        ],
        [
          0.151,
          0.8517
        ],
        [
          0.1611,
          0.8521
        ],
        [
          0.1715,
          0.8526
        ],
        [
          0.1815,
          0.8535
        ],
        [
          0.1915,
          0.854
        ],
        [
          0.2015,
          0.8535
        ],
        [
          0.2116,
          0.8535
        ],
        [
          0.222,
          0.854
        ],
        [
          0.232,
          0.8544
        ],
        [
          0.242,
          0.8544
        ],
        [
          0.252,
          0.8535
        ],
        [
          0.2621,
          0.8535
        ],
        [
          0.2725,
          0.8544
        ],
        [
          0.2825,
          0.8553
        ],
        [
          0.2925,
          0.8567
        ],
        [
          0.3025,
          0.8562
        ],
        [
          0.3126,
          0.8567
        ],
        [
          0.323,
          0.8535
        ],
        [
          0.333,
          0.8544
        ],
        [
          0.343,
          0.8544
        ],
        [
          0.353,
          0.854
        ],
        [
          0.3631,
          0.854
        ],
        [
          0.3735,
          0.854
        ],
        [
          0.3835,
          0.8549
        ],
        [
          0.3935,
          0.8544
        ],
        [
          0.4035,
          0.854
        ],
        [
          0.4136,
          0.853
        ],
        [
          0.424,
          0.8526
        ],
        [
          0.434,
          0.853
        ],
        [
          0.444,
          0.8512
        ],
        [
          0.454,
          0.8485
        ],
        [
          0.4641,
          0.8462
        ],
        [
          0.4745,
          0.8462
        ],
        [
          0.4845,
          0.8449
        ],
        [
          0.4945,
          0.8412
        ],
        [
          0.5045,
          0.8403
        ],
        [
          0.5146,
          0.8385
        ],
        [
          0.525,
          0.8367
        ],
        [
          0.535,
          0.8353
        ],
        [
          0.545,
          0.8335
        ],
        [
          0.5551,
          0.8298
        ],
        [
          0.5655,
          0.8285
        ],
        [
          0.5755,
          0.8276
        ],
        [
          0.5855,
          0.8262
        ],
        [
          0.5955,
          0.8221
        ],
        [
          0.6056,
          0.8207
        ],
        [
          0.616,
          0.8185
        ],
        [
          0.626,
          0.8157
        ],
        [
          0.636,
          0.8135
        ],
        [
          0.646,
          0.8121
        ],
        [
          0.6561,
          0.8103
        ],
        [
          0.6665,
          0.8071
        ],
        [
          0.6765,
          0.8039
        ],
        [
          0.6865,
          0.8025
        ],
        [
          0.6965,
          0.7998
        ],
        [
          0.7066,
          0.7971
        ],
        [
          0.717,
          0.7948
        ],
        [
          0.727,
          0.7916
        ],
        [
          0.737,
          0.7884
        ],
        [
          0.747,
          0.7848
        ],
        [
          0.7571,
          0.7807
        ],
        [
          0.7675,
          0.7771
        ],
        [
          0.7775,
          0.7725
        ],
        [
          0.7875,
          0.7702
        ],
        [
          0.7975,
          0.7648
        ],
        [
          0.8076,
          0.7611
        ],
        [
          0.818,
          0.7566
        ],
        [
          0.828,
          0.7525
        ],
        [
          0.838,
          0.7475
        ],
        [
          0.848,
          0.7429
        ],
        [
          0.8581,
          0.7389
        ],
        [
          0.8685,
          0.7352
        ],
        [
          0.8785,
          0.7298
        ],
        [
          0.8885,
          0.7284
        ],
        [
          0.8985,
          0.7252
        ],
        [
          0.9086,
          0.722
        ],
        [
          0.919,
          0.7166
        ],
        [
          0.929,
          0.7106
        ],
        [
          0.939,
          0.7079
        ],
        [
          0.949,
          0.7025
        ],
        [
          0.9591,
          0.6993
        ],
        [
          0.9695,
          0.692
        ],
        [
          0.9795,
          0.6861
        ],
        [
          0.9895,
          0.682
        ],
        [
          0.9995,
          0.6793
        ],
        [
          1.0,
          0.6788
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.44559700273109903,
        "difference": -0.0020289540154920546,
        "worse": 0.5066666666666667,
        "kept_right": 0.4058367517432226,
        "kept_wrong": 0.039760250987876514,
        "handed_off": 0.5544029972689012
      },
      "usd_per_1000": 2.186052652025729
    },
    {
      "decision": 259,
      "frontier": 140,
      "frontier_accuracy": 0.9808917197452229,
      "in_sample": {
        "threshold": 0.9557880066229854,
        "answered": 0.21201091901728844,
        "accuracy": 0.9808917197452229
      },
      "curve": [
        [
          0.0,
          0.9809
        ],
        [
          0.01,
          0.9809
        ],
        [
          0.02,
          0.9809
        ],
        [
          0.03,
          0.9809
        ],
        [
          0.04,
          0.9809
        ],
        [
          0.05,
          0.9809
        ],
        [
          0.0601,
          0.9809
        ],
        [
          0.0705,
          0.9809
        ],
        [
          0.0805,
          0.9809
        ],
        [
          0.0905,
          0.9809
        ],
        [
          0.1005,
          0.9809
        ],
        [
          0.1106,
          0.9809
        ],
        [
          0.121,
          0.9809
        ],
        [
          0.131,
          0.9809
        ],
        [
          0.141,
          0.9809
        ],
        [
          0.151,
          0.9809
        ],
        [
          0.1611,
          0.9809
        ],
        [
          0.1715,
          0.9809
        ],
        [
          0.1815,
          0.9809
        ],
        [
          0.1915,
          0.9809
        ],
        [
          0.2015,
          0.9809
        ],
        [
          0.2116,
          0.9809
        ],
        [
          0.222,
          0.98
        ],
        [
          0.232,
          0.9791
        ],
        [
          0.242,
          0.9786
        ],
        [
          0.252,
          0.9786
        ],
        [
          0.2621,
          0.9786
        ],
        [
          0.2725,
          0.9786
        ],
        [
          0.2825,
          0.9786
        ],
        [
          0.2925,
          0.9786
        ],
        [
          0.3025,
          0.9786
        ],
        [
          0.3126,
          0.9782
        ],
        [
          0.323,
          0.9777
        ],
        [
          0.333,
          0.9768
        ],
        [
          0.343,
          0.9763
        ],
        [
          0.353,
          0.9754
        ],
        [
          0.3631,
          0.9741
        ],
        [
          0.3735,
          0.9732
        ],
        [
          0.3835,
          0.9709
        ],
        [
          0.3935,
          0.97
        ],
        [
          0.4035,
          0.9682
        ],
        [
          0.4136,
          0.9663
        ],
        [
          0.424,
          0.9641
        ],
        [
          0.434,
          0.9627
        ],
        [
          0.444,
          0.9613
        ],
        [
          0.454,
          0.9581
        ],
        [
          0.4641,
          0.9554
        ],
        [
          0.4745,
          0.9509
        ],
        [
          0.4845,
          0.9463
        ],
        [
          0.4945,
          0.9431
        ],
        [
          0.5045,
          0.9395
        ],
        [
          0.5146,
          0.9359
        ],
        [
          0.525,
          0.9331
        ],
        [
          0.535,
          0.929
        ],
        [
          0.545,
          0.9249
        ],
        [
          0.5551,
          0.9213
        ],
        [
          0.5655,
          0.9186
        ],
        [
          0.5755,
          0.9145
        ],
        [
          0.5855,
          0.9126
        ],
        [
          0.5955,
          0.9072
        ],
        [
          0.6056,
          0.904
        ],
        [
          0.616,
          0.8995
        ],
        [
          0.626,
          0.8944
        ],
        [
          0.636,
          0.889
        ],
        [
          0.646,
          0.8835
        ],
        [
          0.6561,
          0.879
        ],
        [
          0.6665,
          0.8758
        ],
        [
          0.6765,
          0.8703
        ],
        [
          0.6865,
          0.8653
        ],
        [
          0.6965,
          0.8585
        ],
        [
          0.7066,
          0.8535
        ],
        [
          0.717,
          0.8485
        ],
        [
          0.727,
          0.8439
        ],
        [
          0.737,
          0.8367
        ],
        [
          0.747,
          0.8317
        ],
        [
          0.7571,
          0.8258
        ],
        [
          0.7675,
          0.8194
        ],
        [
          0.7775,
          0.8139
        ],
        [
          0.7875,
          0.8066
        ],
        [
          0.7975,
          0.8021
        ],
        [
          0.8076,
          0.7975
        ],
        [
          0.818,
          0.7907
        ],
        [
          0.828,
          0.7853
        ],
        [
          0.838,
          0.7798
        ],
        [
          0.848,
          0.7748
        ],
        [
          0.8581,
          0.7689
        ],
        [
          0.8685,
          0.7625
        ],
        [
          0.8785,
          0.7571
        ],
        [
          0.8885,
          0.7498
        ],
        [
          0.8985,
          0.7434
        ],
        [
          0.9086,
          0.737
        ],
        [
          0.919,
          0.7307
        ],
        [
          0.929,
          0.7257
        ],
        [
          0.939,
          0.7188
        ],
        [
          0.949,
          0.712
        ],
        [
          0.9591,
          0.7075
        ],
        [
          0.9695,
          0.6993
        ],
        [
          0.9795,
          0.6934
        ],
        [
          0.9895,
          0.6856
        ],
        [
          0.9995,
          0.6783
        ],
        [
          1.0,
          0.6779
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.21886512233495523,
        "difference": -0.0008549076829409613,
        "worse": 0.5,
        "kept_right": 0.21799502999361575,
        "kept_wrong": 0.0008700923413393802,
        "handed_off": 0.7811348776650442
      },
      "usd_per_1000": 1.8896458524809834
    },
    {
      "decision": 259,
      "frontier": 277,
      "frontier_accuracy": 0.9795268425841674,
      "in_sample": {
        "threshold": 0.9552899958153984,
        "answered": 0.21519563239308462,
        "accuracy": 0.9795268425841674
      },
      "curve": [
        [
          0.0,
          0.9795
        ],
        [
          0.01,
          0.9795
        ],
        [
          0.02,
          0.9795
        ],
        [
          0.03,
          0.9795
        ],
        [
          0.04,
          0.9795
        ],
        [
          0.05,
          0.9795
        ],
        [
          0.0601,
          0.9795
        ],
        [
          0.0705,
          0.9795
        ],
        [
          0.0805,
          0.9795
        ],
        [
          0.0905,
          0.9795
        ],
        [
          0.1005,
          0.9795
        ],
        [
          0.1106,
          0.9795
        ],
        [
          0.121,
          0.9795
        ],
        [
          0.131,
          0.9795
        ],
        [
          0.141,
          0.9795
        ],
        [
          0.151,
          0.9795
        ],
        [
          0.1611,
          0.98
        ],
        [
          0.1715,
          0.98
        ],
        [
          0.1815,
          0.98
        ],
        [
          0.1915,
          0.98
        ],
        [
          0.2015,
          0.98
        ],
        [
          0.2116,
          0.98
        ],
        [
          0.222,
          0.9791
        ],
        [
          0.232,
          0.9782
        ],
        [
          0.242,
          0.9777
        ],
        [
          0.252,
          0.9777
        ],
        [
          0.2621,
          0.9777
        ],
        [
          0.2725,
          0.9777
        ],
        [
          0.2825,
          0.9777
        ],
        [
          0.2925,
          0.9777
        ],
        [
          0.3025,
          0.9777
        ],
        [
          0.3126,
          0.9773
        ],
        [
          0.323,
          0.9768
        ],
        [
          0.333,
          0.9759
        ],
        [
          0.343,
          0.975
        ],
        [
          0.353,
          0.9741
        ],
        [
          0.3631,
          0.9727
        ],
        [
          0.3735,
          0.9718
        ],
        [
          0.3835,
          0.9695
        ],
        [
          0.3935,
          0.9686
        ],
        [
          0.4035,
          0.9668
        ],
        [
          0.4136,
          0.965
        ],
        [
          0.424,
          0.9627
        ],
        [
          0.434,
          0.9613
        ],
        [
          0.444,
          0.9604
        ],
        [
          0.454,
          0.9577
        ],
        [
          0.4641,
          0.9554
        ],
        [
          0.4745,
          0.9513
        ],
        [
          0.4845,
          0.9472
        ],
        [
          0.4945,
          0.944
        ],
        [
          0.5045,
          0.9404
        ],
        [
          0.5146,
          0.9363
        ],
        [
          0.525,
          0.9336
        ],
        [
          0.535,
          0.9295
        ],
        [
          0.545,
          0.9249
        ],
        [
          0.5551,
          0.9213
        ],
        [
          0.5655,
          0.9186
        ],
        [
          0.5755,
          0.9145
        ],
        [
          0.5855,
          0.9126
        ],
        [
          0.5955,
          0.9076
        ],
        [
          0.6056,
          0.9035
        ],
        [
          0.616,
          0.8995
        ],
        [
          0.626,
          0.8944
        ],
        [
          0.636,
          0.889
        ],
        [
          0.646,
          0.8835
        ],
        [
          0.6561,
          0.879
        ],
        [
          0.6665,
          0.8758
        ],
        [
          0.6765,
          0.8703
        ],
        [
          0.6865,
          0.8653
        ],
        [
          0.6965,
          0.8585
        ],
        [
          0.7066,
          0.853
        ],
        [
          0.717,
          0.848
        ],
        [
          0.727,
          0.8439
        ],
        [
          0.737,
          0.8367
        ],
        [
          0.747,
          0.8317
        ],
        [
          0.7571,
          0.8258
        ],
        [
          0.7675,
          0.8194
        ],
        [
          0.7775,
          0.8135
        ],
        [
          0.7875,
          0.8062
        ],
        [
          0.7975,
          0.8016
        ],
        [
          0.8076,
          0.7971
        ],
        [
          0.818,
          0.7903
        ],
        [
          0.828,
          0.7848
        ],
        [
          0.838,
          0.7798
        ],
        [
          0.848,
          0.7748
        ],
        [
          0.8581,
          0.7689
        ],
        [
          0.8685,
          0.763
        ],
        [
          0.8785,
          0.7575
        ],
        [
          0.8885,
          0.7502
        ],
        [
          0.8985,
          0.7439
        ],
        [
          0.9086,
          0.737
        ],
        [
          0.919,
          0.7307
        ],
        [
          0.929,
          0.7257
        ],
        [
          0.939,
          0.7188
        ],
        [
          0.949,
          0.7116
        ],
        [
          0.9591,
          0.7061
        ],
        [
          0.9695,
          0.6988
        ],
        [
          0.9795,
          0.6934
        ],
        [
          0.9895,
          0.6856
        ],
        [
          0.9995,
          0.6783
        ],
        [
          1.0,
          0.6779
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.229350786472039,
        "difference": -0.0008793156327140651,
        "worse": 0.49333333333333335,
        "kept_right": 0.22790754146610676,
        "kept_wrong": 0.0014432450059322624,
        "handed_off": 0.7706492135279607
      },
      "usd_per_1000": 4.674415228928114
    },
    {
      "decision": 259,
      "frontier": 276,
      "frontier_accuracy": 0.9777070063694268,
      "in_sample": {
        "threshold": 0.952133926963857,
        "answered": 0.22611464968152867,
        "accuracy": 0.9777070063694268
      },
      "curve": [
        [
          0.0,
          0.9777
        ],
        [
          0.01,
          0.9777
        ],
        [
          0.02,
          0.9777
        ],
        [
          0.03,
          0.9777
        ],
        [
          0.04,
          0.9777
        ],
        [
          0.05,
          0.9777
        ],
        [
          0.0601,
          0.9777
        ],
        [
          0.0705,
          0.9777
        ],
        [
          0.0805,
          0.9777
        ],
        [
          0.0905,
          0.9777
        ],
        [
          0.1005,
          0.9777
        ],
        [
          0.1106,
          0.9777
        ],
        [
          0.121,
          0.9777
        ],
        [
          0.131,
          0.9777
        ],
        [
          0.141,
          0.9777
        ],
        [
          0.151,
          0.9777
        ],
        [
          0.1611,
          0.9782
        ],
        [
          0.1715,
          0.9782
        ],
        [
          0.1815,
          0.9782
        ],
        [
          0.1915,
          0.9782
        ],
        [
          0.2015,
          0.9782
        ],
        [
          0.2116,
          0.9782
        ],
        [
          0.222,
          0.9773
        ],
        [
          0.232,
          0.9768
        ],
        [
          0.242,
          0.9763
        ],
        [
          0.252,
          0.9763
        ],
        [
          0.2621,
          0.9763
        ],
        [
          0.2725,
          0.9763
        ],
        [
          0.2825,
          0.9763
        ],
        [
          0.2925,
          0.9763
        ],
        [
          0.3025,
          0.9763
        ],
        [
          0.3126,
          0.9759
        ],
        [
          0.323,
          0.9754
        ],
        [
          0.333,
          0.9745
        ],
        [
          0.343,
          0.9736
        ],
        [
          0.353,
          0.9727
        ],
        [
          0.3631,
          0.9713
        ],
        [
          0.3735,
          0.9709
        ],
        [
          0.3835,
          0.9686
        ],
        [
          0.3935,
          0.9677
        ],
        [
          0.4035,
          0.9659
        ],
        [
          0.4136,
          0.9641
        ],
        [
          0.424,
          0.9618
        ],
        [
          0.434,
          0.9604
        ],
        [
          0.444,
          0.9595
        ],
        [
          0.454,
          0.9568
        ],
        [
          0.4641,
          0.955
        ],
        [
          0.4745,
          0.9509
        ],
        [
          0.4845,
          0.9468
        ],
        [
          0.4945,
          0.9436
        ],
        [
          0.5045,
          0.9399
        ],
        [
          0.5146,
          0.9359
        ],
        [
          0.525,
          0.9336
        ],
        [
          0.535,
          0.9295
        ],
        [
          0.545,
          0.9249
        ],
        [
          0.5551,
          0.9213
        ],
        [
          0.5655,
          0.9186
        ],
        [
          0.5755,
          0.9145
        ],
        [
          0.5855,
          0.9131
        ],
        [
          0.5955,
          0.9076
        ],
        [
          0.6056,
          0.9035
        ],
        [
          0.616,
          0.8999
        ],
        [
          0.626,
          0.8949
        ],
        [
          0.636,
          0.8894
        ],
        [
          0.646,
          0.8844
        ],
        [
          0.6561,
          0.8799
        ],
        [
          0.6665,
          0.8767
        ],
        [
          0.6765,
          0.8712
        ],
        [
          0.6865,
          0.8662
        ],
        [
          0.6965,
          0.8599
        ],
        [
          0.7066,
          0.8544
        ],
        [
          0.717,
          0.8494
        ],
        [
          0.727,
          0.8449
        ],
        [
          0.737,
          0.8376
        ],
        [
          0.747,
          0.8326
        ],
        [
          0.7571,
          0.8267
        ],
        [
          0.7675,
          0.8203
        ],
        [
          0.7775,
          0.8144
        ],
        [
          0.7875,
          0.8071
        ],
        [
          0.7975,
          0.8025
        ],
        [
          0.8076,
          0.798
        ],
        [
          0.818,
          0.7912
        ],
        [
          0.828,
          0.7857
        ],
        [
          0.838,
          0.7803
        ],
        [
          0.848,
          0.7753
        ],
        [
          0.8581,
          0.7693
        ],
        [
          0.8685,
          0.763
        ],
        [
          0.8785,
          0.7575
        ],
        [
          0.8885,
          0.7502
        ],
        [
          0.8985,
          0.7439
        ],
        [
          0.9086,
          0.737
        ],
        [
          0.919,
          0.7307
        ],
        [
          0.929,
          0.7257
        ],
        [
          0.939,
          0.7188
        ],
        [
          0.949,
          0.712
        ],
        [
          0.9591,
          0.7066
        ],
        [
          0.9695,
          0.6993
        ],
        [
          0.9795,
          0.6934
        ],
        [
          0.9895,
          0.6856
        ],
        [
          0.9995,
          0.6783
        ],
        [
          1.0,
          0.6779
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.24349919111151622,
        "difference": -0.0011370349270355027,
        "worse": 0.53,
        "kept_right": 0.241504121616227,
        "kept_wrong": 0.001995069495289001,
        "handed_off": 0.7565008088884837
      },
      "usd_per_1000": 1.2271881779580283
    },
    {
      "decision": 259,
      "frontier": 48,
      "frontier_accuracy": 0.9758871701546861,
      "in_sample": {
        "threshold": 0.9552899958153984,
        "answered": 0.21519563239308462,
        "accuracy": 0.9758871701546861
      },
      "curve": [
        [
          0.0,
          0.9759
        ],
        [
          0.01,
          0.9759
        ],
        [
          0.02,
          0.9759
        ],
        [
          0.03,
          0.9759
        ],
        [
          0.04,
          0.9759
        ],
        [
          0.05,
          0.9759
        ],
        [
          0.0601,
          0.9759
        ],
        [
          0.0705,
          0.9759
        ],
        [
          0.0805,
          0.9759
        ],
        [
          0.0905,
          0.9759
        ],
        [
          0.1005,
          0.9759
        ],
        [
          0.1106,
          0.9759
        ],
        [
          0.121,
          0.9759
        ],
        [
          0.131,
          0.9759
        ],
        [
          0.141,
          0.9759
        ],
        [
          0.151,
          0.9759
        ],
        [
          0.1611,
          0.9763
        ],
        [
          0.1715,
          0.9763
        ],
        [
          0.1815,
          0.9763
        ],
        [
          0.1915,
          0.9763
        ],
        [
          0.2015,
          0.9763
        ],
        [
          0.2116,
          0.9763
        ],
        [
          0.222,
          0.9754
        ],
        [
          0.232,
          0.9745
        ],
        [
          0.242,
          0.9741
        ],
        [
          0.252,
          0.9741
        ],
        [
          0.2621,
          0.9741
        ],
        [
          0.2725,
          0.9741
        ],
        [
          0.2825,
          0.9741
        ],
        [
          0.2925,
          0.9741
        ],
        [
          0.3025,
          0.9741
        ],
        [
          0.3126,
          0.9736
        ],
        [
          0.323,
          0.9736
        ],
        [
          0.333,
          0.9727
        ],
        [
          0.343,
          0.9718
        ],
        [
          0.353,
          0.9709
        ],
        [
          0.3631,
          0.9695
        ],
        [
          0.3735,
          0.9691
        ],
        [
          0.3835,
          0.9668
        ],
        [
          0.3935,
          0.9659
        ],
        [
          0.4035,
          0.9641
        ],
        [
          0.4136,
          0.9622
        ],
        [
          0.424,
          0.96
        ],
        [
          0.434,
          0.9586
        ],
        [
          0.444,
          0.9581
        ],
        [
          0.454,
          0.9554
        ],
        [
          0.4641,
          0.9531
        ],
        [
          0.4745,
          0.949
        ],
        [
          0.4845,
          0.9449
        ],
        [
          0.4945,
          0.9418
        ],
        [
          0.5045,
          0.9381
        ],
        [
          0.5146,
          0.934
        ],
        [
          0.525,
          0.9313
        ],
        [
          0.535,
          0.9272
        ],
        [
          0.545,
          0.9227
        ],
        [
          0.5551,
          0.919
        ],
        [
          0.5655,
          0.9163
        ],
        [
          0.5755,
          0.9122
        ],
        [
          0.5855,
          0.9104
        ],
        [
          0.5955,
          0.9054
        ],
        [
          0.6056,
          0.9013
        ],
        [
          0.616,
          0.8972
        ],
        [
          0.626,
          0.8922
        ],
        [
          0.636,
          0.8867
        ],
        [
          0.646,
          0.8813
        ],
        [
          0.6561,
          0.8767
        ],
        [
          0.6665,
          0.8735
        ],
        [
          0.6765,
          0.8685
        ],
        [
          0.6865,
          0.8635
        ],
        [
          0.6965,
          0.8571
        ],
        [
          0.7066,
          0.8517
        ],
        [
          0.717,
          0.8462
        ],
        [
          0.727,
          0.8417
        ],
        [
          0.737,
          0.8344
        ],
        [
          0.747,
          0.8298
        ],
        [
          0.7571,
          0.8244
        ],
        [
          0.7675,
          0.8185
        ],
        [
          0.7775,
          0.813
        ],
        [
          0.7875,
          0.8057
        ],
        [
          0.7975,
          0.8012
        ],
        [
          0.8076,
          0.7966
        ],
        [
          0.818,
          0.7898
        ],
        [
          0.828,
          0.7848
        ],
        [
          0.838,
          0.7798
        ],
        [
          0.848,
          0.7748
        ],
        [
          0.8581,
          0.7689
        ],
        [
          0.8685,
          0.7625
        ],
        [
          0.8785,
          0.7575
        ],
        [
          0.8885,
          0.7502
        ],
        [
          0.8985,
          0.7439
        ],
        [
          0.9086,
          0.737
        ],
        [
          0.919,
          0.7307
        ],
        [
          0.929,
          0.7261
        ],
        [
          0.939,
          0.7188
        ],
        [
          0.949,
          0.712
        ],
        [
          0.9591,
          0.7066
        ],
        [
          0.9695,
          0.6988
        ],
        [
          0.9795,
          0.6934
        ],
        [
          0.9895,
          0.6856
        ],
        [
          0.9995,
          0.6783
        ],
        [
          1.0,
          0.6779
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.23432464895868535,
        "difference": -0.0010429492838416876,
        "worse": 0.5066666666666667,
        "kept_right": 0.232629833539484,
        "kept_wrong": 0.001694815419201372,
        "handed_off": 0.7656753510413143
      },
      "usd_per_1000": 4.117975267617621
    },
    {
      "decision": 259,
      "frontier": 191,
      "frontier_accuracy": 0.9740673339399454,
      "in_sample": {
        "threshold": 0.9557880066229854,
        "answered": 0.21201091901728844,
        "accuracy": 0.9740673339399454
      },
      "curve": [
        [
          0.0,
          0.9741
        ],
        [
          0.01,
          0.9741
        ],
        [
          0.02,
          0.9741
        ],
        [
          0.03,
          0.9741
        ],
        [
          0.04,
          0.9741
        ],
        [
          0.05,
          0.9741
        ],
        [
          0.0601,
          0.9741
        ],
        [
          0.0705,
          0.9741
        ],
        [
          0.0805,
          0.9741
        ],
        [
          0.0905,
          0.9741
        ],
        [
          0.1005,
          0.9741
        ],
        [
          0.1106,
          0.9741
        ],
        [
          0.121,
          0.9741
        ],
        [
          0.131,
          0.9741
        ],
        [
          0.141,
          0.9741
        ],
        [
          0.151,
          0.9741
        ],
        [
          0.1611,
          0.9741
        ],
        [
          0.1715,
          0.9741
        ],
        [
          0.1815,
          0.9741
        ],
        [
          0.1915,
          0.9741
        ],
        [
          0.2015,
          0.9741
        ],
        [
          0.2116,
          0.9741
        ],
        [
          0.222,
          0.9732
        ],
        [
          0.232,
          0.9722
        ],
        [
          0.242,
          0.9718
        ],
        [
          0.252,
          0.9718
        ],
        [
          0.2621,
          0.9718
        ],
        [
          0.2725,
          0.9718
        ],
        [
          0.2825,
          0.9718
        ],
        [
          0.2925,
          0.9718
        ],
        [
          0.3025,
          0.9718
        ],
        [
          0.3126,
          0.9713
        ],
        [
          0.323,
          0.9709
        ],
        [
          0.333,
          0.97
        ],
        [
          0.343,
          0.9691
        ],
        [
          0.353,
          0.9682
        ],
        [
          0.3631,
          0.9672
        ],
        [
          0.3735,
          0.9668
        ],
        [
          0.3835,
          0.9645
        ],
        [
          0.3935,
          0.9636
        ],
        [
          0.4035,
          0.9618
        ],
        [
          0.4136,
          0.96
        ],
        [
          0.424,
          0.9577
        ],
        [
          0.434,
          0.9563
        ],
        [
          0.444,
          0.9554
        ],
        [
          0.454,
          0.9527
        ],
        [
          0.4641,
          0.9504
        ],
        [
          0.4745,
          0.9463
        ],
        [
          0.4845,
          0.9422
        ],
        [
          0.4945,
          0.939
        ],
        [
          0.5045,
          0.9359
        ],
        [
          0.5146,
          0.9318
        ],
        [
          0.525,
          0.9295
        ],
        [
          0.535,
          0.9254
        ],
        [
          0.545,
          0.9208
        ],
        [
          0.5551,
          0.9172
        ],
        [
          0.5655,
          0.914
        ],
        [
          0.5755,
          0.9099
        ],
        [
          0.5855,
          0.9086
        ],
        [
          0.5955,
          0.9035
        ],
        [
          0.6056,
          0.8999
        ],
        [
          0.616,
          0.8958
        ],
        [
          0.626,
          0.8913
        ],
        [
          0.636,
          0.8858
        ],
        [
          0.646,
          0.8808
        ],
        [
          0.6561,
          0.8767
        ],
        [
          0.6665,
          0.874
        ],
        [
          0.6765,
          0.869
        ],
        [
          0.6865,
          0.864
        ],
        [
          0.6965,
          0.8576
        ],
        [
          0.7066,
          0.8521
        ],
        [
          0.717,
          0.8467
        ],
        [
          0.727,
          0.8421
        ],
        [
          0.737,
          0.8348
        ],
        [
          0.747,
          0.8298
        ],
        [
          0.7571,
          0.8239
        ],
        [
          0.7675,
          0.8176
        ],
        [
          0.7775,
          0.8121
        ],
        [
          0.7875,
          0.8048
        ],
        [
          0.7975,
          0.8003
        ],
        [
          0.8076,
          0.7957
        ],
        [
          0.818,
          0.7889
        ],
        [
          0.828,
          0.7839
        ],
        [
          0.838,
          0.7789
        ],
        [
          0.848,
          0.7739
        ],
        [
          0.8581,
          0.768
        ],
        [
          0.8685,
          0.7616
        ],
        [
          0.8785,
          0.7566
        ],
        [
          0.8885,
          0.7493
        ],
        [
          0.8985,
          0.7429
        ],
        [
          0.9086,
          0.7357
        ],
        [
          0.919,
          0.7293
        ],
        [
          0.929,
          0.7243
        ],
        [
          0.939,
          0.7175
        ],
        [
          0.949,
          0.7106
        ],
        [
          0.9591,
          0.7052
        ],
        [
          0.9695,
          0.6979
        ],
        [
          0.9795,
          0.6924
        ],
        [
          0.9895,
          0.6852
        ],
        [
          0.9995,
          0.6783
        ],
        [
          1.0,
          0.6779
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.21886512233495523,
        "difference": -0.0008549076829409613,
        "worse": 0.5,
        "kept_right": 0.21799502999361575,
        "kept_wrong": 0.0008700923413393802,
        "handed_off": 0.7811348776650442
      },
      "usd_per_1000": 20.885746962238958
    },
    {
      "decision": 259,
      "frontier": 188,
      "frontier_accuracy": 0.9713375796178344,
      "in_sample": {
        "threshold": 0.9557880066229854,
        "answered": 0.21201091901728844,
        "accuracy": 0.9713375796178344
      },
      "curve": [
        [
          0.0,
          0.9713
        ],
        [
          0.01,
          0.9713
        ],
        [
          0.02,
          0.9713
        ],
        [
          0.03,
          0.9713
        ],
        [
          0.04,
          0.9713
        ],
        [
          0.05,
          0.9713
        ],
        [
          0.0601,
          0.9713
        ],
        [
          0.0705,
          0.9713
        ],
        [
          0.0805,
          0.9713
        ],
        [
          0.0905,
          0.9713
        ],
        [
          0.1005,
          0.9713
        ],
        [
          0.1106,
          0.9713
        ],
        [
          0.121,
          0.9713
        ],
        [
          0.131,
          0.9713
        ],
        [
          0.141,
          0.9713
        ],
        [
          0.151,
          0.9713
        ],
        [
          0.1611,
          0.9713
        ],
        [
          0.1715,
          0.9713
        ],
        [
          0.1815,
          0.9713
        ],
        [
          0.1915,
          0.9713
        ],
        [
          0.2015,
          0.9713
        ],
        [
          0.2116,
          0.9713
        ],
        [
          0.222,
          0.9704
        ],
        [
          0.232,
          0.9695
        ],
        [
          0.242,
          0.9691
        ],
        [
          0.252,
          0.9695
        ],
        [
          0.2621,
          0.9695
        ],
        [
          0.2725,
          0.9695
        ],
        [
          0.2825,
          0.9695
        ],
        [
          0.2925,
          0.9695
        ],
        [
          0.3025,
          0.9695
        ],
        [
          0.3126,
          0.9691
        ],
        [
          0.323,
          0.9691
        ],
        [
          0.333,
          0.9682
        ],
        [
          0.343,
          0.9677
        ],
        [
          0.353,
          0.9668
        ],
        [
          0.3631,
          0.9654
        ],
        [
          0.3735,
          0.965
        ],
        [
          0.3835,
          0.9627
        ],
        [
          0.3935,
          0.9613
        ],
        [
          0.4035,
          0.9595
        ],
        [
          0.4136,
          0.9577
        ],
        [
          0.424,
          0.9554
        ],
        [
          0.434,
          0.954
        ],
        [
          0.444,
          0.9531
        ],
        [
          0.454,
          0.95
        ],
        [
          0.4641,
          0.9472
        ],
        [
          0.4745,
          0.9431
        ],
        [
          0.4845,
          0.9386
        ],
        [
          0.4945,
          0.9359
        ],
        [
          0.5045,
          0.9322
        ],
        [
          0.5146,
          0.9286
        ],
        [
          0.525,
          0.9268
        ],
        [
          0.535,
          0.9227
        ],
        [
          0.545,
          0.919
        ],
        [
          0.5551,
          0.9154
        ],
        [
          0.5655,
          0.9136
        ],
        [
          0.5755,
          0.9095
        ],
        [
          0.5855,
          0.9081
        ],
        [
          0.5955,
          0.9026
        ],
        [
          0.6056,
          0.8995
        ],
        [
          0.616,
          0.8958
        ],
        [
          0.626,
          0.8908
        ],
        [
          0.636,
          0.8854
        ],
        [
          0.646,
          0.8799
        ],
        [
          0.6561,
          0.8753
        ],
        [
          0.6665,
          0.8722
        ],
        [
          0.6765,
          0.8667
        ],
        [
          0.6865,
          0.8617
        ],
        [
          0.6965,
          0.8549
        ],
        [
          0.7066,
          0.8499
        ],
        [
          0.717,
          0.8453
        ],
        [
          0.727,
          0.8408
        ],
        [
          0.737,
          0.8335
        ],
        [
          0.747,
          0.8285
        ],
        [
          0.7571,
          0.8226
        ],
        [
          0.7675,
          0.8162
        ],
        [
          0.7775,
          0.8107
        ],
        [
          0.7875,
          0.8035
        ],
        [
          0.7975,
          0.7989
        ],
        [
          0.8076,
          0.7944
        ],
        [
          0.818,
          0.788
        ],
        [
          0.828,
          0.7825
        ],
        [
          0.838,
          0.7766
        ],
        [
          0.848,
          0.7716
        ],
        [
          0.8581,
          0.7657
        ],
        [
          0.8685,
          0.7598
        ],
        [
          0.8785,
          0.7543
        ],
        [
          0.8885,
          0.7475
        ],
        [
          0.8985,
          0.7411
        ],
        [
          0.9086,
          0.7352
        ],
        [
          0.919,
          0.7288
        ],
        [
          0.929,
          0.7238
        ],
        [
          0.939,
          0.717
        ],
        [
          0.949,
          0.7102
        ],
        [
          0.9591,
          0.7052
        ],
        [
          0.9695,
          0.6975
        ],
        [
          0.9795,
          0.6929
        ],
        [
          0.9895,
          0.6852
        ],
        [
          0.9995,
          0.6783
        ],
        [
          1.0,
          0.6779
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.23023298657444458,
        "difference": -0.0012369581327469714,
        "worse": 0.5266666666666666,
        "kept_right": 0.2287836890187278,
        "kept_wrong": 0.0014492975557168022,
        "handed_off": 0.7697670134255548
      },
      "usd_per_1000": 0.7416242018361024
    },
    {
      "decision": 259,
      "frontier": 190,
      "frontier_accuracy": 0.9672429481346679,
      "in_sample": {
        "threshold": 0.9552899958153984,
        "answered": 0.21519563239308462,
        "accuracy": 0.9672429481346679
      },
      "curve": [
        [
          0.0,
          0.9672
        ],
        [
          0.01,
          0.9672
        ],
        [
          0.02,
          0.9672
        ],
        [
          0.03,
          0.9672
        ],
        [
          0.04,
          0.9672
        ],
        [
          0.05,
          0.9672
        ],
        [
          0.0601,
          0.9672
        ],
        [
          0.0705,
          0.9672
        ],
        [
          0.0805,
          0.9672
        ],
        [
          0.0905,
          0.9672
        ],
        [
          0.1005,
          0.9672
        ],
        [
          0.1106,
          0.9672
        ],
        [
          0.121,
          0.9672
        ],
        [
          0.131,
          0.9672
        ],
        [
          0.141,
          0.9672
        ],
        [
          0.151,
          0.9672
        ],
        [
          0.1611,
          0.9677
        ],
        [
          0.1715,
          0.9677
        ],
        [
          0.1815,
          0.9677
        ],
        [
          0.1915,
          0.9677
        ],
        [
          0.2015,
          0.9677
        ],
        [
          0.2116,
          0.9677
        ],
        [
          0.222,
          0.9668
        ],
        [
          0.232,
          0.9659
        ],
        [
          0.242,
          0.9654
        ],
        [
          0.252,
          0.9654
        ],
        [
          0.2621,
          0.9659
        ],
        [
          0.2725,
          0.9659
        ],
        [
          0.2825,
          0.9659
        ],
        [
          0.2925,
          0.9659
        ],
        [
          0.3025,
          0.9663
        ],
        [
          0.3126,
          0.9659
        ],
        [
          0.323,
          0.9654
        ],
        [
          0.333,
          0.9645
        ],
        [
          0.343,
          0.9636
        ],
        [
          0.353,
          0.9627
        ],
        [
          0.3631,
          0.9613
        ],
        [
          0.3735,
          0.9604
        ],
        [
          0.3835,
          0.9586
        ],
        [
          0.3935,
          0.9577
        ],
        [
          0.4035,
          0.9559
        ],
        [
          0.4136,
          0.954
        ],
        [
          0.424,
          0.9518
        ],
        [
          0.434,
          0.9504
        ],
        [
          0.444,
          0.949
        ],
        [
          0.454,
          0.9463
        ],
        [
          0.4641,
          0.9445
        ],
        [
          0.4745,
          0.9404
        ],
        [
          0.4845,
          0.9363
        ],
        [
          0.4945,
          0.9336
        ],
        [
          0.5045,
          0.9304
        ],
        [
          0.5146,
          0.9263
        ],
        [
          0.525,
          0.9236
        ],
        [
          0.535,
          0.9199
        ],
        [
          0.545,
          0.9154
        ],
        [
          0.5551,
          0.9117
        ],
        [
          0.5655,
          0.909
        ],
        [
          0.5755,
          0.9049
        ],
        [
          0.5855,
          0.9035
        ],
        [
          0.5955,
          0.899
        ],
        [
          0.6056,
          0.8949
        ],
        [
          0.616,
          0.8913
        ],
        [
          0.626,
          0.8863
        ],
        [
          0.636,
          0.8808
        ],
        [
          0.646,
          0.8753
        ],
        [
          0.6561,
          0.8712
        ],
        [
          0.6665,
          0.8685
        ],
        [
          0.6765,
          0.8635
        ],
        [
          0.6865,
          0.8585
        ],
        [
          0.6965,
          0.8521
        ],
        [
          0.7066,
          0.8467
        ],
        [
          0.717,
          0.8421
        ],
        [
          0.727,
          0.8371
        ],
        [
          0.737,
          0.8298
        ],
        [
          0.747,
          0.8248
        ],
        [
          0.7571,
          0.8194
        ],
        [
          0.7675,
          0.813
        ],
        [
          0.7775,
          0.8076
        ],
        [
          0.7875,
          0.8003
        ],
        [
          0.7975,
          0.7962
        ],
        [
          0.8076,
          0.7921
        ],
        [
          0.818,
          0.7853
        ],
        [
          0.828,
          0.7803
        ],
        [
          0.838,
          0.7753
        ],
        [
          0.848,
          0.7712
        ],
        [
          0.8581,
          0.7652
        ],
        [
          0.8685,
          0.7593
        ],
        [
          0.8785,
          0.7543
        ],
        [
          0.8885,
          0.747
        ],
        [
          0.8985,
          0.7411
        ],
        [
          0.9086,
          0.7343
        ],
        [
          0.919,
          0.7279
        ],
        [
          0.929,
          0.7229
        ],
        [
          0.939,
          0.7161
        ],
        [
          0.949,
          0.7097
        ],
        [
          0.9591,
          0.7043
        ],
        [
          0.9695,
          0.697
        ],
        [
          0.9795,
          0.6915
        ],
        [
          0.9895,
          0.6847
        ],
        [
          0.9995,
          0.6783
        ],
        [
          1.0,
          0.6779
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.25612702059923775,
        "difference": -0.001355090378945414,
        "worse": 0.5733333333333334,
        "kept_right": 0.2537592452132056,
        "kept_wrong": 0.00236777538603237,
        "handed_off": 0.743872979400762
      },
      "usd_per_1000": 4.14227083224016
    },
    {
      "decision": 259,
      "frontier": 124,
      "frontier_accuracy": 0.9667879890809827,
      "in_sample": {
        "threshold": 0.9552899958153984,
        "answered": 0.21519563239308462,
        "accuracy": 0.9667879890809827
      },
      "curve": [
        [
          0.0,
          0.9668
        ],
        [
          0.01,
          0.9668
        ],
        [
          0.02,
          0.9668
        ],
        [
          0.03,
          0.9668
        ],
        [
          0.04,
          0.9668
        ],
        [
          0.05,
          0.9668
        ],
        [
          0.0601,
          0.9668
        ],
        [
          0.0705,
          0.9668
        ],
        [
          0.0805,
          0.9668
        ],
        [
          0.0905,
          0.9668
        ],
        [
          0.1005,
          0.9668
        ],
        [
          0.1106,
          0.9668
        ],
        [
          0.121,
          0.9668
        ],
        [
          0.131,
          0.9668
        ],
        [
          0.141,
          0.9668
        ],
        [
          0.151,
          0.9668
        ],
        [
          0.1611,
          0.9672
        ],
        [
          0.1715,
          0.9672
        ],
        [
          0.1815,
          0.9672
        ],
        [
          0.1915,
          0.9672
        ],
        [
          0.2015,
          0.9672
        ],
        [
          0.2116,
          0.9672
        ],
        [
          0.222,
          0.9663
        ],
        [
          0.232,
          0.9654
        ],
        [
          0.242,
          0.965
        ],
        [
          0.252,
          0.965
        ],
        [
          0.2621,
          0.965
        ],
        [
          0.2725,
          0.965
        ],
        [
          0.2825,
          0.965
        ],
        [
          0.2925,
          0.965
        ],
        [
          0.3025,
          0.965
        ],
        [
          0.3126,
          0.9645
        ],
        [
          0.323,
          0.9645
        ],
        [
          0.333,
          0.9636
        ],
        [
          0.343,
          0.9627
        ],
        [
          0.353,
          0.9618
        ],
        [
          0.3631,
          0.9604
        ],
        [
          0.3735,
          0.96
        ],
        [
          0.3835,
          0.9577
        ],
        [
          0.3935,
          0.9568
        ],
        [
          0.4035,
          0.955
        ],
        [
          0.4136,
          0.9536
        ],
        [
          0.424,
          0.9513
        ],
        [
          0.434,
          0.95
        ],
        [
          0.444,
          0.949
        ],
        [
          0.454,
          0.9463
        ],
        [
          0.4641,
          0.9445
        ],
        [
          0.4745,
          0.9404
        ],
        [
          0.4845,
          0.9363
        ],
        [
          0.4945,
          0.9331
        ],
        [
          0.5045,
          0.9295
        ],
        [
          0.5146,
          0.9254
        ],
        [
          0.525,
          0.9236
        ],
        [
          0.535,
          0.9199
        ],
        [
          0.545,
          0.9154
        ],
        [
          0.5551,
          0.9117
        ],
        [
          0.5655,
          0.9095
        ],
        [
          0.5755,
          0.9054
        ],
        [
          0.5855,
          0.9035
        ],
        [
          0.5955,
          0.8985
        ],
        [
          0.6056,
          0.8944
        ],
        [
          0.616,
          0.8899
        ],
        [
          0.626,
          0.8854
        ],
        [
          0.636,
          0.8799
        ],
        [
          0.646,
          0.8749
        ],
        [
          0.6561,
          0.8703
        ],
        [
          0.6665,
          0.8672
        ],
        [
          0.6765,
          0.8626
        ],
        [
          0.6865,
          0.8576
        ],
        [
          0.6965,
          0.8517
        ],
        [
          0.7066,
          0.8467
        ],
        [
          0.717,
          0.8421
        ],
        [
          0.727,
          0.8376
        ],
        [
          0.737,
          0.8303
        ],
        [
          0.747,
          0.8258
        ],
        [
          0.7571,
          0.8198
        ],
        [
          0.7675,
          0.8139
        ],
        [
          0.7775,
          0.8085
        ],
        [
          0.7875,
          0.8012
        ],
        [
          0.7975,
          0.7966
        ],
        [
          0.8076,
          0.7921
        ],
        [
          0.818,
          0.7853
        ],
        [
          0.828,
          0.7803
        ],
        [
          0.838,
          0.7753
        ],
        [
          0.848,
          0.7702
        ],
        [
          0.8581,
          0.7643
        ],
        [
          0.8685,
          0.7584
        ],
        [
          0.8785,
          0.7534
        ],
        [
          0.8885,
          0.7466
        ],
        [
          0.8985,
          0.7407
        ],
        [
          0.9086,
          0.7343
        ],
        [
          0.919,
          0.7284
        ],
        [
          0.929,
          0.7234
        ],
        [
          0.939,
          0.7166
        ],
        [
          0.949,
          0.7102
        ],
        [
          0.9591,
          0.7047
        ],
        [
          0.9695,
          0.6975
        ],
        [
          0.9795,
          0.692
        ],
        [
          0.9895,
          0.6847
        ],
        [
          0.9995,
          0.6783
        ],
        [
          1.0,
          0.6779
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.23432464895868535,
        "difference": -0.0010429492838416876,
        "worse": 0.5066666666666667,
        "kept_right": 0.232629833539484,
        "kept_wrong": 0.001694815419201372,
        "handed_off": 0.7656753510413143
      },
      "usd_per_1000": 0.8654987944418603
    },
    {
      "decision": 259,
      "frontier": 278,
      "frontier_accuracy": 0.9499545040946314,
      "in_sample": {
        "threshold": 0.9557880066229854,
        "answered": 0.21201091901728844,
        "accuracy": 0.9499545040946314
      },
      "curve": [
        [
          0.0,
          0.95
        ],
        [
          0.01,
          0.95
        ],
        [
          0.02,
          0.95
        ],
        [
          0.03,
          0.95
        ],
        [
          0.04,
          0.95
        ],
        [
          0.05,
          0.95
        ],
        [
          0.0601,
          0.95
        ],
        [
          0.0705,
          0.95
        ],
        [
          0.0805,
          0.95
        ],
        [
          0.0905,
          0.95
        ],
        [
          0.1005,
          0.95
        ],
        [
          0.1106,
          0.95
        ],
        [
          0.121,
          0.95
        ],
        [
          0.131,
          0.95
        ],
        [
          0.141,
          0.95
        ],
        [
          0.151,
          0.95
        ],
        [
          0.1611,
          0.95
        ],
        [
          0.1715,
          0.95
        ],
        [
          0.1815,
          0.95
        ],
        [
          0.1915,
          0.95
        ],
        [
          0.2015,
          0.95
        ],
        [
          0.2116,
          0.95
        ],
        [
          0.222,
          0.949
        ],
        [
          0.232,
          0.9481
        ],
        [
          0.242,
          0.9477
        ],
        [
          0.252,
          0.9477
        ],
        [
          0.2621,
          0.9477
        ],
        [
          0.2725,
          0.9481
        ],
        [
          0.2825,
          0.9481
        ],
        [
          0.2925,
          0.9481
        ],
        [
          0.3025,
          0.9481
        ],
        [
          0.3126,
          0.9477
        ],
        [
          0.323,
          0.9481
        ],
        [
          0.333,
          0.9477
        ],
        [
          0.343,
          0.9477
        ],
        [
          0.353,
          0.9468
        ],
        [
          0.3631,
          0.9454
        ],
        [
          0.3735,
          0.9454
        ],
        [
          0.3835,
          0.9431
        ],
        [
          0.3935,
          0.9422
        ],
        [
          0.4035,
          0.9404
        ],
        [
          0.4136,
          0.9386
        ],
        [
          0.424,
          0.9363
        ],
        [
          0.434,
          0.9349
        ],
        [
          0.444,
          0.934
        ],
        [
          0.454,
          0.9318
        ],
        [
          0.4641,
          0.9299
        ],
        [
          0.4745,
          0.9258
        ],
        [
          0.4845,
          0.9217
        ],
        [
          0.4945,
          0.919
        ],
        [
          0.5045,
          0.9163
        ],
        [
          0.5146,
          0.9122
        ],
        [
          0.525,
          0.9104
        ],
        [
          0.535,
          0.9067
        ],
        [
          0.545,
          0.9022
        ],
        [
          0.5551,
          0.899
        ],
        [
          0.5655,
          0.8967
        ],
        [
          0.5755,
          0.8935
        ],
        [
          0.5855,
          0.8922
        ],
        [
          0.5955,
          0.8872
        ],
        [
          0.6056,
          0.884
        ],
        [
          0.616,
          0.8808
        ],
        [
          0.626,
          0.8767
        ],
        [
          0.636,
          0.8722
        ],
        [
          0.646,
          0.8676
        ],
        [
          0.6561,
          0.8631
        ],
        [
          0.6665,
          0.8599
        ],
        [
          0.6765,
          0.8549
        ],
        [
          0.6865,
          0.8503
        ],
        [
          0.6965,
          0.8439
        ],
        [
          0.7066,
          0.8389
        ],
        [
          0.717,
          0.8344
        ],
        [
          0.727,
          0.8308
        ],
        [
          0.737,
          0.8239
        ],
        [
          0.747,
          0.8194
        ],
        [
          0.7571,
          0.8139
        ],
        [
          0.7675,
          0.808
        ],
        [
          0.7775,
          0.8025
        ],
        [
          0.7875,
          0.7953
        ],
        [
          0.7975,
          0.7912
        ],
        [
          0.8076,
          0.7866
        ],
        [
          0.818,
          0.7812
        ],
        [
          0.828,
          0.7762
        ],
        [
          0.838,
          0.7716
        ],
        [
          0.848,
          0.7666
        ],
        [
          0.8581,
          0.7611
        ],
        [
          0.8685,
          0.7552
        ],
        [
          0.8785,
          0.7507
        ],
        [
          0.8885,
          0.7439
        ],
        [
          0.8985,
          0.7375
        ],
        [
          0.9086,
          0.7316
        ],
        [
          0.919,
          0.7261
        ],
        [
          0.929,
          0.7216
        ],
        [
          0.939,
          0.7152
        ],
        [
          0.949,
          0.7088
        ],
        [
          0.9591,
          0.7034
        ],
        [
          0.9695,
          0.6961
        ],
        [
          0.9795,
          0.6915
        ],
        [
          0.9895,
          0.6843
        ],
        [
          0.9995,
          0.6783
        ],
        [
          1.0,
          0.6779
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.24501985153240544,
        "difference": -0.0017825260791099467,
        "worse": 0.5733333333333334,
        "kept_right": 0.24250659415386225,
        "kept_wrong": 0.002513257378543141,
        "handed_off": 0.7549801484675941
      },
      "usd_per_1000": 4.914034643781994
    },
    {
      "decision": 259,
      "frontier": 275,
      "frontier_accuracy": 0.9426751592356688,
      "in_sample": {
        "threshold": 0.9552899958153984,
        "answered": 0.21519563239308462,
        "accuracy": 0.9426751592356688
      },
      "curve": [
        [
          0.0,
          0.9427
        ],
        [
          0.01,
          0.9427
        ],
        [
          0.02,
          0.9427
        ],
        [
          0.03,
          0.9427
        ],
        [
          0.04,
          0.9427
        ],
        [
          0.05,
          0.9427
        ],
        [
          0.0601,
          0.9427
        ],
        [
          0.0705,
          0.9427
        ],
        [
          0.0805,
          0.9427
        ],
        [
          0.0905,
          0.9427
        ],
        [
          0.1005,
          0.9427
        ],
        [
          0.1106,
          0.9427
        ],
        [
          0.121,
          0.9427
        ],
        [
          0.131,
          0.9427
        ],
        [
          0.141,
          0.9427
        ],
        [
          0.151,
          0.9427
        ],
        [
          0.1611,
          0.9431
        ],
        [
          0.1715,
          0.9431
        ],
        [
          0.1815,
          0.9431
        ],
        [
          0.1915,
          0.9431
        ],
        [
          0.2015,
          0.9431
        ],
        [
          0.2116,
          0.9431
        ],
        [
          0.222,
          0.9422
        ],
        [
          0.232,
          0.9413
        ],
        [
          0.242,
          0.9409
        ],
        [
          0.252,
          0.9409
        ],
        [
          0.2621,
          0.9409
        ],
        [
          0.2725,
          0.9413
        ],
        [
          0.2825,
          0.9413
        ],
        [
          0.2925,
          0.9413
        ],
        [
          0.3025,
          0.9413
        ],
        [
          0.3126,
          0.9409
        ],
        [
          0.323,
          0.9409
        ],
        [
          0.333,
          0.9399
        ],
        [
          0.343,
          0.9395
        ],
        [
          0.353,
          0.9386
        ],
        [
          0.3631,
          0.9377
        ],
        [
          0.3735,
          0.9372
        ],
        [
          0.3835,
          0.9349
        ],
        [
          0.3935,
          0.934
        ],
        [
          0.4035,
          0.9322
        ],
        [
          0.4136,
          0.9308
        ],
        [
          0.424,
          0.929
        ],
        [
          0.434,
          0.9281
        ],
        [
          0.444,
          0.9272
        ],
        [
          0.454,
          0.9245
        ],
        [
          0.4641,
          0.9227
        ],
        [
          0.4745,
          0.919
        ],
        [
          0.4845,
          0.9154
        ],
        [
          0.4945,
          0.9131
        ],
        [
          0.5045,
          0.9099
        ],
        [
          0.5146,
          0.9063
        ],
        [
          0.525,
          0.904
        ],
        [
          0.535,
          0.9008
        ],
        [
          0.545,
          0.8963
        ],
        [
          0.5551,
          0.8931
        ],
        [
          0.5655,
          0.8913
        ],
        [
          0.5755,
          0.8872
        ],
        [
          0.5855,
          0.8854
        ],
        [
          0.5955,
          0.8808
        ],
        [
          0.6056,
          0.8781
        ],
        [
          0.616,
          0.8749
        ],
        [
          0.626,
          0.8717
        ],
        [
          0.636,
          0.8667
        ],
        [
          0.646,
          0.8617
        ],
        [
          0.6561,
          0.8571
        ],
        [
          0.6665,
          0.8544
        ],
        [
          0.6765,
          0.8499
        ],
        [
          0.6865,
          0.8453
        ],
        [
          0.6965,
          0.8399
        ],
        [
          0.7066,
          0.8353
        ],
        [
          0.717,
          0.8303
        ],
        [
          0.727,
          0.8262
        ],
        [
          0.737,
          0.8194
        ],
        [
          0.747,
          0.8144
        ],
        [
          0.7571,
          0.8094
        ],
        [
          0.7675,
          0.8039
        ],
        [
          0.7775,
          0.7989
        ],
        [
          0.7875,
          0.7925
        ],
        [
          0.7975,
          0.7889
        ],
        [
          0.8076,
          0.7853
        ],
        [
          0.818,
          0.7789
        ],
        [
          0.828,
          0.7739
        ],
        [
          0.838,
          0.7693
        ],
        [
          0.848,
          0.7648
        ],
        [
          0.8581,
          0.7589
        ],
        [
          0.8685,
          0.7534
        ],
        [
          0.8785,
          0.7498
        ],
        [
          0.8885,
          0.7429
        ],
        [
          0.8985,
          0.7366
        ],
        [
          0.9086,
          0.7298
        ],
        [
          0.919,
          0.7243
        ],
        [
          0.929,
          0.7207
        ],
        [
          0.939,
          0.7138
        ],
        [
          0.949,
          0.7075
        ],
        [
          0.9591,
          0.7025
        ],
        [
          0.9695,
          0.6965
        ],
        [
          0.9795,
          0.692
        ],
        [
          0.9895,
          0.6852
        ],
        [
          0.9995,
          0.6783
        ],
        [
          1.0,
          0.6779
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.2424306296590378,
        "difference": -0.0012491511859636737,
        "worse": 0.5166666666666667,
        "kept_right": 0.24031433644181527,
        "kept_wrong": 0.002116293217222533,
        "handed_off": 0.7575693703409622
      },
      "usd_per_1000": 0.38977943626869616
    },
    {
      "decision": 259,
      "frontier": 192,
      "frontier_accuracy": 0.9331210191082803,
      "in_sample": {
        "threshold": 0.9552899958153984,
        "answered": 0.21519563239308462,
        "accuracy": 0.9331210191082803
      },
      "curve": [
        [
          0.0,
          0.9331
        ],
        [
          0.01,
          0.9331
        ],
        [
          0.02,
          0.9331
        ],
        [
          0.03,
          0.9331
        ],
        [
          0.04,
          0.9331
        ],
        [
          0.05,
          0.9331
        ],
        [
          0.0601,
          0.9331
        ],
        [
          0.0705,
          0.9331
        ],
        [
          0.0805,
          0.9331
        ],
        [
          0.0905,
          0.9331
        ],
        [
          0.1005,
          0.9331
        ],
        [
          0.1106,
          0.9331
        ],
        [
          0.121,
          0.9331
        ],
        [
          0.131,
          0.9331
        ],
        [
          0.141,
          0.9331
        ],
        [
          0.151,
          0.9331
        ],
        [
          0.1611,
          0.9336
        ],
        [
          0.1715,
          0.9336
        ],
        [
          0.1815,
          0.9336
        ],
        [
          0.1915,
          0.9336
        ],
        [
          0.2015,
          0.9336
        ],
        [
          0.2116,
          0.9336
        ],
        [
          0.222,
          0.9327
        ],
        [
          0.232,
          0.9318
        ],
        [
          0.242,
          0.9313
        ],
        [
          0.252,
          0.9313
        ],
        [
          0.2621,
          0.9318
        ],
        [
          0.2725,
          0.9322
        ],
        [
          0.2825,
          0.9322
        ],
        [
          0.2925,
          0.9322
        ],
        [
          0.3025,
          0.9322
        ],
        [
          0.3126,
          0.9318
        ],
        [
          0.323,
          0.9322
        ],
        [
          0.333,
          0.9313
        ],
        [
          0.343,
          0.9304
        ],
        [
          0.353,
          0.9295
        ],
        [
          0.3631,
          0.929
        ],
        [
          0.3735,
          0.9286
        ],
        [
          0.3835,
          0.9268
        ],
        [
          0.3935,
          0.9254
        ],
        [
          0.4035,
          0.924
        ],
        [
          0.4136,
          0.9222
        ],
        [
          0.424,
          0.9204
        ],
        [
          0.434,
          0.9195
        ],
        [
          0.444,
          0.919
        ],
        [
          0.454,
          0.9172
        ],
        [
          0.4641,
          0.9145
        ],
        [
          0.4745,
          0.9104
        ],
        [
          0.4845,
          0.9081
        ],
        [
          0.4945,
          0.9054
        ],
        [
          0.5045,
          0.9022
        ],
        [
          0.5146,
          0.8981
        ],
        [
          0.525,
          0.8958
        ],
        [
          0.535,
          0.8935
        ],
        [
          0.545,
          0.8899
        ],
        [
          0.5551,
          0.8876
        ],
        [
          0.5655,
          0.8854
        ],
        [
          0.5755,
          0.8817
        ],
        [
          0.5855,
          0.8813
        ],
        [
          0.5955,
          0.8767
        ],
        [
          0.6056,
          0.8735
        ],
        [
          0.616,
          0.8699
        ],
        [
          0.626,
          0.8662
        ],
        [
          0.636,
          0.8621
        ],
        [
          0.646,
          0.8576
        ],
        [
          0.6561,
          0.8535
        ],
        [
          0.6665,
          0.8512
        ],
        [
          0.6765,
          0.8462
        ],
        [
          0.6865,
          0.8417
        ],
        [
          0.6965,
          0.8358
        ],
        [
          0.7066,
          0.8308
        ],
        [
          0.717,
          0.8262
        ],
        [
          0.727,
          0.8226
        ],
        [
          0.737,
          0.8153
        ],
        [
          0.747,
          0.8112
        ],
        [
          0.7571,
          0.8057
        ],
        [
          0.7675,
          0.7998
        ],
        [
          0.7775,
          0.7944
        ],
        [
          0.7875,
          0.7871
        ],
        [
          0.7975,
          0.7834
        ],
        [
          0.8076,
          0.7793
        ],
        [
          0.818,
          0.7739
        ],
        [
          0.828,
          0.7684
        ],
        [
          0.838,
          0.7648
        ],
        [
          0.848,
          0.7607
        ],
        [
          0.8581,
          0.7552
        ],
        [
          0.8685,
          0.7502
        ],
        [
          0.8785,
          0.7457
        ],
        [
          0.8885,
          0.7398
        ],
        [
          0.8985,
          0.7338
        ],
        [
          0.9086,
          0.7279
        ],
        [
          0.919,
          0.7225
        ],
        [
          0.929,
          0.7179
        ],
        [
          0.939,
          0.7125
        ],
        [
          0.949,
          0.7061
        ],
        [
          0.9591,
          0.7011
        ],
        [
          0.9695,
          0.6952
        ],
        [
          0.9795,
          0.6911
        ],
        [
          0.9895,
          0.6843
        ],
        [
          0.9995,
          0.6783
        ],
        [
          1.0,
          0.6779
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.26657967339473476,
        "difference": -0.0015733027527557346,
        "worse": 0.6,
        "kept_right": 0.26344797670799264,
        "kept_wrong": 0.003131696686742093,
        "handed_off": 0.7334203266052651
      },
      "usd_per_1000": 8.514691197628695
    },
    {
      "decision": 259,
      "frontier": 123,
      "frontier_accuracy": 0.9326660600545951,
      "in_sample": {
        "threshold": 0.9557880066229854,
        "answered": 0.21201091901728844,
        "accuracy": 0.9326660600545951
      },
      "curve": [
        [
          0.0,
          0.9327
        ],
        [
          0.01,
          0.9327
        ],
        [
          0.02,
          0.9327
        ],
        [
          0.03,
          0.9327
        ],
        [
          0.04,
          0.9327
        ],
        [
          0.05,
          0.9327
        ],
        [
          0.0601,
          0.9327
        ],
        [
          0.0705,
          0.9327
        ],
        [
          0.0805,
          0.9327
        ],
        [
          0.0905,
          0.9327
        ],
        [
          0.1005,
          0.9327
        ],
        [
          0.1106,
          0.9327
        ],
        [
          0.121,
          0.9327
        ],
        [
          0.131,
          0.9327
        ],
        [
          0.141,
          0.9327
        ],
        [
          0.151,
          0.9327
        ],
        [
          0.1611,
          0.9327
        ],
        [
          0.1715,
          0.9327
        ],
        [
          0.1815,
          0.9327
        ],
        [
          0.1915,
          0.9327
        ],
        [
          0.2015,
          0.9327
        ],
        [
          0.2116,
          0.9327
        ],
        [
          0.222,
          0.9318
        ],
        [
          0.232,
          0.9308
        ],
        [
          0.242,
          0.9304
        ],
        [
          0.252,
          0.9304
        ],
        [
          0.2621,
          0.9304
        ],
        [
          0.2725,
          0.9308
        ],
        [
          0.2825,
          0.9308
        ],
        [
          0.2925,
          0.9308
        ],
        [
          0.3025,
          0.9308
        ],
        [
          0.3126,
          0.9308
        ],
        [
          0.323,
          0.9308
        ],
        [
          0.333,
          0.9304
        ],
        [
          0.343,
          0.9299
        ],
        [
          0.353,
          0.929
        ],
        [
          0.3631,
          0.9277
        ],
        [
          0.3735,
          0.9272
        ],
        [
          0.3835,
          0.9254
        ],
        [
          0.3935,
          0.9249
        ],
        [
          0.4035,
          0.9231
        ],
        [
          0.4136,
          0.9213
        ],
        [
          0.424,
          0.919
        ],
        [
          0.434,
          0.9177
        ],
        [
          0.444,
          0.9167
        ],
        [
          0.454,
          0.9136
        ],
        [
          0.4641,
          0.9117
        ],
        [
          0.4745,
          0.9081
        ],
        [
          0.4845,
          0.9045
        ],
        [
          0.4945,
          0.9022
        ],
        [
          0.5045,
          0.8985
        ],
        [
          0.5146,
          0.8954
        ],
        [
          0.525,
          0.8935
        ],
        [
          0.535,
          0.8904
        ],
        [
          0.545,
          0.8867
        ],
        [
          0.5551,
          0.8835
        ],
        [
          0.5655,
          0.8817
        ],
        [
          0.5755,
          0.8776
        ],
        [
          0.5855,
          0.8758
        ],
        [
          0.5955,
          0.8712
        ],
        [
          0.6056,
          0.869
        ],
        [
          0.616,
          0.8662
        ],
        [
          0.626,
          0.8621
        ],
        [
          0.636,
          0.8585
        ],
        [
          0.646,
          0.854
        ],
        [
          0.6561,
          0.8499
        ],
        [
          0.6665,
          0.848
        ],
        [
          0.6765,
          0.8439
        ],
        [
          0.6865,
          0.8394
        ],
        [
          0.6965,
          0.833
        ],
        [
          0.7066,
          0.8294
        ],
        [
          0.717,
          0.8244
        ],
        [
          0.727,
          0.8203
        ],
        [
          0.737,
          0.8139
        ],
        [
          0.747,
          0.8089
        ],
        [
          0.7571,
          0.8044
        ],
        [
          0.7675,
          0.7989
        ],
        [
          0.7775,
          0.7934
        ],
        [
          0.7875,
          0.7871
        ],
        [
          0.7975,
          0.783
        ],
        [
          0.8076,
          0.7784
        ],
        [
          0.818,
          0.7716
        ],
        [
          0.828,
          0.7671
        ],
        [
          0.838,
          0.763
        ],
        [
          0.848,
          0.7584
        ],
        [
          0.8581,
          0.7525
        ],
        [
          0.8685,
          0.747
        ],
        [
          0.8785,
          0.7429
        ],
        [
          0.8885,
          0.7375
        ],
        [
          0.8985,
          0.732
        ],
        [
          0.9086,
          0.7266
        ],
        [
          0.919,
          0.7202
        ],
        [
          0.929,
          0.7161
        ],
        [
          0.939,
          0.7097
        ],
        [
          0.949,
          0.7038
        ],
        [
          0.9591,
          0.7006
        ],
        [
          0.9695,
          0.6943
        ],
        [
          0.9795,
          0.6888
        ],
        [
          0.9895,
          0.6838
        ],
        [
          0.9995,
          0.6783
        ],
        [
          1.0,
          0.6779
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.2502112938639453,
        "difference": -0.002037339075024688,
        "worse": 0.59,
        "kept_right": 0.24742804171403326,
        "kept_wrong": 0.0027832521499120476,
        "handed_off": 0.7497887061360544
      },
      "usd_per_1000": 0.201381204820589
    },
    {
      "decision": 259,
      "frontier": 274,
      "frontier_accuracy": 0.9085532302092811,
      "in_sample": {
        "threshold": 0.8748145037583898,
        "answered": 0.3252957233848954,
        "accuracy": 0.9085532302092811
      },
      "curve": [
        [
          0.0,
          0.9086
        ],
        [
          0.01,
          0.9086
        ],
        [
          0.02,
          0.9086
        ],
        [
          0.03,
          0.9086
        ],
        [
          0.04,
          0.9086
        ],
        [
          0.05,
          0.9086
        ],
        [
          0.0601,
          0.9086
        ],
        [
          0.0705,
          0.9086
        ],
        [
          0.0805,
          0.9086
        ],
        [
          0.0905,
          0.9086
        ],
        [
          0.1005,
          0.9086
        ],
        [
          0.1106,
          0.9086
        ],
        [
          0.121,
          0.9086
        ],
        [
          0.131,
          0.9086
        ],
        [
          0.141,
          0.9086
        ],
        [
          0.151,
          0.9086
        ],
        [
          0.1611,
          0.909
        ],
        [
          0.1715,
          0.9095
        ],
        [
          0.1815,
          0.9095
        ],
        [
          0.1915,
          0.9095
        ],
        [
          0.2015,
          0.9095
        ],
        [
          0.2116,
          0.9095
        ],
        [
          0.222,
          0.909
        ],
        [
          0.232,
          0.909
        ],
        [
          0.242,
          0.9086
        ],
        [
          0.252,
          0.9086
        ],
        [
          0.2621,
          0.9086
        ],
        [
          0.2725,
          0.909
        ],
        [
          0.2825,
          0.909
        ],
        [
          0.2925,
          0.909
        ],
        [
          0.3025,
          0.909
        ],
        [
          0.3126,
          0.9086
        ],
        [
          0.323,
          0.909
        ],
        [
          0.333,
          0.9081
        ],
        [
          0.343,
          0.9076
        ],
        [
          0.353,
          0.9072
        ],
        [
          0.3631,
          0.9067
        ],
        [
          0.3735,
          0.9063
        ],
        [
          0.3835,
          0.9049
        ],
        [
          0.3935,
          0.904
        ],
        [
          0.4035,
          0.9022
        ],
        [
          0.4136,
          0.9004
        ],
        [
          0.424,
          0.8981
        ],
        [
          0.434,
          0.8972
        ],
        [
          0.444,
          0.8963
        ],
        [
          0.454,
          0.8944
        ],
        [
          0.4641,
          0.8922
        ],
        [
          0.4745,
          0.8885
        ],
        [
          0.4845,
          0.8849
        ],
        [
          0.4945,
          0.8826
        ],
        [
          0.5045,
          0.8794
        ],
        [
          0.5146,
          0.8758
        ],
        [
          0.525,
          0.8735
        ],
        [
          0.535,
          0.8703
        ],
        [
          0.545,
          0.8667
        ],
        [
          0.5551,
          0.8644
        ],
        [
          0.5655,
          0.8626
        ],
        [
          0.5755,
          0.8585
        ],
        [
          0.5855,
          0.8571
        ],
        [
          0.5955,
          0.8535
        ],
        [
          0.6056,
          0.8508
        ],
        [
          0.616,
          0.8476
        ],
        [
          0.626,
          0.8453
        ],
        [
          0.636,
          0.8417
        ],
        [
          0.646,
          0.8376
        ],
        [
          0.6561,
          0.8339
        ],
        [
          0.6665,
          0.8321
        ],
        [
          0.6765,
          0.8285
        ],
        [
          0.6865,
          0.8239
        ],
        [
          0.6965,
          0.818
        ],
        [
          0.7066,
          0.813
        ],
        [
          0.717,
          0.8089
        ],
        [
          0.727,
          0.8048
        ],
        [
          0.737,
          0.7989
        ],
        [
          0.747,
          0.7939
        ],
        [
          0.7571,
          0.7898
        ],
        [
          0.7675,
          0.7862
        ],
        [
          0.7775,
          0.7812
        ],
        [
          0.7875,
          0.7753
        ],
        [
          0.7975,
          0.7716
        ],
        [
          0.8076,
          0.768
        ],
        [
          0.818,
          0.7611
        ],
        [
          0.828,
          0.7571
        ],
        [
          0.838,
          0.7543
        ],
        [
          0.848,
          0.7502
        ],
        [
          0.8581,
          0.7448
        ],
        [
          0.8685,
          0.7402
        ],
        [
          0.8785,
          0.7366
        ],
        [
          0.8885,
          0.7302
        ],
        [
          0.8985,
          0.7238
        ],
        [
          0.9086,
          0.7184
        ],
        [
          0.919,
          0.7129
        ],
        [
          0.929,
          0.7111
        ],
        [
          0.939,
          0.7047
        ],
        [
          0.949,
          0.6997
        ],
        [
          0.9591,
          0.6956
        ],
        [
          0.9695,
          0.6902
        ],
        [
          0.9795,
          0.6888
        ],
        [
          0.9895,
          0.6833
        ],
        [
          0.9995,
          0.6783
        ],
        [
          1.0,
          0.6779
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.3234365551663004,
        "difference": -0.0016134086023680727,
        "worse": 0.47,
        "kept_right": 0.3164175923032289,
        "kept_wrong": 0.0070189628630717244,
        "handed_off": 0.676563444833699
      },
      "usd_per_1000": 0.2883822441780506
    },
    {
      "decision": 259,
      "frontier": 47,
      "frontier_accuracy": 0.9044585987261147,
      "in_sample": {
        "threshold": 0.8757460292252313,
        "answered": 0.32302092811646954,
        "accuracy": 0.9044585987261147
      },
      "curve": [
        [
          0.0,
          0.9045
        ],
        [
          0.01,
          0.9045
        ],
        [
          0.02,
          0.9045
        ],
        [
          0.03,
          0.9045
        ],
        [
          0.04,
          0.9045
        ],
        [
          0.05,
          0.9045
        ],
        [
          0.0601,
          0.9045
        ],
        [
          0.0705,
          0.9045
        ],
        [
          0.0805,
          0.9045
        ],
        [
          0.0905,
          0.9045
        ],
        [
          0.1005,
          0.9045
        ],
        [
          0.1106,
          0.9045
        ],
        [
          0.121,
          0.9045
        ],
        [
          0.131,
          0.9045
        ],
        [
          0.141,
          0.9045
        ],
        [
          0.151,
          0.9045
        ],
        [
          0.1611,
          0.9049
        ],
        [
          0.1715,
          0.9049
        ],
        [
          0.1815,
          0.9049
        ],
        [
          0.1915,
          0.9049
        ],
        [
          0.2015,
          0.9049
        ],
        [
          0.2116,
          0.9049
        ],
        [
          0.222,
          0.9049
        ],
        [
          0.232,
          0.904
        ],
        [
          0.242,
          0.9035
        ],
        [
          0.252,
          0.904
        ],
        [
          0.2621,
          0.9045
        ],
        [
          0.2725,
          0.9045
        ],
        [
          0.2825,
          0.9045
        ],
        [
          0.2925,
          0.9045
        ],
        [
          0.3025,
          0.9045
        ],
        [
          0.3126,
          0.904
        ],
        [
          0.323,
          0.9045
        ],
        [
          0.333,
          0.9035
        ],
        [
          0.343,
          0.9035
        ],
        [
          0.353,
          0.9026
        ],
        [
          0.3631,
          0.9013
        ],
        [
          0.3735,
          0.9013
        ],
        [
          0.3835,
          0.899
        ],
        [
          0.3935,
          0.899
        ],
        [
          0.4035,
          0.8981
        ],
        [
          0.4136,
          0.8972
        ],
        [
          0.424,
          0.8954
        ],
        [
          0.434,
          0.8944
        ],
        [
          0.444,
          0.8935
        ],
        [
          0.454,
          0.8913
        ],
        [
          0.4641,
          0.889
        ],
        [
          0.4745,
          0.8849
        ],
        [
          0.4845,
          0.8831
        ],
        [
          0.4945,
          0.8817
        ],
        [
          0.5045,
          0.8794
        ],
        [
          0.5146,
          0.8753
        ],
        [
          0.525,
          0.8726
        ],
        [
          0.535,
          0.869
        ],
        [
          0.545,
          0.8653
        ],
        [
          0.5551,
          0.8635
        ],
        [
          0.5655,
          0.8617
        ],
        [
          0.5755,
          0.8581
        ],
        [
          0.5855,
          0.8567
        ],
        [
          0.5955,
          0.8517
        ],
        [
          0.6056,
          0.849
        ],
        [
          0.616,
          0.8458
        ],
        [
          0.626,
          0.8435
        ],
        [
          0.636,
          0.8408
        ],
        [
          0.646,
          0.8376
        ],
        [
          0.6561,
          0.8339
        ],
        [
          0.6665,
          0.8312
        ],
        [
          0.6765,
          0.8262
        ],
        [
          0.6865,
          0.8221
        ],
        [
          0.6965,
          0.8153
        ],
        [
          0.7066,
          0.8098
        ],
        [
          0.717,
          0.8062
        ],
        [
          0.727,
          0.8035
        ],
        [
          0.737,
          0.7966
        ],
        [
          0.747,
          0.7916
        ],
        [
          0.7571,
          0.7875
        ],
        [
          0.7675,
          0.7825
        ],
        [
          0.7775,
          0.7771
        ],
        [
          0.7875,
          0.7707
        ],
        [
          0.7975,
          0.7671
        ],
        [
          0.8076,
          0.7634
        ],
        [
          0.818,
          0.7584
        ],
        [
          0.828,
          0.753
        ],
        [
          0.838,
          0.7498
        ],
        [
          0.848,
          0.7452
        ],
        [
          0.8581,
          0.7402
        ],
        [
          0.8685,
          0.7357
        ],
        [
          0.8785,
          0.7325
        ],
        [
          0.8885,
          0.727
        ],
        [
          0.8985,
          0.7225
        ],
        [
          0.9086,
          0.717
        ],
        [
          0.919,
          0.7138
        ],
        [
          0.929,
          0.7106
        ],
        [
          0.939,
          0.7038
        ],
        [
          0.949,
          0.6984
        ],
        [
          0.9591,
          0.6934
        ],
        [
          0.9695,
          0.6884
        ],
        [
          0.9795,
          0.6865
        ],
        [
          0.9895,
          0.682
        ],
        [
          0.9995,
          0.6783
        ],
        [
          1.0,
          0.6779
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.30809551798738,
        "difference": -0.001952595805880457,
        "worse": 0.55,
        "kept_right": 0.30219565668464893,
        "kept_wrong": 0.005899861302730712,
        "handed_off": 0.6919044820126197
      },
      "usd_per_1000": 3.4114945335639386
    },
    {
      "decision": 259,
      "frontier": 193,
      "frontier_accuracy": 0.89171974522293,
      "in_sample": {
        "threshold": 0.8757460292252313,
        "answered": 0.32302092811646954,
        "accuracy": 0.89171974522293
      },
      "curve": [
        [
          0.0,
          0.8917
        ],
        [
          0.01,
          0.8917
        ],
        [
          0.02,
          0.8917
        ],
        [
          0.03,
          0.8917
        ],
        [
          0.04,
          0.8917
        ],
        [
          0.05,
          0.8917
        ],
        [
          0.0601,
          0.8917
        ],
        [
          0.0705,
          0.8917
        ],
        [
          0.0805,
          0.8917
        ],
        [
          0.0905,
          0.8917
        ],
        [
          0.1005,
          0.8917
        ],
        [
          0.1106,
          0.8917
        ],
        [
          0.121,
          0.8917
        ],
        [
          0.131,
          0.8917
        ],
        [
          0.141,
          0.8917
        ],
        [
          0.151,
          0.8917
        ],
        [
          0.1611,
          0.8922
        ],
        [
          0.1715,
          0.8922
        ],
        [
          0.1815,
          0.8922
        ],
        [
          0.1915,
          0.8926
        ],
        [
          0.2015,
          0.8926
        ],
        [
          0.2116,
          0.8926
        ],
        [
          0.222,
          0.8922
        ],
        [
          0.232,
          0.8917
        ],
        [
          0.242,
          0.8917
        ],
        [
          0.252,
          0.8917
        ],
        [
          0.2621,
          0.8913
        ],
        [
          0.2725,
          0.8917
        ],
        [
          0.2825,
          0.8917
        ],
        [
          0.2925,
          0.8917
        ],
        [
          0.3025,
          0.8917
        ],
        [
          0.3126,
          0.8913
        ],
        [
          0.323,
          0.8917
        ],
        [
          0.333,
          0.8908
        ],
        [
          0.343,
          0.8908
        ],
        [
          0.353,
          0.8899
        ],
        [
          0.3631,
          0.8894
        ],
        [
          0.3735,
          0.889
        ],
        [
          0.3835,
          0.8867
        ],
        [
          0.3935,
          0.8863
        ],
        [
          0.4035,
          0.8844
        ],
        [
          0.4136,
          0.8826
        ],
        [
          0.424,
          0.8813
        ],
        [
          0.434,
          0.8803
        ],
        [
          0.444,
          0.8799
        ],
        [
          0.454,
          0.8785
        ],
        [
          0.4641,
          0.8767
        ],
        [
          0.4745,
          0.8726
        ],
        [
          0.4845,
          0.8694
        ],
        [
          0.4945,
          0.8672
        ],
        [
          0.5045,
          0.8644
        ],
        [
          0.5146,
          0.8621
        ],
        [
          0.525,
          0.8599
        ],
        [
          0.535,
          0.8571
        ],
        [
          0.545,
          0.8544
        ],
        [
          0.5551,
          0.8535
        ],
        [
          0.5655,
          0.8521
        ],
        [
          0.5755,
          0.8494
        ],
        [
          0.5855,
          0.848
        ],
        [
          0.5955,
          0.8449
        ],
        [
          0.6056,
          0.8426
        ],
        [
          0.616,
          0.8403
        ],
        [
          0.626,
          0.8371
        ],
        [
          0.636,
          0.8335
        ],
        [
          0.646,
          0.8303
        ],
        [
          0.6561,
          0.8262
        ],
        [
          0.6665,
          0.8239
        ],
        [
          0.6765,
          0.8207
        ],
        [
          0.6865,
          0.8167
        ],
        [
          0.6965,
          0.8112
        ],
        [
          0.7066,
          0.8071
        ],
        [
          0.717,
          0.8044
        ],
        [
          0.727,
          0.8003
        ],
        [
          0.737,
          0.7948
        ],
        [
          0.747,
          0.7912
        ],
        [
          0.7571,
          0.788
        ],
        [
          0.7675,
          0.7834
        ],
        [
          0.7775,
          0.7784
        ],
        [
          0.7875,
          0.7725
        ],
        [
          0.7975,
          0.7702
        ],
        [
          0.8076,
          0.7671
        ],
        [
          0.818,
          0.7621
        ],
        [
          0.828,
          0.7571
        ],
        [
          0.838,
          0.7525
        ],
        [
          0.848,
          0.7484
        ],
        [
          0.8581,
          0.7434
        ],
        [
          0.8685,
          0.7389
        ],
        [
          0.8785,
          0.7338
        ],
        [
          0.8885,
          0.7279
        ],
        [
          0.8985,
          0.7234
        ],
        [
          0.9086,
          0.7175
        ],
        [
          0.919,
          0.7138
        ],
        [
          0.929,
          0.7093
        ],
        [
          0.939,
          0.7047
        ],
        [
          0.949,
          0.6988
        ],
        [
          0.9591,
          0.6943
        ],
        [
          0.9695,
          0.6897
        ],
        [
          0.9795,
          0.6865
        ],
        [
          0.9895,
          0.6815
        ],
        [
          0.9995,
          0.6783
        ],
        [
          1.0,
          0.6779
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.3161101733280975,
        "difference": -0.0016556035533661936,
        "worse": 0.58,
        "kept_right": 0.3098069091821189,
        "kept_wrong": 0.006303264145978417,
        "handed_off": 0.6838898266719023
      },
      "usd_per_1000": 5.298497976002127
    },
    {
      "decision": 259,
      "frontier": 55,
      "frontier_accuracy": 0.8503184713375797,
      "in_sample": {
        "threshold": 0.6976943187381583,
        "answered": 0.5009099181073703,
        "accuracy": 0.8503184713375797
      },
      "curve": [
        [
          0.0,
          0.8503
        ],
        [
          0.01,
          0.8503
        ],
        [
          0.02,
          0.8503
        ],
        [
          0.03,
          0.8503
        ],
        [
          0.04,
          0.8512
        ],
        [
          0.05,
          0.8512
        ],
        [
          0.0601,
          0.8517
        ],
        [
          0.0705,
          0.8517
        ],
        [
          0.0805,
          0.8517
        ],
        [
          0.0905,
          0.8517
        ],
        [
          0.1005,
          0.8526
        ],
        [
          0.1106,
          0.853
        ],
        [
          0.121,
          0.853
        ],
        [
          0.131,
          0.853
        ],
        [
          0.141,
          0.853
        ],
        [
          0.151,
          0.853
        ],
        [
          0.1611,
          0.853
        ],
        [
          0.1715,
          0.853
        ],
        [
          0.1815,
          0.8535
        ],
        [
          0.1915,
          0.854
        ],
        [
          0.2015,
          0.8544
        ],
        [
          0.2116,
          0.8549
        ],
        [
          0.222,
          0.8558
        ],
        [
          0.232,
          0.8558
        ],
        [
          0.242,
          0.8553
        ],
        [
          0.252,
          0.8558
        ],
        [
          0.2621,
          0.8562
        ],
        [
          0.2725,
          0.8576
        ],
        [
          0.2825,
          0.8585
        ],
        [
          0.2925,
          0.859
        ],
        [
          0.3025,
          0.859
        ],
        [
          0.3126,
          0.8603
        ],
        [
          0.323,
          0.8612
        ],
        [
          0.333,
          0.8612
        ],
        [
          0.343,
          0.8621
        ],
        [
          0.353,
          0.8617
        ],
        [
          0.3631,
          0.8626
        ],
        [
          0.3735,
          0.8631
        ],
        [
          0.3835,
          0.8617
        ],
        [
          0.3935,
          0.8617
        ],
        [
          0.4035,
          0.8612
        ],
        [
          0.4136,
          0.8608
        ],
        [
          0.424,
          0.8599
        ],
        [
          0.434,
          0.8599
        ],
        [
          0.444,
          0.8603
        ],
        [
          0.454,
          0.8603
        ],
        [
          0.4641,
          0.8594
        ],
        [
          0.4745,
          0.8567
        ],
        [
          0.4845,
          0.8535
        ],
        [
          0.4945,
          0.8512
        ],
        [
          0.5045,
          0.849
        ],
        [
          0.5146,
          0.8458
        ],
        [
          0.525,
          0.8439
        ],
        [
          0.535,
          0.8412
        ],
        [
          0.545,
          0.8385
        ],
        [
          0.5551,
          0.8376
        ],
        [
          0.5655,
          0.8358
        ],
        [
          0.5755,
          0.833
        ],
        [
          0.5855,
          0.8335
        ],
        [
          0.5955,
          0.8298
        ],
        [
          0.6056,
          0.8276
        ],
        [
          0.616,
          0.8248
        ],
        [
          0.626,
          0.8221
        ],
        [
          0.636,
          0.8198
        ],
        [
          0.646,
          0.8167
        ],
        [
          0.6561,
          0.813
        ],
        [
          0.6665,
          0.8107
        ],
        [
          0.6765,
          0.8071
        ],
        [
          0.6865,
          0.803
        ],
        [
          0.6965,
          0.7971
        ],
        [
          0.7066,
          0.7934
        ],
        [
          0.717,
          0.7925
        ],
        [
          0.727,
          0.7884
        ],
        [
          0.737,
          0.783
        ],
        [
          0.747,
          0.7793
        ],
        [
          0.7571,
          0.7753
        ],
        [
          0.7675,
          0.7712
        ],
        [
          0.7775,
          0.7675
        ],
        [
          0.7875,
          0.7616
        ],
        [
          0.7975,
          0.7602
        ],
        [
          0.8076,
          0.7584
        ],
        [
          0.818,
          0.7539
        ],
        [
          0.828,
          0.7493
        ],
        [
          0.838,
          0.7466
        ],
        [
          0.848,
          0.7429
        ],
        [
          0.8581,
          0.7379
        ],
        [
          0.8685,
          0.7348
        ],
        [
          0.8785,
          0.7307
        ],
        [
          0.8885,
          0.7247
        ],
        [
          0.8985,
          0.7202
        ],
        [
          0.9086,
          0.7143
        ],
        [
          0.919,
          0.7111
        ],
        [
          0.929,
          0.7075
        ],
        [
          0.939,
          0.7038
        ],
        [
          0.949,
          0.6984
        ],
        [
          0.9591,
          0.6943
        ],
        [
          0.9695,
          0.6888
        ],
        [
          0.9795,
          0.687
        ],
        [
          0.9895,
          0.682
        ],
        [
          0.9995,
          0.6779
        ],
        [
          1.0,
          0.6779
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.5018109580678753,
        "difference": -0.0007016472990503344,
        "worse": 0.45666666666666667,
        "kept_right": 0.45625035301474537,
        "kept_wrong": 0.045560605053129734,
        "handed_off": 0.4981890419321247
      },
      "usd_per_1000": 1.820548965003068
    },
    {
      "decision": 269,
      "frontier": 140,
      "frontier_accuracy": 0.9808917197452229,
      "in_sample": {
        "threshold": 0.9864338454383234,
        "answered": 0.20336669699727025,
        "accuracy": 0.9808917197452229
      },
      "curve": [
        [
          0.0,
          0.9809
        ],
        [
          0.01,
          0.9809
        ],
        [
          0.02,
          0.9809
        ],
        [
          0.03,
          0.9809
        ],
        [
          0.04,
          0.9809
        ],
        [
          0.05,
          0.9809
        ],
        [
          0.0601,
          0.9809
        ],
        [
          0.0705,
          0.9809
        ],
        [
          0.0805,
          0.9809
        ],
        [
          0.0905,
          0.9809
        ],
        [
          0.1005,
          0.9809
        ],
        [
          0.1106,
          0.9809
        ],
        [
          0.121,
          0.9809
        ],
        [
          0.131,
          0.9809
        ],
        [
          0.141,
          0.9809
        ],
        [
          0.151,
          0.9809
        ],
        [
          0.1611,
          0.9809
        ],
        [
          0.1715,
          0.9809
        ],
        [
          0.1815,
          0.9809
        ],
        [
          0.1915,
          0.9809
        ],
        [
          0.2015,
          0.9809
        ],
        [
          0.2116,
          0.9804
        ],
        [
          0.222,
          0.9804
        ],
        [
          0.232,
          0.9804
        ],
        [
          0.242,
          0.98
        ],
        [
          0.252,
          0.98
        ],
        [
          0.2621,
          0.9791
        ],
        [
          0.2725,
          0.9786
        ],
        [
          0.2825,
          0.9786
        ],
        [
          0.2925,
          0.9777
        ],
        [
          0.3025,
          0.9777
        ],
        [
          0.3126,
          0.9773
        ],
        [
          0.323,
          0.9763
        ],
        [
          0.333,
          0.9763
        ],
        [
          0.343,
          0.9759
        ],
        [
          0.353,
          0.9745
        ],
        [
          0.3631,
          0.9727
        ],
        [
          0.3735,
          0.9713
        ],
        [
          0.3835,
          0.97
        ],
        [
          0.3935,
          0.9686
        ],
        [
          0.4035,
          0.9663
        ],
        [
          0.4136,
          0.9654
        ],
        [
          0.424,
          0.9627
        ],
        [
          0.434,
          0.96
        ],
        [
          0.444,
          0.9586
        ],
        [
          0.454,
          0.9554
        ],
        [
          0.4641,
          0.9527
        ],
        [
          0.4745,
          0.9495
        ],
        [
          0.4845,
          0.9468
        ],
        [
          0.4945,
          0.9436
        ],
        [
          0.5045,
          0.939
        ],
        [
          0.5146,
          0.9368
        ],
        [
          0.525,
          0.9336
        ],
        [
          0.535,
          0.9313
        ],
        [
          0.545,
          0.9272
        ],
        [
          0.5551,
          0.9222
        ],
        [
          0.5655,
          0.9186
        ],
        [
          0.5755,
          0.9158
        ],
        [
          0.5855,
          0.9126
        ],
        [
          0.5955,
          0.9095
        ],
        [
          0.6056,
          0.9058
        ],
        [
          0.616,
          0.9022
        ],
        [
          0.626,
          0.8985
        ],
        [
          0.636,
          0.8931
        ],
        [
          0.646,
          0.889
        ],
        [
          0.6561,
          0.8835
        ],
        [
          0.6665,
          0.8781
        ],
        [
          0.6765,
          0.8731
        ],
        [
          0.6865,
          0.8676
        ],
        [
          0.6965,
          0.8621
        ],
        [
          0.7066,
          0.8562
        ],
        [
          0.717,
          0.8512
        ],
        [
          0.727,
          0.8449
        ],
        [
          0.737,
          0.8394
        ],
        [
          0.747,
          0.8358
        ],
        [
          0.7571,
          0.8289
        ],
        [
          0.7675,
          0.8221
        ],
        [
          0.7775,
          0.8157
        ],
        [
          0.7875,
          0.8089
        ],
        [
          0.7975,
          0.8021
        ],
        [
          0.8076,
          0.7966
        ],
        [
          0.818,
          0.7903
        ],
        [
          0.828,
          0.7825
        ],
        [
          0.838,
          0.7771
        ],
        [
          0.848,
          0.7721
        ],
        [
          0.8581,
          0.768
        ],
        [
          0.8685,
          0.7616
        ],
        [
          0.8785,
          0.7557
        ],
        [
          0.8885,
          0.7493
        ],
        [
          0.8985,
          0.7429
        ],
        [
          0.9086,
          0.7361
        ],
        [
          0.919,
          0.7279
        ],
        [
          0.929,
          0.7211
        ],
        [
          0.939,
          0.7138
        ],
        [
          0.949,
          0.707
        ],
        [
          0.9591,
          0.7006
        ],
        [
          0.9695,
          0.6956
        ],
        [
          0.9795,
          0.6906
        ],
        [
          0.9895,
          0.6847
        ],
        [
          0.9995,
          0.6779
        ],
        [
          1.0,
          0.6779
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.22684002002003073,
        "difference": -0.0009731997590825071,
        "worse": 0.49,
        "kept_right": 0.22510285213509965,
        "kept_wrong": 0.0017371678849310956,
        "handed_off": 0.7731599799799691
      },
      "usd_per_1000": 1.8703537522746667
    },
    {
      "decision": 269,
      "frontier": 277,
      "frontier_accuracy": 0.9795268425841674,
      "in_sample": {
        "threshold": 0.9809937576468168,
        "answered": 0.24021838034576887,
        "accuracy": 0.9795268425841674
      },
      "curve": [
        [
          0.0,
          0.9795
        ],
        [
          0.01,
          0.9795
        ],
        [
          0.02,
          0.9795
        ],
        [
          0.03,
          0.9795
        ],
        [
          0.04,
          0.9795
        ],
        [
          0.05,
          0.9795
        ],
        [
          0.0601,
          0.9795
        ],
        [
          0.0705,
          0.9795
        ],
        [
          0.0805,
          0.9795
        ],
        [
          0.0905,
          0.9795
        ],
        [
          0.1005,
          0.9795
        ],
        [
          0.1106,
          0.9795
        ],
        [
          0.121,
          0.9795
        ],
        [
          0.131,
          0.9795
        ],
        [
          0.141,
          0.9795
        ],
        [
          0.151,
          0.9795
        ],
        [
          0.1611,
          0.9795
        ],
        [
          0.1715,
          0.9795
        ],
        [
          0.1815,
          0.9795
        ],
        [
          0.1915,
          0.98
        ],
        [
          0.2015,
          0.98
        ],
        [
          0.2116,
          0.9795
        ],
        [
          0.222,
          0.9795
        ],
        [
          0.232,
          0.9795
        ],
        [
          0.242,
          0.9791
        ],
        [
          0.252,
          0.9791
        ],
        [
          0.2621,
          0.9782
        ],
        [
          0.2725,
          0.9777
        ],
        [
          0.2825,
          0.9777
        ],
        [
          0.2925,
          0.9768
        ],
        [
          0.3025,
          0.9768
        ],
        [
          0.3126,
          0.9759
        ],
        [
          0.323,
          0.975
        ],
        [
          0.333,
          0.975
        ],
        [
          0.343,
          0.9745
        ],
        [
          0.353,
          0.9736
        ],
        [
          0.3631,
          0.9722
        ],
        [
          0.3735,
          0.9709
        ],
        [
          0.3835,
          0.9695
        ],
        [
          0.3935,
          0.9682
        ],
        [
          0.4035,
          0.9654
        ],
        [
          0.4136,
          0.9645
        ],
        [
          0.424,
          0.9618
        ],
        [
          0.434,
          0.9595
        ],
        [
          0.444,
          0.9581
        ],
        [
          0.454,
          0.955
        ],
        [
          0.4641,
          0.9522
        ],
        [
          0.4745,
          0.949
        ],
        [
          0.4845,
          0.9468
        ],
        [
          0.4945,
          0.9436
        ],
        [
          0.5045,
          0.939
        ],
        [
          0.5146,
          0.9368
        ],
        [
          0.525,
          0.9336
        ],
        [
          0.535,
          0.9313
        ],
        [
          0.545,
          0.9272
        ],
        [
          0.5551,
          0.9227
        ],
        [
          0.5655,
          0.919
        ],
        [
          0.5755,
          0.9163
        ],
        [
          0.5855,
          0.9126
        ],
        [
          0.5955,
          0.909
        ],
        [
          0.6056,
          0.9054
        ],
        [
          0.616,
          0.9013
        ],
        [
          0.626,
          0.8981
        ],
        [
          0.636,
          0.8926
        ],
        [
          0.646,
          0.889
        ],
        [
          0.6561,
          0.8835
        ],
        [
          0.6665,
          0.8781
        ],
        [
          0.6765,
          0.8731
        ],
        [
          0.6865,
          0.8676
        ],
        [
          0.6965,
          0.8617
        ],
        [
          0.7066,
          0.8558
        ],
        [
          0.717,
          0.8508
        ],
        [
          0.727,
          0.8449
        ],
        [
          0.737,
          0.8394
        ],
        [
          0.747,
          0.8358
        ],
        [
          0.7571,
          0.8289
        ],
        [
          0.7675,
          0.8221
        ],
        [
          0.7775,
          0.8157
        ],
        [
          0.7875,
          0.8089
        ],
        [
          0.7975,
          0.8016
        ],
        [
          0.8076,
          0.7962
        ],
        [
          0.818,
          0.7898
        ],
        [
          0.828,
          0.7821
        ],
        [
          0.838,
          0.7766
        ],
        [
          0.848,
          0.7716
        ],
        [
          0.8581,
          0.7675
        ],
        [
          0.8685,
          0.7611
        ],
        [
          0.8785,
          0.7548
        ],
        [
          0.8885,
          0.748
        ],
        [
          0.8985,
          0.7425
        ],
        [
          0.9086,
          0.7357
        ],
        [
          0.919,
          0.7284
        ],
        [
          0.929,
          0.7216
        ],
        [
          0.939,
          0.7143
        ],
        [
          0.949,
          0.707
        ],
        [
          0.9591,
          0.7002
        ],
        [
          0.9695,
          0.6952
        ],
        [
          0.9795,
          0.6902
        ],
        [
          0.9895,
          0.6847
        ],
        [
          0.9995,
          0.6779
        ],
        [
          1.0,
          0.6779
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.24372446841194248,
        "difference": -0.0009975278259613555,
        "worse": 0.4866666666666667,
        "kept_right": 0.24138395780321492,
        "kept_wrong": 0.0023405106087274665,
        "handed_off": 0.7562755315880575
      },
      "usd_per_1000": 4.587230869849784
    },
    {
      "decision": 269,
      "frontier": 276,
      "frontier_accuracy": 0.9777070063694268,
      "in_sample": {
        "threshold": 0.9794597561579199,
        "answered": 0.25204731574158323,
        "accuracy": 0.9777070063694268
      },
      "curve": [
        [
          0.0,
          0.9777
        ],
        [
          0.01,
          0.9777
        ],
        [
          0.02,
          0.9777
        ],
        [
          0.03,
          0.9777
        ],
        [
          0.04,
          0.9777
        ],
        [
          0.05,
          0.9777
        ],
        [
          0.0601,
          0.9777
        ],
        [
          0.0705,
          0.9777
        ],
        [
          0.0805,
          0.9777
        ],
        [
          0.0905,
          0.9777
        ],
        [
          0.1005,
          0.9777
        ],
        [
          0.1106,
          0.9782
        ],
        [
          0.121,
          0.9782
        ],
        [
          0.131,
          0.9782
        ],
        [
          0.141,
          0.9782
        ],
        [
          0.151,
          0.9782
        ],
        [
          0.1611,
          0.9782
        ],
        [
          0.1715,
          0.9782
        ],
        [
          0.1815,
          0.9782
        ],
        [
          0.1915,
          0.9786
        ],
        [
          0.2015,
          0.9786
        ],
        [
          0.2116,
          0.9782
        ],
        [
          0.222,
          0.9782
        ],
        [
          0.232,
          0.9782
        ],
        [
          0.242,
          0.9777
        ],
        [
          0.252,
          0.9777
        ],
        [
          0.2621,
          0.9768
        ],
        [
          0.2725,
          0.9763
        ],
        [
          0.2825,
          0.9763
        ],
        [
          0.2925,
          0.9759
        ],
        [
          0.3025,
          0.9759
        ],
        [
          0.3126,
          0.975
        ],
        [
          0.323,
          0.9741
        ],
        [
          0.333,
          0.9741
        ],
        [
          0.343,
          0.9736
        ],
        [
          0.353,
          0.9727
        ],
        [
          0.3631,
          0.9718
        ],
        [
          0.3735,
          0.9704
        ],
        [
          0.3835,
          0.9691
        ],
        [
          0.3935,
          0.9677
        ],
        [
          0.4035,
          0.965
        ],
        [
          0.4136,
          0.9641
        ],
        [
          0.424,
          0.9613
        ],
        [
          0.434,
          0.9591
        ],
        [
          0.444,
          0.9572
        ],
        [
          0.454,
          0.954
        ],
        [
          0.4641,
          0.9513
        ],
        [
          0.4745,
          0.9486
        ],
        [
          0.4845,
          0.9463
        ],
        [
          0.4945,
          0.9431
        ],
        [
          0.5045,
          0.9386
        ],
        [
          0.5146,
          0.9363
        ],
        [
          0.525,
          0.9331
        ],
        [
          0.535,
          0.9308
        ],
        [
          0.545,
          0.9268
        ],
        [
          0.5551,
          0.9222
        ],
        [
          0.5655,
          0.9186
        ],
        [
          0.5755,
          0.9158
        ],
        [
          0.5855,
          0.9122
        ],
        [
          0.5955,
          0.9086
        ],
        [
          0.6056,
          0.9049
        ],
        [
          0.616,
          0.9008
        ],
        [
          0.626,
          0.8972
        ],
        [
          0.636,
          0.8917
        ],
        [
          0.646,
          0.8881
        ],
        [
          0.6561,
          0.8826
        ],
        [
          0.6665,
          0.8772
        ],
        [
          0.6765,
          0.8722
        ],
        [
          0.6865,
          0.8667
        ],
        [
          0.6965,
          0.8608
        ],
        [
          0.7066,
          0.8549
        ],
        [
          0.717,
          0.8499
        ],
        [
          0.727,
          0.8444
        ],
        [
          0.737,
          0.8389
        ],
        [
          0.747,
          0.8353
        ],
        [
          0.7571,
          0.8285
        ],
        [
          0.7675,
          0.8217
        ],
        [
          0.7775,
          0.8153
        ],
        [
          0.7875,
          0.8089
        ],
        [
          0.7975,
          0.8016
        ],
        [
          0.8076,
          0.7962
        ],
        [
          0.818,
          0.7898
        ],
        [
          0.828,
          0.7821
        ],
        [
          0.838,
          0.7766
        ],
        [
          0.848,
          0.7716
        ],
        [
          0.8581,
          0.7675
        ],
        [
          0.8685,
          0.7611
        ],
        [
          0.8785,
          0.7552
        ],
        [
          0.8885,
          0.7484
        ],
        [
          0.8985,
          0.7429
        ],
        [
          0.9086,
          0.7361
        ],
        [
          0.919,
          0.7284
        ],
        [
          0.929,
          0.7216
        ],
        [
          0.939,
          0.7143
        ],
        [
          0.949,
          0.707
        ],
        [
          0.9591,
          0.7002
        ],
        [
          0.9695,
          0.6947
        ],
        [
          0.9795,
          0.6902
        ],
        [
          0.9895,
          0.6847
        ],
        [
          0.9995,
          0.6779
        ],
        [
          1.0,
          0.6779
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.2633625061459235,
        "difference": -0.0013979255646301332,
        "worse": 0.5166666666666667,
        "kept_right": 0.26000614494683794,
        "kept_wrong": 0.0033563611990855254,
        "handed_off": 0.7366374938540768
      },
      "usd_per_1000": 1.194966103508305
    },
    {
      "decision": 269,
      "frontier": 48,
      "frontier_accuracy": 0.9758871701546861,
      "in_sample": {
        "threshold": 0.9809937576468168,
        "answered": 0.24021838034576887,
        "accuracy": 0.9758871701546861
      },
      "curve": [
        [
          0.0,
          0.9759
        ],
        [
          0.01,
          0.9759
        ],
        [
          0.02,
          0.9759
        ],
        [
          0.03,
          0.9759
        ],
        [
          0.04,
          0.9759
        ],
        [
          0.05,
          0.9759
        ],
        [
          0.0601,
          0.9759
        ],
        [
          0.0705,
          0.9759
        ],
        [
          0.0805,
          0.9759
        ],
        [
          0.0905,
          0.9759
        ],
        [
          0.1005,
          0.9759
        ],
        [
          0.1106,
          0.9759
        ],
        [
          0.121,
          0.9759
        ],
        [
          0.131,
          0.9759
        ],
        [
          0.141,
          0.9759
        ],
        [
          0.151,
          0.9759
        ],
        [
          0.1611,
          0.9759
        ],
        [
          0.1715,
          0.9759
        ],
        [
          0.1815,
          0.9759
        ],
        [
          0.1915,
          0.9763
        ],
        [
          0.2015,
          0.9763
        ],
        [
          0.2116,
          0.9759
        ],
        [
          0.222,
          0.9759
        ],
        [
          0.232,
          0.9759
        ],
        [
          0.242,
          0.9754
        ],
        [
          0.252,
          0.9754
        ],
        [
          0.2621,
          0.975
        ],
        [
          0.2725,
          0.9745
        ],
        [
          0.2825,
          0.9745
        ],
        [
          0.2925,
          0.9741
        ],
        [
          0.3025,
          0.9741
        ],
        [
          0.3126,
          0.9732
        ],
        [
          0.323,
          0.9722
        ],
        [
          0.333,
          0.9722
        ],
        [
          0.343,
          0.9718
        ],
        [
          0.353,
          0.9709
        ],
        [
          0.3631,
          0.9695
        ],
        [
          0.3735,
          0.9682
        ],
        [
          0.3835,
          0.9668
        ],
        [
          0.3935,
          0.9654
        ],
        [
          0.4035,
          0.9627
        ],
        [
          0.4136,
          0.9618
        ],
        [
          0.424,
          0.9591
        ],
        [
          0.434,
          0.9563
        ],
        [
          0.444,
          0.9545
        ],
        [
          0.454,
          0.9513
        ],
        [
          0.4641,
          0.9486
        ],
        [
          0.4745,
          0.9454
        ],
        [
          0.4845,
          0.9436
        ],
        [
          0.4945,
          0.9404
        ],
        [
          0.5045,
          0.9359
        ],
        [
          0.5146,
          0.9336
        ],
        [
          0.525,
          0.9299
        ],
        [
          0.535,
          0.9277
        ],
        [
          0.545,
          0.9236
        ],
        [
          0.5551,
          0.919
        ],
        [
          0.5655,
          0.9154
        ],
        [
          0.5755,
          0.9126
        ],
        [
          0.5855,
          0.909
        ],
        [
          0.5955,
          0.9058
        ],
        [
          0.6056,
          0.9022
        ],
        [
          0.616,
          0.8981
        ],
        [
          0.626,
          0.8944
        ],
        [
          0.636,
          0.889
        ],
        [
          0.646,
          0.8854
        ],
        [
          0.6561,
          0.8803
        ],
        [
          0.6665,
          0.8749
        ],
        [
          0.6765,
          0.8699
        ],
        [
          0.6865,
          0.8644
        ],
        [
          0.6965,
          0.8585
        ],
        [
          0.7066,
          0.853
        ],
        [
          0.717,
          0.848
        ],
        [
          0.727,
          0.8421
        ],
        [
          0.737,
          0.8367
        ],
        [
          0.747,
          0.833
        ],
        [
          0.7571,
          0.8262
        ],
        [
          0.7675,
          0.8194
        ],
        [
          0.7775,
          0.813
        ],
        [
          0.7875,
          0.8062
        ],
        [
          0.7975,
          0.7994
        ],
        [
          0.8076,
          0.7944
        ],
        [
          0.818,
          0.7884
        ],
        [
          0.828,
          0.7807
        ],
        [
          0.838,
          0.7753
        ],
        [
          0.848,
          0.7698
        ],
        [
          0.8581,
          0.7657
        ],
        [
          0.8685,
          0.7593
        ],
        [
          0.8785,
          0.7539
        ],
        [
          0.8885,
          0.747
        ],
        [
          0.8985,
          0.7416
        ],
        [
          0.9086,
          0.7348
        ],
        [
          0.919,
          0.727
        ],
        [
          0.929,
          0.7202
        ],
        [
          0.939,
          0.7129
        ],
        [
          0.949,
          0.7061
        ],
        [
          0.9591,
          0.6997
        ],
        [
          0.9695,
          0.6947
        ],
        [
          0.9795,
          0.6902
        ],
        [
          0.9895,
          0.6847
        ],
        [
          0.9995,
          0.6779
        ],
        [
          1.0,
          0.6779
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.25518176147237426,
        "difference": -0.001537214744664546,
        "worse": 0.5466666666666666,
        "kept_right": 0.2520953597670627,
        "kept_wrong": 0.00308640170531161,
        "handed_off": 0.7448182385276255
      },
      "usd_per_1000": 4.005800997715266
    },
    {
      "decision": 269,
      "frontier": 191,
      "frontier_accuracy": 0.9740673339399454,
      "in_sample": {
        "threshold": 0.9864338454383234,
        "answered": 0.20336669699727025,
        "accuracy": 0.9740673339399454
      },
      "curve": [
        [
          0.0,
          0.9741
        ],
        [
          0.01,
          0.9741
        ],
        [
          0.02,
          0.9741
        ],
        [
          0.03,
          0.9741
        ],
        [
          0.04,
          0.9741
        ],
        [
          0.05,
          0.9741
        ],
        [
          0.0601,
          0.9741
        ],
        [
          0.0705,
          0.9741
        ],
        [
          0.0805,
          0.9741
        ],
        [
          0.0905,
          0.9741
        ],
        [
          0.1005,
          0.9741
        ],
        [
          0.1106,
          0.9741
        ],
        [
          0.121,
          0.9741
        ],
        [
          0.131,
          0.9741
        ],
        [
          0.141,
          0.9741
        ],
        [
          0.151,
          0.9741
        ],
        [
          0.1611,
          0.9741
        ],
        [
          0.1715,
          0.9741
        ],
        [
          0.1815,
          0.9741
        ],
        [
          0.1915,
          0.9741
        ],
        [
          0.2015,
          0.9741
        ],
        [
          0.2116,
          0.9736
        ],
        [
          0.222,
          0.9736
        ],
        [
          0.232,
          0.9736
        ],
        [
          0.242,
          0.9732
        ],
        [
          0.252,
          0.9732
        ],
        [
          0.2621,
          0.9722
        ],
        [
          0.2725,
          0.9718
        ],
        [
          0.2825,
          0.9718
        ],
        [
          0.2925,
          0.9713
        ],
        [
          0.3025,
          0.9713
        ],
        [
          0.3126,
          0.9709
        ],
        [
          0.323,
          0.97
        ],
        [
          0.333,
          0.97
        ],
        [
          0.343,
          0.9695
        ],
        [
          0.353,
          0.9686
        ],
        [
          0.3631,
          0.9672
        ],
        [
          0.3735,
          0.9659
        ],
        [
          0.3835,
          0.9645
        ],
        [
          0.3935,
          0.9631
        ],
        [
          0.4035,
          0.9604
        ],
        [
          0.4136,
          0.9595
        ],
        [
          0.424,
          0.9568
        ],
        [
          0.434,
          0.954
        ],
        [
          0.444,
          0.9522
        ],
        [
          0.454,
          0.949
        ],
        [
          0.4641,
          0.9463
        ],
        [
          0.4745,
          0.9431
        ],
        [
          0.4845,
          0.9413
        ],
        [
          0.4945,
          0.9381
        ],
        [
          0.5045,
          0.9336
        ],
        [
          0.5146,
          0.9308
        ],
        [
          0.525,
          0.9272
        ],
        [
          0.535,
          0.9249
        ],
        [
          0.545,
          0.9208
        ],
        [
          0.5551,
          0.9167
        ],
        [
          0.5655,
          0.9131
        ],
        [
          0.5755,
          0.9108
        ],
        [
          0.5855,
          0.9076
        ],
        [
          0.5955,
          0.904
        ],
        [
          0.6056,
          0.9004
        ],
        [
          0.616,
          0.8972
        ],
        [
          0.626,
          0.8935
        ],
        [
          0.636,
          0.8881
        ],
        [
          0.646,
          0.8844
        ],
        [
          0.6561,
          0.879
        ],
        [
          0.6665,
          0.8735
        ],
        [
          0.6765,
          0.8685
        ],
        [
          0.6865,
          0.8631
        ],
        [
          0.6965,
          0.8567
        ],
        [
          0.7066,
          0.8512
        ],
        [
          0.717,
          0.8467
        ],
        [
          0.727,
          0.8408
        ],
        [
          0.737,
          0.8353
        ],
        [
          0.747,
          0.8317
        ],
        [
          0.7571,
          0.8248
        ],
        [
          0.7675,
          0.818
        ],
        [
          0.7775,
          0.8116
        ],
        [
          0.7875,
          0.8053
        ],
        [
          0.7975,
          0.798
        ],
        [
          0.8076,
          0.7925
        ],
        [
          0.818,
          0.7866
        ],
        [
          0.828,
          0.7793
        ],
        [
          0.838,
          0.7748
        ],
        [
          0.848,
          0.7698
        ],
        [
          0.8581,
          0.7652
        ],
        [
          0.8685,
          0.7589
        ],
        [
          0.8785,
          0.7525
        ],
        [
          0.8885,
          0.7461
        ],
        [
          0.8985,
          0.7407
        ],
        [
          0.9086,
          0.7343
        ],
        [
          0.919,
          0.727
        ],
        [
          0.929,
          0.7202
        ],
        [
          0.939,
          0.7129
        ],
        [
          0.949,
          0.7061
        ],
        [
          0.9591,
          0.6997
        ],
        [
          0.9695,
          0.6943
        ],
        [
          0.9795,
          0.6902
        ],
        [
          0.9895,
          0.6847
        ],
        [
          0.9995,
          0.6779
        ],
        [
          1.0,
          0.6779
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.22998678905174325,
        "difference": -0.0011581340525695868,
        "worse": 0.49666666666666665,
        "kept_right": 0.2280010173464069,
        "kept_wrong": 0.0019857717053364168,
        "handed_off": 0.7700132109482566
      },
      "usd_per_1000": 20.58837921758066
    },
    {
      "decision": 269,
      "frontier": 188,
      "frontier_accuracy": 0.9713375796178344,
      "in_sample": {
        "threshold": 0.9809937576468168,
        "answered": 0.24021838034576887,
        "accuracy": 0.9713375796178344
      },
      "curve": [
        [
          0.0,
          0.9713
        ],
        [
          0.01,
          0.9713
        ],
        [
          0.02,
          0.9713
        ],
        [
          0.03,
          0.9713
        ],
        [
          0.04,
          0.9713
        ],
        [
          0.05,
          0.9713
        ],
        [
          0.0601,
          0.9713
        ],
        [
          0.0705,
          0.9713
        ],
        [
          0.0805,
          0.9713
        ],
        [
          0.0905,
          0.9713
        ],
        [
          0.1005,
          0.9713
        ],
        [
          0.1106,
          0.9713
        ],
        [
          0.121,
          0.9713
        ],
        [
          0.131,
          0.9713
        ],
        [
          0.141,
          0.9713
        ],
        [
          0.151,
          0.9713
        ],
        [
          0.1611,
          0.9713
        ],
        [
          0.1715,
          0.9713
        ],
        [
          0.1815,
          0.9713
        ],
        [
          0.1915,
          0.9718
        ],
        [
          0.2015,
          0.9718
        ],
        [
          0.2116,
          0.9713
        ],
        [
          0.222,
          0.9713
        ],
        [
          0.232,
          0.9713
        ],
        [
          0.242,
          0.9709
        ],
        [
          0.252,
          0.9709
        ],
        [
          0.2621,
          0.97
        ],
        [
          0.2725,
          0.9695
        ],
        [
          0.2825,
          0.9695
        ],
        [
          0.2925,
          0.9691
        ],
        [
          0.3025,
          0.9691
        ],
        [
          0.3126,
          0.9686
        ],
        [
          0.323,
          0.9677
        ],
        [
          0.333,
          0.9677
        ],
        [
          0.343,
          0.9672
        ],
        [
          0.353,
          0.9659
        ],
        [
          0.3631,
          0.9645
        ],
        [
          0.3735,
          0.9631
        ],
        [
          0.3835,
          0.9618
        ],
        [
          0.3935,
          0.9604
        ],
        [
          0.4035,
          0.9581
        ],
        [
          0.4136,
          0.9572
        ],
        [
          0.424,
          0.9545
        ],
        [
          0.434,
          0.9522
        ],
        [
          0.444,
          0.9509
        ],
        [
          0.454,
          0.9481
        ],
        [
          0.4641,
          0.9454
        ],
        [
          0.4745,
          0.9422
        ],
        [
          0.4845,
          0.9399
        ],
        [
          0.4945,
          0.9368
        ],
        [
          0.5045,
          0.9322
        ],
        [
          0.5146,
          0.9299
        ],
        [
          0.525,
          0.9268
        ],
        [
          0.535,
          0.9245
        ],
        [
          0.545,
          0.9204
        ],
        [
          0.5551,
          0.9158
        ],
        [
          0.5655,
          0.9122
        ],
        [
          0.5755,
          0.909
        ],
        [
          0.5855,
          0.9058
        ],
        [
          0.5955,
          0.9026
        ],
        [
          0.6056,
          0.899
        ],
        [
          0.616,
          0.8954
        ],
        [
          0.626,
          0.8926
        ],
        [
          0.636,
          0.8876
        ],
        [
          0.646,
          0.8835
        ],
        [
          0.6561,
          0.8781
        ],
        [
          0.6665,
          0.8731
        ],
        [
          0.6765,
          0.869
        ],
        [
          0.6865,
          0.8635
        ],
        [
          0.6965,
          0.8576
        ],
        [
          0.7066,
          0.8517
        ],
        [
          0.717,
          0.8467
        ],
        [
          0.727,
          0.8412
        ],
        [
          0.737,
          0.8358
        ],
        [
          0.747,
          0.8321
        ],
        [
          0.7571,
          0.8258
        ],
        [
          0.7675,
          0.8194
        ],
        [
          0.7775,
          0.813
        ],
        [
          0.7875,
          0.8071
        ],
        [
          0.7975,
          0.8003
        ],
        [
          0.8076,
          0.7948
        ],
        [
          0.818,
          0.7884
        ],
        [
          0.828,
          0.7812
        ],
        [
          0.838,
          0.7757
        ],
        [
          0.848,
          0.7702
        ],
        [
          0.8581,
          0.7662
        ],
        [
          0.8685,
          0.7598
        ],
        [
          0.8785,
          0.7539
        ],
        [
          0.8885,
          0.7475
        ],
        [
          0.8985,
          0.7411
        ],
        [
          0.9086,
          0.7343
        ],
        [
          0.919,
          0.727
        ],
        [
          0.929,
          0.7202
        ],
        [
          0.939,
          0.7134
        ],
        [
          0.949,
          0.707
        ],
        [
          0.9591,
          0.7006
        ],
        [
          0.9695,
          0.6956
        ],
        [
          0.9795,
          0.6902
        ],
        [
          0.9895,
          0.6847
        ],
        [
          0.9995,
          0.6779
        ],
        [
          1.0,
          0.6779
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.251430566407898,
        "difference": -0.0014766910079189901,
        "worse": 0.53,
        "kept_right": 0.24846231660637216,
        "kept_wrong": 0.002968249801525727,
        "handed_off": 0.7485694335921016
      },
      "usd_per_1000": 0.7212016090896516
    },
    {
      "decision": 269,
      "frontier": 190,
      "frontier_accuracy": 0.9672429481346679,
      "in_sample": {
        "threshold": 0.9794597561579199,
        "answered": 0.25204731574158323,
        "accuracy": 0.9672429481346679
      },
      "curve": [
        [
          0.0,
          0.9672
        ],
        [
          0.01,
          0.9672
        ],
        [
          0.02,
          0.9672
        ],
        [
          0.03,
          0.9672
        ],
        [
          0.04,
          0.9672
        ],
        [
          0.05,
          0.9672
        ],
        [
          0.0601,
          0.9672
        ],
        [
          0.0705,
          0.9672
        ],
        [
          0.0805,
          0.9672
        ],
        [
          0.0905,
          0.9672
        ],
        [
          0.1005,
          0.9672
        ],
        [
          0.1106,
          0.9672
        ],
        [
          0.121,
          0.9677
        ],
        [
          0.131,
          0.9677
        ],
        [
          0.141,
          0.9677
        ],
        [
          0.151,
          0.9677
        ],
        [
          0.1611,
          0.9677
        ],
        [
          0.1715,
          0.9677
        ],
        [
          0.1815,
          0.9677
        ],
        [
          0.1915,
          0.9682
        ],
        [
          0.2015,
          0.9682
        ],
        [
          0.2116,
          0.9677
        ],
        [
          0.222,
          0.9677
        ],
        [
          0.232,
          0.9677
        ],
        [
          0.242,
          0.9672
        ],
        [
          0.252,
          0.9672
        ],
        [
          0.2621,
          0.9663
        ],
        [
          0.2725,
          0.9659
        ],
        [
          0.2825,
          0.9659
        ],
        [
          0.2925,
          0.965
        ],
        [
          0.3025,
          0.965
        ],
        [
          0.3126,
          0.965
        ],
        [
          0.323,
          0.9641
        ],
        [
          0.333,
          0.9641
        ],
        [
          0.343,
          0.9636
        ],
        [
          0.353,
          0.9627
        ],
        [
          0.3631,
          0.9613
        ],
        [
          0.3735,
          0.96
        ],
        [
          0.3835,
          0.9591
        ],
        [
          0.3935,
          0.9577
        ],
        [
          0.4035,
          0.955
        ],
        [
          0.4136,
          0.954
        ],
        [
          0.424,
          0.9513
        ],
        [
          0.434,
          0.949
        ],
        [
          0.444,
          0.9472
        ],
        [
          0.454,
          0.944
        ],
        [
          0.4641,
          0.9413
        ],
        [
          0.4745,
          0.9386
        ],
        [
          0.4845,
          0.9363
        ],
        [
          0.4945,
          0.9331
        ],
        [
          0.5045,
          0.9286
        ],
        [
          0.5146,
          0.9263
        ],
        [
          0.525,
          0.9231
        ],
        [
          0.535,
          0.9208
        ],
        [
          0.545,
          0.9167
        ],
        [
          0.5551,
          0.9122
        ],
        [
          0.5655,
          0.9086
        ],
        [
          0.5755,
          0.9058
        ],
        [
          0.5855,
          0.9022
        ],
        [
          0.5955,
          0.8985
        ],
        [
          0.6056,
          0.8954
        ],
        [
          0.616,
          0.8922
        ],
        [
          0.626,
          0.8894
        ],
        [
          0.636,
          0.884
        ],
        [
          0.646,
          0.8803
        ],
        [
          0.6561,
          0.8749
        ],
        [
          0.6665,
          0.8694
        ],
        [
          0.6765,
          0.8644
        ],
        [
          0.6865,
          0.8594
        ],
        [
          0.6965,
          0.8535
        ],
        [
          0.7066,
          0.848
        ],
        [
          0.717,
          0.8435
        ],
        [
          0.727,
          0.8376
        ],
        [
          0.737,
          0.8321
        ],
        [
          0.747,
          0.8285
        ],
        [
          0.7571,
          0.8217
        ],
        [
          0.7675,
          0.8153
        ],
        [
          0.7775,
          0.8089
        ],
        [
          0.7875,
          0.8025
        ],
        [
          0.7975,
          0.7953
        ],
        [
          0.8076,
          0.7903
        ],
        [
          0.818,
          0.7843
        ],
        [
          0.828,
          0.7775
        ],
        [
          0.838,
          0.7721
        ],
        [
          0.848,
          0.7671
        ],
        [
          0.8581,
          0.7625
        ],
        [
          0.8685,
          0.7561
        ],
        [
          0.8785,
          0.7502
        ],
        [
          0.8885,
          0.7439
        ],
        [
          0.8985,
          0.7379
        ],
        [
          0.9086,
          0.7316
        ],
        [
          0.919,
          0.7247
        ],
        [
          0.929,
          0.7179
        ],
        [
          0.939,
          0.7106
        ],
        [
          0.949,
          0.7038
        ],
        [
          0.9591,
          0.6993
        ],
        [
          0.9695,
          0.6943
        ],
        [
          0.9795,
          0.6902
        ],
        [
          0.9895,
          0.6847
        ],
        [
          0.9995,
          0.6779
        ],
        [
          1.0,
          0.6779
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.26453955656349437,
        "difference": -0.0014250026915587205,
        "worse": 0.5466666666666666,
        "kept_right": 0.26099541004524507,
        "kept_wrong": 0.0035441465182491426,
        "handed_off": 0.7354604434365052
      },
      "usd_per_1000": 4.095425465739574
    },
    {
      "decision": 269,
      "frontier": 124,
      "frontier_accuracy": 0.9667879890809827,
      "in_sample": {
        "threshold": 0.9809937576468168,
        "answered": 0.24021838034576887,
        "accuracy": 0.9667879890809827
      },
      "curve": [
        [
          0.0,
          0.9668
        ],
        [
          0.01,
          0.9668
        ],
        [
          0.02,
          0.9668
        ],
        [
          0.03,
          0.9668
        ],
        [
          0.04,
          0.9668
        ],
        [
          0.05,
          0.9668
        ],
        [
          0.0601,
          0.9668
        ],
        [
          0.0705,
          0.9668
        ],
        [
          0.0805,
          0.9668
        ],
        [
          0.0905,
          0.9668
        ],
        [
          0.1005,
          0.9668
        ],
        [
          0.1106,
          0.9668
        ],
        [
          0.121,
          0.9668
        ],
        [
          0.131,
          0.9668
        ],
        [
          0.141,
          0.9668
        ],
        [
          0.151,
          0.9668
        ],
        [
          0.1611,
          0.9668
        ],
        [
          0.1715,
          0.9668
        ],
        [
          0.1815,
          0.9668
        ],
        [
          0.1915,
          0.9672
        ],
        [
          0.2015,
          0.9672
        ],
        [
          0.2116,
          0.9668
        ],
        [
          0.222,
          0.9668
        ],
        [
          0.232,
          0.9668
        ],
        [
          0.242,
          0.9663
        ],
        [
          0.252,
          0.9663
        ],
        [
          0.2621,
          0.9659
        ],
        [
          0.2725,
          0.9654
        ],
        [
          0.2825,
          0.9654
        ],
        [
          0.2925,
          0.965
        ],
        [
          0.3025,
          0.965
        ],
        [
          0.3126,
          0.9641
        ],
        [
          0.323,
          0.9631
        ],
        [
          0.333,
          0.9631
        ],
        [
          0.343,
          0.9627
        ],
        [
          0.353,
          0.9622
        ],
        [
          0.3631,
          0.9613
        ],
        [
          0.3735,
          0.96
        ],
        [
          0.3835,
          0.9586
        ],
        [
          0.3935,
          0.9572
        ],
        [
          0.4035,
          0.9545
        ],
        [
          0.4136,
          0.9536
        ],
        [
          0.424,
          0.9509
        ],
        [
          0.434,
          0.9486
        ],
        [
          0.444,
          0.9472
        ],
        [
          0.454,
          0.944
        ],
        [
          0.4641,
          0.9413
        ],
        [
          0.4745,
          0.9386
        ],
        [
          0.4845,
          0.9363
        ],
        [
          0.4945,
          0.9336
        ],
        [
          0.5045,
          0.929
        ],
        [
          0.5146,
          0.9268
        ],
        [
          0.525,
          0.9236
        ],
        [
          0.535,
          0.9213
        ],
        [
          0.545,
          0.9172
        ],
        [
          0.5551,
          0.9131
        ],
        [
          0.5655,
          0.9095
        ],
        [
          0.5755,
          0.9072
        ],
        [
          0.5855,
          0.9035
        ],
        [
          0.5955,
          0.8999
        ],
        [
          0.6056,
          0.8967
        ],
        [
          0.616,
          0.8935
        ],
        [
          0.626,
          0.8904
        ],
        [
          0.636,
          0.8849
        ],
        [
          0.646,
          0.8813
        ],
        [
          0.6561,
          0.8763
        ],
        [
          0.6665,
          0.8708
        ],
        [
          0.6765,
          0.8658
        ],
        [
          0.6865,
          0.8608
        ],
        [
          0.6965,
          0.8549
        ],
        [
          0.7066,
          0.8494
        ],
        [
          0.717,
          0.8444
        ],
        [
          0.727,
          0.8389
        ],
        [
          0.737,
          0.8335
        ],
        [
          0.747,
          0.8298
        ],
        [
          0.7571,
          0.823
        ],
        [
          0.7675,
          0.8162
        ],
        [
          0.7775,
          0.8107
        ],
        [
          0.7875,
          0.8044
        ],
        [
          0.7975,
          0.798
        ],
        [
          0.8076,
          0.7925
        ],
        [
          0.818,
          0.7866
        ],
        [
          0.828,
          0.7789
        ],
        [
          0.838,
          0.7734
        ],
        [
          0.848,
          0.768
        ],
        [
          0.8581,
          0.7643
        ],
        [
          0.8685,
          0.758
        ],
        [
          0.8785,
          0.752
        ],
        [
          0.8885,
          0.7457
        ],
        [
          0.8985,
          0.7398
        ],
        [
          0.9086,
          0.7329
        ],
        [
          0.919,
          0.7261
        ],
        [
          0.929,
          0.7193
        ],
        [
          0.939,
          0.712
        ],
        [
          0.949,
          0.7052
        ],
        [
          0.9591,
          0.6993
        ],
        [
          0.9695,
          0.6943
        ],
        [
          0.9795,
          0.6902
        ],
        [
          0.9895,
          0.6847
        ],
        [
          0.9995,
          0.6779
        ],
        [
          1.0,
          0.6779
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.2556851533769763,
        "difference": -0.0015705591374156795,
        "worse": 0.5466666666666666,
        "kept_right": 0.2525199195902945,
        "kept_wrong": 0.003165233786681882,
        "handed_off": 0.7443148466230236
      },
      "usd_per_1000": 0.8413534555621929
    },
    {
      "decision": 269,
      "frontier": 278,
      "frontier_accuracy": 0.9499545040946314,
      "in_sample": {
        "threshold": 0.9794597561579199,
        "answered": 0.25204731574158323,
        "accuracy": 0.9499545040946314
      },
      "curve": [
        [
          0.0,
          0.95
        ],
        [
          0.01,
          0.95
        ],
        [
          0.02,
          0.95
        ],
        [
          0.03,
          0.95
        ],
        [
          0.04,
          0.95
        ],
        [
          0.05,
          0.95
        ],
        [
          0.0601,
          0.95
        ],
        [
          0.0705,
          0.95
        ],
        [
          0.0805,
          0.95
        ],
        [
          0.0905,
          0.95
        ],
        [
          0.1005,
          0.95
        ],
        [
          0.1106,
          0.95
        ],
        [
          0.121,
          0.95
        ],
        [
          0.131,
          0.95
        ],
        [
          0.141,
          0.95
        ],
        [
          0.151,
          0.95
        ],
        [
          0.1611,
          0.95
        ],
        [
          0.1715,
          0.95
        ],
        [
          0.1815,
          0.95
        ],
        [
          0.1915,
          0.95
        ],
        [
          0.2015,
          0.95
        ],
        [
          0.2116,
          0.9495
        ],
        [
          0.222,
          0.9495
        ],
        [
          0.232,
          0.9504
        ],
        [
          0.242,
          0.95
        ],
        [
          0.252,
          0.95
        ],
        [
          0.2621,
          0.9495
        ],
        [
          0.2725,
          0.949
        ],
        [
          0.2825,
          0.949
        ],
        [
          0.2925,
          0.9486
        ],
        [
          0.3025,
          0.9486
        ],
        [
          0.3126,
          0.9481
        ],
        [
          0.323,
          0.9477
        ],
        [
          0.333,
          0.9477
        ],
        [
          0.343,
          0.9472
        ],
        [
          0.353,
          0.9463
        ],
        [
          0.3631,
          0.9449
        ],
        [
          0.3735,
          0.9445
        ],
        [
          0.3835,
          0.9431
        ],
        [
          0.3935,
          0.9422
        ],
        [
          0.4035,
          0.9395
        ],
        [
          0.4136,
          0.9386
        ],
        [
          0.424,
          0.9363
        ],
        [
          0.434,
          0.9349
        ],
        [
          0.444,
          0.934
        ],
        [
          0.454,
          0.9318
        ],
        [
          0.4641,
          0.9295
        ],
        [
          0.4745,
          0.9268
        ],
        [
          0.4845,
          0.9254
        ],
        [
          0.4945,
          0.9227
        ],
        [
          0.5045,
          0.9181
        ],
        [
          0.5146,
          0.9158
        ],
        [
          0.525,
          0.9126
        ],
        [
          0.535,
          0.9104
        ],
        [
          0.545,
          0.9063
        ],
        [
          0.5551,
          0.9017
        ],
        [
          0.5655,
          0.8981
        ],
        [
          0.5755,
          0.8958
        ],
        [
          0.5855,
          0.8922
        ],
        [
          0.5955,
          0.889
        ],
        [
          0.6056,
          0.8858
        ],
        [
          0.616,
          0.8831
        ],
        [
          0.626,
          0.8808
        ],
        [
          0.636,
          0.8758
        ],
        [
          0.646,
          0.8722
        ],
        [
          0.6561,
          0.8672
        ],
        [
          0.6665,
          0.8621
        ],
        [
          0.6765,
          0.8571
        ],
        [
          0.6865,
          0.8526
        ],
        [
          0.6965,
          0.848
        ],
        [
          0.7066,
          0.843
        ],
        [
          0.717,
          0.8376
        ],
        [
          0.727,
          0.8321
        ],
        [
          0.737,
          0.8271
        ],
        [
          0.747,
          0.8235
        ],
        [
          0.7571,
          0.8167
        ],
        [
          0.7675,
          0.8112
        ],
        [
          0.7775,
          0.8048
        ],
        [
          0.7875,
          0.7985
        ],
        [
          0.7975,
          0.7912
        ],
        [
          0.8076,
          0.7875
        ],
        [
          0.818,
          0.7816
        ],
        [
          0.828,
          0.7739
        ],
        [
          0.838,
          0.7684
        ],
        [
          0.848,
          0.7634
        ],
        [
          0.8581,
          0.7598
        ],
        [
          0.8685,
          0.7539
        ],
        [
          0.8785,
          0.748
        ],
        [
          0.8885,
          0.7425
        ],
        [
          0.8985,
          0.7366
        ],
        [
          0.9086,
          0.7307
        ],
        [
          0.919,
          0.7238
        ],
        [
          0.929,
          0.717
        ],
        [
          0.939,
          0.7097
        ],
        [
          0.949,
          0.7038
        ],
        [
          0.9591,
          0.6979
        ],
        [
          0.9695,
          0.6929
        ],
        [
          0.9795,
          0.6884
        ],
        [
          0.9895,
          0.6829
        ],
        [
          0.9995,
          0.6774
        ],
        [
          1.0,
          0.6779
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.27953394994211717,
        "difference": -0.0020282180462457737,
        "worse": 0.63,
        "kept_right": 0.2749530584289234,
        "kept_wrong": 0.004580891513193941,
        "handed_off": 0.7204660500578828
      },
      "usd_per_1000": 4.689388372448272
    },
    {
      "decision": 269,
      "frontier": 275,
      "frontier_accuracy": 0.9426751592356688,
      "in_sample": {
        "threshold": 0.9809937576468168,
        "answered": 0.24021838034576887,
        "accuracy": 0.9426751592356688
      },
      "curve": [
        [
          0.0,
          0.9427
        ],
        [
          0.01,
          0.9427
        ],
        [
          0.02,
          0.9427
        ],
        [
          0.03,
          0.9427
        ],
        [
          0.04,
          0.9427
        ],
        [
          0.05,
          0.9427
        ],
        [
          0.0601,
          0.9427
        ],
        [
          0.0705,
          0.9427
        ],
        [
          0.0805,
          0.9427
        ],
        [
          0.0905,
          0.9427
        ],
        [
          0.1005,
          0.9427
        ],
        [
          0.1106,
          0.9427
        ],
        [
          0.121,
          0.9427
        ],
        [
          0.131,
          0.9427
        ],
        [
          0.141,
          0.9427
        ],
        [
          0.151,
          0.9427
        ],
        [
          0.1611,
          0.9427
        ],
        [
          0.1715,
          0.9427
        ],
        [
          0.1815,
          0.9427
        ],
        [
          0.1915,
          0.9431
        ],
        [
          0.2015,
          0.9431
        ],
        [
          0.2116,
          0.9427
        ],
        [
          0.222,
          0.9427
        ],
        [
          0.232,
          0.9427
        ],
        [
          0.242,
          0.9422
        ],
        [
          0.252,
          0.9422
        ],
        [
          0.2621,
          0.9413
        ],
        [
          0.2725,
          0.9409
        ],
        [
          0.2825,
          0.9409
        ],
        [
          0.2925,
          0.9404
        ],
        [
          0.3025,
          0.9404
        ],
        [
          0.3126,
          0.9399
        ],
        [
          0.323,
          0.9399
        ],
        [
          0.333,
          0.9399
        ],
        [
          0.343,
          0.9395
        ],
        [
          0.353,
          0.9386
        ],
        [
          0.3631,
          0.9372
        ],
        [
          0.3735,
          0.9368
        ],
        [
          0.3835,
          0.9354
        ],
        [
          0.3935,
          0.9349
        ],
        [
          0.4035,
          0.9327
        ],
        [
          0.4136,
          0.9322
        ],
        [
          0.424,
          0.9304
        ],
        [
          0.434,
          0.9286
        ],
        [
          0.444,
          0.9272
        ],
        [
          0.454,
          0.924
        ],
        [
          0.4641,
          0.9208
        ],
        [
          0.4745,
          0.9181
        ],
        [
          0.4845,
          0.9167
        ],
        [
          0.4945,
          0.914
        ],
        [
          0.5045,
          0.9099
        ],
        [
          0.5146,
          0.9076
        ],
        [
          0.525,
          0.9045
        ],
        [
          0.535,
          0.9022
        ],
        [
          0.545,
          0.899
        ],
        [
          0.5551,
          0.8954
        ],
        [
          0.5655,
          0.8926
        ],
        [
          0.5755,
          0.8908
        ],
        [
          0.5855,
          0.8876
        ],
        [
          0.5955,
          0.8844
        ],
        [
          0.6056,
          0.8817
        ],
        [
          0.616,
          0.8785
        ],
        [
          0.626,
          0.8758
        ],
        [
          0.636,
          0.8703
        ],
        [
          0.646,
          0.8667
        ],
        [
          0.6561,
          0.8617
        ],
        [
          0.6665,
          0.8567
        ],
        [
          0.6765,
          0.8517
        ],
        [
          0.6865,
          0.848
        ],
        [
          0.6965,
          0.8426
        ],
        [
          0.7066,
          0.8371
        ],
        [
          0.717,
          0.8335
        ],
        [
          0.727,
          0.8285
        ],
        [
          0.737,
          0.8235
        ],
        [
          0.747,
          0.8198
        ],
        [
          0.7571,
          0.813
        ],
        [
          0.7675,
          0.8066
        ],
        [
          0.7775,
          0.8007
        ],
        [
          0.7875,
          0.7948
        ],
        [
          0.7975,
          0.788
        ],
        [
          0.8076,
          0.783
        ],
        [
          0.818,
          0.778
        ],
        [
          0.828,
          0.7707
        ],
        [
          0.838,
          0.7662
        ],
        [
          0.848,
          0.7611
        ],
        [
          0.8581,
          0.758
        ],
        [
          0.8685,
          0.752
        ],
        [
          0.8785,
          0.7475
        ],
        [
          0.8885,
          0.7416
        ],
        [
          0.8985,
          0.7357
        ],
        [
          0.9086,
          0.7298
        ],
        [
          0.919,
          0.7234
        ],
        [
          0.929,
          0.7184
        ],
        [
          0.939,
          0.7111
        ],
        [
          0.949,
          0.7038
        ],
        [
          0.9591,
          0.6979
        ],
        [
          0.9695,
          0.6924
        ],
        [
          0.9795,
          0.6888
        ],
        [
          0.9895,
          0.6843
        ],
        [
          0.9995,
          0.6779
        ],
        [
          1.0,
          0.6779
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.25896081549091665,
        "difference": -0.001764708434406224,
        "worse": 0.54,
        "kept_right": 0.2554468067696206,
        "kept_wrong": 0.0035140087212960705,
        "handed_off": 0.7410391845090833
      },
      "usd_per_1000": 0.3812744375620211
    },
    {
      "decision": 269,
      "frontier": 192,
      "frontier_accuracy": 0.9331210191082803,
      "in_sample": {
        "threshold": 0.9794597561579199,
        "answered": 0.25204731574158323,
        "accuracy": 0.9331210191082803
      },
      "curve": [
        [
          0.0,
          0.9331
        ],
        [
          0.01,
          0.9331
        ],
        [
          0.02,
          0.9331
        ],
        [
          0.03,
          0.9331
        ],
        [
          0.04,
          0.9331
        ],
        [
          0.05,
          0.9331
        ],
        [
          0.0601,
          0.9331
        ],
        [
          0.0705,
          0.9331
        ],
        [
          0.0805,
          0.9331
        ],
        [
          0.0905,
          0.9331
        ],
        [
          0.1005,
          0.9331
        ],
        [
          0.1106,
          0.9331
        ],
        [
          0.121,
          0.9336
        ],
        [
          0.131,
          0.9336
        ],
        [
          0.141,
          0.9336
        ],
        [
          0.151,
          0.9336
        ],
        [
          0.1611,
          0.9336
        ],
        [
          0.1715,
          0.9336
        ],
        [
          0.1815,
          0.9336
        ],
        [
          0.1915,
          0.934
        ],
        [
          0.2015,
          0.934
        ],
        [
          0.2116,
          0.9336
        ],
        [
          0.222,
          0.9336
        ],
        [
          0.232,
          0.9336
        ],
        [
          0.242,
          0.9331
        ],
        [
          0.252,
          0.9331
        ],
        [
          0.2621,
          0.9327
        ],
        [
          0.2725,
          0.9322
        ],
        [
          0.2825,
          0.9322
        ],
        [
          0.2925,
          0.9313
        ],
        [
          0.3025,
          0.9313
        ],
        [
          0.3126,
          0.9304
        ],
        [
          0.323,
          0.9299
        ],
        [
          0.333,
          0.9299
        ],
        [
          0.343,
          0.9295
        ],
        [
          0.353,
          0.9281
        ],
        [
          0.3631,
          0.9272
        ],
        [
          0.3735,
          0.9268
        ],
        [
          0.3835,
          0.9258
        ],
        [
          0.3935,
          0.9249
        ],
        [
          0.4035,
          0.9227
        ],
        [
          0.4136,
          0.9217
        ],
        [
          0.424,
          0.9195
        ],
        [
          0.434,
          0.9177
        ],
        [
          0.444,
          0.9158
        ],
        [
          0.454,
          0.9131
        ],
        [
          0.4641,
          0.9104
        ],
        [
          0.4745,
          0.9076
        ],
        [
          0.4845,
          0.9063
        ],
        [
          0.4945,
          0.904
        ],
        [
          0.5045,
          0.8995
        ],
        [
          0.5146,
          0.8972
        ],
        [
          0.525,
          0.894
        ],
        [
          0.535,
          0.8922
        ],
        [
          0.545,
          0.889
        ],
        [
          0.5551,
          0.8854
        ],
        [
          0.5655,
          0.8817
        ],
        [
          0.5755,
          0.8799
        ],
        [
          0.5855,
          0.8772
        ],
        [
          0.5955,
          0.8735
        ],
        [
          0.6056,
          0.8717
        ],
        [
          0.616,
          0.8699
        ],
        [
          0.626,
          0.8672
        ],
        [
          0.636,
          0.8626
        ],
        [
          0.646,
          0.8599
        ],
        [
          0.6561,
          0.8549
        ],
        [
          0.6665,
          0.8508
        ],
        [
          0.6765,
          0.8462
        ],
        [
          0.6865,
          0.8421
        ],
        [
          0.6965,
          0.8362
        ],
        [
          0.7066,
          0.8312
        ],
        [
          0.717,
          0.8267
        ],
        [
          0.727,
          0.8212
        ],
        [
          0.737,
          0.8162
        ],
        [
          0.747,
          0.8126
        ],
        [
          0.7571,
          0.8062
        ],
        [
          0.7675,
          0.8007
        ],
        [
          0.7775,
          0.7948
        ],
        [
          0.7875,
          0.7884
        ],
        [
          0.7975,
          0.7816
        ],
        [
          0.8076,
          0.778
        ],
        [
          0.818,
          0.7721
        ],
        [
          0.828,
          0.7662
        ],
        [
          0.838,
          0.7611
        ],
        [
          0.848,
          0.7571
        ],
        [
          0.8581,
          0.7534
        ],
        [
          0.8685,
          0.748
        ],
        [
          0.8785,
          0.742
        ],
        [
          0.8885,
          0.7361
        ],
        [
          0.8985,
          0.7325
        ],
        [
          0.9086,
          0.7266
        ],
        [
          0.919,
          0.7193
        ],
        [
          0.929,
          0.7134
        ],
        [
          0.939,
          0.7075
        ],
        [
          0.949,
          0.7011
        ],
        [
          0.9591,
          0.6956
        ],
        [
          0.9695,
          0.6911
        ],
        [
          0.9795,
          0.6874
        ],
        [
          0.9895,
          0.6824
        ],
        [
          0.9995,
          0.6774
        ],
        [
          1.0,
          0.6779
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.26532661678900343,
        "difference": -0.0013005971078114175,
        "worse": 0.59,
        "kept_right": 0.2618826388495965,
        "kept_wrong": 0.003443977939406773,
        "handed_off": 0.7346733832109962
      },
      "usd_per_1000": 8.529238640157779
    },
    {
      "decision": 269,
      "frontier": 123,
      "frontier_accuracy": 0.9326660600545951,
      "in_sample": {
        "threshold": 0.9809937576468168,
        "answered": 0.24021838034576887,
        "accuracy": 0.9326660600545951
      },
      "curve": [
        [
          0.0,
          0.9327
        ],
        [
          0.01,
          0.9327
        ],
        [
          0.02,
          0.9327
        ],
        [
          0.03,
          0.9327
        ],
        [
          0.04,
          0.9327
        ],
        [
          0.05,
          0.9327
        ],
        [
          0.0601,
          0.9327
        ],
        [
          0.0705,
          0.9327
        ],
        [
          0.0805,
          0.9327
        ],
        [
          0.0905,
          0.9327
        ],
        [
          0.1005,
          0.9327
        ],
        [
          0.1106,
          0.9327
        ],
        [
          0.121,
          0.9327
        ],
        [
          0.131,
          0.9327
        ],
        [
          0.141,
          0.9327
        ],
        [
          0.151,
          0.9327
        ],
        [
          0.1611,
          0.9327
        ],
        [
          0.1715,
          0.9327
        ],
        [
          0.1815,
          0.9327
        ],
        [
          0.1915,
          0.9327
        ],
        [
          0.2015,
          0.9327
        ],
        [
          0.2116,
          0.9322
        ],
        [
          0.222,
          0.9322
        ],
        [
          0.232,
          0.9322
        ],
        [
          0.242,
          0.9322
        ],
        [
          0.252,
          0.9322
        ],
        [
          0.2621,
          0.9318
        ],
        [
          0.2725,
          0.9313
        ],
        [
          0.2825,
          0.9313
        ],
        [
          0.2925,
          0.9308
        ],
        [
          0.3025,
          0.9308
        ],
        [
          0.3126,
          0.9308
        ],
        [
          0.323,
          0.9299
        ],
        [
          0.333,
          0.9299
        ],
        [
          0.343,
          0.9295
        ],
        [
          0.353,
          0.9281
        ],
        [
          0.3631,
          0.9268
        ],
        [
          0.3735,
          0.9258
        ],
        [
          0.3835,
          0.9245
        ],
        [
          0.3935,
          0.9231
        ],
        [
          0.4035,
          0.9208
        ],
        [
          0.4136,
          0.9199
        ],
        [
          0.424,
          0.9177
        ],
        [
          0.434,
          0.9154
        ],
        [
          0.444,
          0.9149
        ],
        [
          0.454,
          0.9117
        ],
        [
          0.4641,
          0.9086
        ],
        [
          0.4745,
          0.9067
        ],
        [
          0.4845,
          0.9049
        ],
        [
          0.4945,
          0.9031
        ],
        [
          0.5045,
          0.8995
        ],
        [
          0.5146,
          0.8981
        ],
        [
          0.525,
          0.8949
        ],
        [
          0.535,
          0.8931
        ],
        [
          0.545,
          0.8899
        ],
        [
          0.5551,
          0.8854
        ],
        [
          0.5655,
          0.8826
        ],
        [
          0.5755,
          0.8808
        ],
        [
          0.5855,
          0.8776
        ],
        [
          0.5955,
          0.8749
        ],
        [
          0.6056,
          0.8722
        ],
        [
          0.616,
          0.8694
        ],
        [
          0.626,
          0.8658
        ],
        [
          0.636,
          0.8617
        ],
        [
          0.646,
          0.8581
        ],
        [
          0.6561,
          0.853
        ],
        [
          0.6665,
          0.8485
        ],
        [
          0.6765,
          0.8444
        ],
        [
          0.6865,
          0.8389
        ],
        [
          0.6965,
          0.833
        ],
        [
          0.7066,
          0.828
        ],
        [
          0.717,
          0.8248
        ],
        [
          0.727,
          0.8189
        ],
        [
          0.737,
          0.8148
        ],
        [
          0.747,
          0.8116
        ],
        [
          0.7571,
          0.8062
        ],
        [
          0.7675,
          0.8003
        ],
        [
          0.7775,
          0.7939
        ],
        [
          0.7875,
          0.7889
        ],
        [
          0.7975,
          0.7834
        ],
        [
          0.8076,
          0.7784
        ],
        [
          0.818,
          0.773
        ],
        [
          0.828,
          0.7666
        ],
        [
          0.838,
          0.7625
        ],
        [
          0.848,
          0.7575
        ],
        [
          0.8581,
          0.7552
        ],
        [
          0.8685,
          0.7498
        ],
        [
          0.8785,
          0.7448
        ],
        [
          0.8885,
          0.7384
        ],
        [
          0.8985,
          0.732
        ],
        [
          0.9086,
          0.7266
        ],
        [
          0.919,
          0.7207
        ],
        [
          0.929,
          0.7152
        ],
        [
          0.939,
          0.7093
        ],
        [
          0.949,
          0.7025
        ],
        [
          0.9591,
          0.697
        ],
        [
          0.9695,
          0.6929
        ],
        [
          0.9795,
          0.6897
        ],
        [
          0.9895,
          0.6843
        ],
        [
          0.9995,
          0.6779
        ],
        [
          1.0,
          0.6779
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.26430125173924374,
        "difference": -0.0021948380333699233,
        "worse": 0.6266666666666667,
        "kept_right": 0.26043300629971133,
        "kept_wrong": 0.0038682454395324545,
        "handed_off": 0.7356987482607559
      },
      "usd_per_1000": 0.19759686842077656
    },
    {
      "decision": 269,
      "frontier": 274,
      "frontier_accuracy": 0.9085532302092811,
      "in_sample": {
        "threshold": 0.9610674971418556,
        "answered": 0.3471337579617834,
        "accuracy": 0.9085532302092811
      },
      "curve": [
        [
          0.0,
          0.9086
        ],
        [
          0.01,
          0.9086
        ],
        [
          0.02,
          0.9086
        ],
        [
          0.03,
          0.9086
        ],
        [
          0.04,
          0.9086
        ],
        [
          0.05,
          0.9086
        ],
        [
          0.0601,
          0.9086
        ],
        [
          0.0705,
          0.9086
        ],
        [
          0.0805,
          0.9086
        ],
        [
          0.0905,
          0.9086
        ],
        [
          0.1005,
          0.9086
        ],
        [
          0.1106,
          0.909
        ],
        [
          0.121,
          0.909
        ],
        [
          0.131,
          0.909
        ],
        [
          0.141,
          0.909
        ],
        [
          0.151,
          0.909
        ],
        [
          0.1611,
          0.909
        ],
        [
          0.1715,
          0.909
        ],
        [
          0.1815,
          0.909
        ],
        [
          0.1915,
          0.9095
        ],
        [
          0.2015,
          0.9099
        ],
        [
          0.2116,
          0.9095
        ],
        [
          0.222,
          0.9095
        ],
        [
          0.232,
          0.9095
        ],
        [
          0.242,
          0.9095
        ],
        [
          0.252,
          0.9095
        ],
        [
          0.2621,
          0.9086
        ],
        [
          0.2725,
          0.9081
        ],
        [
          0.2825,
          0.9081
        ],
        [
          0.2925,
          0.9081
        ],
        [
          0.3025,
          0.9081
        ],
        [
          0.3126,
          0.9081
        ],
        [
          0.323,
          0.9081
        ],
        [
          0.333,
          0.9086
        ],
        [
          0.343,
          0.9081
        ],
        [
          0.353,
          0.9067
        ],
        [
          0.3631,
          0.9058
        ],
        [
          0.3735,
          0.9054
        ],
        [
          0.3835,
          0.904
        ],
        [
          0.3935,
          0.9035
        ],
        [
          0.4035,
          0.9013
        ],
        [
          0.4136,
          0.9004
        ],
        [
          0.424,
          0.899
        ],
        [
          0.434,
          0.8967
        ],
        [
          0.444,
          0.8963
        ],
        [
          0.454,
          0.8935
        ],
        [
          0.4641,
          0.8904
        ],
        [
          0.4745,
          0.8876
        ],
        [
          0.4845,
          0.8872
        ],
        [
          0.4945,
          0.8849
        ],
        [
          0.5045,
          0.8803
        ],
        [
          0.5146,
          0.8785
        ],
        [
          0.525,
          0.8758
        ],
        [
          0.535,
          0.8735
        ],
        [
          0.545,
          0.8703
        ],
        [
          0.5551,
          0.8676
        ],
        [
          0.5655,
          0.8653
        ],
        [
          0.5755,
          0.8635
        ],
        [
          0.5855,
          0.8608
        ],
        [
          0.5955,
          0.8576
        ],
        [
          0.6056,
          0.8553
        ],
        [
          0.616,
          0.853
        ],
        [
          0.626,
          0.8512
        ],
        [
          0.636,
          0.8467
        ],
        [
          0.646,
          0.8439
        ],
        [
          0.6561,
          0.8394
        ],
        [
          0.6665,
          0.8348
        ],
        [
          0.6765,
          0.8308
        ],
        [
          0.6865,
          0.8267
        ],
        [
          0.6965,
          0.8212
        ],
        [
          0.7066,
          0.8162
        ],
        [
          0.717,
          0.8126
        ],
        [
          0.727,
          0.8071
        ],
        [
          0.737,
          0.8021
        ],
        [
          0.747,
          0.7985
        ],
        [
          0.7571,
          0.7921
        ],
        [
          0.7675,
          0.7866
        ],
        [
          0.7775,
          0.7816
        ],
        [
          0.7875,
          0.7762
        ],
        [
          0.7975,
          0.7702
        ],
        [
          0.8076,
          0.7648
        ],
        [
          0.818,
          0.7602
        ],
        [
          0.828,
          0.7534
        ],
        [
          0.838,
          0.7502
        ],
        [
          0.848,
          0.7457
        ],
        [
          0.8581,
          0.7416
        ],
        [
          0.8685,
          0.7361
        ],
        [
          0.8785,
          0.7311
        ],
        [
          0.8885,
          0.7261
        ],
        [
          0.8985,
          0.7207
        ],
        [
          0.9086,
          0.7147
        ],
        [
          0.919,
          0.7097
        ],
        [
          0.929,
          0.7056
        ],
        [
          0.939,
          0.7011
        ],
        [
          0.949,
          0.6965
        ],
        [
          0.9591,
          0.6915
        ],
        [
          0.9695,
          0.6879
        ],
        [
          0.9795,
          0.6843
        ],
        [
          0.9895,
          0.6815
        ],
        [
          0.9995,
          0.6779
        ],
        [
          1.0,
          0.6779
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.32608908718219154,
        "difference": -0.0018225598958788907,
        "worse": 0.54,
        "kept_right": 0.3181758424510377,
        "kept_wrong": 0.007913244731153903,
        "handed_off": 0.6739109128178083
      },
      "usd_per_1000": 0.28725161387081477
    },
    {
      "decision": 269,
      "frontier": 47,
      "frontier_accuracy": 0.9044585987261147,
      "in_sample": {
        "threshold": 0.9587413257899685,
        "answered": 0.3525932666060055,
        "accuracy": 0.9044585987261147
      },
      "curve": [
        [
          0.0,
          0.9045
        ],
        [
          0.01,
          0.9045
        ],
        [
          0.02,
          0.9045
        ],
        [
          0.03,
          0.9045
        ],
        [
          0.04,
          0.9045
        ],
        [
          0.05,
          0.9045
        ],
        [
          0.0601,
          0.9045
        ],
        [
          0.0705,
          0.9045
        ],
        [
          0.0805,
          0.9045
        ],
        [
          0.0905,
          0.9045
        ],
        [
          0.1005,
          0.9045
        ],
        [
          0.1106,
          0.9045
        ],
        [
          0.121,
          0.9045
        ],
        [
          0.131,
          0.9045
        ],
        [
          0.141,
          0.9045
        ],
        [
          0.151,
          0.9045
        ],
        [
          0.1611,
          0.9049
        ],
        [
          0.1715,
          0.9049
        ],
        [
          0.1815,
          0.9049
        ],
        [
          0.1915,
          0.9054
        ],
        [
          0.2015,
          0.9054
        ],
        [
          0.2116,
          0.9049
        ],
        [
          0.222,
          0.9049
        ],
        [
          0.232,
          0.9054
        ],
        [
          0.242,
          0.9049
        ],
        [
          0.252,
          0.9049
        ],
        [
          0.2621,
          0.9045
        ],
        [
          0.2725,
          0.904
        ],
        [
          0.2825,
          0.904
        ],
        [
          0.2925,
          0.9035
        ],
        [
          0.3025,
          0.9035
        ],
        [
          0.3126,
          0.9035
        ],
        [
          0.323,
          0.9031
        ],
        [
          0.333,
          0.9035
        ],
        [
          0.343,
          0.904
        ],
        [
          0.353,
          0.904
        ],
        [
          0.3631,
          0.9031
        ],
        [
          0.3735,
          0.9022
        ],
        [
          0.3835,
          0.9008
        ],
        [
          0.3935,
          0.8995
        ],
        [
          0.4035,
          0.8967
        ],
        [
          0.4136,
          0.8963
        ],
        [
          0.424,
          0.894
        ],
        [
          0.434,
          0.8922
        ],
        [
          0.444,
          0.8913
        ],
        [
          0.454,
          0.889
        ],
        [
          0.4641,
          0.8863
        ],
        [
          0.4745,
          0.8831
        ],
        [
          0.4845,
          0.8822
        ],
        [
          0.4945,
          0.8799
        ],
        [
          0.5045,
          0.8753
        ],
        [
          0.5146,
          0.8744
        ],
        [
          0.525,
          0.8712
        ],
        [
          0.535,
          0.8708
        ],
        [
          0.545,
          0.8681
        ],
        [
          0.5551,
          0.8653
        ],
        [
          0.5655,
          0.8631
        ],
        [
          0.5755,
          0.8612
        ],
        [
          0.5855,
          0.8585
        ],
        [
          0.5955,
          0.8558
        ],
        [
          0.6056,
          0.853
        ],
        [
          0.616,
          0.8499
        ],
        [
          0.626,
          0.8471
        ],
        [
          0.636,
          0.8421
        ],
        [
          0.646,
          0.8389
        ],
        [
          0.6561,
          0.8344
        ],
        [
          0.6665,
          0.8312
        ],
        [
          0.6765,
          0.8262
        ],
        [
          0.6865,
          0.8212
        ],
        [
          0.6965,
          0.8167
        ],
        [
          0.7066,
          0.8121
        ],
        [
          0.717,
          0.808
        ],
        [
          0.727,
          0.8035
        ],
        [
          0.737,
          0.798
        ],
        [
          0.747,
          0.7953
        ],
        [
          0.7571,
          0.7903
        ],
        [
          0.7675,
          0.7848
        ],
        [
          0.7775,
          0.7784
        ],
        [
          0.7875,
          0.7725
        ],
        [
          0.7975,
          0.7657
        ],
        [
          0.8076,
          0.763
        ],
        [
          0.818,
          0.7575
        ],
        [
          0.828,
          0.7516
        ],
        [
          0.838,
          0.747
        ],
        [
          0.848,
          0.742
        ],
        [
          0.8581,
          0.7398
        ],
        [
          0.8685,
          0.7352
        ],
        [
          0.8785,
          0.7311
        ],
        [
          0.8885,
          0.7247
        ],
        [
          0.8985,
          0.7211
        ],
        [
          0.9086,
          0.7157
        ],
        [
          0.919,
          0.7102
        ],
        [
          0.929,
          0.7061
        ],
        [
          0.939,
          0.7011
        ],
        [
          0.949,
          0.6961
        ],
        [
          0.9591,
          0.692
        ],
        [
          0.9695,
          0.6879
        ],
        [
          0.9795,
          0.6856
        ],
        [
          0.9895,
          0.682
        ],
        [
          0.9995,
          0.6774
        ],
        [
          1.0,
          0.6779
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.33172464821168757,
        "difference": -0.0019373890641329133,
        "worse": 0.5133333333333333,
        "kept_right": 0.3226202499232796,
        "kept_wrong": 0.009104398288408118,
        "handed_off": 0.6682753517883123
      },
      "usd_per_1000": 3.2949890755293056
    },
    {
      "decision": 269,
      "frontier": 193,
      "frontier_accuracy": 0.89171974522293,
      "in_sample": {
        "threshold": 0.9794597561579199,
        "answered": 0.25204731574158323,
        "accuracy": 0.89171974522293
      },
      "curve": [
        [
          0.0,
          0.8917
        ],
        [
          0.01,
          0.8917
        ],
        [
          0.02,
          0.8917
        ],
        [
          0.03,
          0.8917
        ],
        [
          0.04,
          0.8917
        ],
        [
          0.05,
          0.8917
        ],
        [
          0.0601,
          0.8917
        ],
        [
          0.0705,
          0.8917
        ],
        [
          0.0805,
          0.8917
        ],
        [
          0.0905,
          0.8917
        ],
        [
          0.1005,
          0.8917
        ],
        [
          0.1106,
          0.8917
        ],
        [
          0.121,
          0.8917
        ],
        [
          0.131,
          0.8913
        ],
        [
          0.141,
          0.8913
        ],
        [
          0.151,
          0.8913
        ],
        [
          0.1611,
          0.8913
        ],
        [
          0.1715,
          0.8913
        ],
        [
          0.1815,
          0.8913
        ],
        [
          0.1915,
          0.8917
        ],
        [
          0.2015,
          0.8917
        ],
        [
          0.2116,
          0.8913
        ],
        [
          0.222,
          0.8917
        ],
        [
          0.232,
          0.8922
        ],
        [
          0.242,
          0.8917
        ],
        [
          0.252,
          0.8917
        ],
        [
          0.2621,
          0.8913
        ],
        [
          0.2725,
          0.8908
        ],
        [
          0.2825,
          0.8908
        ],
        [
          0.2925,
          0.8908
        ],
        [
          0.3025,
          0.8908
        ],
        [
          0.3126,
          0.8908
        ],
        [
          0.323,
          0.8904
        ],
        [
          0.333,
          0.8904
        ],
        [
          0.343,
          0.8899
        ],
        [
          0.353,
          0.889
        ],
        [
          0.3631,
          0.8885
        ],
        [
          0.3735,
          0.8894
        ],
        [
          0.3835,
          0.8885
        ],
        [
          0.3935,
          0.8876
        ],
        [
          0.4035,
          0.8863
        ],
        [
          0.4136,
          0.8863
        ],
        [
          0.424,
          0.8854
        ],
        [
          0.434,
          0.8835
        ],
        [
          0.444,
          0.8826
        ],
        [
          0.454,
          0.8808
        ],
        [
          0.4641,
          0.8781
        ],
        [
          0.4745,
          0.8763
        ],
        [
          0.4845,
          0.8744
        ],
        [
          0.4945,
          0.8731
        ],
        [
          0.5045,
          0.8699
        ],
        [
          0.5146,
          0.8694
        ],
        [
          0.525,
          0.8662
        ],
        [
          0.535,
          0.8644
        ],
        [
          0.545,
          0.8626
        ],
        [
          0.5551,
          0.8585
        ],
        [
          0.5655,
          0.8558
        ],
        [
          0.5755,
          0.8562
        ],
        [
          0.5855,
          0.854
        ],
        [
          0.5955,
          0.8517
        ],
        [
          0.6056,
          0.8494
        ],
        [
          0.616,
          0.848
        ],
        [
          0.626,
          0.8471
        ],
        [
          0.636,
          0.843
        ],
        [
          0.646,
          0.8399
        ],
        [
          0.6561,
          0.8353
        ],
        [
          0.6665,
          0.8303
        ],
        [
          0.6765,
          0.8253
        ],
        [
          0.6865,
          0.8207
        ],
        [
          0.6965,
          0.8153
        ],
        [
          0.7066,
          0.8107
        ],
        [
          0.717,
          0.8071
        ],
        [
          0.727,
          0.8021
        ],
        [
          0.737,
          0.7994
        ],
        [
          0.747,
          0.7962
        ],
        [
          0.7571,
          0.7912
        ],
        [
          0.7675,
          0.7862
        ],
        [
          0.7775,
          0.7807
        ],
        [
          0.7875,
          0.7753
        ],
        [
          0.7975,
          0.7689
        ],
        [
          0.8076,
          0.7648
        ],
        [
          0.818,
          0.7602
        ],
        [
          0.828,
          0.7543
        ],
        [
          0.838,
          0.7502
        ],
        [
          0.848,
          0.7457
        ],
        [
          0.8581,
          0.7425
        ],
        [
          0.8685,
          0.7379
        ],
        [
          0.8785,
          0.7325
        ],
        [
          0.8885,
          0.727
        ],
        [
          0.8985,
          0.7234
        ],
        [
          0.9086,
          0.7179
        ],
        [
          0.919,
          0.7116
        ],
        [
          0.929,
          0.707
        ],
        [
          0.939,
          0.702
        ],
        [
          0.949,
          0.6965
        ],
        [
          0.9591,
          0.6915
        ],
        [
          0.9695,
          0.6879
        ],
        [
          0.9795,
          0.6847
        ],
        [
          0.9895,
          0.6811
        ],
        [
          0.9995,
          0.6774
        ],
        [
          1.0,
          0.6779
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.29411076988631873,
        "difference": -0.0025739492576829187,
        "worse": 0.6666666666666666,
        "kept_right": 0.2870165077735764,
        "kept_wrong": 0.007094262112742262,
        "handed_off": 0.7058892301136819
      },
      "usd_per_1000": 5.468940333913442
    },
    {
      "decision": 269,
      "frontier": 55,
      "frontier_accuracy": 0.8503184713375797,
      "in_sample": {
        "threshold": 0.8695209326804485,
        "answered": 0.5209281164695178,
        "accuracy": 0.8503184713375797
      },
      "curve": [
        [
          0.0,
          0.8503
        ],
        [
          0.01,
          0.8503
        ],
        [
          0.02,
          0.8508
        ],
        [
          0.03,
          0.8512
        ],
        [
          0.04,
          0.8512
        ],
        [
          0.05,
          0.8512
        ],
        [
          0.0601,
          0.8517
        ],
        [
          0.0705,
          0.8521
        ],
        [
          0.0805,
          0.8521
        ],
        [
          0.0905,
          0.8521
        ],
        [
          0.1005,
          0.8521
        ],
        [
          0.1106,
          0.8526
        ],
        [
          0.121,
          0.8526
        ],
        [
          0.131,
          0.8526
        ],
        [
          0.141,
          0.853
        ],
        [
          0.151,
          0.8535
        ],
        [
          0.1611,
          0.8535
        ],
        [
          0.1715,
          0.854
        ],
        [
          0.1815,
          0.8553
        ],
        [
          0.1915,
          0.8576
        ],
        [
          0.2015,
          0.8576
        ],
        [
          0.2116,
          0.8576
        ],
        [
          0.222,
          0.859
        ],
        [
          0.232,
          0.859
        ],
        [
          0.242,
          0.859
        ],
        [
          0.252,
          0.8599
        ],
        [
          0.2621,
          0.8594
        ],
        [
          0.2725,
          0.8603
        ],
        [
          0.2825,
          0.8612
        ],
        [
          0.2925,
          0.8617
        ],
        [
          0.3025,
          0.8626
        ],
        [
          0.3126,
          0.8631
        ],
        [
          0.323,
          0.8631
        ],
        [
          0.333,
          0.8631
        ],
        [
          0.343,
          0.8644
        ],
        [
          0.353,
          0.8644
        ],
        [
          0.3631,
          0.864
        ],
        [
          0.3735,
          0.8649
        ],
        [
          0.3835,
          0.8644
        ],
        [
          0.3935,
          0.8649
        ],
        [
          0.4035,
          0.8635
        ],
        [
          0.4136,
          0.8635
        ],
        [
          0.424,
          0.8626
        ],
        [
          0.434,
          0.8612
        ],
        [
          0.444,
          0.8608
        ],
        [
          0.454,
          0.8594
        ],
        [
          0.4641,
          0.8581
        ],
        [
          0.4745,
          0.8562
        ],
        [
          0.4845,
          0.8553
        ],
        [
          0.4945,
          0.8549
        ],
        [
          0.5045,
          0.8521
        ],
        [
          0.5146,
          0.8521
        ],
        [
          0.525,
          0.8494
        ],
        [
          0.535,
          0.848
        ],
        [
          0.545,
          0.8462
        ],
        [
          0.5551,
          0.8426
        ],
        [
          0.5655,
          0.8403
        ],
        [
          0.5755,
          0.8399
        ],
        [
          0.5855,
          0.8385
        ],
        [
          0.5955,
          0.8385
        ],
        [
          0.6056,
          0.8358
        ],
        [
          0.616,
          0.8344
        ],
        [
          0.626,
          0.8335
        ],
        [
          0.636,
          0.8303
        ],
        [
          0.646,
          0.8271
        ],
        [
          0.6561,
          0.8239
        ],
        [
          0.6665,
          0.8198
        ],
        [
          0.6765,
          0.8167
        ],
        [
          0.6865,
          0.813
        ],
        [
          0.6965,
          0.8094
        ],
        [
          0.7066,
          0.8053
        ],
        [
          0.717,
          0.8012
        ],
        [
          0.727,
          0.7962
        ],
        [
          0.737,
          0.7934
        ],
        [
          0.747,
          0.7912
        ],
        [
          0.7571,
          0.7862
        ],
        [
          0.7675,
          0.7807
        ],
        [
          0.7775,
          0.7748
        ],
        [
          0.7875,
          0.7702
        ],
        [
          0.7975,
          0.7648
        ],
        [
          0.8076,
          0.7616
        ],
        [
          0.818,
          0.7571
        ],
        [
          0.828,
          0.752
        ],
        [
          0.838,
          0.7475
        ],
        [
          0.848,
          0.7425
        ],
        [
          0.8581,
          0.7398
        ],
        [
          0.8685,
          0.7348
        ],
        [
          0.8785,
          0.7302
        ],
        [
          0.8885,
          0.7252
        ],
        [
          0.8985,
          0.722
        ],
        [
          0.9086,
          0.7161
        ],
        [
          0.919,
          0.7106
        ],
        [
          0.929,
          0.7056
        ],
        [
          0.939,
          0.7006
        ],
        [
          0.949,
          0.6961
        ],
        [
          0.9591,
          0.6911
        ],
        [
          0.9695,
          0.6888
        ],
        [
          0.9795,
          0.6852
        ],
        [
          0.9895,
          0.6806
        ],
        [
          0.9995,
          0.6774
        ],
        [
          1.0,
          0.6779
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.5261459121070838,
        "difference": -0.0014953912046142635,
        "worse": 0.49,
        "kept_right": 0.47242771182077803,
        "kept_wrong": 0.05371820028630638,
        "handed_off": 0.47385408789291594
      },
      "usd_per_1000": 1.7316209243186358
    },
    {
      "decision": 322,
      "frontier": 140,
      "frontier_accuracy": 0.9808917197452229,
      "in_sample": {
        "threshold": 0.996594163565176,
        "answered": 0.12283894449499545,
        "accuracy": 0.9808917197452229
      },
      "curve": [
        [
          0.0,
          0.9809
        ],
        [
          0.01,
          0.9809
        ],
        [
          0.02,
          0.9809
        ],
        [
          0.03,
          0.9809
        ],
        [
          0.04,
          0.9809
        ],
        [
          0.05,
          0.9809
        ],
        [
          0.0601,
          0.9809
        ],
        [
          0.0705,
          0.9809
        ],
        [
          0.0805,
          0.9809
        ],
        [
          0.0905,
          0.9809
        ],
        [
          0.1005,
          0.9809
        ],
        [
          0.1106,
          0.9809
        ],
        [
          0.121,
          0.9809
        ],
        [
          0.131,
          0.9804
        ],
        [
          0.141,
          0.9804
        ],
        [
          0.151,
          0.9804
        ],
        [
          0.1611,
          0.9804
        ],
        [
          0.1715,
          0.9804
        ],
        [
          0.1815,
          0.9804
        ],
        [
          0.1915,
          0.9804
        ],
        [
          0.2015,
          0.9804
        ],
        [
          0.2116,
          0.9791
        ],
        [
          0.222,
          0.9791
        ],
        [
          0.232,
          0.9791
        ],
        [
          0.242,
          0.9786
        ],
        [
          0.252,
          0.9777
        ],
        [
          0.2621,
          0.9773
        ],
        [
          0.2725,
          0.9759
        ],
        [
          0.2825,
          0.9759
        ],
        [
          0.2925,
          0.975
        ],
        [
          0.3025,
          0.9745
        ],
        [
          0.3126,
          0.9741
        ],
        [
          0.323,
          0.9736
        ],
        [
          0.333,
          0.9732
        ],
        [
          0.343,
          0.9718
        ],
        [
          0.353,
          0.9695
        ],
        [
          0.3631,
          0.9695
        ],
        [
          0.3735,
          0.9672
        ],
        [
          0.3835,
          0.9668
        ],
        [
          0.3935,
          0.9663
        ],
        [
          0.4035,
          0.9645
        ],
        [
          0.4136,
          0.9627
        ],
        [
          0.424,
          0.96
        ],
        [
          0.434,
          0.9577
        ],
        [
          0.444,
          0.9563
        ],
        [
          0.454,
          0.9559
        ],
        [
          0.4641,
          0.9518
        ],
        [
          0.4745,
          0.9477
        ],
        [
          0.4845,
          0.944
        ],
        [
          0.4945,
          0.9386
        ],
        [
          0.5045,
          0.9345
        ],
        [
          0.5146,
          0.9308
        ],
        [
          0.525,
          0.929
        ],
        [
          0.535,
          0.9249
        ],
        [
          0.545,
          0.9217
        ],
        [
          0.5551,
          0.9195
        ],
        [
          0.5655,
          0.9158
        ],
        [
          0.5755,
          0.9113
        ],
        [
          0.5855,
          0.9072
        ],
        [
          0.5955,
          0.9031
        ],
        [
          0.6056,
          0.8999
        ],
        [
          0.616,
          0.8958
        ],
        [
          0.626,
          0.8917
        ],
        [
          0.636,
          0.8867
        ],
        [
          0.646,
          0.8831
        ],
        [
          0.6561,
          0.8772
        ],
        [
          0.6665,
          0.8712
        ],
        [
          0.6765,
          0.8658
        ],
        [
          0.6865,
          0.8603
        ],
        [
          0.6965,
          0.8553
        ],
        [
          0.7066,
          0.8517
        ],
        [
          0.717,
          0.8462
        ],
        [
          0.727,
          0.8408
        ],
        [
          0.737,
          0.8371
        ],
        [
          0.747,
          0.8321
        ],
        [
          0.7571,
          0.828
        ],
        [
          0.7675,
          0.8244
        ],
        [
          0.7775,
          0.8194
        ],
        [
          0.7875,
          0.8116
        ],
        [
          0.7975,
          0.8071
        ],
        [
          0.8076,
          0.803
        ],
        [
          0.818,
          0.7957
        ],
        [
          0.828,
          0.7903
        ],
        [
          0.838,
          0.783
        ],
        [
          0.848,
          0.7748
        ],
        [
          0.8581,
          0.7689
        ],
        [
          0.8685,
          0.7634
        ],
        [
          0.8785,
          0.758
        ],
        [
          0.8885,
          0.752
        ],
        [
          0.8985,
          0.7429
        ],
        [
          0.9086,
          0.7375
        ],
        [
          0.919,
          0.7307
        ],
        [
          0.929,
          0.7243
        ],
        [
          0.939,
          0.7184
        ],
        [
          0.949,
          0.7106
        ],
        [
          0.9591,
          0.7066
        ],
        [
          0.9695,
          0.7011
        ],
        [
          0.9795,
          0.6938
        ],
        [
          0.9895,
          0.6861
        ],
        [
          0.9995,
          0.6779
        ],
        [
          1.0,
          0.6774
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.16664395165424695,
        "difference": -0.0008822437834908637,
        "worse": 0.5066666666666667,
        "kept_right": 0.1657556417500403,
        "kept_wrong": 0.0008883099042066659,
        "handed_off": 0.8333560483457525
      },
      "usd_per_1000": 2.0392685009137823
    },
    {
      "decision": 322,
      "frontier": 277,
      "frontier_accuracy": 0.9795268425841674,
      "in_sample": {
        "threshold": 0.9899266448524215,
        "answered": 0.2024567788898999,
        "accuracy": 0.9795268425841674
      },
      "curve": [
        [
          0.0,
          0.9795
        ],
        [
          0.01,
          0.9795
        ],
        [
          0.02,
          0.9795
        ],
        [
          0.03,
          0.9795
        ],
        [
          0.04,
          0.9795
        ],
        [
          0.05,
          0.9795
        ],
        [
          0.0601,
          0.9795
        ],
        [
          0.0705,
          0.9795
        ],
        [
          0.0805,
          0.9795
        ],
        [
          0.0905,
          0.9795
        ],
        [
          0.1005,
          0.9795
        ],
        [
          0.1106,
          0.98
        ],
        [
          0.121,
          0.98
        ],
        [
          0.131,
          0.9795
        ],
        [
          0.141,
          0.9795
        ],
        [
          0.151,
          0.9795
        ],
        [
          0.1611,
          0.9795
        ],
        [
          0.1715,
          0.9795
        ],
        [
          0.1815,
          0.9795
        ],
        [
          0.1915,
          0.9795
        ],
        [
          0.2015,
          0.9795
        ],
        [
          0.2116,
          0.9782
        ],
        [
          0.222,
          0.9782
        ],
        [
          0.232,
          0.9782
        ],
        [
          0.242,
          0.9777
        ],
        [
          0.252,
          0.9768
        ],
        [
          0.2621,
          0.9763
        ],
        [
          0.2725,
          0.975
        ],
        [
          0.2825,
          0.975
        ],
        [
          0.2925,
          0.9741
        ],
        [
          0.3025,
          0.9736
        ],
        [
          0.3126,
          0.9732
        ],
        [
          0.323,
          0.9727
        ],
        [
          0.333,
          0.9722
        ],
        [
          0.343,
          0.9709
        ],
        [
          0.353,
          0.9686
        ],
        [
          0.3631,
          0.9682
        ],
        [
          0.3735,
          0.9659
        ],
        [
          0.3835,
          0.9654
        ],
        [
          0.3935,
          0.965
        ],
        [
          0.4035,
          0.9631
        ],
        [
          0.4136,
          0.9613
        ],
        [
          0.424,
          0.9586
        ],
        [
          0.434,
          0.9568
        ],
        [
          0.444,
          0.9554
        ],
        [
          0.454,
          0.9545
        ],
        [
          0.4641,
          0.95
        ],
        [
          0.4745,
          0.9459
        ],
        [
          0.4845,
          0.9422
        ],
        [
          0.4945,
          0.9372
        ],
        [
          0.5045,
          0.9331
        ],
        [
          0.5146,
          0.9295
        ],
        [
          0.525,
          0.9277
        ],
        [
          0.535,
          0.9245
        ],
        [
          0.545,
          0.9208
        ],
        [
          0.5551,
          0.9186
        ],
        [
          0.5655,
          0.9149
        ],
        [
          0.5755,
          0.9104
        ],
        [
          0.5855,
          0.9063
        ],
        [
          0.5955,
          0.9022
        ],
        [
          0.6056,
          0.899
        ],
        [
          0.616,
          0.8954
        ],
        [
          0.626,
          0.8913
        ],
        [
          0.636,
          0.8867
        ],
        [
          0.646,
          0.8826
        ],
        [
          0.6561,
          0.8767
        ],
        [
          0.6665,
          0.8703
        ],
        [
          0.6765,
          0.8649
        ],
        [
          0.6865,
          0.8594
        ],
        [
          0.6965,
          0.8544
        ],
        [
          0.7066,
          0.8512
        ],
        [
          0.717,
          0.8458
        ],
        [
          0.727,
          0.8403
        ],
        [
          0.737,
          0.8362
        ],
        [
          0.747,
          0.8312
        ],
        [
          0.7571,
          0.8271
        ],
        [
          0.7675,
          0.823
        ],
        [
          0.7775,
          0.818
        ],
        [
          0.7875,
          0.8103
        ],
        [
          0.7975,
          0.8057
        ],
        [
          0.8076,
          0.8012
        ],
        [
          0.818,
          0.7939
        ],
        [
          0.828,
          0.7884
        ],
        [
          0.838,
          0.7816
        ],
        [
          0.848,
          0.7734
        ],
        [
          0.8581,
          0.7675
        ],
        [
          0.8685,
          0.7625
        ],
        [
          0.8785,
          0.7571
        ],
        [
          0.8885,
          0.7507
        ],
        [
          0.8985,
          0.7416
        ],
        [
          0.9086,
          0.7366
        ],
        [
          0.919,
          0.7298
        ],
        [
          0.929,
          0.7234
        ],
        [
          0.939,
          0.7179
        ],
        [
          0.949,
          0.7102
        ],
        [
          0.9591,
          0.7066
        ],
        [
          0.9695,
          0.7006
        ],
        [
          0.9795,
          0.6934
        ],
        [
          0.9895,
          0.6861
        ],
        [
          0.9995,
          0.6779
        ],
        [
          1.0,
          0.6774
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.19029168936834254,
        "difference": -0.0008186101871446448,
        "worse": 0.5033333333333333,
        "kept_right": 0.18900018045250122,
        "kept_wrong": 0.001291508915841301,
        "handed_off": 0.8097083106316569
      },
      "usd_per_1000": 4.934624581933218
    },
    {
      "decision": 322,
      "frontier": 276,
      "frontier_accuracy": 0.9777070063694268,
      "in_sample": {
        "threshold": 0.9899266448524215,
        "answered": 0.2024567788898999,
        "accuracy": 0.9777070063694268
      },
      "curve": [
        [
          0.0,
          0.9777
        ],
        [
          0.01,
          0.9777
        ],
        [
          0.02,
          0.9777
        ],
        [
          0.03,
          0.9777
        ],
        [
          0.04,
          0.9777
        ],
        [
          0.05,
          0.9777
        ],
        [
          0.0601,
          0.9777
        ],
        [
          0.0705,
          0.9777
        ],
        [
          0.0805,
          0.9777
        ],
        [
          0.0905,
          0.9777
        ],
        [
          0.1005,
          0.9777
        ],
        [
          0.1106,
          0.9782
        ],
        [
          0.121,
          0.9782
        ],
        [
          0.131,
          0.9777
        ],
        [
          0.141,
          0.9777
        ],
        [
          0.151,
          0.9777
        ],
        [
          0.1611,
          0.9777
        ],
        [
          0.1715,
          0.9777
        ],
        [
          0.1815,
          0.9777
        ],
        [
          0.1915,
          0.9777
        ],
        [
          0.2015,
          0.9777
        ],
        [
          0.2116,
          0.9768
        ],
        [
          0.222,
          0.9768
        ],
        [
          0.232,
          0.9773
        ],
        [
          0.242,
          0.9768
        ],
        [
          0.252,
          0.9759
        ],
        [
          0.2621,
          0.9754
        ],
        [
          0.2725,
          0.9741
        ],
        [
          0.2825,
          0.9741
        ],
        [
          0.2925,
          0.9732
        ],
        [
          0.3025,
          0.9727
        ],
        [
          0.3126,
          0.9722
        ],
        [
          0.323,
          0.9722
        ],
        [
          0.333,
          0.9718
        ],
        [
          0.343,
          0.9709
        ],
        [
          0.353,
          0.9686
        ],
        [
          0.3631,
          0.9682
        ],
        [
          0.3735,
          0.9659
        ],
        [
          0.3835,
          0.9654
        ],
        [
          0.3935,
          0.965
        ],
        [
          0.4035,
          0.9631
        ],
        [
          0.4136,
          0.9613
        ],
        [
          0.424,
          0.9586
        ],
        [
          0.434,
          0.9568
        ],
        [
          0.444,
          0.9554
        ],
        [
          0.454,
          0.9545
        ],
        [
          0.4641,
          0.95
        ],
        [
          0.4745,
          0.9459
        ],
        [
          0.4845,
          0.9422
        ],
        [
          0.4945,
          0.9372
        ],
        [
          0.5045,
          0.9331
        ],
        [
          0.5146,
          0.9295
        ],
        [
          0.525,
          0.9277
        ],
        [
          0.535,
          0.9245
        ],
        [
          0.545,
          0.9208
        ],
        [
          0.5551,
          0.9186
        ],
        [
          0.5655,
          0.9149
        ],
        [
          0.5755,
          0.9104
        ],
        [
          0.5855,
          0.9067
        ],
        [
          0.5955,
          0.9031
        ],
        [
          0.6056,
          0.8999
        ],
        [
          0.616,
          0.8958
        ],
        [
          0.626,
          0.8917
        ],
        [
          0.636,
          0.8872
        ],
        [
          0.646,
          0.8826
        ],
        [
          0.6561,
          0.8767
        ],
        [
          0.6665,
          0.8703
        ],
        [
          0.6765,
          0.8649
        ],
        [
          0.6865,
          0.8599
        ],
        [
          0.6965,
          0.8549
        ],
        [
          0.7066,
          0.8517
        ],
        [
          0.717,
          0.8462
        ],
        [
          0.727,
          0.8408
        ],
        [
          0.737,
          0.8367
        ],
        [
          0.747,
          0.8317
        ],
        [
          0.7571,
          0.8276
        ],
        [
          0.7675,
          0.8235
        ],
        [
          0.7775,
          0.818
        ],
        [
          0.7875,
          0.8103
        ],
        [
          0.7975,
          0.8057
        ],
        [
          0.8076,
          0.8012
        ],
        [
          0.818,
          0.7939
        ],
        [
          0.828,
          0.7884
        ],
        [
          0.838,
          0.7812
        ],
        [
          0.848,
          0.773
        ],
        [
          0.8581,
          0.7671
        ],
        [
          0.8685,
          0.7621
        ],
        [
          0.8785,
          0.7571
        ],
        [
          0.8885,
          0.7507
        ],
        [
          0.8985,
          0.7416
        ],
        [
          0.9086,
          0.7361
        ],
        [
          0.919,
          0.7293
        ],
        [
          0.929,
          0.7229
        ],
        [
          0.939,
          0.7179
        ],
        [
          0.949,
          0.7102
        ],
        [
          0.9591,
          0.7066
        ],
        [
          0.9695,
          0.7006
        ],
        [
          0.9795,
          0.6934
        ],
        [
          0.9895,
          0.6861
        ],
        [
          0.9995,
          0.6779
        ],
        [
          1.0,
          0.6774
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.20642620262404054,
        "difference": -0.0011369496248990025,
        "worse": 0.5933333333333334,
        "kept_right": 0.20436467585487128,
        "kept_wrong": 0.0020615267691693278,
        "handed_off": 0.7935737973759593
      },
      "usd_per_1000": 1.3106218648794308
    },
    {
      "decision": 322,
      "frontier": 48,
      "frontier_accuracy": 0.9758871701546861,
      "in_sample": {
        "threshold": 0.9899266448524215,
        "answered": 0.2024567788898999,
        "accuracy": 0.9758871701546861
      },
      "curve": [
        [
          0.0,
          0.9759
        ],
        [
          0.01,
          0.9759
        ],
        [
          0.02,
          0.9759
        ],
        [
          0.03,
          0.9759
        ],
        [
          0.04,
          0.9759
        ],
        [
          0.05,
          0.9759
        ],
        [
          0.0601,
          0.9759
        ],
        [
          0.0705,
          0.9759
        ],
        [
          0.0805,
          0.9759
        ],
        [
          0.0905,
          0.9759
        ],
        [
          0.1005,
          0.9759
        ],
        [
          0.1106,
          0.9763
        ],
        [
          0.121,
          0.9763
        ],
        [
          0.131,
          0.9759
        ],
        [
          0.141,
          0.9759
        ],
        [
          0.151,
          0.9759
        ],
        [
          0.1611,
          0.9759
        ],
        [
          0.1715,
          0.9759
        ],
        [
          0.1815,
          0.9759
        ],
        [
          0.1915,
          0.9759
        ],
        [
          0.2015,
          0.9759
        ],
        [
          0.2116,
          0.9745
        ],
        [
          0.222,
          0.9745
        ],
        [
          0.232,
          0.9745
        ],
        [
          0.242,
          0.9741
        ],
        [
          0.252,
          0.9732
        ],
        [
          0.2621,
          0.9727
        ],
        [
          0.2725,
          0.9713
        ],
        [
          0.2825,
          0.9713
        ],
        [
          0.2925,
          0.9704
        ],
        [
          0.3025,
          0.97
        ],
        [
          0.3126,
          0.9695
        ],
        [
          0.323,
          0.9695
        ],
        [
          0.333,
          0.9691
        ],
        [
          0.343,
          0.9677
        ],
        [
          0.353,
          0.9654
        ],
        [
          0.3631,
          0.965
        ],
        [
          0.3735,
          0.9627
        ],
        [
          0.3835,
          0.9622
        ],
        [
          0.3935,
          0.9618
        ],
        [
          0.4035,
          0.96
        ],
        [
          0.4136,
          0.9581
        ],
        [
          0.424,
          0.9559
        ],
        [
          0.434,
          0.954
        ],
        [
          0.444,
          0.9527
        ],
        [
          0.454,
          0.9518
        ],
        [
          0.4641,
          0.9472
        ],
        [
          0.4745,
          0.9431
        ],
        [
          0.4845,
          0.9395
        ],
        [
          0.4945,
          0.9345
        ],
        [
          0.5045,
          0.9304
        ],
        [
          0.5146,
          0.9268
        ],
        [
          0.525,
          0.9249
        ],
        [
          0.535,
          0.9217
        ],
        [
          0.545,
          0.9186
        ],
        [
          0.5551,
          0.9163
        ],
        [
          0.5655,
          0.9126
        ],
        [
          0.5755,
          0.9081
        ],
        [
          0.5855,
          0.904
        ],
        [
          0.5955,
          0.8999
        ],
        [
          0.6056,
          0.8967
        ],
        [
          0.616,
          0.8931
        ],
        [
          0.626,
          0.889
        ],
        [
          0.636,
          0.8844
        ],
        [
          0.646,
          0.8803
        ],
        [
          0.6561,
          0.8744
        ],
        [
          0.6665,
          0.8681
        ],
        [
          0.6765,
          0.8631
        ],
        [
          0.6865,
          0.8576
        ],
        [
          0.6965,
          0.8526
        ],
        [
          0.7066,
          0.8494
        ],
        [
          0.717,
          0.8439
        ],
        [
          0.727,
          0.8385
        ],
        [
          0.737,
          0.8344
        ],
        [
          0.747,
          0.8294
        ],
        [
          0.7571,
          0.8253
        ],
        [
          0.7675,
          0.8207
        ],
        [
          0.7775,
          0.8153
        ],
        [
          0.7875,
          0.8085
        ],
        [
          0.7975,
          0.8039
        ],
        [
          0.8076,
          0.7994
        ],
        [
          0.818,
          0.7921
        ],
        [
          0.828,
          0.7866
        ],
        [
          0.838,
          0.7803
        ],
        [
          0.848,
          0.7725
        ],
        [
          0.8581,
          0.7666
        ],
        [
          0.8685,
          0.7616
        ],
        [
          0.8785,
          0.7566
        ],
        [
          0.8885,
          0.7502
        ],
        [
          0.8985,
          0.7411
        ],
        [
          0.9086,
          0.7366
        ],
        [
          0.919,
          0.7293
        ],
        [
          0.929,
          0.7229
        ],
        [
          0.939,
          0.7179
        ],
        [
          0.949,
          0.7102
        ],
        [
          0.9591,
          0.7061
        ],
        [
          0.9695,
          0.7002
        ],
        [
          0.9795,
          0.6929
        ],
        [
          0.9895,
          0.6861
        ],
        [
          0.9995,
          0.6779
        ],
        [
          1.0,
          0.6774
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.19054926512591833,
        "difference": -0.0008489132174476755,
        "worse": 0.5033333333333333,
        "kept_right": 0.18922139257371332,
        "kept_wrong": 0.0013278725522049377,
        "handed_off": 0.8094507348740811
      },
      "usd_per_1000": 4.376703435120796
    },
    {
      "decision": 322,
      "frontier": 191,
      "frontier_accuracy": 0.9740673339399454,
      "in_sample": {
        "threshold": 0.996594163565176,
        "answered": 0.12283894449499545,
        "accuracy": 0.9740673339399454
      },
      "curve": [
        [
          0.0,
          0.9741
        ],
        [
          0.01,
          0.9741
        ],
        [
          0.02,
          0.9741
        ],
        [
          0.03,
          0.9741
        ],
        [
          0.04,
          0.9741
        ],
        [
          0.05,
          0.9741
        ],
        [
          0.0601,
          0.9741
        ],
        [
          0.0705,
          0.9741
        ],
        [
          0.0805,
          0.9741
        ],
        [
          0.0905,
          0.9741
        ],
        [
          0.1005,
          0.9741
        ],
        [
          0.1106,
          0.9741
        ],
        [
          0.121,
          0.9741
        ],
        [
          0.131,
          0.9736
        ],
        [
          0.141,
          0.9736
        ],
        [
          0.151,
          0.9736
        ],
        [
          0.1611,
          0.9736
        ],
        [
          0.1715,
          0.9736
        ],
        [
          0.1815,
          0.9736
        ],
        [
          0.1915,
          0.9736
        ],
        [
          0.2015,
          0.9736
        ],
        [
          0.2116,
          0.9722
        ],
        [
          0.222,
          0.9722
        ],
        [
          0.232,
          0.9722
        ],
        [
          0.242,
          0.9718
        ],
        [
          0.252,
          0.9709
        ],
        [
          0.2621,
          0.9704
        ],
        [
          0.2725,
          0.9691
        ],
        [
          0.2825,
          0.9691
        ],
        [
          0.2925,
          0.9682
        ],
        [
          0.3025,
          0.9677
        ],
        [
          0.3126,
          0.9672
        ],
        [
          0.323,
          0.9672
        ],
        [
          0.333,
          0.9668
        ],
        [
          0.343,
          0.9654
        ],
        [
          0.353,
          0.9631
        ],
        [
          0.3631,
          0.9627
        ],
        [
          0.3735,
          0.9613
        ],
        [
          0.3835,
          0.9609
        ],
        [
          0.3935,
          0.9604
        ],
        [
          0.4035,
          0.9586
        ],
        [
          0.4136,
          0.9568
        ],
        [
          0.424,
          0.9545
        ],
        [
          0.434,
          0.9522
        ],
        [
          0.444,
          0.9509
        ],
        [
          0.454,
          0.95
        ],
        [
          0.4641,
          0.9454
        ],
        [
          0.4745,
          0.9413
        ],
        [
          0.4845,
          0.9377
        ],
        [
          0.4945,
          0.9327
        ],
        [
          0.5045,
          0.9286
        ],
        [
          0.5146,
          0.9249
        ],
        [
          0.525,
          0.9231
        ],
        [
          0.535,
          0.9195
        ],
        [
          0.545,
          0.9158
        ],
        [
          0.5551,
          0.9136
        ],
        [
          0.5655,
          0.9104
        ],
        [
          0.5755,
          0.9058
        ],
        [
          0.5855,
          0.9022
        ],
        [
          0.5955,
          0.8985
        ],
        [
          0.6056,
          0.8954
        ],
        [
          0.616,
          0.8917
        ],
        [
          0.626,
          0.8876
        ],
        [
          0.636,
          0.8835
        ],
        [
          0.646,
          0.8794
        ],
        [
          0.6561,
          0.8735
        ],
        [
          0.6665,
          0.8672
        ],
        [
          0.6765,
          0.8626
        ],
        [
          0.6865,
          0.8576
        ],
        [
          0.6965,
          0.8526
        ],
        [
          0.7066,
          0.849
        ],
        [
          0.717,
          0.8435
        ],
        [
          0.727,
          0.838
        ],
        [
          0.737,
          0.8339
        ],
        [
          0.747,
          0.8289
        ],
        [
          0.7571,
          0.8248
        ],
        [
          0.7675,
          0.8203
        ],
        [
          0.7775,
          0.8148
        ],
        [
          0.7875,
          0.8076
        ],
        [
          0.7975,
          0.803
        ],
        [
          0.8076,
          0.7985
        ],
        [
          0.818,
          0.7916
        ],
        [
          0.828,
          0.7862
        ],
        [
          0.838,
          0.7793
        ],
        [
          0.848,
          0.7712
        ],
        [
          0.8581,
          0.7652
        ],
        [
          0.8685,
          0.7602
        ],
        [
          0.8785,
          0.7552
        ],
        [
          0.8885,
          0.7493
        ],
        [
          0.8985,
          0.7407
        ],
        [
          0.9086,
          0.7361
        ],
        [
          0.919,
          0.7293
        ],
        [
          0.929,
          0.7229
        ],
        [
          0.939,
          0.7175
        ],
        [
          0.949,
          0.7097
        ],
        [
          0.9591,
          0.7061
        ],
        [
          0.9695,
          0.7006
        ],
        [
          0.9795,
          0.6934
        ],
        [
          0.9895,
          0.6861
        ],
        [
          0.9995,
          0.6779
        ],
        [
          1.0,
          0.6774
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.16664395165424695,
        "difference": -0.0008822437834908637,
        "worse": 0.5066666666666667,
        "kept_right": 0.1657556417500403,
        "kept_wrong": 0.0008883099042066659,
        "handed_off": 0.8333560483457525
      },
      "usd_per_1000": 22.305315002543043
    },
    {
      "decision": 322,
      "frontier": 188,
      "frontier_accuracy": 0.9713375796178344,
      "in_sample": {
        "threshold": 0.996594163565176,
        "answered": 0.12283894449499545,
        "accuracy": 0.9713375796178344
      },
      "curve": [
        [
          0.0,
          0.9713
        ],
        [
          0.01,
          0.9713
        ],
        [
          0.02,
          0.9713
        ],
        [
          0.03,
          0.9713
        ],
        [
          0.04,
          0.9713
        ],
        [
          0.05,
          0.9713
        ],
        [
          0.0601,
          0.9713
        ],
        [
          0.0705,
          0.9713
        ],
        [
          0.0805,
          0.9713
        ],
        [
          0.0905,
          0.9713
        ],
        [
          0.1005,
          0.9713
        ],
        [
          0.1106,
          0.9713
        ],
        [
          0.121,
          0.9713
        ],
        [
          0.131,
          0.9709
        ],
        [
          0.141,
          0.9709
        ],
        [
          0.151,
          0.9709
        ],
        [
          0.1611,
          0.9709
        ],
        [
          0.1715,
          0.9709
        ],
        [
          0.1815,
          0.9709
        ],
        [
          0.1915,
          0.9709
        ],
        [
          0.2015,
          0.9709
        ],
        [
          0.2116,
          0.9695
        ],
        [
          0.222,
          0.9695
        ],
        [
          0.232,
          0.9695
        ],
        [
          0.242,
          0.9691
        ],
        [
          0.252,
          0.9686
        ],
        [
          0.2621,
          0.9682
        ],
        [
          0.2725,
          0.9668
        ],
        [
          0.2825,
          0.9668
        ],
        [
          0.2925,
          0.9659
        ],
        [
          0.3025,
          0.9654
        ],
        [
          0.3126,
          0.9654
        ],
        [
          0.323,
          0.9654
        ],
        [
          0.333,
          0.965
        ],
        [
          0.343,
          0.9636
        ],
        [
          0.353,
          0.9613
        ],
        [
          0.3631,
          0.9613
        ],
        [
          0.3735,
          0.9586
        ],
        [
          0.3835,
          0.9581
        ],
        [
          0.3935,
          0.9577
        ],
        [
          0.4035,
          0.9559
        ],
        [
          0.4136,
          0.9545
        ],
        [
          0.424,
          0.9522
        ],
        [
          0.434,
          0.9504
        ],
        [
          0.444,
          0.949
        ],
        [
          0.454,
          0.9486
        ],
        [
          0.4641,
          0.9445
        ],
        [
          0.4745,
          0.9409
        ],
        [
          0.4845,
          0.9377
        ],
        [
          0.4945,
          0.9327
        ],
        [
          0.5045,
          0.9286
        ],
        [
          0.5146,
          0.9249
        ],
        [
          0.525,
          0.9231
        ],
        [
          0.535,
          0.9195
        ],
        [
          0.545,
          0.9163
        ],
        [
          0.5551,
          0.914
        ],
        [
          0.5655,
          0.9108
        ],
        [
          0.5755,
          0.9063
        ],
        [
          0.5855,
          0.9026
        ],
        [
          0.5955,
          0.899
        ],
        [
          0.6056,
          0.8958
        ],
        [
          0.616,
          0.8917
        ],
        [
          0.626,
          0.8876
        ],
        [
          0.636,
          0.8831
        ],
        [
          0.646,
          0.8794
        ],
        [
          0.6561,
          0.8735
        ],
        [
          0.6665,
          0.8676
        ],
        [
          0.6765,
          0.8626
        ],
        [
          0.6865,
          0.8571
        ],
        [
          0.6965,
          0.8526
        ],
        [
          0.7066,
          0.8494
        ],
        [
          0.717,
          0.8439
        ],
        [
          0.727,
          0.8385
        ],
        [
          0.737,
          0.8344
        ],
        [
          0.747,
          0.8294
        ],
        [
          0.7571,
          0.8253
        ],
        [
          0.7675,
          0.8217
        ],
        [
          0.7775,
          0.8167
        ],
        [
          0.7875,
          0.8089
        ],
        [
          0.7975,
          0.8044
        ],
        [
          0.8076,
          0.7998
        ],
        [
          0.818,
          0.7925
        ],
        [
          0.828,
          0.7871
        ],
        [
          0.838,
          0.7798
        ],
        [
          0.848,
          0.7716
        ],
        [
          0.8581,
          0.7662
        ],
        [
          0.8685,
          0.7607
        ],
        [
          0.8785,
          0.7552
        ],
        [
          0.8885,
          0.7489
        ],
        [
          0.8985,
          0.7402
        ],
        [
          0.9086,
          0.7352
        ],
        [
          0.919,
          0.7279
        ],
        [
          0.929,
          0.7216
        ],
        [
          0.939,
          0.7166
        ],
        [
          0.949,
          0.7093
        ],
        [
          0.9591,
          0.7061
        ],
        [
          0.9695,
          0.7006
        ],
        [
          0.9795,
          0.6934
        ],
        [
          0.9895,
          0.6861
        ],
        [
          0.9995,
          0.6779
        ],
        [
          1.0,
          0.6774
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1694515743306792,
        "difference": -0.0010095581014098506,
        "worse": 0.5166666666666667,
        "kept_right": 0.16840563602889144,
        "kept_wrong": 0.0010459383017878388,
        "handed_off": 0.8305484256693201
      },
      "usd_per_1000": 0.8234776847341673
    },
    {
      "decision": 322,
      "frontier": 190,
      "frontier_accuracy": 0.9672429481346679,
      "in_sample": {
        "threshold": 0.9890878122232423,
        "answered": 0.20746132848043677,
        "accuracy": 0.9672429481346679
      },
      "curve": [
        [
          0.0,
          0.9672
        ],
        [
          0.01,
          0.9672
        ],
        [
          0.02,
          0.9672
        ],
        [
          0.03,
          0.9672
        ],
        [
          0.04,
          0.9672
        ],
        [
          0.05,
          0.9672
        ],
        [
          0.0601,
          0.9672
        ],
        [
          0.0705,
          0.9672
        ],
        [
          0.0805,
          0.9672
        ],
        [
          0.0905,
          0.9672
        ],
        [
          0.1005,
          0.9672
        ],
        [
          0.1106,
          0.9677
        ],
        [
          0.121,
          0.9677
        ],
        [
          0.131,
          0.9672
        ],
        [
          0.141,
          0.9672
        ],
        [
          0.151,
          0.9672
        ],
        [
          0.1611,
          0.9672
        ],
        [
          0.1715,
          0.9672
        ],
        [
          0.1815,
          0.9677
        ],
        [
          0.1915,
          0.9677
        ],
        [
          0.2015,
          0.9677
        ],
        [
          0.2116,
          0.9663
        ],
        [
          0.222,
          0.9663
        ],
        [
          0.232,
          0.9663
        ],
        [
          0.242,
          0.9659
        ],
        [
          0.252,
          0.965
        ],
        [
          0.2621,
          0.9645
        ],
        [
          0.2725,
          0.9631
        ],
        [
          0.2825,
          0.9631
        ],
        [
          0.2925,
          0.9622
        ],
        [
          0.3025,
          0.9618
        ],
        [
          0.3126,
          0.9618
        ],
        [
          0.323,
          0.9618
        ],
        [
          0.333,
          0.9613
        ],
        [
          0.343,
          0.9604
        ],
        [
          0.353,
          0.9581
        ],
        [
          0.3631,
          0.9577
        ],
        [
          0.3735,
          0.9559
        ],
        [
          0.3835,
          0.9559
        ],
        [
          0.3935,
          0.9554
        ],
        [
          0.4035,
          0.9536
        ],
        [
          0.4136,
          0.9518
        ],
        [
          0.424,
          0.949
        ],
        [
          0.434,
          0.9468
        ],
        [
          0.444,
          0.9454
        ],
        [
          0.454,
          0.9445
        ],
        [
          0.4641,
          0.9399
        ],
        [
          0.4745,
          0.9359
        ],
        [
          0.4845,
          0.9322
        ],
        [
          0.4945,
          0.9272
        ],
        [
          0.5045,
          0.9231
        ],
        [
          0.5146,
          0.9195
        ],
        [
          0.525,
          0.9177
        ],
        [
          0.535,
          0.9145
        ],
        [
          0.545,
          0.9113
        ],
        [
          0.5551,
          0.9095
        ],
        [
          0.5655,
          0.9058
        ],
        [
          0.5755,
          0.9013
        ],
        [
          0.5855,
          0.8976
        ],
        [
          0.5955,
          0.894
        ],
        [
          0.6056,
          0.8908
        ],
        [
          0.616,
          0.8872
        ],
        [
          0.626,
          0.8831
        ],
        [
          0.636,
          0.879
        ],
        [
          0.646,
          0.8749
        ],
        [
          0.6561,
          0.869
        ],
        [
          0.6665,
          0.8621
        ],
        [
          0.6765,
          0.8576
        ],
        [
          0.6865,
          0.8521
        ],
        [
          0.6965,
          0.8471
        ],
        [
          0.7066,
          0.8439
        ],
        [
          0.717,
          0.8385
        ],
        [
          0.727,
          0.8335
        ],
        [
          0.737,
          0.8294
        ],
        [
          0.747,
          0.8248
        ],
        [
          0.7571,
          0.8207
        ],
        [
          0.7675,
          0.8171
        ],
        [
          0.7775,
          0.8121
        ],
        [
          0.7875,
          0.8048
        ],
        [
          0.7975,
          0.8003
        ],
        [
          0.8076,
          0.7962
        ],
        [
          0.818,
          0.7894
        ],
        [
          0.828,
          0.7843
        ],
        [
          0.838,
          0.778
        ],
        [
          0.848,
          0.7698
        ],
        [
          0.8581,
          0.7639
        ],
        [
          0.8685,
          0.7589
        ],
        [
          0.8785,
          0.7539
        ],
        [
          0.8885,
          0.748
        ],
        [
          0.8985,
          0.7398
        ],
        [
          0.9086,
          0.7352
        ],
        [
          0.919,
          0.7284
        ],
        [
          0.929,
          0.722
        ],
        [
          0.939,
          0.717
        ],
        [
          0.949,
          0.7093
        ],
        [
          0.9591,
          0.7056
        ],
        [
          0.9695,
          0.7002
        ],
        [
          0.9795,
          0.6929
        ],
        [
          0.9895,
          0.6856
        ],
        [
          0.9995,
          0.6779
        ],
        [
          1.0,
          0.6774
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.20956996775256573,
        "difference": -0.001067051364594293,
        "worse": 0.48,
        "kept_right": 0.20766316308786745,
        "kept_wrong": 0.0019068046646981429,
        "handed_off": 0.7904300322474339
      },
      "usd_per_1000": 4.424818915723318
    },
    {
      "decision": 322,
      "frontier": 124,
      "frontier_accuracy": 0.9667879890809827,
      "in_sample": {
        "threshold": 0.9899266448524215,
        "answered": 0.2024567788898999,
        "accuracy": 0.9667879890809827
      },
      "curve": [
        [
          0.0,
          0.9668
        ],
        [
          0.01,
          0.9668
        ],
        [
          0.02,
          0.9668
        ],
        [
          0.03,
          0.9668
        ],
        [
          0.04,
          0.9668
        ],
        [
          0.05,
          0.9668
        ],
        [
          0.0601,
          0.9668
        ],
        [
          0.0705,
          0.9668
        ],
        [
          0.0805,
          0.9668
        ],
        [
          0.0905,
          0.9668
        ],
        [
          0.1005,
          0.9668
        ],
        [
          0.1106,
          0.9672
        ],
        [
          0.121,
          0.9672
        ],
        [
          0.131,
          0.9668
        ],
        [
          0.141,
          0.9668
        ],
        [
          0.151,
          0.9668
        ],
        [
          0.1611,
          0.9668
        ],
        [
          0.1715,
          0.9668
        ],
        [
          0.1815,
          0.9668
        ],
        [
          0.1915,
          0.9668
        ],
        [
          0.2015,
          0.9668
        ],
        [
          0.2116,
          0.9659
        ],
        [
          0.222,
          0.9659
        ],
        [
          0.232,
          0.9659
        ],
        [
          0.242,
          0.9654
        ],
        [
          0.252,
          0.9645
        ],
        [
          0.2621,
          0.9641
        ],
        [
          0.2725,
          0.9627
        ],
        [
          0.2825,
          0.9627
        ],
        [
          0.2925,
          0.9618
        ],
        [
          0.3025,
          0.9613
        ],
        [
          0.3126,
          0.9609
        ],
        [
          0.323,
          0.9609
        ],
        [
          0.333,
          0.9604
        ],
        [
          0.343,
          0.9595
        ],
        [
          0.353,
          0.9572
        ],
        [
          0.3631,
          0.9568
        ],
        [
          0.3735,
          0.9545
        ],
        [
          0.3835,
          0.954
        ],
        [
          0.3935,
          0.9536
        ],
        [
          0.4035,
          0.9518
        ],
        [
          0.4136,
          0.95
        ],
        [
          0.424,
          0.9472
        ],
        [
          0.434,
          0.9454
        ],
        [
          0.444,
          0.944
        ],
        [
          0.454,
          0.9431
        ],
        [
          0.4641,
          0.9386
        ],
        [
          0.4745,
          0.9349
        ],
        [
          0.4845,
          0.9318
        ],
        [
          0.4945,
          0.9268
        ],
        [
          0.5045,
          0.9227
        ],
        [
          0.5146,
          0.919
        ],
        [
          0.525,
          0.9172
        ],
        [
          0.535,
          0.9145
        ],
        [
          0.545,
          0.9108
        ],
        [
          0.5551,
          0.909
        ],
        [
          0.5655,
          0.9058
        ],
        [
          0.5755,
          0.9013
        ],
        [
          0.5855,
          0.8972
        ],
        [
          0.5955,
          0.8935
        ],
        [
          0.6056,
          0.8904
        ],
        [
          0.616,
          0.8867
        ],
        [
          0.626,
          0.8826
        ],
        [
          0.636,
          0.8781
        ],
        [
          0.646,
          0.8744
        ],
        [
          0.6561,
          0.8685
        ],
        [
          0.6665,
          0.8626
        ],
        [
          0.6765,
          0.8576
        ],
        [
          0.6865,
          0.8526
        ],
        [
          0.6965,
          0.8476
        ],
        [
          0.7066,
          0.8439
        ],
        [
          0.717,
          0.8389
        ],
        [
          0.727,
          0.8335
        ],
        [
          0.737,
          0.8298
        ],
        [
          0.747,
          0.8248
        ],
        [
          0.7571,
          0.8207
        ],
        [
          0.7675,
          0.8167
        ],
        [
          0.7775,
          0.8116
        ],
        [
          0.7875,
          0.8048
        ],
        [
          0.7975,
          0.8003
        ],
        [
          0.8076,
          0.7957
        ],
        [
          0.818,
          0.7889
        ],
        [
          0.828,
          0.7834
        ],
        [
          0.838,
          0.7771
        ],
        [
          0.848,
          0.7693
        ],
        [
          0.8581,
          0.7634
        ],
        [
          0.8685,
          0.7584
        ],
        [
          0.8785,
          0.7534
        ],
        [
          0.8885,
          0.7475
        ],
        [
          0.8985,
          0.7393
        ],
        [
          0.9086,
          0.7338
        ],
        [
          0.919,
          0.727
        ],
        [
          0.929,
          0.7207
        ],
        [
          0.939,
          0.7161
        ],
        [
          0.949,
          0.7088
        ],
        [
          0.9591,
          0.7061
        ],
        [
          0.9695,
          0.7006
        ],
        [
          0.9795,
          0.6934
        ],
        [
          0.9895,
          0.6861
        ],
        [
          0.9995,
          0.6779
        ],
        [
          1.0,
          0.6774
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1956829186333568,
        "difference": -0.0008853542347296138,
        "worse": 0.5066666666666667,
        "kept_right": 0.19411853040545093,
        "kept_wrong": 0.0015643882279059182,
        "handed_off": 0.8043170813666429
      },
      "usd_per_1000": 0.9324726149591933
    },
    {
      "decision": 322,
      "frontier": 278,
      "frontier_accuracy": 0.9499545040946314,
      "in_sample": {
        "threshold": 0.996594163565176,
        "answered": 0.12283894449499545,
        "accuracy": 0.9499545040946314
      },
      "curve": [
        [
          0.0,
          0.95
        ],
        [
          0.01,
          0.95
        ],
        [
          0.02,
          0.95
        ],
        [
          0.03,
          0.95
        ],
        [
          0.04,
          0.95
        ],
        [
          0.05,
          0.95
        ],
        [
          0.0601,
          0.95
        ],
        [
          0.0705,
          0.95
        ],
        [
          0.0805,
          0.95
        ],
        [
          0.0905,
          0.95
        ],
        [
          0.1005,
          0.95
        ],
        [
          0.1106,
          0.95
        ],
        [
          0.121,
          0.95
        ],
        [
          0.131,
          0.9495
        ],
        [
          0.141,
          0.9495
        ],
        [
          0.151,
          0.9495
        ],
        [
          0.1611,
          0.9495
        ],
        [
          0.1715,
          0.9495
        ],
        [
          0.1815,
          0.9495
        ],
        [
          0.1915,
          0.9495
        ],
        [
          0.2015,
          0.9495
        ],
        [
          0.2116,
          0.9481
        ],
        [
          0.222,
          0.9481
        ],
        [
          0.232,
          0.9481
        ],
        [
          0.242,
          0.9477
        ],
        [
          0.252,
          0.9468
        ],
        [
          0.2621,
          0.9463
        ],
        [
          0.2725,
          0.9449
        ],
        [
          0.2825,
          0.9449
        ],
        [
          0.2925,
          0.944
        ],
        [
          0.3025,
          0.944
        ],
        [
          0.3126,
          0.9436
        ],
        [
          0.323,
          0.9436
        ],
        [
          0.333,
          0.9436
        ],
        [
          0.343,
          0.9422
        ],
        [
          0.353,
          0.9399
        ],
        [
          0.3631,
          0.9395
        ],
        [
          0.3735,
          0.9372
        ],
        [
          0.3835,
          0.9368
        ],
        [
          0.3935,
          0.9363
        ],
        [
          0.4035,
          0.9349
        ],
        [
          0.4136,
          0.9336
        ],
        [
          0.424,
          0.9318
        ],
        [
          0.434,
          0.9299
        ],
        [
          0.444,
          0.929
        ],
        [
          0.454,
          0.9281
        ],
        [
          0.4641,
          0.9236
        ],
        [
          0.4745,
          0.9195
        ],
        [
          0.4845,
          0.9163
        ],
        [
          0.4945,
          0.9122
        ],
        [
          0.5045,
          0.9081
        ],
        [
          0.5146,
          0.9049
        ],
        [
          0.525,
          0.9031
        ],
        [
          0.535,
          0.9004
        ],
        [
          0.545,
          0.8972
        ],
        [
          0.5551,
          0.8958
        ],
        [
          0.5655,
          0.8931
        ],
        [
          0.5755,
          0.8885
        ],
        [
          0.5855,
          0.8854
        ],
        [
          0.5955,
          0.8826
        ],
        [
          0.6056,
          0.8799
        ],
        [
          0.616,
          0.8767
        ],
        [
          0.626,
          0.8731
        ],
        [
          0.636,
          0.8685
        ],
        [
          0.646,
          0.8653
        ],
        [
          0.6561,
          0.8603
        ],
        [
          0.6665,
          0.8558
        ],
        [
          0.6765,
          0.8512
        ],
        [
          0.6865,
          0.8462
        ],
        [
          0.6965,
          0.8412
        ],
        [
          0.7066,
          0.8385
        ],
        [
          0.717,
          0.8335
        ],
        [
          0.727,
          0.8285
        ],
        [
          0.737,
          0.8253
        ],
        [
          0.747,
          0.8207
        ],
        [
          0.7571,
          0.8171
        ],
        [
          0.7675,
          0.813
        ],
        [
          0.7775,
          0.808
        ],
        [
          0.7875,
          0.8012
        ],
        [
          0.7975,
          0.7971
        ],
        [
          0.8076,
          0.7925
        ],
        [
          0.818,
          0.7857
        ],
        [
          0.828,
          0.7803
        ],
        [
          0.838,
          0.7734
        ],
        [
          0.848,
          0.7652
        ],
        [
          0.8581,
          0.7598
        ],
        [
          0.8685,
          0.7543
        ],
        [
          0.8785,
          0.7489
        ],
        [
          0.8885,
          0.7429
        ],
        [
          0.8985,
          0.7348
        ],
        [
          0.9086,
          0.7311
        ],
        [
          0.919,
          0.7247
        ],
        [
          0.929,
          0.7193
        ],
        [
          0.939,
          0.7138
        ],
        [
          0.949,
          0.7061
        ],
        [
          0.9591,
          0.7034
        ],
        [
          0.9695,
          0.6988
        ],
        [
          0.9795,
          0.692
        ],
        [
          0.9895,
          0.6847
        ],
        [
          0.9995,
          0.6779
        ],
        [
          1.0,
          0.6774
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.16664395165424695,
        "difference": -0.0008822437834908637,
        "worse": 0.5066666666666667,
        "kept_right": 0.1657556417500403,
        "kept_wrong": 0.0008883099042066659,
        "handed_off": 0.8333560483457525
      },
      "usd_per_1000": 5.447464011248812
    },
    {
      "decision": 322,
      "frontier": 275,
      "frontier_accuracy": 0.9426751592356688,
      "in_sample": {
        "threshold": 0.9899266448524215,
        "answered": 0.2024567788898999,
        "accuracy": 0.9426751592356688
      },
      "curve": [
        [
          0.0,
          0.9427
        ],
        [
          0.01,
          0.9427
        ],
        [
          0.02,
          0.9427
        ],
        [
          0.03,
          0.9427
        ],
        [
          0.04,
          0.9427
        ],
        [
          0.05,
          0.9427
        ],
        [
          0.0601,
          0.9427
        ],
        [
          0.0705,
          0.9427
        ],
        [
          0.0805,
          0.9427
        ],
        [
          0.0905,
          0.9427
        ],
        [
          0.1005,
          0.9427
        ],
        [
          0.1106,
          0.9431
        ],
        [
          0.121,
          0.9431
        ],
        [
          0.131,
          0.9427
        ],
        [
          0.141,
          0.9427
        ],
        [
          0.151,
          0.9427
        ],
        [
          0.1611,
          0.9427
        ],
        [
          0.1715,
          0.9427
        ],
        [
          0.1815,
          0.9427
        ],
        [
          0.1915,
          0.9427
        ],
        [
          0.2015,
          0.9427
        ],
        [
          0.2116,
          0.9413
        ],
        [
          0.222,
          0.9413
        ],
        [
          0.232,
          0.9413
        ],
        [
          0.242,
          0.9409
        ],
        [
          0.252,
          0.9399
        ],
        [
          0.2621,
          0.9395
        ],
        [
          0.2725,
          0.9386
        ],
        [
          0.2825,
          0.9386
        ],
        [
          0.2925,
          0.9377
        ],
        [
          0.3025,
          0.9377
        ],
        [
          0.3126,
          0.9372
        ],
        [
          0.323,
          0.9372
        ],
        [
          0.333,
          0.9377
        ],
        [
          0.343,
          0.9363
        ],
        [
          0.353,
          0.934
        ],
        [
          0.3631,
          0.9336
        ],
        [
          0.3735,
          0.9313
        ],
        [
          0.3835,
          0.9308
        ],
        [
          0.3935,
          0.9304
        ],
        [
          0.4035,
          0.929
        ],
        [
          0.4136,
          0.9272
        ],
        [
          0.424,
          0.9254
        ],
        [
          0.434,
          0.9236
        ],
        [
          0.444,
          0.9222
        ],
        [
          0.454,
          0.9213
        ],
        [
          0.4641,
          0.9167
        ],
        [
          0.4745,
          0.9131
        ],
        [
          0.4845,
          0.9099
        ],
        [
          0.4945,
          0.9045
        ],
        [
          0.5045,
          0.9008
        ],
        [
          0.5146,
          0.8976
        ],
        [
          0.525,
          0.8963
        ],
        [
          0.535,
          0.8935
        ],
        [
          0.545,
          0.8899
        ],
        [
          0.5551,
          0.889
        ],
        [
          0.5655,
          0.8867
        ],
        [
          0.5755,
          0.8831
        ],
        [
          0.5855,
          0.879
        ],
        [
          0.5955,
          0.8758
        ],
        [
          0.6056,
          0.8735
        ],
        [
          0.616,
          0.8699
        ],
        [
          0.626,
          0.8672
        ],
        [
          0.636,
          0.8626
        ],
        [
          0.646,
          0.859
        ],
        [
          0.6561,
          0.854
        ],
        [
          0.6665,
          0.848
        ],
        [
          0.6765,
          0.8439
        ],
        [
          0.6865,
          0.8399
        ],
        [
          0.6965,
          0.8353
        ],
        [
          0.7066,
          0.8321
        ],
        [
          0.717,
          0.8271
        ],
        [
          0.727,
          0.8226
        ],
        [
          0.737,
          0.8185
        ],
        [
          0.747,
          0.8139
        ],
        [
          0.7571,
          0.8098
        ],
        [
          0.7675,
          0.8062
        ],
        [
          0.7775,
          0.8012
        ],
        [
          0.7875,
          0.7953
        ],
        [
          0.7975,
          0.7912
        ],
        [
          0.8076,
          0.7866
        ],
        [
          0.818,
          0.7807
        ],
        [
          0.828,
          0.7757
        ],
        [
          0.838,
          0.7698
        ],
        [
          0.848,
          0.7625
        ],
        [
          0.8581,
          0.7575
        ],
        [
          0.8685,
          0.7525
        ],
        [
          0.8785,
          0.748
        ],
        [
          0.8885,
          0.742
        ],
        [
          0.8985,
          0.7338
        ],
        [
          0.9086,
          0.7293
        ],
        [
          0.919,
          0.7234
        ],
        [
          0.929,
          0.7179
        ],
        [
          0.939,
          0.7129
        ],
        [
          0.949,
          0.7066
        ],
        [
          0.9591,
          0.7043
        ],
        [
          0.9695,
          0.6997
        ],
        [
          0.9795,
          0.6929
        ],
        [
          0.9895,
          0.6856
        ],
        [
          0.9995,
          0.6779
        ],
        [
          1.0,
          0.6774
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.19079770876976104,
        "difference": -0.0008701198440396787,
        "worse": 0.5033333333333333,
        "kept_right": 0.18944256898490341,
        "kept_wrong": 0.0013551397848575468,
        "handed_off": 0.8092022912302385
      },
      "usd_per_1000": 0.43963950461887047
    },
    {
      "decision": 322,
      "frontier": 192,
      "frontier_accuracy": 0.9331210191082803,
      "in_sample": {
        "threshold": 0.9890878122232423,
        "answered": 0.20746132848043677,
        "accuracy": 0.9331210191082803
      },
      "curve": [
        [
          0.0,
          0.9331
        ],
        [
          0.01,
          0.9331
        ],
        [
          0.02,
          0.9331
        ],
        [
          0.03,
          0.9331
        ],
        [
          0.04,
          0.9331
        ],
        [
          0.05,
          0.9331
        ],
        [
          0.0601,
          0.9331
        ],
        [
          0.0705,
          0.9331
        ],
        [
          0.0805,
          0.9331
        ],
        [
          0.0905,
          0.9331
        ],
        [
          0.1005,
          0.9331
        ],
        [
          0.1106,
          0.9336
        ],
        [
          0.121,
          0.9336
        ],
        [
          0.131,
          0.9331
        ],
        [
          0.141,
          0.9331
        ],
        [
          0.151,
          0.9331
        ],
        [
          0.1611,
          0.9331
        ],
        [
          0.1715,
          0.9331
        ],
        [
          0.1815,
          0.9336
        ],
        [
          0.1915,
          0.9336
        ],
        [
          0.2015,
          0.9336
        ],
        [
          0.2116,
          0.9322
        ],
        [
          0.222,
          0.9322
        ],
        [
          0.232,
          0.9322
        ],
        [
          0.242,
          0.9318
        ],
        [
          0.252,
          0.9308
        ],
        [
          0.2621,
          0.9304
        ],
        [
          0.2725,
          0.929
        ],
        [
          0.2825,
          0.929
        ],
        [
          0.2925,
          0.9281
        ],
        [
          0.3025,
          0.9281
        ],
        [
          0.3126,
          0.9277
        ],
        [
          0.323,
          0.9272
        ],
        [
          0.333,
          0.9268
        ],
        [
          0.343,
          0.9254
        ],
        [
          0.353,
          0.9231
        ],
        [
          0.3631,
          0.9227
        ],
        [
          0.3735,
          0.9204
        ],
        [
          0.3835,
          0.9199
        ],
        [
          0.3935,
          0.9204
        ],
        [
          0.4035,
          0.9186
        ],
        [
          0.4136,
          0.9177
        ],
        [
          0.424,
          0.9158
        ],
        [
          0.434,
          0.914
        ],
        [
          0.444,
          0.9131
        ],
        [
          0.454,
          0.9122
        ],
        [
          0.4641,
          0.9076
        ],
        [
          0.4745,
          0.904
        ],
        [
          0.4845,
          0.9013
        ],
        [
          0.4945,
          0.8972
        ],
        [
          0.5045,
          0.8935
        ],
        [
          0.5146,
          0.8904
        ],
        [
          0.525,
          0.8885
        ],
        [
          0.535,
          0.8858
        ],
        [
          0.545,
          0.8822
        ],
        [
          0.5551,
          0.8808
        ],
        [
          0.5655,
          0.8781
        ],
        [
          0.5755,
          0.8735
        ],
        [
          0.5855,
          0.8703
        ],
        [
          0.5955,
          0.8681
        ],
        [
          0.6056,
          0.8662
        ],
        [
          0.616,
          0.8631
        ],
        [
          0.626,
          0.8594
        ],
        [
          0.636,
          0.8562
        ],
        [
          0.646,
          0.8521
        ],
        [
          0.6561,
          0.8471
        ],
        [
          0.6665,
          0.8421
        ],
        [
          0.6765,
          0.8385
        ],
        [
          0.6865,
          0.8339
        ],
        [
          0.6965,
          0.8289
        ],
        [
          0.7066,
          0.8267
        ],
        [
          0.717,
          0.8221
        ],
        [
          0.727,
          0.8176
        ],
        [
          0.737,
          0.8144
        ],
        [
          0.747,
          0.8098
        ],
        [
          0.7571,
          0.8066
        ],
        [
          0.7675,
          0.8035
        ],
        [
          0.7775,
          0.7989
        ],
        [
          0.7875,
          0.7916
        ],
        [
          0.7975,
          0.7875
        ],
        [
          0.8076,
          0.783
        ],
        [
          0.818,
          0.7766
        ],
        [
          0.828,
          0.7721
        ],
        [
          0.838,
          0.7671
        ],
        [
          0.848,
          0.7598
        ],
        [
          0.8581,
          0.7543
        ],
        [
          0.8685,
          0.7489
        ],
        [
          0.8785,
          0.7452
        ],
        [
          0.8885,
          0.7398
        ],
        [
          0.8985,
          0.732
        ],
        [
          0.9086,
          0.7288
        ],
        [
          0.919,
          0.722
        ],
        [
          0.929,
          0.7166
        ],
        [
          0.939,
          0.712
        ],
        [
          0.949,
          0.7047
        ],
        [
          0.9591,
          0.7015
        ],
        [
          0.9695,
          0.6975
        ],
        [
          0.9795,
          0.6911
        ],
        [
          0.9895,
          0.6843
        ],
        [
          0.9995,
          0.6779
        ],
        [
          1.0,
          0.6774
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.20901827584157567,
        "difference": -0.0010246160678158108,
        "worse": 0.48,
        "kept_right": 0.20716906626579895,
        "kept_wrong": 0.0018492095757765832,
        "handed_off": 0.790981724158424
      },
      "usd_per_1000": 9.206248252447795
    },
    {
      "decision": 322,
      "frontier": 123,
      "frontier_accuracy": 0.9326660600545951,
      "in_sample": {
        "threshold": 0.996594163565176,
        "answered": 0.12283894449499545,
        "accuracy": 0.9326660600545951
      },
      "curve": [
        [
          0.0,
          0.9327
        ],
        [
          0.01,
          0.9327
        ],
        [
          0.02,
          0.9327
        ],
        [
          0.03,
          0.9327
        ],
        [
          0.04,
          0.9327
        ],
        [
          0.05,
          0.9327
        ],
        [
          0.0601,
          0.9327
        ],
        [
          0.0705,
          0.9327
        ],
        [
          0.0805,
          0.9327
        ],
        [
          0.0905,
          0.9327
        ],
        [
          0.1005,
          0.9327
        ],
        [
          0.1106,
          0.9327
        ],
        [
          0.121,
          0.9327
        ],
        [
          0.131,
          0.9322
        ],
        [
          0.141,
          0.9322
        ],
        [
          0.151,
          0.9322
        ],
        [
          0.1611,
          0.9322
        ],
        [
          0.1715,
          0.9322
        ],
        [
          0.1815,
          0.9322
        ],
        [
          0.1915,
          0.9322
        ],
        [
          0.2015,
          0.9322
        ],
        [
          0.2116,
          0.9308
        ],
        [
          0.222,
          0.9308
        ],
        [
          0.232,
          0.9308
        ],
        [
          0.242,
          0.9304
        ],
        [
          0.252,
          0.9295
        ],
        [
          0.2621,
          0.929
        ],
        [
          0.2725,
          0.9277
        ],
        [
          0.2825,
          0.9277
        ],
        [
          0.2925,
          0.9268
        ],
        [
          0.3025,
          0.9263
        ],
        [
          0.3126,
          0.9258
        ],
        [
          0.323,
          0.9263
        ],
        [
          0.333,
          0.9258
        ],
        [
          0.343,
          0.9249
        ],
        [
          0.353,
          0.9227
        ],
        [
          0.3631,
          0.9227
        ],
        [
          0.3735,
          0.9204
        ],
        [
          0.3835,
          0.9199
        ],
        [
          0.3935,
          0.919
        ],
        [
          0.4035,
          0.9172
        ],
        [
          0.4136,
          0.9167
        ],
        [
          0.424,
          0.9149
        ],
        [
          0.434,
          0.9136
        ],
        [
          0.444,
          0.9122
        ],
        [
          0.454,
          0.9117
        ],
        [
          0.4641,
          0.9086
        ],
        [
          0.4745,
          0.9049
        ],
        [
          0.4845,
          0.9013
        ],
        [
          0.4945,
          0.8972
        ],
        [
          0.5045,
          0.8935
        ],
        [
          0.5146,
          0.8904
        ],
        [
          0.525,
          0.889
        ],
        [
          0.535,
          0.8854
        ],
        [
          0.545,
          0.8831
        ],
        [
          0.5551,
          0.8817
        ],
        [
          0.5655,
          0.8794
        ],
        [
          0.5755,
          0.8749
        ],
        [
          0.5855,
          0.8712
        ],
        [
          0.5955,
          0.8681
        ],
        [
          0.6056,
          0.8662
        ],
        [
          0.616,
          0.8626
        ],
        [
          0.626,
          0.8603
        ],
        [
          0.636,
          0.8558
        ],
        [
          0.646,
          0.8526
        ],
        [
          0.6561,
          0.8476
        ],
        [
          0.6665,
          0.8421
        ],
        [
          0.6765,
          0.8371
        ],
        [
          0.6865,
          0.8321
        ],
        [
          0.6965,
          0.8271
        ],
        [
          0.7066,
          0.8248
        ],
        [
          0.717,
          0.8198
        ],
        [
          0.727,
          0.8144
        ],
        [
          0.737,
          0.8116
        ],
        [
          0.747,
          0.8071
        ],
        [
          0.7571,
          0.8039
        ],
        [
          0.7675,
          0.8003
        ],
        [
          0.7775,
          0.7957
        ],
        [
          0.7875,
          0.7884
        ],
        [
          0.7975,
          0.7843
        ],
        [
          0.8076,
          0.7812
        ],
        [
          0.818,
          0.7762
        ],
        [
          0.828,
          0.7716
        ],
        [
          0.838,
          0.7657
        ],
        [
          0.848,
          0.7575
        ],
        [
          0.8581,
          0.7516
        ],
        [
          0.8685,
          0.7475
        ],
        [
          0.8785,
          0.7429
        ],
        [
          0.8885,
          0.7375
        ],
        [
          0.8985,
          0.7293
        ],
        [
          0.9086,
          0.7252
        ],
        [
          0.919,
          0.7193
        ],
        [
          0.929,
          0.7138
        ],
        [
          0.939,
          0.7097
        ],
        [
          0.949,
          0.7025
        ],
        [
          0.9591,
          0.7002
        ],
        [
          0.9695,
          0.697
        ],
        [
          0.9795,
          0.6911
        ],
        [
          0.9895,
          0.6843
        ],
        [
          0.9995,
          0.6774
        ],
        [
          1.0,
          0.6774
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.16664395165424695,
        "difference": -0.0008822437834908637,
        "worse": 0.5066666666666667,
        "kept_right": 0.1657556417500403,
        "kept_wrong": 0.0008883099042066659,
        "handed_off": 0.8333560483457525
      },
      "usd_per_1000": 0.24712030015959716
    },
    {
      "decision": 322,
      "frontier": 274,
      "frontier_accuracy": 0.9085532302092811,
      "in_sample": {
        "threshold": 0.9899266448524215,
        "answered": 0.2024567788898999,
        "accuracy": 0.9085532302092811
      },
      "curve": [
        [
          0.0,
          0.9086
        ],
        [
          0.01,
          0.9086
        ],
        [
          0.02,
          0.9086
        ],
        [
          0.03,
          0.9086
        ],
        [
          0.04,
          0.9086
        ],
        [
          0.05,
          0.9086
        ],
        [
          0.0601,
          0.9086
        ],
        [
          0.0705,
          0.9086
        ],
        [
          0.0805,
          0.9086
        ],
        [
          0.0905,
          0.9086
        ],
        [
          0.1005,
          0.9086
        ],
        [
          0.1106,
          0.909
        ],
        [
          0.121,
          0.909
        ],
        [
          0.131,
          0.9086
        ],
        [
          0.141,
          0.9086
        ],
        [
          0.151,
          0.9086
        ],
        [
          0.1611,
          0.9086
        ],
        [
          0.1715,
          0.9086
        ],
        [
          0.1815,
          0.9086
        ],
        [
          0.1915,
          0.9086
        ],
        [
          0.2015,
          0.9086
        ],
        [
          0.2116,
          0.9072
        ],
        [
          0.222,
          0.9072
        ],
        [
          0.232,
          0.9076
        ],
        [
          0.242,
          0.9072
        ],
        [
          0.252,
          0.9067
        ],
        [
          0.2621,
          0.9067
        ],
        [
          0.2725,
          0.9063
        ],
        [
          0.2825,
          0.9063
        ],
        [
          0.2925,
          0.9054
        ],
        [
          0.3025,
          0.9054
        ],
        [
          0.3126,
          0.9049
        ],
        [
          0.323,
          0.9049
        ],
        [
          0.333,
          0.9049
        ],
        [
          0.343,
          0.904
        ],
        [
          0.353,
          0.9022
        ],
        [
          0.3631,
          0.9017
        ],
        [
          0.3735,
          0.8999
        ],
        [
          0.3835,
          0.8995
        ],
        [
          0.3935,
          0.8999
        ],
        [
          0.4035,
          0.8985
        ],
        [
          0.4136,
          0.8981
        ],
        [
          0.424,
          0.8963
        ],
        [
          0.434,
          0.8958
        ],
        [
          0.444,
          0.8944
        ],
        [
          0.454,
          0.894
        ],
        [
          0.4641,
          0.8894
        ],
        [
          0.4745,
          0.8863
        ],
        [
          0.4845,
          0.8826
        ],
        [
          0.4945,
          0.8776
        ],
        [
          0.5045,
          0.8744
        ],
        [
          0.5146,
          0.8717
        ],
        [
          0.525,
          0.8699
        ],
        [
          0.535,
          0.8667
        ],
        [
          0.545,
          0.8635
        ],
        [
          0.5551,
          0.8621
        ],
        [
          0.5655,
          0.8599
        ],
        [
          0.5755,
          0.8558
        ],
        [
          0.5855,
          0.8521
        ],
        [
          0.5955,
          0.8494
        ],
        [
          0.6056,
          0.848
        ],
        [
          0.616,
          0.8449
        ],
        [
          0.626,
          0.8426
        ],
        [
          0.636,
          0.8389
        ],
        [
          0.646,
          0.8371
        ],
        [
          0.6561,
          0.8321
        ],
        [
          0.6665,
          0.8267
        ],
        [
          0.6765,
          0.823
        ],
        [
          0.6865,
          0.8194
        ],
        [
          0.6965,
          0.8157
        ],
        [
          0.7066,
          0.8126
        ],
        [
          0.717,
          0.8076
        ],
        [
          0.727,
          0.803
        ],
        [
          0.737,
          0.7989
        ],
        [
          0.747,
          0.7944
        ],
        [
          0.7571,
          0.7912
        ],
        [
          0.7675,
          0.788
        ],
        [
          0.7775,
          0.783
        ],
        [
          0.7875,
          0.7766
        ],
        [
          0.7975,
          0.7734
        ],
        [
          0.8076,
          0.7689
        ],
        [
          0.818,
          0.7639
        ],
        [
          0.828,
          0.7584
        ],
        [
          0.838,
          0.7534
        ],
        [
          0.848,
          0.7461
        ],
        [
          0.8581,
          0.7416
        ],
        [
          0.8685,
          0.7366
        ],
        [
          0.8785,
          0.7325
        ],
        [
          0.8885,
          0.7275
        ],
        [
          0.8985,
          0.7207
        ],
        [
          0.9086,
          0.7175
        ],
        [
          0.919,
          0.7125
        ],
        [
          0.929,
          0.7075
        ],
        [
          0.939,
          0.7038
        ],
        [
          0.949,
          0.6975
        ],
        [
          0.9591,
          0.6965
        ],
        [
          0.9695,
          0.6956
        ],
        [
          0.9795,
          0.6897
        ],
        [
          0.9895,
          0.6833
        ],
        [
          0.9995,
          0.6779
        ],
        [
          1.0,
          0.6774
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.20712232470045464,
        "difference": -0.0014339775865031764,
        "worse": 0.5733333333333334,
        "kept_right": 0.2046515659923795,
        "kept_wrong": 0.0024707587080750344,
        "handed_off": 0.7928776752995456
      },
      "usd_per_1000": 0.3612549315086626
    },
    {
      "decision": 322,
      "frontier": 47,
      "frontier_accuracy": 0.9044585987261147,
      "in_sample": {
        "threshold": 0.9899266448524215,
        "answered": 0.2024567788898999,
        "accuracy": 0.9044585987261147
      },
      "curve": [
        [
          0.0,
          0.9045
        ],
        [
          0.01,
          0.9045
        ],
        [
          0.02,
          0.9045
        ],
        [
          0.03,
          0.9045
        ],
        [
          0.04,
          0.9045
        ],
        [
          0.05,
          0.9045
        ],
        [
          0.0601,
          0.9045
        ],
        [
          0.0705,
          0.9045
        ],
        [
          0.0805,
          0.9045
        ],
        [
          0.0905,
          0.9045
        ],
        [
          0.1005,
          0.9045
        ],
        [
          0.1106,
          0.9049
        ],
        [
          0.121,
          0.9049
        ],
        [
          0.131,
          0.9045
        ],
        [
          0.141,
          0.9045
        ],
        [
          0.151,
          0.9045
        ],
        [
          0.1611,
          0.9045
        ],
        [
          0.1715,
          0.9045
        ],
        [
          0.1815,
          0.9045
        ],
        [
          0.1915,
          0.9045
        ],
        [
          0.2015,
          0.9045
        ],
        [
          0.2116,
          0.9031
        ],
        [
          0.222,
          0.9031
        ],
        [
          0.232,
          0.9031
        ],
        [
          0.242,
          0.9026
        ],
        [
          0.252,
          0.9017
        ],
        [
          0.2621,
          0.9022
        ],
        [
          0.2725,
          0.9008
        ],
        [
          0.2825,
          0.9008
        ],
        [
          0.2925,
          0.9004
        ],
        [
          0.3025,
          0.9008
        ],
        [
          0.3126,
          0.9004
        ],
        [
          0.323,
          0.9004
        ],
        [
          0.333,
          0.9004
        ],
        [
          0.343,
          0.8995
        ],
        [
          0.353,
          0.8972
        ],
        [
          0.3631,
          0.8972
        ],
        [
          0.3735,
          0.8949
        ],
        [
          0.3835,
          0.8944
        ],
        [
          0.3935,
          0.8935
        ],
        [
          0.4035,
          0.8917
        ],
        [
          0.4136,
          0.8908
        ],
        [
          0.424,
          0.889
        ],
        [
          0.434,
          0.8872
        ],
        [
          0.444,
          0.8863
        ],
        [
          0.454,
          0.8858
        ],
        [
          0.4641,
          0.8826
        ],
        [
          0.4745,
          0.879
        ],
        [
          0.4845,
          0.8776
        ],
        [
          0.4945,
          0.874
        ],
        [
          0.5045,
          0.8708
        ],
        [
          0.5146,
          0.8681
        ],
        [
          0.525,
          0.8662
        ],
        [
          0.535,
          0.8631
        ],
        [
          0.545,
          0.8603
        ],
        [
          0.5551,
          0.8585
        ],
        [
          0.5655,
          0.8558
        ],
        [
          0.5755,
          0.8512
        ],
        [
          0.5855,
          0.8471
        ],
        [
          0.5955,
          0.8444
        ],
        [
          0.6056,
          0.8421
        ],
        [
          0.616,
          0.8389
        ],
        [
          0.626,
          0.8371
        ],
        [
          0.636,
          0.8344
        ],
        [
          0.646,
          0.8308
        ],
        [
          0.6561,
          0.8258
        ],
        [
          0.6665,
          0.8207
        ],
        [
          0.6765,
          0.8176
        ],
        [
          0.6865,
          0.813
        ],
        [
          0.6965,
          0.8089
        ],
        [
          0.7066,
          0.8066
        ],
        [
          0.717,
          0.8021
        ],
        [
          0.727,
          0.798
        ],
        [
          0.737,
          0.7948
        ],
        [
          0.747,
          0.7907
        ],
        [
          0.7571,
          0.788
        ],
        [
          0.7675,
          0.7843
        ],
        [
          0.7775,
          0.7807
        ],
        [
          0.7875,
          0.7743
        ],
        [
          0.7975,
          0.7721
        ],
        [
          0.8076,
          0.7689
        ],
        [
          0.818,
          0.763
        ],
        [
          0.828,
          0.7593
        ],
        [
          0.838,
          0.7539
        ],
        [
          0.848,
          0.747
        ],
        [
          0.8581,
          0.742
        ],
        [
          0.8685,
          0.7379
        ],
        [
          0.8785,
          0.7329
        ],
        [
          0.8885,
          0.7279
        ],
        [
          0.8985,
          0.7197
        ],
        [
          0.9086,
          0.7166
        ],
        [
          0.919,
          0.7106
        ],
        [
          0.929,
          0.707
        ],
        [
          0.939,
          0.7038
        ],
        [
          0.949,
          0.6988
        ],
        [
          0.9591,
          0.6975
        ],
        [
          0.9695,
          0.6938
        ],
        [
          0.9795,
          0.6893
        ],
        [
          0.9895,
          0.6833
        ],
        [
          0.9995,
          0.6774
        ],
        [
          1.0,
          0.6774
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.20105352276967756,
        "difference": -0.0013673300505020426,
        "worse": 0.5266666666666666,
        "kept_right": 0.19881304807642794,
        "kept_wrong": 0.0022404746932495247,
        "handed_off": 0.7989464772303222
      },
      "usd_per_1000": 3.962568543007853
    },
    {
      "decision": 322,
      "frontier": 193,
      "frontier_accuracy": 0.89171974522293,
      "in_sample": {
        "threshold": 0.9890878122232423,
        "answered": 0.20746132848043677,
        "accuracy": 0.89171974522293
      },
      "curve": [
        [
          0.0,
          0.8917
        ],
        [
          0.01,
          0.8917
        ],
        [
          0.02,
          0.8917
        ],
        [
          0.03,
          0.8917
        ],
        [
          0.04,
          0.8917
        ],
        [
          0.05,
          0.8917
        ],
        [
          0.0601,
          0.8917
        ],
        [
          0.0705,
          0.8917
        ],
        [
          0.0805,
          0.8917
        ],
        [
          0.0905,
          0.8917
        ],
        [
          0.1005,
          0.8917
        ],
        [
          0.1106,
          0.8922
        ],
        [
          0.121,
          0.8922
        ],
        [
          0.131,
          0.8922
        ],
        [
          0.141,
          0.8922
        ],
        [
          0.151,
          0.8922
        ],
        [
          0.1611,
          0.8922
        ],
        [
          0.1715,
          0.8922
        ],
        [
          0.1815,
          0.8922
        ],
        [
          0.1915,
          0.8922
        ],
        [
          0.2015,
          0.8922
        ],
        [
          0.2116,
          0.8908
        ],
        [
          0.222,
          0.8908
        ],
        [
          0.232,
          0.8908
        ],
        [
          0.242,
          0.8904
        ],
        [
          0.252,
          0.8894
        ],
        [
          0.2621,
          0.8899
        ],
        [
          0.2725,
          0.889
        ],
        [
          0.2825,
          0.8894
        ],
        [
          0.2925,
          0.889
        ],
        [
          0.3025,
          0.889
        ],
        [
          0.3126,
          0.8885
        ],
        [
          0.323,
          0.8885
        ],
        [
          0.333,
          0.8894
        ],
        [
          0.343,
          0.8885
        ],
        [
          0.353,
          0.8863
        ],
        [
          0.3631,
          0.8858
        ],
        [
          0.3735,
          0.8844
        ],
        [
          0.3835,
          0.8844
        ],
        [
          0.3935,
          0.8844
        ],
        [
          0.4035,
          0.8826
        ],
        [
          0.4136,
          0.8822
        ],
        [
          0.424,
          0.8799
        ],
        [
          0.434,
          0.8785
        ],
        [
          0.444,
          0.8776
        ],
        [
          0.454,
          0.8776
        ],
        [
          0.4641,
          0.8735
        ],
        [
          0.4745,
          0.8699
        ],
        [
          0.4845,
          0.8667
        ],
        [
          0.4945,
          0.8626
        ],
        [
          0.5045,
          0.8608
        ],
        [
          0.5146,
          0.859
        ],
        [
          0.525,
          0.8571
        ],
        [
          0.535,
          0.8549
        ],
        [
          0.545,
          0.8517
        ],
        [
          0.5551,
          0.8512
        ],
        [
          0.5655,
          0.848
        ],
        [
          0.5755,
          0.8439
        ],
        [
          0.5855,
          0.8417
        ],
        [
          0.5955,
          0.8399
        ],
        [
          0.6056,
          0.838
        ],
        [
          0.616,
          0.8358
        ],
        [
          0.626,
          0.8321
        ],
        [
          0.636,
          0.8285
        ],
        [
          0.646,
          0.8244
        ],
        [
          0.6561,
          0.8194
        ],
        [
          0.6665,
          0.8162
        ],
        [
          0.6765,
          0.8121
        ],
        [
          0.6865,
          0.8094
        ],
        [
          0.6965,
          0.8053
        ],
        [
          0.7066,
          0.8035
        ],
        [
          0.717,
          0.7989
        ],
        [
          0.727,
          0.7944
        ],
        [
          0.737,
          0.793
        ],
        [
          0.747,
          0.7889
        ],
        [
          0.7571,
          0.7862
        ],
        [
          0.7675,
          0.7843
        ],
        [
          0.7775,
          0.7812
        ],
        [
          0.7875,
          0.7753
        ],
        [
          0.7975,
          0.7716
        ],
        [
          0.8076,
          0.7684
        ],
        [
          0.818,
          0.763
        ],
        [
          0.828,
          0.7584
        ],
        [
          0.838,
          0.7534
        ],
        [
          0.848,
          0.7466
        ],
        [
          0.8581,
          0.742
        ],
        [
          0.8685,
          0.7375
        ],
        [
          0.8785,
          0.7338
        ],
        [
          0.8885,
          0.7293
        ],
        [
          0.8985,
          0.722
        ],
        [
          0.9086,
          0.7188
        ],
        [
          0.919,
          0.7138
        ],
        [
          0.929,
          0.7093
        ],
        [
          0.939,
          0.7047
        ],
        [
          0.949,
          0.6988
        ],
        [
          0.9591,
          0.697
        ],
        [
          0.9695,
          0.6938
        ],
        [
          0.9795,
          0.6884
        ],
        [
          0.9895,
          0.682
        ],
        [
          0.9995,
          0.6774
        ],
        [
          1.0,
          0.6774
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.24126766495288735,
        "difference": -0.0017008595207356758,
        "worse": 0.51,
        "kept_right": 0.23725044347251217,
        "kept_wrong": 0.004017221480375298,
        "handed_off": 0.7587323350471118
      },
      "usd_per_1000": 5.901641294101378
    },
    {
      "decision": 322,
      "frontier": 55,
      "frontier_accuracy": 0.8503184713375797,
      "in_sample": {
        "threshold": 0.8272945046106192,
        "answered": 0.4822565969062784,
        "accuracy": 0.8503184713375797
      },
      "curve": [
        [
          0.0,
          0.8503
        ],
        [
          0.01,
          0.8503
        ],
        [
          0.02,
          0.8503
        ],
        [
          0.03,
          0.8503
        ],
        [
          0.04,
          0.8503
        ],
        [
          0.05,
          0.8512
        ],
        [
          0.0601,
          0.8517
        ],
        [
          0.0705,
          0.8517
        ],
        [
          0.0805,
          0.8517
        ],
        [
          0.0905,
          0.8521
        ],
        [
          0.1005,
          0.8521
        ],
        [
          0.1106,
          0.8521
        ],
        [
          0.121,
          0.8526
        ],
        [
          0.131,
          0.8526
        ],
        [
          0.141,
          0.853
        ],
        [
          0.151,
          0.853
        ],
        [
          0.1611,
          0.8535
        ],
        [
          0.1715,
          0.854
        ],
        [
          0.1815,
          0.8544
        ],
        [
          0.1915,
          0.8544
        ],
        [
          0.2015,
          0.8549
        ],
        [
          0.2116,
          0.8544
        ],
        [
          0.222,
          0.8558
        ],
        [
          0.232,
          0.8567
        ],
        [
          0.242,
          0.8562
        ],
        [
          0.252,
          0.8562
        ],
        [
          0.2621,
          0.8567
        ],
        [
          0.2725,
          0.8558
        ],
        [
          0.2825,
          0.8571
        ],
        [
          0.2925,
          0.8567
        ],
        [
          0.3025,
          0.8571
        ],
        [
          0.3126,
          0.8571
        ],
        [
          0.323,
          0.859
        ],
        [
          0.333,
          0.8603
        ],
        [
          0.343,
          0.8612
        ],
        [
          0.353,
          0.8603
        ],
        [
          0.3631,
          0.8608
        ],
        [
          0.3735,
          0.8603
        ],
        [
          0.3835,
          0.8603
        ],
        [
          0.3935,
          0.8617
        ],
        [
          0.4035,
          0.8608
        ],
        [
          0.4136,
          0.8608
        ],
        [
          0.424,
          0.859
        ],
        [
          0.434,
          0.8581
        ],
        [
          0.444,
          0.8576
        ],
        [
          0.454,
          0.8571
        ],
        [
          0.4641,
          0.8549
        ],
        [
          0.4745,
          0.8512
        ],
        [
          0.4845,
          0.8494
        ],
        [
          0.4945,
          0.8458
        ],
        [
          0.5045,
          0.8435
        ],
        [
          0.5146,
          0.8403
        ],
        [
          0.525,
          0.8394
        ],
        [
          0.535,
          0.8367
        ],
        [
          0.545,
          0.8353
        ],
        [
          0.5551,
          0.8344
        ],
        [
          0.5655,
          0.8317
        ],
        [
          0.5755,
          0.8271
        ],
        [
          0.5855,
          0.8253
        ],
        [
          0.5955,
          0.823
        ],
        [
          0.6056,
          0.8212
        ],
        [
          0.616,
          0.8185
        ],
        [
          0.626,
          0.8153
        ],
        [
          0.636,
          0.8126
        ],
        [
          0.646,
          0.8094
        ],
        [
          0.6561,
          0.8048
        ],
        [
          0.6665,
          0.8021
        ],
        [
          0.6765,
          0.7985
        ],
        [
          0.6865,
          0.7944
        ],
        [
          0.6965,
          0.7903
        ],
        [
          0.7066,
          0.7884
        ],
        [
          0.717,
          0.7848
        ],
        [
          0.727,
          0.7803
        ],
        [
          0.737,
          0.778
        ],
        [
          0.747,
          0.7753
        ],
        [
          0.7571,
          0.7725
        ],
        [
          0.7675,
          0.7707
        ],
        [
          0.7775,
          0.7689
        ],
        [
          0.7875,
          0.7634
        ],
        [
          0.7975,
          0.7621
        ],
        [
          0.8076,
          0.7598
        ],
        [
          0.818,
          0.7548
        ],
        [
          0.828,
          0.7516
        ],
        [
          0.838,
          0.748
        ],
        [
          0.848,
          0.7411
        ],
        [
          0.8581,
          0.7384
        ],
        [
          0.8685,
          0.7334
        ],
        [
          0.8785,
          0.7288
        ],
        [
          0.8885,
          0.7234
        ],
        [
          0.8985,
          0.7179
        ],
        [
          0.9086,
          0.7152
        ],
        [
          0.919,
          0.7106
        ],
        [
          0.929,
          0.707
        ],
        [
          0.939,
          0.7034
        ],
        [
          0.949,
          0.6988
        ],
        [
          0.9591,
          0.6975
        ],
        [
          0.9695,
          0.6947
        ],
        [
          0.9795,
          0.6879
        ],
        [
          0.9895,
          0.682
        ],
        [
          0.9995,
          0.6774
        ],
        [
          1.0,
          0.6774
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.4801862557707065,
        "difference": -0.00039850398163423394,
        "worse": 0.46,
        "kept_right": 0.43736638178164344,
        "kept_wrong": 0.04281987398906354,
        "handed_off": 0.5198137442292929
      },
      "usd_per_1000": 1.9228670909457077
    },
    {
      "decision": 326,
      "frontier": 140,
      "frontier_accuracy": 0.9808917197452229,
      "in_sample": {
        "threshold": 0.9768268404588278,
        "answered": 0.1565059144676979,
        "accuracy": 0.9808917197452229
      },
      "curve": [
        [
          0.0,
          0.9809
        ],
        [
          0.01,
          0.9809
        ],
        [
          0.02,
          0.9809
        ],
        [
          0.03,
          0.9809
        ],
        [
          0.04,
          0.9809
        ],
        [
          0.05,
          0.9809
        ],
        [
          0.0601,
          0.9809
        ],
        [
          0.0705,
          0.9809
        ],
        [
          0.0805,
          0.9809
        ],
        [
          0.0905,
          0.9809
        ],
        [
          0.1005,
          0.9809
        ],
        [
          0.1106,
          0.9809
        ],
        [
          0.121,
          0.9809
        ],
        [
          0.131,
          0.9809
        ],
        [
          0.141,
          0.9809
        ],
        [
          0.151,
          0.9809
        ],
        [
          0.1611,
          0.9804
        ],
        [
          0.1715,
          0.9804
        ],
        [
          0.1815,
          0.98
        ],
        [
          0.1915,
          0.98
        ],
        [
          0.2015,
          0.98
        ],
        [
          0.2116,
          0.9795
        ],
        [
          0.222,
          0.9795
        ],
        [
          0.232,
          0.9786
        ],
        [
          0.242,
          0.9786
        ],
        [
          0.252,
          0.9782
        ],
        [
          0.2621,
          0.9782
        ],
        [
          0.2725,
          0.9786
        ],
        [
          0.2825,
          0.9786
        ],
        [
          0.2925,
          0.9786
        ],
        [
          0.3025,
          0.9782
        ],
        [
          0.3126,
          0.9777
        ],
        [
          0.323,
          0.9773
        ],
        [
          0.333,
          0.9754
        ],
        [
          0.343,
          0.9745
        ],
        [
          0.353,
          0.9736
        ],
        [
          0.3631,
          0.9713
        ],
        [
          0.3735,
          0.9713
        ],
        [
          0.3835,
          0.9709
        ],
        [
          0.3935,
          0.9686
        ],
        [
          0.4035,
          0.9668
        ],
        [
          0.4136,
          0.9641
        ],
        [
          0.424,
          0.9613
        ],
        [
          0.434,
          0.9595
        ],
        [
          0.444,
          0.9572
        ],
        [
          0.454,
          0.955
        ],
        [
          0.4641,
          0.9509
        ],
        [
          0.4745,
          0.9481
        ],
        [
          0.4845,
          0.9449
        ],
        [
          0.4945,
          0.9409
        ],
        [
          0.5045,
          0.9372
        ],
        [
          0.5146,
          0.9349
        ],
        [
          0.525,
          0.9308
        ],
        [
          0.535,
          0.929
        ],
        [
          0.545,
          0.9268
        ],
        [
          0.5551,
          0.9245
        ],
        [
          0.5655,
          0.9181
        ],
        [
          0.5755,
          0.9136
        ],
        [
          0.5855,
          0.9081
        ],
        [
          0.5955,
          0.9035
        ],
        [
          0.6056,
          0.899
        ],
        [
          0.616,
          0.8944
        ],
        [
          0.626,
          0.8894
        ],
        [
          0.636,
          0.8844
        ],
        [
          0.646,
          0.8785
        ],
        [
          0.6561,
          0.8731
        ],
        [
          0.6665,
          0.8681
        ],
        [
          0.6765,
          0.8631
        ],
        [
          0.6865,
          0.8585
        ],
        [
          0.6965,
          0.854
        ],
        [
          0.7066,
          0.8503
        ],
        [
          0.717,
          0.8471
        ],
        [
          0.727,
          0.8412
        ],
        [
          0.737,
          0.8367
        ],
        [
          0.747,
          0.8326
        ],
        [
          0.7571,
          0.8253
        ],
        [
          0.7675,
          0.8189
        ],
        [
          0.7775,
          0.8148
        ],
        [
          0.7875,
          0.8089
        ],
        [
          0.7975,
          0.8053
        ],
        [
          0.8076,
          0.7998
        ],
        [
          0.818,
          0.7921
        ],
        [
          0.828,
          0.7875
        ],
        [
          0.838,
          0.7825
        ],
        [
          0.848,
          0.7766
        ],
        [
          0.8581,
          0.7693
        ],
        [
          0.8685,
          0.7643
        ],
        [
          0.8785,
          0.7575
        ],
        [
          0.8885,
          0.7507
        ],
        [
          0.8985,
          0.7443
        ],
        [
          0.9086,
          0.7384
        ],
        [
          0.919,
          0.732
        ],
        [
          0.929,
          0.7261
        ],
        [
          0.939,
          0.7202
        ],
        [
          0.949,
          0.7134
        ],
        [
          0.9591,
          0.7056
        ],
        [
          0.9695,
          0.6984
        ],
        [
          0.9795,
          0.6906
        ],
        [
          0.9895,
          0.6829
        ],
        [
          0.9995,
          0.6742
        ],
        [
          1.0,
          0.6738
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.18162845542724193,
        "difference": -0.0009701090359244177,
        "worse": 0.5033333333333333,
        "kept_right": 0.18061286972196386,
        "kept_wrong": 0.0010155857052782228,
        "handed_off": 0.8183715445727584
      },
      "usd_per_1000": 2.0028394404560474
    },
    {
      "decision": 326,
      "frontier": 277,
      "frontier_accuracy": 0.9795268425841674,
      "in_sample": {
        "threshold": 0.9701587024993364,
        "answered": 0.18016378525932666,
        "accuracy": 0.9795268425841674
      },
      "curve": [
        [
          0.0,
          0.9795
        ],
        [
          0.01,
          0.9795
        ],
        [
          0.02,
          0.9795
        ],
        [
          0.03,
          0.9795
        ],
        [
          0.04,
          0.9795
        ],
        [
          0.05,
          0.9795
        ],
        [
          0.0601,
          0.9795
        ],
        [
          0.0705,
          0.9795
        ],
        [
          0.0805,
          0.9795
        ],
        [
          0.0905,
          0.9795
        ],
        [
          0.1005,
          0.9795
        ],
        [
          0.1106,
          0.9795
        ],
        [
          0.121,
          0.9795
        ],
        [
          0.131,
          0.9795
        ],
        [
          0.141,
          0.9795
        ],
        [
          0.151,
          0.9795
        ],
        [
          0.1611,
          0.9791
        ],
        [
          0.1715,
          0.9791
        ],
        [
          0.1815,
          0.9791
        ],
        [
          0.1915,
          0.9791
        ],
        [
          0.2015,
          0.9791
        ],
        [
          0.2116,
          0.9786
        ],
        [
          0.222,
          0.9786
        ],
        [
          0.232,
          0.9777
        ],
        [
          0.242,
          0.9777
        ],
        [
          0.252,
          0.9773
        ],
        [
          0.2621,
          0.9773
        ],
        [
          0.2725,
          0.9777
        ],
        [
          0.2825,
          0.9777
        ],
        [
          0.2925,
          0.9777
        ],
        [
          0.3025,
          0.9773
        ],
        [
          0.3126,
          0.9768
        ],
        [
          0.323,
          0.9763
        ],
        [
          0.333,
          0.9745
        ],
        [
          0.343,
          0.9736
        ],
        [
          0.353,
          0.9727
        ],
        [
          0.3631,
          0.9704
        ],
        [
          0.3735,
          0.9704
        ],
        [
          0.3835,
          0.9695
        ],
        [
          0.3935,
          0.9677
        ],
        [
          0.4035,
          0.9659
        ],
        [
          0.4136,
          0.9631
        ],
        [
          0.424,
          0.9604
        ],
        [
          0.434,
          0.9586
        ],
        [
          0.444,
          0.9563
        ],
        [
          0.454,
          0.954
        ],
        [
          0.4641,
          0.9495
        ],
        [
          0.4745,
          0.9468
        ],
        [
          0.4845,
          0.944
        ],
        [
          0.4945,
          0.9395
        ],
        [
          0.5045,
          0.9359
        ],
        [
          0.5146,
          0.9336
        ],
        [
          0.525,
          0.9295
        ],
        [
          0.535,
          0.9281
        ],
        [
          0.545,
          0.9258
        ],
        [
          0.5551,
          0.9236
        ],
        [
          0.5655,
          0.9172
        ],
        [
          0.5755,
          0.9126
        ],
        [
          0.5855,
          0.9072
        ],
        [
          0.5955,
          0.9026
        ],
        [
          0.6056,
          0.8981
        ],
        [
          0.616,
          0.8935
        ],
        [
          0.626,
          0.8881
        ],
        [
          0.636,
          0.8826
        ],
        [
          0.646,
          0.8767
        ],
        [
          0.6561,
          0.8712
        ],
        [
          0.6665,
          0.8662
        ],
        [
          0.6765,
          0.8612
        ],
        [
          0.6865,
          0.8567
        ],
        [
          0.6965,
          0.8535
        ],
        [
          0.7066,
          0.8503
        ],
        [
          0.717,
          0.8471
        ],
        [
          0.727,
          0.8412
        ],
        [
          0.737,
          0.8367
        ],
        [
          0.747,
          0.8326
        ],
        [
          0.7571,
          0.8253
        ],
        [
          0.7675,
          0.8189
        ],
        [
          0.7775,
          0.8148
        ],
        [
          0.7875,
          0.8085
        ],
        [
          0.7975,
          0.8039
        ],
        [
          0.8076,
          0.798
        ],
        [
          0.818,
          0.7903
        ],
        [
          0.828,
          0.7857
        ],
        [
          0.838,
          0.7807
        ],
        [
          0.848,
          0.7753
        ],
        [
          0.8581,
          0.768
        ],
        [
          0.8685,
          0.763
        ],
        [
          0.8785,
          0.7561
        ],
        [
          0.8885,
          0.7493
        ],
        [
          0.8985,
          0.7429
        ],
        [
          0.9086,
          0.7375
        ],
        [
          0.919,
          0.7311
        ],
        [
          0.929,
          0.7252
        ],
        [
          0.939,
          0.7193
        ],
        [
          0.949,
          0.7125
        ],
        [
          0.9591,
          0.7047
        ],
        [
          0.9695,
          0.6975
        ],
        [
          0.9795,
          0.6893
        ],
        [
          0.9895,
          0.6829
        ],
        [
          0.9995,
          0.6742
        ],
        [
          1.0,
          0.6738
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.2082876666309146,
        "difference": -0.0013491732297221273,
        "worse": 0.5033333333333333,
        "kept_right": 0.20655344038222823,
        "kept_wrong": 0.0017342262486864793,
        "handed_off": 0.7917123333690852
      },
      "usd_per_1000": 4.825288991167721
    },
    {
      "decision": 326,
      "frontier": 276,
      "frontier_accuracy": 0.9777070063694268,
      "in_sample": {
        "threshold": 0.9701587024993364,
        "answered": 0.18016378525932666,
        "accuracy": 0.9777070063694268
      },
      "curve": [
        [
          0.0,
          0.9777
        ],
        [
          0.01,
          0.9777
        ],
        [
          0.02,
          0.9777
        ],
        [
          0.03,
          0.9777
        ],
        [
          0.04,
          0.9777
        ],
        [
          0.05,
          0.9777
        ],
        [
          0.0601,
          0.9777
        ],
        [
          0.0705,
          0.9777
        ],
        [
          0.0805,
          0.9777
        ],
        [
          0.0905,
          0.9777
        ],
        [
          0.1005,
          0.9777
        ],
        [
          0.1106,
          0.9777
        ],
        [
          0.121,
          0.9777
        ],
        [
          0.131,
          0.9777
        ],
        [
          0.141,
          0.9777
        ],
        [
          0.151,
          0.9777
        ],
        [
          0.1611,
          0.9773
        ],
        [
          0.1715,
          0.9773
        ],
        [
          0.1815,
          0.9773
        ],
        [
          0.1915,
          0.9773
        ],
        [
          0.2015,
          0.9773
        ],
        [
          0.2116,
          0.9768
        ],
        [
          0.222,
          0.9768
        ],
        [
          0.232,
          0.9763
        ],
        [
          0.242,
          0.9763
        ],
        [
          0.252,
          0.9759
        ],
        [
          0.2621,
          0.9759
        ],
        [
          0.2725,
          0.9763
        ],
        [
          0.2825,
          0.9763
        ],
        [
          0.2925,
          0.9768
        ],
        [
          0.3025,
          0.9763
        ],
        [
          0.3126,
          0.9763
        ],
        [
          0.323,
          0.9759
        ],
        [
          0.333,
          0.9741
        ],
        [
          0.343,
          0.9732
        ],
        [
          0.353,
          0.9722
        ],
        [
          0.3631,
          0.9704
        ],
        [
          0.3735,
          0.9704
        ],
        [
          0.3835,
          0.9695
        ],
        [
          0.3935,
          0.9677
        ],
        [
          0.4035,
          0.9659
        ],
        [
          0.4136,
          0.9631
        ],
        [
          0.424,
          0.9604
        ],
        [
          0.434,
          0.9586
        ],
        [
          0.444,
          0.9563
        ],
        [
          0.454,
          0.954
        ],
        [
          0.4641,
          0.9495
        ],
        [
          0.4745,
          0.9472
        ],
        [
          0.4845,
          0.9445
        ],
        [
          0.4945,
          0.9399
        ],
        [
          0.5045,
          0.9363
        ],
        [
          0.5146,
          0.934
        ],
        [
          0.525,
          0.9299
        ],
        [
          0.535,
          0.9286
        ],
        [
          0.545,
          0.9263
        ],
        [
          0.5551,
          0.924
        ],
        [
          0.5655,
          0.9177
        ],
        [
          0.5755,
          0.9131
        ],
        [
          0.5855,
          0.9076
        ],
        [
          0.5955,
          0.9031
        ],
        [
          0.6056,
          0.8985
        ],
        [
          0.616,
          0.894
        ],
        [
          0.626,
          0.8885
        ],
        [
          0.636,
          0.8831
        ],
        [
          0.646,
          0.8772
        ],
        [
          0.6561,
          0.8717
        ],
        [
          0.6665,
          0.8667
        ],
        [
          0.6765,
          0.8612
        ],
        [
          0.6865,
          0.8567
        ],
        [
          0.6965,
          0.853
        ],
        [
          0.7066,
          0.8499
        ],
        [
          0.717,
          0.8467
        ],
        [
          0.727,
          0.8408
        ],
        [
          0.737,
          0.8362
        ],
        [
          0.747,
          0.8321
        ],
        [
          0.7571,
          0.8248
        ],
        [
          0.7675,
          0.8185
        ],
        [
          0.7775,
          0.8144
        ],
        [
          0.7875,
          0.808
        ],
        [
          0.7975,
          0.8035
        ],
        [
          0.8076,
          0.7975
        ],
        [
          0.818,
          0.7907
        ],
        [
          0.828,
          0.7862
        ],
        [
          0.838,
          0.7812
        ],
        [
          0.848,
          0.7757
        ],
        [
          0.8581,
          0.7689
        ],
        [
          0.8685,
          0.7643
        ],
        [
          0.8785,
          0.7575
        ],
        [
          0.8885,
          0.7507
        ],
        [
          0.8985,
          0.7443
        ],
        [
          0.9086,
          0.7379
        ],
        [
          0.919,
          0.7316
        ],
        [
          0.929,
          0.7257
        ],
        [
          0.939,
          0.7197
        ],
        [
          0.949,
          0.7129
        ],
        [
          0.9591,
          0.7052
        ],
        [
          0.9695,
          0.6979
        ],
        [
          0.9795,
          0.6897
        ],
        [
          0.9895,
          0.6829
        ],
        [
          0.9995,
          0.6742
        ],
        [
          1.0,
          0.6738
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.23547427643067134,
        "difference": -0.0018646559005851904,
        "worse": 0.59,
        "kept_right": 0.23243620535258577,
        "kept_wrong": 0.0030380710780856736,
        "handed_off": 0.7645257235693282
      },
      "usd_per_1000": 1.2633203650090923
    },
    {
      "decision": 326,
      "frontier": 48,
      "frontier_accuracy": 0.9758871701546861,
      "in_sample": {
        "threshold": 0.9701587024993364,
        "answered": 0.18016378525932666,
        "accuracy": 0.9758871701546861
      },
      "curve": [
        [
          0.0,
          0.9759
        ],
        [
          0.01,
          0.9759
        ],
        [
          0.02,
          0.9759
        ],
        [
          0.03,
          0.9759
        ],
        [
          0.04,
          0.9759
        ],
        [
          0.05,
          0.9759
        ],
        [
          0.0601,
          0.9759
        ],
        [
          0.0705,
          0.9759
        ],
        [
          0.0805,
          0.9759
        ],
        [
          0.0905,
          0.9759
        ],
        [
          0.1005,
          0.9759
        ],
        [
          0.1106,
          0.9759
        ],
        [
          0.121,
          0.9759
        ],
        [
          0.131,
          0.9759
        ],
        [
          0.141,
          0.9759
        ],
        [
          0.151,
          0.9759
        ],
        [
          0.1611,
          0.9754
        ],
        [
          0.1715,
          0.9754
        ],
        [
          0.1815,
          0.9754
        ],
        [
          0.1915,
          0.9754
        ],
        [
          0.2015,
          0.9754
        ],
        [
          0.2116,
          0.975
        ],
        [
          0.222,
          0.975
        ],
        [
          0.232,
          0.9741
        ],
        [
          0.242,
          0.9741
        ],
        [
          0.252,
          0.9736
        ],
        [
          0.2621,
          0.9736
        ],
        [
          0.2725,
          0.9741
        ],
        [
          0.2825,
          0.9741
        ],
        [
          0.2925,
          0.9745
        ],
        [
          0.3025,
          0.9741
        ],
        [
          0.3126,
          0.9736
        ],
        [
          0.323,
          0.9732
        ],
        [
          0.333,
          0.9713
        ],
        [
          0.343,
          0.9709
        ],
        [
          0.353,
          0.97
        ],
        [
          0.3631,
          0.9677
        ],
        [
          0.3735,
          0.9677
        ],
        [
          0.3835,
          0.9668
        ],
        [
          0.3935,
          0.965
        ],
        [
          0.4035,
          0.9636
        ],
        [
          0.4136,
          0.9609
        ],
        [
          0.424,
          0.9581
        ],
        [
          0.434,
          0.9563
        ],
        [
          0.444,
          0.954
        ],
        [
          0.454,
          0.9518
        ],
        [
          0.4641,
          0.9472
        ],
        [
          0.4745,
          0.9445
        ],
        [
          0.4845,
          0.9418
        ],
        [
          0.4945,
          0.9377
        ],
        [
          0.5045,
          0.934
        ],
        [
          0.5146,
          0.9318
        ],
        [
          0.525,
          0.9277
        ],
        [
          0.535,
          0.9263
        ],
        [
          0.545,
          0.924
        ],
        [
          0.5551,
          0.9217
        ],
        [
          0.5655,
          0.9154
        ],
        [
          0.5755,
          0.9108
        ],
        [
          0.5855,
          0.9058
        ],
        [
          0.5955,
          0.9013
        ],
        [
          0.6056,
          0.8967
        ],
        [
          0.616,
          0.8922
        ],
        [
          0.626,
          0.8872
        ],
        [
          0.636,
          0.8817
        ],
        [
          0.646,
          0.8758
        ],
        [
          0.6561,
          0.8703
        ],
        [
          0.6665,
          0.8649
        ],
        [
          0.6765,
          0.8594
        ],
        [
          0.6865,
          0.8549
        ],
        [
          0.6965,
          0.8512
        ],
        [
          0.7066,
          0.848
        ],
        [
          0.717,
          0.8449
        ],
        [
          0.727,
          0.8394
        ],
        [
          0.737,
          0.8348
        ],
        [
          0.747,
          0.8308
        ],
        [
          0.7571,
          0.8239
        ],
        [
          0.7675,
          0.8171
        ],
        [
          0.7775,
          0.813
        ],
        [
          0.7875,
          0.8076
        ],
        [
          0.7975,
          0.803
        ],
        [
          0.8076,
          0.7971
        ],
        [
          0.818,
          0.7898
        ],
        [
          0.828,
          0.7853
        ],
        [
          0.838,
          0.7803
        ],
        [
          0.848,
          0.7748
        ],
        [
          0.8581,
          0.768
        ],
        [
          0.8685,
          0.763
        ],
        [
          0.8785,
          0.7561
        ],
        [
          0.8885,
          0.7498
        ],
        [
          0.8985,
          0.7434
        ],
        [
          0.9086,
          0.7379
        ],
        [
          0.919,
          0.7316
        ],
        [
          0.929,
          0.7257
        ],
        [
          0.939,
          0.7197
        ],
        [
          0.949,
          0.7129
        ],
        [
          0.9591,
          0.7052
        ],
        [
          0.9695,
          0.6979
        ],
        [
          0.9795,
          0.6897
        ],
        [
          0.9895,
          0.6829
        ],
        [
          0.9995,
          0.6742
        ],
        [
          1.0,
          0.6738
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.21776728503105958,
        "difference": -0.00156139106888964,
        "worse": 0.53,
        "kept_right": 0.21562679108040328,
        "kept_wrong": 0.002140493950656498,
        "handed_off": 0.7822327149689401
      },
      "usd_per_1000": 4.230138766147892
    },
    {
      "decision": 326,
      "frontier": 191,
      "frontier_accuracy": 0.9740673339399454,
      "in_sample": {
        "threshold": 0.9768268404588278,
        "answered": 0.1565059144676979,
        "accuracy": 0.9740673339399454
      },
      "curve": [
        [
          0.0,
          0.9741
        ],
        [
          0.01,
          0.9741
        ],
        [
          0.02,
          0.9741
        ],
        [
          0.03,
          0.9741
        ],
        [
          0.04,
          0.9741
        ],
        [
          0.05,
          0.9741
        ],
        [
          0.0601,
          0.9741
        ],
        [
          0.0705,
          0.9741
        ],
        [
          0.0805,
          0.9741
        ],
        [
          0.0905,
          0.9741
        ],
        [
          0.1005,
          0.9741
        ],
        [
          0.1106,
          0.9741
        ],
        [
          0.121,
          0.9741
        ],
        [
          0.131,
          0.9741
        ],
        [
          0.141,
          0.9741
        ],
        [
          0.151,
          0.9741
        ],
        [
          0.1611,
          0.9736
        ],
        [
          0.1715,
          0.9736
        ],
        [
          0.1815,
          0.9732
        ],
        [
          0.1915,
          0.9732
        ],
        [
          0.2015,
          0.9732
        ],
        [
          0.2116,
          0.9727
        ],
        [
          0.222,
          0.9727
        ],
        [
          0.232,
          0.9718
        ],
        [
          0.242,
          0.9718
        ],
        [
          0.252,
          0.9713
        ],
        [
          0.2621,
          0.9713
        ],
        [
          0.2725,
          0.9718
        ],
        [
          0.2825,
          0.9718
        ],
        [
          0.2925,
          0.9722
        ],
        [
          0.3025,
          0.9718
        ],
        [
          0.3126,
          0.9713
        ],
        [
          0.323,
          0.9709
        ],
        [
          0.333,
          0.9695
        ],
        [
          0.343,
          0.9686
        ],
        [
          0.353,
          0.9677
        ],
        [
          0.3631,
          0.9654
        ],
        [
          0.3735,
          0.9654
        ],
        [
          0.3835,
          0.9645
        ],
        [
          0.3935,
          0.9627
        ],
        [
          0.4035,
          0.9613
        ],
        [
          0.4136,
          0.9586
        ],
        [
          0.424,
          0.9559
        ],
        [
          0.434,
          0.954
        ],
        [
          0.444,
          0.9518
        ],
        [
          0.454,
          0.949
        ],
        [
          0.4641,
          0.9449
        ],
        [
          0.4745,
          0.9431
        ],
        [
          0.4845,
          0.9404
        ],
        [
          0.4945,
          0.9363
        ],
        [
          0.5045,
          0.9327
        ],
        [
          0.5146,
          0.9304
        ],
        [
          0.525,
          0.9258
        ],
        [
          0.535,
          0.924
        ],
        [
          0.545,
          0.9217
        ],
        [
          0.5551,
          0.9195
        ],
        [
          0.5655,
          0.9131
        ],
        [
          0.5755,
          0.9086
        ],
        [
          0.5855,
          0.9031
        ],
        [
          0.5955,
          0.899
        ],
        [
          0.6056,
          0.8944
        ],
        [
          0.616,
          0.8904
        ],
        [
          0.626,
          0.8854
        ],
        [
          0.636,
          0.8799
        ],
        [
          0.646,
          0.874
        ],
        [
          0.6561,
          0.8685
        ],
        [
          0.6665,
          0.8635
        ],
        [
          0.6765,
          0.8581
        ],
        [
          0.6865,
          0.8535
        ],
        [
          0.6965,
          0.8499
        ],
        [
          0.7066,
          0.8462
        ],
        [
          0.717,
          0.843
        ],
        [
          0.727,
          0.8371
        ],
        [
          0.737,
          0.833
        ],
        [
          0.747,
          0.8289
        ],
        [
          0.7571,
          0.8221
        ],
        [
          0.7675,
          0.8153
        ],
        [
          0.7775,
          0.8112
        ],
        [
          0.7875,
          0.8053
        ],
        [
          0.7975,
          0.8007
        ],
        [
          0.8076,
          0.7953
        ],
        [
          0.818,
          0.7884
        ],
        [
          0.828,
          0.7839
        ],
        [
          0.838,
          0.7789
        ],
        [
          0.848,
          0.7734
        ],
        [
          0.8581,
          0.7662
        ],
        [
          0.8685,
          0.7621
        ],
        [
          0.8785,
          0.7557
        ],
        [
          0.8885,
          0.7489
        ],
        [
          0.8985,
          0.7425
        ],
        [
          0.9086,
          0.737
        ],
        [
          0.919,
          0.7307
        ],
        [
          0.929,
          0.7247
        ],
        [
          0.939,
          0.7193
        ],
        [
          0.949,
          0.7125
        ],
        [
          0.9591,
          0.7047
        ],
        [
          0.9695,
          0.6975
        ],
        [
          0.9795,
          0.6897
        ],
        [
          0.9895,
          0.6829
        ],
        [
          0.9995,
          0.6742
        ],
        [
          1.0,
          0.6738
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.19561632507281027,
        "difference": -0.0013824292425302807,
        "worse": 0.5366666666666666,
        "kept_right": 0.1940155510685605,
        "kept_wrong": 0.0016007740042499829,
        "handed_off": 0.8043836749271898
      },
      "usd_per_1000": 21.530480487148218
    },
    {
      "decision": 326,
      "frontier": 188,
      "frontier_accuracy": 0.9713375796178344,
      "in_sample": {
        "threshold": 0.9701587024993364,
        "answered": 0.18016378525932666,
        "accuracy": 0.9713375796178344
      },
      "curve": [
        [
          0.0,
          0.9713
        ],
        [
          0.01,
          0.9713
        ],
        [
          0.02,
          0.9713
        ],
        [
          0.03,
          0.9713
        ],
        [
          0.04,
          0.9713
        ],
        [
          0.05,
          0.9713
        ],
        [
          0.0601,
          0.9713
        ],
        [
          0.0705,
          0.9713
        ],
        [
          0.0805,
          0.9713
        ],
        [
          0.0905,
          0.9713
        ],
        [
          0.1005,
          0.9713
        ],
        [
          0.1106,
          0.9713
        ],
        [
          0.121,
          0.9713
        ],
        [
          0.131,
          0.9718
        ],
        [
          0.141,
          0.9718
        ],
        [
          0.151,
          0.9718
        ],
        [
          0.1611,
          0.9713
        ],
        [
          0.1715,
          0.9713
        ],
        [
          0.1815,
          0.9709
        ],
        [
          0.1915,
          0.9709
        ],
        [
          0.2015,
          0.9709
        ],
        [
          0.2116,
          0.9704
        ],
        [
          0.222,
          0.9704
        ],
        [
          0.232,
          0.9695
        ],
        [
          0.242,
          0.9695
        ],
        [
          0.252,
          0.9691
        ],
        [
          0.2621,
          0.9691
        ],
        [
          0.2725,
          0.9695
        ],
        [
          0.2825,
          0.9695
        ],
        [
          0.2925,
          0.97
        ],
        [
          0.3025,
          0.9695
        ],
        [
          0.3126,
          0.9691
        ],
        [
          0.323,
          0.9686
        ],
        [
          0.333,
          0.9668
        ],
        [
          0.343,
          0.9668
        ],
        [
          0.353,
          0.9659
        ],
        [
          0.3631,
          0.9636
        ],
        [
          0.3735,
          0.9636
        ],
        [
          0.3835,
          0.9631
        ],
        [
          0.3935,
          0.9609
        ],
        [
          0.4035,
          0.96
        ],
        [
          0.4136,
          0.9572
        ],
        [
          0.424,
          0.9545
        ],
        [
          0.434,
          0.9527
        ],
        [
          0.444,
          0.9504
        ],
        [
          0.454,
          0.9481
        ],
        [
          0.4641,
          0.944
        ],
        [
          0.4745,
          0.9418
        ],
        [
          0.4845,
          0.9395
        ],
        [
          0.4945,
          0.9354
        ],
        [
          0.5045,
          0.9322
        ],
        [
          0.5146,
          0.9299
        ],
        [
          0.525,
          0.9254
        ],
        [
          0.535,
          0.9236
        ],
        [
          0.545,
          0.9213
        ],
        [
          0.5551,
          0.919
        ],
        [
          0.5655,
          0.9126
        ],
        [
          0.5755,
          0.9081
        ],
        [
          0.5855,
          0.9026
        ],
        [
          0.5955,
          0.8981
        ],
        [
          0.6056,
          0.8935
        ],
        [
          0.616,
          0.8894
        ],
        [
          0.626,
          0.8844
        ],
        [
          0.636,
          0.8794
        ],
        [
          0.646,
          0.8735
        ],
        [
          0.6561,
          0.8681
        ],
        [
          0.6665,
          0.8626
        ],
        [
          0.6765,
          0.8576
        ],
        [
          0.6865,
          0.853
        ],
        [
          0.6965,
          0.8494
        ],
        [
          0.7066,
          0.8462
        ],
        [
          0.717,
          0.843
        ],
        [
          0.727,
          0.8371
        ],
        [
          0.737,
          0.8326
        ],
        [
          0.747,
          0.8285
        ],
        [
          0.7571,
          0.8217
        ],
        [
          0.7675,
          0.8153
        ],
        [
          0.7775,
          0.8116
        ],
        [
          0.7875,
          0.8053
        ],
        [
          0.7975,
          0.8012
        ],
        [
          0.8076,
          0.7957
        ],
        [
          0.818,
          0.788
        ],
        [
          0.828,
          0.7834
        ],
        [
          0.838,
          0.7784
        ],
        [
          0.848,
          0.7725
        ],
        [
          0.8581,
          0.7657
        ],
        [
          0.8685,
          0.7616
        ],
        [
          0.8785,
          0.7548
        ],
        [
          0.8885,
          0.748
        ],
        [
          0.8985,
          0.742
        ],
        [
          0.9086,
          0.7366
        ],
        [
          0.919,
          0.7311
        ],
        [
          0.929,
          0.7252
        ],
        [
          0.939,
          0.7193
        ],
        [
          0.949,
          0.7125
        ],
        [
          0.9591,
          0.7047
        ],
        [
          0.9695,
          0.697
        ],
        [
          0.9795,
          0.6902
        ],
        [
          0.9895,
          0.6829
        ],
        [
          0.9995,
          0.6742
        ],
        [
          1.0,
          0.6738
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.21535951130460826,
        "difference": -0.0012674125610025945,
        "worse": 0.5166666666666667,
        "kept_right": 0.21327351885148238,
        "kept_wrong": 0.0020859924531259335,
        "handed_off": 0.7846404886953919
      },
      "usd_per_1000": 0.7790681496639108
    },
    {
      "decision": 326,
      "frontier": 190,
      "frontier_accuracy": 0.9672429481346679,
      "in_sample": {
        "threshold": 0.9577099244045593,
        "answered": 0.21064604185623295,
        "accuracy": 0.9672429481346679
      },
      "curve": [
        [
          0.0,
          0.9672
        ],
        [
          0.01,
          0.9672
        ],
        [
          0.02,
          0.9672
        ],
        [
          0.03,
          0.9672
        ],
        [
          0.04,
          0.9672
        ],
        [
          0.05,
          0.9672
        ],
        [
          0.0601,
          0.9672
        ],
        [
          0.0705,
          0.9672
        ],
        [
          0.0805,
          0.9672
        ],
        [
          0.0905,
          0.9672
        ],
        [
          0.1005,
          0.9672
        ],
        [
          0.1106,
          0.9672
        ],
        [
          0.121,
          0.9672
        ],
        [
          0.131,
          0.9672
        ],
        [
          0.141,
          0.9672
        ],
        [
          0.151,
          0.9672
        ],
        [
          0.1611,
          0.9668
        ],
        [
          0.1715,
          0.9668
        ],
        [
          0.1815,
          0.9668
        ],
        [
          0.1915,
          0.9668
        ],
        [
          0.2015,
          0.9672
        ],
        [
          0.2116,
          0.9668
        ],
        [
          0.222,
          0.9668
        ],
        [
          0.232,
          0.9659
        ],
        [
          0.242,
          0.9659
        ],
        [
          0.252,
          0.9654
        ],
        [
          0.2621,
          0.9654
        ],
        [
          0.2725,
          0.9663
        ],
        [
          0.2825,
          0.9663
        ],
        [
          0.2925,
          0.9668
        ],
        [
          0.3025,
          0.9663
        ],
        [
          0.3126,
          0.9663
        ],
        [
          0.323,
          0.9659
        ],
        [
          0.333,
          0.9645
        ],
        [
          0.343,
          0.9641
        ],
        [
          0.353,
          0.9631
        ],
        [
          0.3631,
          0.9609
        ],
        [
          0.3735,
          0.9609
        ],
        [
          0.3835,
          0.96
        ],
        [
          0.3935,
          0.9581
        ],
        [
          0.4035,
          0.9563
        ],
        [
          0.4136,
          0.9536
        ],
        [
          0.424,
          0.9509
        ],
        [
          0.434,
          0.949
        ],
        [
          0.444,
          0.9468
        ],
        [
          0.454,
          0.944
        ],
        [
          0.4641,
          0.9395
        ],
        [
          0.4745,
          0.9372
        ],
        [
          0.4845,
          0.9345
        ],
        [
          0.4945,
          0.9304
        ],
        [
          0.5045,
          0.9268
        ],
        [
          0.5146,
          0.9245
        ],
        [
          0.525,
          0.9204
        ],
        [
          0.535,
          0.9186
        ],
        [
          0.545,
          0.9163
        ],
        [
          0.5551,
          0.914
        ],
        [
          0.5655,
          0.9076
        ],
        [
          0.5755,
          0.9031
        ],
        [
          0.5855,
          0.8981
        ],
        [
          0.5955,
          0.894
        ],
        [
          0.6056,
          0.8904
        ],
        [
          0.616,
          0.8863
        ],
        [
          0.626,
          0.8817
        ],
        [
          0.636,
          0.8763
        ],
        [
          0.646,
          0.8708
        ],
        [
          0.6561,
          0.8653
        ],
        [
          0.6665,
          0.8603
        ],
        [
          0.6765,
          0.8549
        ],
        [
          0.6865,
          0.8508
        ],
        [
          0.6965,
          0.848
        ],
        [
          0.7066,
          0.8449
        ],
        [
          0.717,
          0.8417
        ],
        [
          0.727,
          0.8358
        ],
        [
          0.737,
          0.8312
        ],
        [
          0.747,
          0.8271
        ],
        [
          0.7571,
          0.8203
        ],
        [
          0.7675,
          0.8139
        ],
        [
          0.7775,
          0.8098
        ],
        [
          0.7875,
          0.8035
        ],
        [
          0.7975,
          0.7989
        ],
        [
          0.8076,
          0.793
        ],
        [
          0.818,
          0.7857
        ],
        [
          0.828,
          0.7816
        ],
        [
          0.838,
          0.7766
        ],
        [
          0.848,
          0.7712
        ],
        [
          0.8581,
          0.7643
        ],
        [
          0.8685,
          0.7598
        ],
        [
          0.8785,
          0.7534
        ],
        [
          0.8885,
          0.7466
        ],
        [
          0.8985,
          0.7402
        ],
        [
          0.9086,
          0.7352
        ],
        [
          0.919,
          0.7293
        ],
        [
          0.929,
          0.7234
        ],
        [
          0.939,
          0.7179
        ],
        [
          0.949,
          0.7111
        ],
        [
          0.9591,
          0.7034
        ],
        [
          0.9695,
          0.6961
        ],
        [
          0.9795,
          0.6884
        ],
        [
          0.9895,
          0.682
        ],
        [
          0.9995,
          0.6742
        ],
        [
          1.0,
          0.6738
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.26202798662014043,
        "difference": -0.002155410946282903,
        "worse": 0.6633333333333333,
        "kept_right": 0.25777458596899044,
        "kept_wrong": 0.004253400651150013,
        "handed_off": 0.7379720133798596
      },
      "usd_per_1000": 4.132525440049952
    },
    {
      "decision": 326,
      "frontier": 124,
      "frontier_accuracy": 0.9667879890809827,
      "in_sample": {
        "threshold": 0.9701587024993364,
        "answered": 0.18016378525932666,
        "accuracy": 0.9667879890809827
      },
      "curve": [
        [
          0.0,
          0.9668
        ],
        [
          0.01,
          0.9668
        ],
        [
          0.02,
          0.9668
        ],
        [
          0.03,
          0.9668
        ],
        [
          0.04,
          0.9668
        ],
        [
          0.05,
          0.9668
        ],
        [
          0.0601,
          0.9668
        ],
        [
          0.0705,
          0.9668
        ],
        [
          0.0805,
          0.9668
        ],
        [
          0.0905,
          0.9668
        ],
        [
          0.1005,
          0.9668
        ],
        [
          0.1106,
          0.9668
        ],
        [
          0.121,
          0.9668
        ],
        [
          0.131,
          0.9668
        ],
        [
          0.141,
          0.9668
        ],
        [
          0.151,
          0.9668
        ],
        [
          0.1611,
          0.9663
        ],
        [
          0.1715,
          0.9663
        ],
        [
          0.1815,
          0.9663
        ],
        [
          0.1915,
          0.9663
        ],
        [
          0.2015,
          0.9663
        ],
        [
          0.2116,
          0.9659
        ],
        [
          0.222,
          0.9659
        ],
        [
          0.232,
          0.965
        ],
        [
          0.242,
          0.965
        ],
        [
          0.252,
          0.9645
        ],
        [
          0.2621,
          0.9645
        ],
        [
          0.2725,
          0.965
        ],
        [
          0.2825,
          0.965
        ],
        [
          0.2925,
          0.9654
        ],
        [
          0.3025,
          0.965
        ],
        [
          0.3126,
          0.965
        ],
        [
          0.323,
          0.9645
        ],
        [
          0.333,
          0.9627
        ],
        [
          0.343,
          0.9622
        ],
        [
          0.353,
          0.9613
        ],
        [
          0.3631,
          0.9595
        ],
        [
          0.3735,
          0.9595
        ],
        [
          0.3835,
          0.9586
        ],
        [
          0.3935,
          0.9568
        ],
        [
          0.4035,
          0.955
        ],
        [
          0.4136,
          0.9522
        ],
        [
          0.424,
          0.9495
        ],
        [
          0.434,
          0.9477
        ],
        [
          0.444,
          0.9454
        ],
        [
          0.454,
          0.9436
        ],
        [
          0.4641,
          0.939
        ],
        [
          0.4745,
          0.9363
        ],
        [
          0.4845,
          0.9336
        ],
        [
          0.4945,
          0.9299
        ],
        [
          0.5045,
          0.9263
        ],
        [
          0.5146,
          0.9245
        ],
        [
          0.525,
          0.9204
        ],
        [
          0.535,
          0.919
        ],
        [
          0.545,
          0.9167
        ],
        [
          0.5551,
          0.9145
        ],
        [
          0.5655,
          0.9081
        ],
        [
          0.5755,
          0.9035
        ],
        [
          0.5855,
          0.8981
        ],
        [
          0.5955,
          0.8935
        ],
        [
          0.6056,
          0.889
        ],
        [
          0.616,
          0.8844
        ],
        [
          0.626,
          0.8799
        ],
        [
          0.636,
          0.8744
        ],
        [
          0.646,
          0.8685
        ],
        [
          0.6561,
          0.8631
        ],
        [
          0.6665,
          0.8576
        ],
        [
          0.6765,
          0.8526
        ],
        [
          0.6865,
          0.848
        ],
        [
          0.6965,
          0.8449
        ],
        [
          0.7066,
          0.8412
        ],
        [
          0.717,
          0.838
        ],
        [
          0.727,
          0.8326
        ],
        [
          0.737,
          0.8285
        ],
        [
          0.747,
          0.8244
        ],
        [
          0.7571,
          0.8176
        ],
        [
          0.7675,
          0.8112
        ],
        [
          0.7775,
          0.8071
        ],
        [
          0.7875,
          0.8012
        ],
        [
          0.7975,
          0.7971
        ],
        [
          0.8076,
          0.7912
        ],
        [
          0.818,
          0.7843
        ],
        [
          0.828,
          0.7798
        ],
        [
          0.838,
          0.7753
        ],
        [
          0.848,
          0.7702
        ],
        [
          0.8581,
          0.7639
        ],
        [
          0.8685,
          0.7598
        ],
        [
          0.8785,
          0.753
        ],
        [
          0.8885,
          0.7461
        ],
        [
          0.8985,
          0.7402
        ],
        [
          0.9086,
          0.7348
        ],
        [
          0.919,
          0.7288
        ],
        [
          0.929,
          0.7229
        ],
        [
          0.939,
          0.7175
        ],
        [
          0.949,
          0.7111
        ],
        [
          0.9591,
          0.7034
        ],
        [
          0.9695,
          0.6961
        ],
        [
          0.9795,
          0.6888
        ],
        [
          0.9895,
          0.6824
        ],
        [
          0.9995,
          0.6742
        ],
        [
          1.0,
          0.6738
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.22031609419699724,
        "difference": -0.0016160634963028736,
        "worse": 0.53,
        "kept_right": 0.21794504869989054,
        "kept_wrong": 0.002371045497106859,
        "handed_off": 0.7796839058030024
      },
      "usd_per_1000": 0.9044479367615599
    },
    {
      "decision": 326,
      "frontier": 278,
      "frontier_accuracy": 0.9499545040946314,
      "in_sample": {
        "threshold": 0.9768268404588278,
        "answered": 0.1565059144676979,
        "accuracy": 0.9499545040946314
      },
      "curve": [
        [
          0.0,
          0.95
        ],
        [
          0.01,
          0.95
        ],
        [
          0.02,
          0.95
        ],
        [
          0.03,
          0.95
        ],
        [
          0.04,
          0.95
        ],
        [
          0.05,
          0.95
        ],
        [
          0.0601,
          0.95
        ],
        [
          0.0705,
          0.95
        ],
        [
          0.0805,
          0.95
        ],
        [
          0.0905,
          0.95
        ],
        [
          0.1005,
          0.95
        ],
        [
          0.1106,
          0.95
        ],
        [
          0.121,
          0.95
        ],
        [
          0.131,
          0.95
        ],
        [
          0.141,
          0.95
        ],
        [
          0.151,
          0.95
        ],
        [
          0.1611,
          0.9495
        ],
        [
          0.1715,
          0.9495
        ],
        [
          0.1815,
          0.949
        ],
        [
          0.1915,
          0.949
        ],
        [
          0.2015,
          0.9495
        ],
        [
          0.2116,
          0.9495
        ],
        [
          0.222,
          0.9495
        ],
        [
          0.232,
          0.9486
        ],
        [
          0.242,
          0.9486
        ],
        [
          0.252,
          0.9481
        ],
        [
          0.2621,
          0.9481
        ],
        [
          0.2725,
          0.9486
        ],
        [
          0.2825,
          0.9486
        ],
        [
          0.2925,
          0.949
        ],
        [
          0.3025,
          0.9486
        ],
        [
          0.3126,
          0.9481
        ],
        [
          0.323,
          0.9477
        ],
        [
          0.333,
          0.9463
        ],
        [
          0.343,
          0.9463
        ],
        [
          0.353,
          0.9454
        ],
        [
          0.3631,
          0.9431
        ],
        [
          0.3735,
          0.9431
        ],
        [
          0.3835,
          0.9427
        ],
        [
          0.3935,
          0.9413
        ],
        [
          0.4035,
          0.9399
        ],
        [
          0.4136,
          0.9372
        ],
        [
          0.424,
          0.9349
        ],
        [
          0.434,
          0.9336
        ],
        [
          0.444,
          0.9313
        ],
        [
          0.454,
          0.9295
        ],
        [
          0.4641,
          0.9254
        ],
        [
          0.4745,
          0.9236
        ],
        [
          0.4845,
          0.9208
        ],
        [
          0.4945,
          0.9177
        ],
        [
          0.5045,
          0.914
        ],
        [
          0.5146,
          0.9117
        ],
        [
          0.525,
          0.9086
        ],
        [
          0.535,
          0.9072
        ],
        [
          0.545,
          0.9058
        ],
        [
          0.5551,
          0.904
        ],
        [
          0.5655,
          0.8981
        ],
        [
          0.5755,
          0.894
        ],
        [
          0.5855,
          0.8885
        ],
        [
          0.5955,
          0.8844
        ],
        [
          0.6056,
          0.8808
        ],
        [
          0.616,
          0.8767
        ],
        [
          0.626,
          0.8722
        ],
        [
          0.636,
          0.8672
        ],
        [
          0.646,
          0.8612
        ],
        [
          0.6561,
          0.8558
        ],
        [
          0.6665,
          0.8512
        ],
        [
          0.6765,
          0.8467
        ],
        [
          0.6865,
          0.8426
        ],
        [
          0.6965,
          0.8394
        ],
        [
          0.7066,
          0.8362
        ],
        [
          0.717,
          0.8335
        ],
        [
          0.727,
          0.828
        ],
        [
          0.737,
          0.8239
        ],
        [
          0.747,
          0.8198
        ],
        [
          0.7571,
          0.813
        ],
        [
          0.7675,
          0.8076
        ],
        [
          0.7775,
          0.8035
        ],
        [
          0.7875,
          0.7994
        ],
        [
          0.7975,
          0.7962
        ],
        [
          0.8076,
          0.7907
        ],
        [
          0.818,
          0.7834
        ],
        [
          0.828,
          0.7789
        ],
        [
          0.838,
          0.7734
        ],
        [
          0.848,
          0.7675
        ],
        [
          0.8581,
          0.7602
        ],
        [
          0.8685,
          0.7561
        ],
        [
          0.8785,
          0.7498
        ],
        [
          0.8885,
          0.7429
        ],
        [
          0.8985,
          0.7366
        ],
        [
          0.9086,
          0.7307
        ],
        [
          0.919,
          0.7247
        ],
        [
          0.929,
          0.7193
        ],
        [
          0.939,
          0.7147
        ],
        [
          0.949,
          0.7084
        ],
        [
          0.9591,
          0.702
        ],
        [
          0.9695,
          0.6947
        ],
        [
          0.9795,
          0.6874
        ],
        [
          0.9895,
          0.6811
        ],
        [
          0.9995,
          0.6742
        ],
        [
          1.0,
          0.6738
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.23181139171241363,
        "difference": -0.0018949288361219438,
        "worse": 0.66,
        "kept_right": 0.22886140609655703,
        "kept_wrong": 0.0029499856158568625,
        "handed_off": 0.768188608287586
      },
      "usd_per_1000": 5.023120463713943
    },
    {
      "decision": 326,
      "frontier": 275,
      "frontier_accuracy": 0.9426751592356688,
      "in_sample": {
        "threshold": 0.9701587024993364,
        "answered": 0.18016378525932666,
        "accuracy": 0.9426751592356688
      },
      "curve": [
        [
          0.0,
          0.9427
        ],
        [
          0.01,
          0.9427
        ],
        [
          0.02,
          0.9427
        ],
        [
          0.03,
          0.9427
        ],
        [
          0.04,
          0.9427
        ],
        [
          0.05,
          0.9427
        ],
        [
          0.0601,
          0.9427
        ],
        [
          0.0705,
          0.9427
        ],
        [
          0.0805,
          0.9427
        ],
        [
          0.0905,
          0.9427
        ],
        [
          0.1005,
          0.9427
        ],
        [
          0.1106,
          0.9427
        ],
        [
          0.121,
          0.9427
        ],
        [
          0.131,
          0.9427
        ],
        [
          0.141,
          0.9427
        ],
        [
          0.151,
          0.9427
        ],
        [
          0.1611,
          0.9422
        ],
        [
          0.1715,
          0.9422
        ],
        [
          0.1815,
          0.9422
        ],
        [
          0.1915,
          0.9422
        ],
        [
          0.2015,
          0.9422
        ],
        [
          0.2116,
          0.9418
        ],
        [
          0.222,
          0.9418
        ],
        [
          0.232,
          0.9409
        ],
        [
          0.242,
          0.9409
        ],
        [
          0.252,
          0.9404
        ],
        [
          0.2621,
          0.9404
        ],
        [
          0.2725,
          0.9409
        ],
        [
          0.2825,
          0.9409
        ],
        [
          0.2925,
          0.9413
        ],
        [
          0.3025,
          0.9409
        ],
        [
          0.3126,
          0.9404
        ],
        [
          0.323,
          0.9399
        ],
        [
          0.333,
          0.9386
        ],
        [
          0.343,
          0.9386
        ],
        [
          0.353,
          0.9377
        ],
        [
          0.3631,
          0.9359
        ],
        [
          0.3735,
          0.9363
        ],
        [
          0.3835,
          0.9354
        ],
        [
          0.3935,
          0.9336
        ],
        [
          0.4035,
          0.9322
        ],
        [
          0.4136,
          0.9299
        ],
        [
          0.424,
          0.9272
        ],
        [
          0.434,
          0.9258
        ],
        [
          0.444,
          0.9236
        ],
        [
          0.454,
          0.9227
        ],
        [
          0.4641,
          0.9186
        ],
        [
          0.4745,
          0.9172
        ],
        [
          0.4845,
          0.9149
        ],
        [
          0.4945,
          0.9117
        ],
        [
          0.5045,
          0.9081
        ],
        [
          0.5146,
          0.9058
        ],
        [
          0.525,
          0.9022
        ],
        [
          0.535,
          0.9008
        ],
        [
          0.545,
          0.899
        ],
        [
          0.5551,
          0.8972
        ],
        [
          0.5655,
          0.8917
        ],
        [
          0.5755,
          0.8876
        ],
        [
          0.5855,
          0.8826
        ],
        [
          0.5955,
          0.879
        ],
        [
          0.6056,
          0.8758
        ],
        [
          0.616,
          0.8712
        ],
        [
          0.626,
          0.8662
        ],
        [
          0.636,
          0.8608
        ],
        [
          0.646,
          0.8553
        ],
        [
          0.6561,
          0.8503
        ],
        [
          0.6665,
          0.8462
        ],
        [
          0.6765,
          0.8412
        ],
        [
          0.6865,
          0.8371
        ],
        [
          0.6965,
          0.8344
        ],
        [
          0.7066,
          0.8317
        ],
        [
          0.717,
          0.8285
        ],
        [
          0.727,
          0.823
        ],
        [
          0.737,
          0.8194
        ],
        [
          0.747,
          0.8153
        ],
        [
          0.7571,
          0.8089
        ],
        [
          0.7675,
          0.8035
        ],
        [
          0.7775,
          0.7998
        ],
        [
          0.7875,
          0.7939
        ],
        [
          0.7975,
          0.7907
        ],
        [
          0.8076,
          0.7848
        ],
        [
          0.818,
          0.7784
        ],
        [
          0.828,
          0.7739
        ],
        [
          0.838,
          0.7702
        ],
        [
          0.848,
          0.7652
        ],
        [
          0.8581,
          0.7589
        ],
        [
          0.8685,
          0.7548
        ],
        [
          0.8785,
          0.748
        ],
        [
          0.8885,
          0.7416
        ],
        [
          0.8985,
          0.7361
        ],
        [
          0.9086,
          0.7311
        ],
        [
          0.919,
          0.7247
        ],
        [
          0.929,
          0.7193
        ],
        [
          0.939,
          0.7143
        ],
        [
          0.949,
          0.7079
        ],
        [
          0.9591,
          0.7006
        ],
        [
          0.9695,
          0.6947
        ],
        [
          0.9795,
          0.687
        ],
        [
          0.9895,
          0.682
        ],
        [
          0.9995,
          0.6742
        ],
        [
          1.0,
          0.6738
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.21996644162977635,
        "difference": -0.0017312315801628992,
        "worse": 0.53,
        "kept_right": 0.217546958219434,
        "kept_wrong": 0.0024194834103426275,
        "handed_off": 0.7800335583702233
      },
      "usd_per_1000": 0.4244518067155987
    },
    {
      "decision": 326,
      "frontier": 192,
      "frontier_accuracy": 0.9331210191082803,
      "in_sample": {
        "threshold": 0.94916747469274,
        "answered": 0.22611464968152867,
        "accuracy": 0.9331210191082803
      },
      "curve": [
        [
          0.0,
          0.9331
        ],
        [
          0.01,
          0.9331
        ],
        [
          0.02,
          0.9331
        ],
        [
          0.03,
          0.9331
        ],
        [
          0.04,
          0.9331
        ],
        [
          0.05,
          0.9331
        ],
        [
          0.0601,
          0.9331
        ],
        [
          0.0705,
          0.9331
        ],
        [
          0.0805,
          0.9331
        ],
        [
          0.0905,
          0.9331
        ],
        [
          0.1005,
          0.9331
        ],
        [
          0.1106,
          0.9331
        ],
        [
          0.121,
          0.9331
        ],
        [
          0.131,
          0.9331
        ],
        [
          0.141,
          0.9331
        ],
        [
          0.151,
          0.9331
        ],
        [
          0.1611,
          0.9327
        ],
        [
          0.1715,
          0.9327
        ],
        [
          0.1815,
          0.9327
        ],
        [
          0.1915,
          0.9327
        ],
        [
          0.2015,
          0.9331
        ],
        [
          0.2116,
          0.9331
        ],
        [
          0.222,
          0.9331
        ],
        [
          0.232,
          0.9322
        ],
        [
          0.242,
          0.9322
        ],
        [
          0.252,
          0.9318
        ],
        [
          0.2621,
          0.9318
        ],
        [
          0.2725,
          0.9322
        ],
        [
          0.2825,
          0.9322
        ],
        [
          0.2925,
          0.9322
        ],
        [
          0.3025,
          0.9318
        ],
        [
          0.3126,
          0.9313
        ],
        [
          0.323,
          0.9304
        ],
        [
          0.333,
          0.9286
        ],
        [
          0.343,
          0.929
        ],
        [
          0.353,
          0.929
        ],
        [
          0.3631,
          0.9272
        ],
        [
          0.3735,
          0.9272
        ],
        [
          0.3835,
          0.9272
        ],
        [
          0.3935,
          0.9258
        ],
        [
          0.4035,
          0.9245
        ],
        [
          0.4136,
          0.9217
        ],
        [
          0.424,
          0.919
        ],
        [
          0.434,
          0.9181
        ],
        [
          0.444,
          0.9163
        ],
        [
          0.454,
          0.9158
        ],
        [
          0.4641,
          0.9117
        ],
        [
          0.4745,
          0.9095
        ],
        [
          0.4845,
          0.9072
        ],
        [
          0.4945,
          0.904
        ],
        [
          0.5045,
          0.8999
        ],
        [
          0.5146,
          0.8976
        ],
        [
          0.525,
          0.894
        ],
        [
          0.535,
          0.8922
        ],
        [
          0.545,
          0.8904
        ],
        [
          0.5551,
          0.8881
        ],
        [
          0.5655,
          0.8826
        ],
        [
          0.5755,
          0.8785
        ],
        [
          0.5855,
          0.874
        ],
        [
          0.5955,
          0.8712
        ],
        [
          0.6056,
          0.8676
        ],
        [
          0.616,
          0.8644
        ],
        [
          0.626,
          0.8599
        ],
        [
          0.636,
          0.8544
        ],
        [
          0.646,
          0.849
        ],
        [
          0.6561,
          0.8435
        ],
        [
          0.6665,
          0.8385
        ],
        [
          0.6765,
          0.8339
        ],
        [
          0.6865,
          0.8312
        ],
        [
          0.6965,
          0.8294
        ],
        [
          0.7066,
          0.8262
        ],
        [
          0.717,
          0.8239
        ],
        [
          0.727,
          0.818
        ],
        [
          0.737,
          0.8139
        ],
        [
          0.747,
          0.8103
        ],
        [
          0.7571,
          0.8039
        ],
        [
          0.7675,
          0.798
        ],
        [
          0.7775,
          0.7944
        ],
        [
          0.7875,
          0.7889
        ],
        [
          0.7975,
          0.7857
        ],
        [
          0.8076,
          0.7798
        ],
        [
          0.818,
          0.773
        ],
        [
          0.828,
          0.7689
        ],
        [
          0.838,
          0.7643
        ],
        [
          0.848,
          0.7584
        ],
        [
          0.8581,
          0.7516
        ],
        [
          0.8685,
          0.7489
        ],
        [
          0.8785,
          0.7434
        ],
        [
          0.8885,
          0.737
        ],
        [
          0.8985,
          0.7307
        ],
        [
          0.9086,
          0.7266
        ],
        [
          0.919,
          0.7216
        ],
        [
          0.929,
          0.7166
        ],
        [
          0.939,
          0.712
        ],
        [
          0.949,
          0.7056
        ],
        [
          0.9591,
          0.6993
        ],
        [
          0.9695,
          0.6934
        ],
        [
          0.9795,
          0.6865
        ],
        [
          0.9895,
          0.6802
        ],
        [
          0.9995,
          0.6742
        ],
        [
          1.0,
          0.6738
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.2427227488746045,
        "difference": -0.0018190442722232484,
        "worse": 0.51,
        "kept_right": 0.23953943249281945,
        "kept_wrong": 0.0031833163817851307,
        "handed_off": 0.7572772511253953
      },
      "usd_per_1000": 8.814773970786169
    },
    {
      "decision": 326,
      "frontier": 123,
      "frontier_accuracy": 0.9326660600545951,
      "in_sample": {
        "threshold": 0.9768268404588278,
        "answered": 0.1565059144676979,
        "accuracy": 0.9326660600545951
      },
      "curve": [
        [
          0.0,
          0.9327
        ],
        [
          0.01,
          0.9327
        ],
        [
          0.02,
          0.9327
        ],
        [
          0.03,
          0.9327
        ],
        [
          0.04,
          0.9327
        ],
        [
          0.05,
          0.9327
        ],
        [
          0.0601,
          0.9327
        ],
        [
          0.0705,
          0.9327
        ],
        [
          0.0805,
          0.9327
        ],
        [
          0.0905,
          0.9327
        ],
        [
          0.1005,
          0.9327
        ],
        [
          0.1106,
          0.9327
        ],
        [
          0.121,
          0.9327
        ],
        [
          0.131,
          0.9327
        ],
        [
          0.141,
          0.9327
        ],
        [
          0.151,
          0.9327
        ],
        [
          0.1611,
          0.9322
        ],
        [
          0.1715,
          0.9322
        ],
        [
          0.1815,
          0.9318
        ],
        [
          0.1915,
          0.9318
        ],
        [
          0.2015,
          0.9318
        ],
        [
          0.2116,
          0.9313
        ],
        [
          0.222,
          0.9313
        ],
        [
          0.232,
          0.9304
        ],
        [
          0.242,
          0.9304
        ],
        [
          0.252,
          0.9299
        ],
        [
          0.2621,
          0.9299
        ],
        [
          0.2725,
          0.9304
        ],
        [
          0.2825,
          0.9304
        ],
        [
          0.2925,
          0.9308
        ],
        [
          0.3025,
          0.9304
        ],
        [
          0.3126,
          0.9299
        ],
        [
          0.323,
          0.9295
        ],
        [
          0.333,
          0.9277
        ],
        [
          0.343,
          0.9277
        ],
        [
          0.353,
          0.9268
        ],
        [
          0.3631,
          0.9249
        ],
        [
          0.3735,
          0.9254
        ],
        [
          0.3835,
          0.9258
        ],
        [
          0.3935,
          0.9236
        ],
        [
          0.4035,
          0.9222
        ],
        [
          0.4136,
          0.9204
        ],
        [
          0.424,
          0.9177
        ],
        [
          0.434,
          0.9163
        ],
        [
          0.444,
          0.914
        ],
        [
          0.454,
          0.9122
        ],
        [
          0.4641,
          0.9086
        ],
        [
          0.4745,
          0.9067
        ],
        [
          0.4845,
          0.9054
        ],
        [
          0.4945,
          0.9026
        ],
        [
          0.5045,
          0.899
        ],
        [
          0.5146,
          0.8972
        ],
        [
          0.525,
          0.8926
        ],
        [
          0.535,
          0.8913
        ],
        [
          0.545,
          0.8899
        ],
        [
          0.5551,
          0.8876
        ],
        [
          0.5655,
          0.8822
        ],
        [
          0.5755,
          0.8776
        ],
        [
          0.5855,
          0.8735
        ],
        [
          0.5955,
          0.8703
        ],
        [
          0.6056,
          0.8667
        ],
        [
          0.616,
          0.8635
        ],
        [
          0.626,
          0.8603
        ],
        [
          0.636,
          0.8553
        ],
        [
          0.646,
          0.8503
        ],
        [
          0.6561,
          0.8453
        ],
        [
          0.6665,
          0.8399
        ],
        [
          0.6765,
          0.8348
        ],
        [
          0.6865,
          0.8326
        ],
        [
          0.6965,
          0.8285
        ],
        [
          0.7066,
          0.8253
        ],
        [
          0.717,
          0.8235
        ],
        [
          0.727,
          0.8185
        ],
        [
          0.737,
          0.8148
        ],
        [
          0.747,
          0.8116
        ],
        [
          0.7571,
          0.8044
        ],
        [
          0.7675,
          0.7985
        ],
        [
          0.7775,
          0.7953
        ],
        [
          0.7875,
          0.7894
        ],
        [
          0.7975,
          0.7866
        ],
        [
          0.8076,
          0.7825
        ],
        [
          0.818,
          0.7753
        ],
        [
          0.828,
          0.7707
        ],
        [
          0.838,
          0.7666
        ],
        [
          0.848,
          0.7611
        ],
        [
          0.8581,
          0.7548
        ],
        [
          0.8685,
          0.7507
        ],
        [
          0.8785,
          0.7443
        ],
        [
          0.8885,
          0.7384
        ],
        [
          0.8985,
          0.7325
        ],
        [
          0.9086,
          0.7266
        ],
        [
          0.919,
          0.7225
        ],
        [
          0.929,
          0.7184
        ],
        [
          0.939,
          0.7125
        ],
        [
          0.949,
          0.7061
        ],
        [
          0.9591,
          0.6988
        ],
        [
          0.9695,
          0.6924
        ],
        [
          0.9795,
          0.6865
        ],
        [
          0.9895,
          0.6815
        ],
        [
          0.9995,
          0.6742
        ],
        [
          1.0,
          0.6738
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1929922676417336,
        "difference": -0.0012854704495421578,
        "worse": 0.53,
        "kept_right": 0.19149147171422312,
        "kept_wrong": 0.0015007959275106755,
        "handed_off": 0.8070077323582667
      },
      "usd_per_1000": 0.23986356564075148
    },
    {
      "decision": 326,
      "frontier": 274,
      "frontier_accuracy": 0.9085532302092811,
      "in_sample": {
        "threshold": 0.8818270219776835,
        "answered": 0.316196542311192,
        "accuracy": 0.9085532302092811
      },
      "curve": [
        [
          0.0,
          0.9086
        ],
        [
          0.01,
          0.9086
        ],
        [
          0.02,
          0.9086
        ],
        [
          0.03,
          0.9086
        ],
        [
          0.04,
          0.9086
        ],
        [
          0.05,
          0.9086
        ],
        [
          0.0601,
          0.9086
        ],
        [
          0.0705,
          0.9086
        ],
        [
          0.0805,
          0.9086
        ],
        [
          0.0905,
          0.9086
        ],
        [
          0.1005,
          0.9086
        ],
        [
          0.1106,
          0.9086
        ],
        [
          0.121,
          0.9086
        ],
        [
          0.131,
          0.9086
        ],
        [
          0.141,
          0.9086
        ],
        [
          0.151,
          0.9086
        ],
        [
          0.1611,
          0.9086
        ],
        [
          0.1715,
          0.9086
        ],
        [
          0.1815,
          0.9086
        ],
        [
          0.1915,
          0.9086
        ],
        [
          0.2015,
          0.9086
        ],
        [
          0.2116,
          0.9086
        ],
        [
          0.222,
          0.9086
        ],
        [
          0.232,
          0.9086
        ],
        [
          0.242,
          0.9086
        ],
        [
          0.252,
          0.9081
        ],
        [
          0.2621,
          0.9081
        ],
        [
          0.2725,
          0.909
        ],
        [
          0.2825,
          0.909
        ],
        [
          0.2925,
          0.909
        ],
        [
          0.3025,
          0.9086
        ],
        [
          0.3126,
          0.9086
        ],
        [
          0.323,
          0.9076
        ],
        [
          0.333,
          0.9067
        ],
        [
          0.343,
          0.9063
        ],
        [
          0.353,
          0.9054
        ],
        [
          0.3631,
          0.9035
        ],
        [
          0.3735,
          0.904
        ],
        [
          0.3835,
          0.9031
        ],
        [
          0.3935,
          0.9017
        ],
        [
          0.4035,
          0.9008
        ],
        [
          0.4136,
          0.8981
        ],
        [
          0.424,
          0.8963
        ],
        [
          0.434,
          0.8949
        ],
        [
          0.444,
          0.8926
        ],
        [
          0.454,
          0.8908
        ],
        [
          0.4641,
          0.8876
        ],
        [
          0.4745,
          0.8867
        ],
        [
          0.4845,
          0.8849
        ],
        [
          0.4945,
          0.8808
        ],
        [
          0.5045,
          0.8772
        ],
        [
          0.5146,
          0.8753
        ],
        [
          0.525,
          0.8717
        ],
        [
          0.535,
          0.8703
        ],
        [
          0.545,
          0.869
        ],
        [
          0.5551,
          0.8676
        ],
        [
          0.5655,
          0.8626
        ],
        [
          0.5755,
          0.859
        ],
        [
          0.5855,
          0.854
        ],
        [
          0.5955,
          0.8512
        ],
        [
          0.6056,
          0.848
        ],
        [
          0.616,
          0.8435
        ],
        [
          0.626,
          0.8399
        ],
        [
          0.636,
          0.8348
        ],
        [
          0.646,
          0.8298
        ],
        [
          0.6561,
          0.8253
        ],
        [
          0.6665,
          0.8207
        ],
        [
          0.6765,
          0.8157
        ],
        [
          0.6865,
          0.8116
        ],
        [
          0.6965,
          0.8094
        ],
        [
          0.7066,
          0.8066
        ],
        [
          0.717,
          0.8035
        ],
        [
          0.727,
          0.7994
        ],
        [
          0.737,
          0.7957
        ],
        [
          0.747,
          0.7916
        ],
        [
          0.7571,
          0.7853
        ],
        [
          0.7675,
          0.7803
        ],
        [
          0.7775,
          0.7762
        ],
        [
          0.7875,
          0.7712
        ],
        [
          0.7975,
          0.7689
        ],
        [
          0.8076,
          0.7643
        ],
        [
          0.818,
          0.7575
        ],
        [
          0.828,
          0.7539
        ],
        [
          0.838,
          0.7507
        ],
        [
          0.848,
          0.7457
        ],
        [
          0.8581,
          0.7407
        ],
        [
          0.8685,
          0.7379
        ],
        [
          0.8785,
          0.7316
        ],
        [
          0.8885,
          0.7257
        ],
        [
          0.8985,
          0.7211
        ],
        [
          0.9086,
          0.717
        ],
        [
          0.919,
          0.7143
        ],
        [
          0.929,
          0.7097
        ],
        [
          0.939,
          0.7052
        ],
        [
          0.949,
          0.6993
        ],
        [
          0.9591,
          0.6934
        ],
        [
          0.9695,
          0.6884
        ],
        [
          0.9795,
          0.6838
        ],
        [
          0.9895,
          0.6806
        ],
        [
          0.9995,
          0.6742
        ],
        [
          1.0,
          0.6738
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.2973791737235052,
        "difference": -0.0017252591432509164,
        "worse": 0.46,
        "kept_right": 0.29171884205645277,
        "kept_wrong": 0.005660331667052246,
        "handed_off": 0.7026208262764949
      },
      "usd_per_1000": 0.3226033397127082
    },
    {
      "decision": 326,
      "frontier": 47,
      "frontier_accuracy": 0.9044585987261147,
      "in_sample": {
        "threshold": 0.8644803214961087,
        "answered": 0.33166515013648773,
        "accuracy": 0.9044585987261147
      },
      "curve": [
        [
          0.0,
          0.9045
        ],
        [
          0.01,
          0.9045
        ],
        [
          0.02,
          0.9045
        ],
        [
          0.03,
          0.9045
        ],
        [
          0.04,
          0.9045
        ],
        [
          0.05,
          0.9045
        ],
        [
          0.0601,
          0.9045
        ],
        [
          0.0705,
          0.9045
        ],
        [
          0.0805,
          0.9045
        ],
        [
          0.0905,
          0.9045
        ],
        [
          0.1005,
          0.9045
        ],
        [
          0.1106,
          0.9045
        ],
        [
          0.121,
          0.9045
        ],
        [
          0.131,
          0.9045
        ],
        [
          0.141,
          0.9045
        ],
        [
          0.151,
          0.9045
        ],
        [
          0.1611,
          0.904
        ],
        [
          0.1715,
          0.904
        ],
        [
          0.1815,
          0.904
        ],
        [
          0.1915,
          0.904
        ],
        [
          0.2015,
          0.9045
        ],
        [
          0.2116,
          0.9045
        ],
        [
          0.222,
          0.9049
        ],
        [
          0.232,
          0.904
        ],
        [
          0.242,
          0.904
        ],
        [
          0.252,
          0.904
        ],
        [
          0.2621,
          0.904
        ],
        [
          0.2725,
          0.9045
        ],
        [
          0.2825,
          0.9045
        ],
        [
          0.2925,
          0.9049
        ],
        [
          0.3025,
          0.9054
        ],
        [
          0.3126,
          0.9049
        ],
        [
          0.323,
          0.9049
        ],
        [
          0.333,
          0.904
        ],
        [
          0.343,
          0.904
        ],
        [
          0.353,
          0.9035
        ],
        [
          0.3631,
          0.9013
        ],
        [
          0.3735,
          0.9017
        ],
        [
          0.3835,
          0.9022
        ],
        [
          0.3935,
          0.9008
        ],
        [
          0.4035,
          0.8999
        ],
        [
          0.4136,
          0.8976
        ],
        [
          0.424,
          0.8949
        ],
        [
          0.434,
          0.8935
        ],
        [
          0.444,
          0.8913
        ],
        [
          0.454,
          0.889
        ],
        [
          0.4641,
          0.8858
        ],
        [
          0.4745,
          0.8835
        ],
        [
          0.4845,
          0.8808
        ],
        [
          0.4945,
          0.8772
        ],
        [
          0.5045,
          0.8735
        ],
        [
          0.5146,
          0.8717
        ],
        [
          0.525,
          0.869
        ],
        [
          0.535,
          0.8676
        ],
        [
          0.545,
          0.8662
        ],
        [
          0.5551,
          0.8653
        ],
        [
          0.5655,
          0.8603
        ],
        [
          0.5755,
          0.8571
        ],
        [
          0.5855,
          0.853
        ],
        [
          0.5955,
          0.8508
        ],
        [
          0.6056,
          0.848
        ],
        [
          0.616,
          0.8453
        ],
        [
          0.626,
          0.8408
        ],
        [
          0.636,
          0.8367
        ],
        [
          0.646,
          0.8317
        ],
        [
          0.6561,
          0.8262
        ],
        [
          0.6665,
          0.8217
        ],
        [
          0.6765,
          0.8171
        ],
        [
          0.6865,
          0.8135
        ],
        [
          0.6965,
          0.8112
        ],
        [
          0.7066,
          0.808
        ],
        [
          0.717,
          0.8066
        ],
        [
          0.727,
          0.8021
        ],
        [
          0.737,
          0.7985
        ],
        [
          0.747,
          0.7953
        ],
        [
          0.7571,
          0.7903
        ],
        [
          0.7675,
          0.7839
        ],
        [
          0.7775,
          0.7803
        ],
        [
          0.7875,
          0.7757
        ],
        [
          0.7975,
          0.7725
        ],
        [
          0.8076,
          0.768
        ],
        [
          0.818,
          0.7611
        ],
        [
          0.828,
          0.7566
        ],
        [
          0.838,
          0.7534
        ],
        [
          0.848,
          0.748
        ],
        [
          0.8581,
          0.7425
        ],
        [
          0.8685,
          0.7384
        ],
        [
          0.8785,
          0.7329
        ],
        [
          0.8885,
          0.7275
        ],
        [
          0.8985,
          0.7216
        ],
        [
          0.9086,
          0.7179
        ],
        [
          0.919,
          0.7129
        ],
        [
          0.929,
          0.7088
        ],
        [
          0.939,
          0.7047
        ],
        [
          0.949,
          0.6993
        ],
        [
          0.9591,
          0.6929
        ],
        [
          0.9695,
          0.6893
        ],
        [
          0.9795,
          0.6838
        ],
        [
          0.9895,
          0.6788
        ],
        [
          0.9995,
          0.6742
        ],
        [
          1.0,
          0.6738
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.33843304305448946,
        "difference": -0.0023523776996001635,
        "worse": 0.5466666666666666,
        "kept_right": 0.3283654109203543,
        "kept_wrong": 0.010067632134135068,
        "handed_off": 0.6615669569455104
      },
      "usd_per_1000": 3.285027007334604
    },
    {
      "decision": 326,
      "frontier": 193,
      "frontier_accuracy": 0.89171974522293,
      "in_sample": {
        "threshold": 0.846631045847093,
        "answered": 0.3489535941765241,
        "accuracy": 0.89171974522293
      },
      "curve": [
        [
          0.0,
          0.8917
        ],
        [
          0.01,
          0.8917
        ],
        [
          0.02,
          0.8917
        ],
        [
          0.03,
          0.8917
        ],
        [
          0.04,
          0.8917
        ],
        [
          0.05,
          0.8917
        ],
        [
          0.0601,
          0.8917
        ],
        [
          0.0705,
          0.8917
        ],
        [
          0.0805,
          0.8917
        ],
        [
          0.0905,
          0.8917
        ],
        [
          0.1005,
          0.8917
        ],
        [
          0.1106,
          0.8917
        ],
        [
          0.121,
          0.8917
        ],
        [
          0.131,
          0.8917
        ],
        [
          0.141,
          0.8917
        ],
        [
          0.151,
          0.8917
        ],
        [
          0.1611,
          0.8917
        ],
        [
          0.1715,
          0.8917
        ],
        [
          0.1815,
          0.8917
        ],
        [
          0.1915,
          0.8917
        ],
        [
          0.2015,
          0.8922
        ],
        [
          0.2116,
          0.8922
        ],
        [
          0.222,
          0.8922
        ],
        [
          0.232,
          0.8917
        ],
        [
          0.242,
          0.8917
        ],
        [
          0.252,
          0.8913
        ],
        [
          0.2621,
          0.8913
        ],
        [
          0.2725,
          0.8917
        ],
        [
          0.2825,
          0.8917
        ],
        [
          0.2925,
          0.8922
        ],
        [
          0.3025,
          0.8922
        ],
        [
          0.3126,
          0.8922
        ],
        [
          0.323,
          0.8917
        ],
        [
          0.333,
          0.8904
        ],
        [
          0.343,
          0.8913
        ],
        [
          0.353,
          0.8913
        ],
        [
          0.3631,
          0.8899
        ],
        [
          0.3735,
          0.8908
        ],
        [
          0.3835,
          0.8908
        ],
        [
          0.3935,
          0.8904
        ],
        [
          0.4035,
          0.889
        ],
        [
          0.4136,
          0.8867
        ],
        [
          0.424,
          0.8844
        ],
        [
          0.434,
          0.8831
        ],
        [
          0.444,
          0.8817
        ],
        [
          0.454,
          0.8813
        ],
        [
          0.4641,
          0.8794
        ],
        [
          0.4745,
          0.8772
        ],
        [
          0.4845,
          0.8749
        ],
        [
          0.4945,
          0.8726
        ],
        [
          0.5045,
          0.869
        ],
        [
          0.5146,
          0.8667
        ],
        [
          0.525,
          0.8635
        ],
        [
          0.535,
          0.8621
        ],
        [
          0.545,
          0.8608
        ],
        [
          0.5551,
          0.8594
        ],
        [
          0.5655,
          0.8549
        ],
        [
          0.5755,
          0.854
        ],
        [
          0.5855,
          0.8499
        ],
        [
          0.5955,
          0.8485
        ],
        [
          0.6056,
          0.8462
        ],
        [
          0.616,
          0.8426
        ],
        [
          0.626,
          0.838
        ],
        [
          0.636,
          0.8335
        ],
        [
          0.646,
          0.8289
        ],
        [
          0.6561,
          0.8244
        ],
        [
          0.6665,
          0.8203
        ],
        [
          0.6765,
          0.8157
        ],
        [
          0.6865,
          0.8139
        ],
        [
          0.6965,
          0.8121
        ],
        [
          0.7066,
          0.8085
        ],
        [
          0.717,
          0.8062
        ],
        [
          0.727,
          0.8007
        ],
        [
          0.737,
          0.7971
        ],
        [
          0.747,
          0.7939
        ],
        [
          0.7571,
          0.788
        ],
        [
          0.7675,
          0.7839
        ],
        [
          0.7775,
          0.7803
        ],
        [
          0.7875,
          0.7766
        ],
        [
          0.7975,
          0.7743
        ],
        [
          0.8076,
          0.7693
        ],
        [
          0.818,
          0.763
        ],
        [
          0.828,
          0.7584
        ],
        [
          0.838,
          0.7543
        ],
        [
          0.848,
          0.7507
        ],
        [
          0.8581,
          0.7443
        ],
        [
          0.8685,
          0.742
        ],
        [
          0.8785,
          0.7357
        ],
        [
          0.8885,
          0.7298
        ],
        [
          0.8985,
          0.7243
        ],
        [
          0.9086,
          0.7188
        ],
        [
          0.919,
          0.7152
        ],
        [
          0.929,
          0.7102
        ],
        [
          0.939,
          0.7061
        ],
        [
          0.949,
          0.7006
        ],
        [
          0.9591,
          0.6938
        ],
        [
          0.9695,
          0.6888
        ],
        [
          0.9795,
          0.6829
        ],
        [
          0.9895,
          0.677
        ],
        [
          0.9995,
          0.6742
        ],
        [
          1.0,
          0.6738
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.35451169923721243,
        "difference": -0.0028226739377892394,
        "worse": 0.6733333333333333,
        "kept_right": 0.3419302087075896,
        "kept_wrong": 0.012581490529622296,
        "handed_off": 0.645488300762788
      },
      "usd_per_1000": 5.024092947550913
    },
    {
      "decision": 326,
      "frontier": 55,
      "frontier_accuracy": 0.8503184713375797,
      "in_sample": {
        "threshold": 0.6951433394285167,
        "answered": 0.4981801637852593,
        "accuracy": 0.8503184713375797
      },
      "curve": [
        [
          0.0,
          0.8503
        ],
        [
          0.01,
          0.8503
        ],
        [
          0.02,
          0.8508
        ],
        [
          0.03,
          0.8508
        ],
        [
          0.04,
          0.8512
        ],
        [
          0.05,
          0.8512
        ],
        [
          0.0601,
          0.8512
        ],
        [
          0.0705,
          0.8512
        ],
        [
          0.0805,
          0.8512
        ],
        [
          0.0905,
          0.8517
        ],
        [
          0.1005,
          0.8521
        ],
        [
          0.1106,
          0.8521
        ],
        [
          0.121,
          0.8521
        ],
        [
          0.131,
          0.8526
        ],
        [
          0.141,
          0.853
        ],
        [
          0.151,
          0.8535
        ],
        [
          0.1611,
          0.854
        ],
        [
          0.1715,
          0.8544
        ],
        [
          0.1815,
          0.854
        ],
        [
          0.1915,
          0.8544
        ],
        [
          0.2015,
          0.8544
        ],
        [
          0.2116,
          0.8549
        ],
        [
          0.222,
          0.8558
        ],
        [
          0.232,
          0.8549
        ],
        [
          0.242,
          0.8553
        ],
        [
          0.252,
          0.8553
        ],
        [
          0.2621,
          0.8558
        ],
        [
          0.2725,
          0.8571
        ],
        [
          0.2825,
          0.8571
        ],
        [
          0.2925,
          0.8576
        ],
        [
          0.3025,
          0.8581
        ],
        [
          0.3126,
          0.859
        ],
        [
          0.323,
          0.8603
        ],
        [
          0.333,
          0.8608
        ],
        [
          0.343,
          0.8608
        ],
        [
          0.353,
          0.8603
        ],
        [
          0.3631,
          0.8599
        ],
        [
          0.3735,
          0.8617
        ],
        [
          0.3835,
          0.8617
        ],
        [
          0.3935,
          0.8612
        ],
        [
          0.4035,
          0.8612
        ],
        [
          0.4136,
          0.8594
        ],
        [
          0.424,
          0.8581
        ],
        [
          0.434,
          0.8576
        ],
        [
          0.444,
          0.8567
        ],
        [
          0.454,
          0.8571
        ],
        [
          0.4641,
          0.8549
        ],
        [
          0.4745,
          0.853
        ],
        [
          0.4845,
          0.8526
        ],
        [
          0.4945,
          0.8508
        ],
        [
          0.5045,
          0.849
        ],
        [
          0.5146,
          0.848
        ],
        [
          0.525,
          0.8462
        ],
        [
          0.535,
          0.8458
        ],
        [
          0.545,
          0.8449
        ],
        [
          0.5551,
          0.8435
        ],
        [
          0.5655,
          0.8399
        ],
        [
          0.5755,
          0.838
        ],
        [
          0.5855,
          0.8335
        ],
        [
          0.5955,
          0.8308
        ],
        [
          0.6056,
          0.8289
        ],
        [
          0.616,
          0.8258
        ],
        [
          0.626,
          0.8217
        ],
        [
          0.636,
          0.8176
        ],
        [
          0.646,
          0.8135
        ],
        [
          0.6561,
          0.8089
        ],
        [
          0.6665,
          0.8044
        ],
        [
          0.6765,
          0.8007
        ],
        [
          0.6865,
          0.7985
        ],
        [
          0.6965,
          0.7957
        ],
        [
          0.7066,
          0.793
        ],
        [
          0.717,
          0.7912
        ],
        [
          0.727,
          0.7857
        ],
        [
          0.737,
          0.783
        ],
        [
          0.747,
          0.7803
        ],
        [
          0.7571,
          0.7748
        ],
        [
          0.7675,
          0.7707
        ],
        [
          0.7775,
          0.768
        ],
        [
          0.7875,
          0.7643
        ],
        [
          0.7975,
          0.7625
        ],
        [
          0.8076,
          0.758
        ],
        [
          0.818,
          0.752
        ],
        [
          0.828,
          0.7489
        ],
        [
          0.838,
          0.7452
        ],
        [
          0.848,
          0.7407
        ],
        [
          0.8581,
          0.7375
        ],
        [
          0.8685,
          0.7352
        ],
        [
          0.8785,
          0.7302
        ],
        [
          0.8885,
          0.7243
        ],
        [
          0.8985,
          0.7193
        ],
        [
          0.9086,
          0.7157
        ],
        [
          0.919,
          0.712
        ],
        [
          0.929,
          0.7079
        ],
        [
          0.939,
          0.7052
        ],
        [
          0.949,
          0.7002
        ],
        [
          0.9591,
          0.6938
        ],
        [
          0.9695,
          0.6888
        ],
        [
          0.9795,
          0.6838
        ],
        [
          0.9895,
          0.6788
        ],
        [
          0.9995,
          0.6742
        ],
        [
          1.0,
          0.6738
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.5040722691577313,
        "difference": -0.0012133057843926912,
        "worse": 0.5,
        "kept_right": 0.45490495088076227,
        "kept_wrong": 0.04916731827696883,
        "handed_off": 0.49592773084226865
      },
      "usd_per_1000": 1.8353996291798944
    },
    {
      "decision": 374,
      "frontier": 140,
      "frontier_accuracy": 0.9808917197452229,
      "in_sample": {
        "threshold": 0.982397,
        "answered": 0.19381255686988172,
        "accuracy": 0.9808917197452229
      },
      "curve": [
        [
          0.0,
          0.9809
        ],
        [
          0.01,
          0.9809
        ],
        [
          0.02,
          0.9809
        ],
        [
          0.03,
          0.9809
        ],
        [
          0.04,
          0.9809
        ],
        [
          0.05,
          0.9809
        ],
        [
          0.0601,
          0.9809
        ],
        [
          0.0705,
          0.9809
        ],
        [
          0.0805,
          0.9809
        ],
        [
          0.0905,
          0.9809
        ],
        [
          0.1005,
          0.9809
        ],
        [
          0.1106,
          0.9809
        ],
        [
          0.121,
          0.9809
        ],
        [
          0.131,
          0.9809
        ],
        [
          0.141,
          0.9809
        ],
        [
          0.151,
          0.9809
        ],
        [
          0.1611,
          0.9809
        ],
        [
          0.1715,
          0.9809
        ],
        [
          0.1815,
          0.9809
        ],
        [
          0.1915,
          0.9809
        ],
        [
          0.2015,
          0.98
        ],
        [
          0.2116,
          0.98
        ],
        [
          0.222,
          0.98
        ],
        [
          0.232,
          0.9795
        ],
        [
          0.242,
          0.9795
        ],
        [
          0.252,
          0.98
        ],
        [
          0.2621,
          0.9795
        ],
        [
          0.2725,
          0.9795
        ],
        [
          0.2825,
          0.9795
        ],
        [
          0.2925,
          0.9791
        ],
        [
          0.3025,
          0.9782
        ],
        [
          0.3126,
          0.9763
        ],
        [
          0.323,
          0.975
        ],
        [
          0.333,
          0.9741
        ],
        [
          0.343,
          0.9732
        ],
        [
          0.353,
          0.9713
        ],
        [
          0.3631,
          0.9718
        ],
        [
          0.3735,
          0.9695
        ],
        [
          0.3835,
          0.9663
        ],
        [
          0.3935,
          0.9636
        ],
        [
          0.4035,
          0.9622
        ],
        [
          0.4136,
          0.96
        ],
        [
          0.424,
          0.9572
        ],
        [
          0.434,
          0.9536
        ],
        [
          0.444,
          0.95
        ],
        [
          0.454,
          0.9468
        ],
        [
          0.4641,
          0.9436
        ],
        [
          0.4745,
          0.9413
        ],
        [
          0.4845,
          0.9386
        ],
        [
          0.4945,
          0.9354
        ],
        [
          0.5045,
          0.9331
        ],
        [
          0.5146,
          0.9304
        ],
        [
          0.525,
          0.9277
        ],
        [
          0.535,
          0.9245
        ],
        [
          0.545,
          0.9208
        ],
        [
          0.5551,
          0.9163
        ],
        [
          0.5655,
          0.914
        ],
        [
          0.5755,
          0.9104
        ],
        [
          0.5855,
          0.9086
        ],
        [
          0.5955,
          0.9054
        ],
        [
          0.6056,
          0.9013
        ],
        [
          0.616,
          0.8954
        ],
        [
          0.626,
          0.8913
        ],
        [
          0.636,
          0.8863
        ],
        [
          0.646,
          0.8813
        ],
        [
          0.6561,
          0.8776
        ],
        [
          0.6665,
          0.8744
        ],
        [
          0.6765,
          0.8699
        ],
        [
          0.6865,
          0.8631
        ],
        [
          0.6965,
          0.8562
        ],
        [
          0.7066,
          0.8517
        ],
        [
          0.717,
          0.848
        ],
        [
          0.727,
          0.8417
        ],
        [
          0.737,
          0.8348
        ],
        [
          0.747,
          0.8289
        ],
        [
          0.7571,
          0.8248
        ],
        [
          0.7675,
          0.8198
        ],
        [
          0.7775,
          0.8139
        ],
        [
          0.7875,
          0.8094
        ],
        [
          0.7975,
          0.803
        ],
        [
          0.8076,
          0.7957
        ],
        [
          0.818,
          0.7898
        ],
        [
          0.828,
          0.7853
        ],
        [
          0.838,
          0.7793
        ],
        [
          0.848,
          0.7734
        ],
        [
          0.8581,
          0.7666
        ],
        [
          0.8685,
          0.7607
        ],
        [
          0.8785,
          0.7548
        ],
        [
          0.8885,
          0.7493
        ],
        [
          0.8985,
          0.7425
        ],
        [
          0.9086,
          0.7361
        ],
        [
          0.919,
          0.7275
        ],
        [
          0.929,
          0.7211
        ],
        [
          0.939,
          0.7152
        ],
        [
          0.949,
          0.7093
        ],
        [
          0.9591,
          0.7002
        ],
        [
          0.9695,
          0.6924
        ],
        [
          0.9795,
          0.6861
        ],
        [
          0.9895,
          0.6797
        ],
        [
          0.9995,
          0.6711
        ],
        [
          1.0,
          0.6711
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.22584932600847787,
        "difference": -0.001273167147244896,
        "worse": 0.58,
        "kept_right": 0.22392136992919678,
        "kept_wrong": 0.0019279560792812022,
        "handed_off": 0.7741506739915224
      },
      "usd_per_1000": 1.9123740921531567
    },
    {
      "decision": 374,
      "frontier": 277,
      "frontier_accuracy": 0.9795268425841674,
      "in_sample": {
        "threshold": 0.982397,
        "answered": 0.19381255686988172,
        "accuracy": 0.9795268425841674
      },
      "curve": [
        [
          0.0,
          0.9795
        ],
        [
          0.01,
          0.9795
        ],
        [
          0.02,
          0.9795
        ],
        [
          0.03,
          0.9795
        ],
        [
          0.04,
          0.9795
        ],
        [
          0.05,
          0.9795
        ],
        [
          0.0601,
          0.9795
        ],
        [
          0.0705,
          0.9795
        ],
        [
          0.0805,
          0.9795
        ],
        [
          0.0905,
          0.9795
        ],
        [
          0.1005,
          0.9795
        ],
        [
          0.1106,
          0.9795
        ],
        [
          0.121,
          0.9795
        ],
        [
          0.131,
          0.9795
        ],
        [
          0.141,
          0.9795
        ],
        [
          0.151,
          0.9795
        ],
        [
          0.1611,
          0.9795
        ],
        [
          0.1715,
          0.9795
        ],
        [
          0.1815,
          0.9795
        ],
        [
          0.1915,
          0.9795
        ],
        [
          0.2015,
          0.9786
        ],
        [
          0.2116,
          0.9786
        ],
        [
          0.222,
          0.9786
        ],
        [
          0.232,
          0.9782
        ],
        [
          0.242,
          0.9782
        ],
        [
          0.252,
          0.9791
        ],
        [
          0.2621,
          0.9786
        ],
        [
          0.2725,
          0.9786
        ],
        [
          0.2825,
          0.9782
        ],
        [
          0.2925,
          0.9777
        ],
        [
          0.3025,
          0.9768
        ],
        [
          0.3126,
          0.975
        ],
        [
          0.323,
          0.9736
        ],
        [
          0.333,
          0.9727
        ],
        [
          0.343,
          0.9722
        ],
        [
          0.353,
          0.9704
        ],
        [
          0.3631,
          0.9704
        ],
        [
          0.3735,
          0.9682
        ],
        [
          0.3835,
          0.965
        ],
        [
          0.3935,
          0.9627
        ],
        [
          0.4035,
          0.9618
        ],
        [
          0.4136,
          0.9595
        ],
        [
          0.424,
          0.9568
        ],
        [
          0.434,
          0.9531
        ],
        [
          0.444,
          0.9495
        ],
        [
          0.454,
          0.9463
        ],
        [
          0.4641,
          0.944
        ],
        [
          0.4745,
          0.9422
        ],
        [
          0.4845,
          0.9395
        ],
        [
          0.4945,
          0.9363
        ],
        [
          0.5045,
          0.934
        ],
        [
          0.5146,
          0.9308
        ],
        [
          0.525,
          0.9281
        ],
        [
          0.535,
          0.9249
        ],
        [
          0.545,
          0.9213
        ],
        [
          0.5551,
          0.9163
        ],
        [
          0.5655,
          0.9145
        ],
        [
          0.5755,
          0.9108
        ],
        [
          0.5855,
          0.909
        ],
        [
          0.5955,
          0.9058
        ],
        [
          0.6056,
          0.9022
        ],
        [
          0.616,
          0.8963
        ],
        [
          0.626,
          0.8922
        ],
        [
          0.636,
          0.8872
        ],
        [
          0.646,
          0.8817
        ],
        [
          0.6561,
          0.8781
        ],
        [
          0.6665,
          0.874
        ],
        [
          0.6765,
          0.869
        ],
        [
          0.6865,
          0.8621
        ],
        [
          0.6965,
          0.8553
        ],
        [
          0.7066,
          0.8508
        ],
        [
          0.717,
          0.8471
        ],
        [
          0.727,
          0.8408
        ],
        [
          0.737,
          0.8339
        ],
        [
          0.747,
          0.8276
        ],
        [
          0.7571,
          0.8239
        ],
        [
          0.7675,
          0.8194
        ],
        [
          0.7775,
          0.8135
        ],
        [
          0.7875,
          0.8085
        ],
        [
          0.7975,
          0.8021
        ],
        [
          0.8076,
          0.7948
        ],
        [
          0.818,
          0.7889
        ],
        [
          0.828,
          0.7843
        ],
        [
          0.838,
          0.7784
        ],
        [
          0.848,
          0.7725
        ],
        [
          0.8581,
          0.7657
        ],
        [
          0.8685,
          0.7598
        ],
        [
          0.8785,
          0.7539
        ],
        [
          0.8885,
          0.748
        ],
        [
          0.8985,
          0.7411
        ],
        [
          0.9086,
          0.7348
        ],
        [
          0.919,
          0.7266
        ],
        [
          0.929,
          0.7202
        ],
        [
          0.939,
          0.7138
        ],
        [
          0.949,
          0.7084
        ],
        [
          0.9591,
          0.6997
        ],
        [
          0.9695,
          0.692
        ],
        [
          0.9795,
          0.6856
        ],
        [
          0.9895,
          0.6797
        ],
        [
          0.9995,
          0.6711
        ],
        [
          1.0,
          0.6711
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.23875482109943771,
        "difference": -0.0014732116456057027,
        "worse": 0.6566666666666666,
        "kept_right": 0.23648434089205575,
        "kept_wrong": 0.002270480207381923,
        "handed_off": 0.7612451789005632
      },
      "usd_per_1000": 4.6569982880656955
    },
    {
      "decision": 374,
      "frontier": 276,
      "frontier_accuracy": 0.9777070063694268,
      "in_sample": {
        "threshold": 0.964977,
        "answered": 0.26160145586897177,
        "accuracy": 0.9777070063694268
      },
      "curve": [
        [
          0.0,
          0.9777
        ],
        [
          0.01,
          0.9777
        ],
        [
          0.02,
          0.9777
        ],
        [
          0.03,
          0.9777
        ],
        [
          0.04,
          0.9777
        ],
        [
          0.05,
          0.9777
        ],
        [
          0.0601,
          0.9777
        ],
        [
          0.0705,
          0.9777
        ],
        [
          0.0805,
          0.9777
        ],
        [
          0.0905,
          0.9777
        ],
        [
          0.1005,
          0.9777
        ],
        [
          0.1106,
          0.9777
        ],
        [
          0.121,
          0.9777
        ],
        [
          0.131,
          0.9777
        ],
        [
          0.141,
          0.9777
        ],
        [
          0.151,
          0.9777
        ],
        [
          0.1611,
          0.9777
        ],
        [
          0.1715,
          0.9777
        ],
        [
          0.1815,
          0.9777
        ],
        [
          0.1915,
          0.9777
        ],
        [
          0.2015,
          0.9773
        ],
        [
          0.2116,
          0.9773
        ],
        [
          0.222,
          0.9773
        ],
        [
          0.232,
          0.9768
        ],
        [
          0.242,
          0.9768
        ],
        [
          0.252,
          0.9777
        ],
        [
          0.2621,
          0.9773
        ],
        [
          0.2725,
          0.9773
        ],
        [
          0.2825,
          0.9768
        ],
        [
          0.2925,
          0.9763
        ],
        [
          0.3025,
          0.9754
        ],
        [
          0.3126,
          0.9736
        ],
        [
          0.323,
          0.9722
        ],
        [
          0.333,
          0.9718
        ],
        [
          0.343,
          0.9713
        ],
        [
          0.353,
          0.9695
        ],
        [
          0.3631,
          0.9695
        ],
        [
          0.3735,
          0.9672
        ],
        [
          0.3835,
          0.9641
        ],
        [
          0.3935,
          0.9618
        ],
        [
          0.4035,
          0.9609
        ],
        [
          0.4136,
          0.9586
        ],
        [
          0.424,
          0.9559
        ],
        [
          0.434,
          0.9522
        ],
        [
          0.444,
          0.9486
        ],
        [
          0.454,
          0.9454
        ],
        [
          0.4641,
          0.9431
        ],
        [
          0.4745,
          0.9413
        ],
        [
          0.4845,
          0.9386
        ],
        [
          0.4945,
          0.9354
        ],
        [
          0.5045,
          0.9331
        ],
        [
          0.5146,
          0.9299
        ],
        [
          0.525,
          0.9272
        ],
        [
          0.535,
          0.9245
        ],
        [
          0.545,
          0.9213
        ],
        [
          0.5551,
          0.9163
        ],
        [
          0.5655,
          0.9145
        ],
        [
          0.5755,
          0.9113
        ],
        [
          0.5855,
          0.9095
        ],
        [
          0.5955,
          0.9063
        ],
        [
          0.6056,
          0.9022
        ],
        [
          0.616,
          0.8963
        ],
        [
          0.626,
          0.8922
        ],
        [
          0.636,
          0.8872
        ],
        [
          0.646,
          0.8817
        ],
        [
          0.6561,
          0.8781
        ],
        [
          0.6665,
          0.874
        ],
        [
          0.6765,
          0.869
        ],
        [
          0.6865,
          0.8621
        ],
        [
          0.6965,
          0.8553
        ],
        [
          0.7066,
          0.8508
        ],
        [
          0.717,
          0.8476
        ],
        [
          0.727,
          0.8417
        ],
        [
          0.737,
          0.8348
        ],
        [
          0.747,
          0.8285
        ],
        [
          0.7571,
          0.8248
        ],
        [
          0.7675,
          0.8203
        ],
        [
          0.7775,
          0.8144
        ],
        [
          0.7875,
          0.8094
        ],
        [
          0.7975,
          0.803
        ],
        [
          0.8076,
          0.7957
        ],
        [
          0.818,
          0.7898
        ],
        [
          0.828,
          0.7857
        ],
        [
          0.838,
          0.7798
        ],
        [
          0.848,
          0.7739
        ],
        [
          0.8581,
          0.7671
        ],
        [
          0.8685,
          0.7611
        ],
        [
          0.8785,
          0.7552
        ],
        [
          0.8885,
          0.7493
        ],
        [
          0.8985,
          0.7425
        ],
        [
          0.9086,
          0.7361
        ],
        [
          0.919,
          0.7275
        ],
        [
          0.929,
          0.7211
        ],
        [
          0.939,
          0.7147
        ],
        [
          0.949,
          0.7088
        ],
        [
          0.9591,
          0.7002
        ],
        [
          0.9695,
          0.6924
        ],
        [
          0.9795,
          0.6856
        ],
        [
          0.9895,
          0.6797
        ],
        [
          0.9995,
          0.6711
        ],
        [
          1.0,
          0.6711
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.2555831467685321,
        "difference": -0.0014944981245321436,
        "worse": 0.5333333333333333,
        "kept_right": 0.2528276245969372,
        "kept_wrong": 0.002755522171594515,
        "handed_off": 0.7444168532314681
      },
      "usd_per_1000": 1.247209452704248
    },
    {
      "decision": 374,
      "frontier": 48,
      "frontier_accuracy": 0.9758871701546861,
      "in_sample": {
        "threshold": 0.982397,
        "answered": 0.19381255686988172,
        "accuracy": 0.9758871701546861
      },
      "curve": [
        [
          0.0,
          0.9759
        ],
        [
          0.01,
          0.9759
        ],
        [
          0.02,
          0.9759
        ],
        [
          0.03,
          0.9759
        ],
        [
          0.04,
          0.9759
        ],
        [
          0.05,
          0.9759
        ],
        [
          0.0601,
          0.9759
        ],
        [
          0.0705,
          0.9759
        ],
        [
          0.0805,
          0.9759
        ],
        [
          0.0905,
          0.9759
        ],
        [
          0.1005,
          0.9759
        ],
        [
          0.1106,
          0.9759
        ],
        [
          0.121,
          0.9759
        ],
        [
          0.131,
          0.9759
        ],
        [
          0.141,
          0.9759
        ],
        [
          0.151,
          0.9759
        ],
        [
          0.1611,
          0.9759
        ],
        [
          0.1715,
          0.9759
        ],
        [
          0.1815,
          0.9759
        ],
        [
          0.1915,
          0.9759
        ],
        [
          0.2015,
          0.975
        ],
        [
          0.2116,
          0.975
        ],
        [
          0.222,
          0.975
        ],
        [
          0.232,
          0.9745
        ],
        [
          0.242,
          0.9745
        ],
        [
          0.252,
          0.9754
        ],
        [
          0.2621,
          0.975
        ],
        [
          0.2725,
          0.975
        ],
        [
          0.2825,
          0.9745
        ],
        [
          0.2925,
          0.9741
        ],
        [
          0.3025,
          0.9732
        ],
        [
          0.3126,
          0.9713
        ],
        [
          0.323,
          0.97
        ],
        [
          0.333,
          0.9691
        ],
        [
          0.343,
          0.9686
        ],
        [
          0.353,
          0.9668
        ],
        [
          0.3631,
          0.9668
        ],
        [
          0.3735,
          0.9645
        ],
        [
          0.3835,
          0.9613
        ],
        [
          0.3935,
          0.9591
        ],
        [
          0.4035,
          0.9581
        ],
        [
          0.4136,
          0.9559
        ],
        [
          0.424,
          0.9536
        ],
        [
          0.434,
          0.95
        ],
        [
          0.444,
          0.9459
        ],
        [
          0.454,
          0.9427
        ],
        [
          0.4641,
          0.9399
        ],
        [
          0.4745,
          0.9381
        ],
        [
          0.4845,
          0.9354
        ],
        [
          0.4945,
          0.9322
        ],
        [
          0.5045,
          0.9299
        ],
        [
          0.5146,
          0.9268
        ],
        [
          0.525,
          0.924
        ],
        [
          0.535,
          0.9213
        ],
        [
          0.545,
          0.9181
        ],
        [
          0.5551,
          0.9131
        ],
        [
          0.5655,
          0.9113
        ],
        [
          0.5755,
          0.9076
        ],
        [
          0.5855,
          0.9058
        ],
        [
          0.5955,
          0.9026
        ],
        [
          0.6056,
          0.899
        ],
        [
          0.616,
          0.8931
        ],
        [
          0.626,
          0.889
        ],
        [
          0.636,
          0.884
        ],
        [
          0.646,
          0.8785
        ],
        [
          0.6561,
          0.8749
        ],
        [
          0.6665,
          0.8717
        ],
        [
          0.6765,
          0.8667
        ],
        [
          0.6865,
          0.8608
        ],
        [
          0.6965,
          0.854
        ],
        [
          0.7066,
          0.8494
        ],
        [
          0.717,
          0.8458
        ],
        [
          0.727,
          0.8394
        ],
        [
          0.737,
          0.8326
        ],
        [
          0.747,
          0.8262
        ],
        [
          0.7571,
          0.8221
        ],
        [
          0.7675,
          0.8176
        ],
        [
          0.7775,
          0.8116
        ],
        [
          0.7875,
          0.8066
        ],
        [
          0.7975,
          0.8007
        ],
        [
          0.8076,
          0.7939
        ],
        [
          0.818,
          0.788
        ],
        [
          0.828,
          0.7839
        ],
        [
          0.838,
          0.778
        ],
        [
          0.848,
          0.7721
        ],
        [
          0.8581,
          0.7657
        ],
        [
          0.8685,
          0.7598
        ],
        [
          0.8785,
          0.7539
        ],
        [
          0.8885,
          0.748
        ],
        [
          0.8985,
          0.7411
        ],
        [
          0.9086,
          0.7348
        ],
        [
          0.919,
          0.7261
        ],
        [
          0.929,
          0.7197
        ],
        [
          0.939,
          0.7134
        ],
        [
          0.949,
          0.7079
        ],
        [
          0.9591,
          0.6997
        ],
        [
          0.9695,
          0.692
        ],
        [
          0.9795,
          0.6856
        ],
        [
          0.9895,
          0.6797
        ],
        [
          0.9995,
          0.6711
        ],
        [
          1.0,
          0.6711
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.23875482109943771,
        "difference": -0.0014732116456057027,
        "worse": 0.6566666666666666,
        "kept_right": 0.23648434089205575,
        "kept_wrong": 0.002270480207381923,
        "handed_off": 0.7612451789005632
      },
      "usd_per_1000": 4.133772548405868
    },
    {
      "decision": 374,
      "frontier": 191,
      "frontier_accuracy": 0.9740673339399454,
      "in_sample": {
        "threshold": 0.981145,
        "answered": 0.19927206551410373,
        "accuracy": 0.9740673339399454
      },
      "curve": [
        [
          0.0,
          0.9741
        ],
        [
          0.01,
          0.9741
        ],
        [
          0.02,
          0.9741
        ],
        [
          0.03,
          0.9741
        ],
        [
          0.04,
          0.9741
        ],
        [
          0.05,
          0.9741
        ],
        [
          0.0601,
          0.9741
        ],
        [
          0.0705,
          0.9741
        ],
        [
          0.0805,
          0.9741
        ],
        [
          0.0905,
          0.9741
        ],
        [
          0.1005,
          0.9741
        ],
        [
          0.1106,
          0.9741
        ],
        [
          0.121,
          0.9741
        ],
        [
          0.131,
          0.9741
        ],
        [
          0.141,
          0.9741
        ],
        [
          0.151,
          0.9741
        ],
        [
          0.1611,
          0.9741
        ],
        [
          0.1715,
          0.9741
        ],
        [
          0.1815,
          0.9741
        ],
        [
          0.1915,
          0.9741
        ],
        [
          0.2015,
          0.9736
        ],
        [
          0.2116,
          0.9736
        ],
        [
          0.222,
          0.9736
        ],
        [
          0.232,
          0.9732
        ],
        [
          0.242,
          0.9732
        ],
        [
          0.252,
          0.9736
        ],
        [
          0.2621,
          0.9732
        ],
        [
          0.2725,
          0.9736
        ],
        [
          0.2825,
          0.9732
        ],
        [
          0.2925,
          0.9727
        ],
        [
          0.3025,
          0.9718
        ],
        [
          0.3126,
          0.97
        ],
        [
          0.323,
          0.9686
        ],
        [
          0.333,
          0.9677
        ],
        [
          0.343,
          0.9672
        ],
        [
          0.353,
          0.9659
        ],
        [
          0.3631,
          0.9659
        ],
        [
          0.3735,
          0.9636
        ],
        [
          0.3835,
          0.9604
        ],
        [
          0.3935,
          0.9581
        ],
        [
          0.4035,
          0.9572
        ],
        [
          0.4136,
          0.955
        ],
        [
          0.424,
          0.9522
        ],
        [
          0.434,
          0.9486
        ],
        [
          0.444,
          0.9449
        ],
        [
          0.454,
          0.9418
        ],
        [
          0.4641,
          0.9386
        ],
        [
          0.4745,
          0.9368
        ],
        [
          0.4845,
          0.934
        ],
        [
          0.4945,
          0.9308
        ],
        [
          0.5045,
          0.9286
        ],
        [
          0.5146,
          0.9249
        ],
        [
          0.525,
          0.9222
        ],
        [
          0.535,
          0.9195
        ],
        [
          0.545,
          0.9167
        ],
        [
          0.5551,
          0.9117
        ],
        [
          0.5655,
          0.9099
        ],
        [
          0.5755,
          0.9063
        ],
        [
          0.5855,
          0.9045
        ],
        [
          0.5955,
          0.9013
        ],
        [
          0.6056,
          0.8976
        ],
        [
          0.616,
          0.8922
        ],
        [
          0.626,
          0.8885
        ],
        [
          0.636,
          0.8835
        ],
        [
          0.646,
          0.8781
        ],
        [
          0.6561,
          0.874
        ],
        [
          0.6665,
          0.8703
        ],
        [
          0.6765,
          0.8653
        ],
        [
          0.6865,
          0.8585
        ],
        [
          0.6965,
          0.8517
        ],
        [
          0.7066,
          0.8471
        ],
        [
          0.717,
          0.8439
        ],
        [
          0.727,
          0.838
        ],
        [
          0.737,
          0.8312
        ],
        [
          0.747,
          0.8248
        ],
        [
          0.7571,
          0.8217
        ],
        [
          0.7675,
          0.8171
        ],
        [
          0.7775,
          0.8112
        ],
        [
          0.7875,
          0.8062
        ],
        [
          0.7975,
          0.7998
        ],
        [
          0.8076,
          0.7925
        ],
        [
          0.818,
          0.7866
        ],
        [
          0.828,
          0.7821
        ],
        [
          0.838,
          0.7766
        ],
        [
          0.848,
          0.7712
        ],
        [
          0.8581,
          0.7648
        ],
        [
          0.8685,
          0.7589
        ],
        [
          0.8785,
          0.753
        ],
        [
          0.8885,
          0.747
        ],
        [
          0.8985,
          0.7402
        ],
        [
          0.9086,
          0.7338
        ],
        [
          0.919,
          0.7252
        ],
        [
          0.929,
          0.7188
        ],
        [
          0.939,
          0.7129
        ],
        [
          0.949,
          0.7075
        ],
        [
          0.9591,
          0.6993
        ],
        [
          0.9695,
          0.6915
        ],
        [
          0.9795,
          0.6852
        ],
        [
          0.9895,
          0.6793
        ],
        [
          0.9995,
          0.6711
        ],
        [
          1.0,
          0.6711
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.2522082043123697,
        "difference": -0.0014701757225166333,
        "worse": 0.6966666666666667,
        "kept_right": 0.24946173176000647,
        "kept_wrong": 0.0027464725523629815,
        "handed_off": 0.7477917956876304
      },
      "usd_per_1000": 20.033853518573785
    },
    {
      "decision": 374,
      "frontier": 188,
      "frontier_accuracy": 0.9713375796178344,
      "in_sample": {
        "threshold": 0.982397,
        "answered": 0.19381255686988172,
        "accuracy": 0.9713375796178344
      },
      "curve": [
        [
          0.0,
          0.9713
        ],
        [
          0.01,
          0.9713
        ],
        [
          0.02,
          0.9713
        ],
        [
          0.03,
          0.9713
        ],
        [
          0.04,
          0.9713
        ],
        [
          0.05,
          0.9713
        ],
        [
          0.0601,
          0.9713
        ],
        [
          0.0705,
          0.9713
        ],
        [
          0.0805,
          0.9713
        ],
        [
          0.0905,
          0.9713
        ],
        [
          0.1005,
          0.9713
        ],
        [
          0.1106,
          0.9713
        ],
        [
          0.121,
          0.9713
        ],
        [
          0.131,
          0.9713
        ],
        [
          0.141,
          0.9713
        ],
        [
          0.151,
          0.9713
        ],
        [
          0.1611,
          0.9713
        ],
        [
          0.1715,
          0.9713
        ],
        [
          0.1815,
          0.9713
        ],
        [
          0.1915,
          0.9713
        ],
        [
          0.2015,
          0.9704
        ],
        [
          0.2116,
          0.9704
        ],
        [
          0.222,
          0.9709
        ],
        [
          0.232,
          0.9704
        ],
        [
          0.242,
          0.9704
        ],
        [
          0.252,
          0.9709
        ],
        [
          0.2621,
          0.9704
        ],
        [
          0.2725,
          0.9704
        ],
        [
          0.2825,
          0.9704
        ],
        [
          0.2925,
          0.97
        ],
        [
          0.3025,
          0.9691
        ],
        [
          0.3126,
          0.9672
        ],
        [
          0.323,
          0.9659
        ],
        [
          0.333,
          0.965
        ],
        [
          0.343,
          0.9641
        ],
        [
          0.353,
          0.9622
        ],
        [
          0.3631,
          0.9622
        ],
        [
          0.3735,
          0.96
        ],
        [
          0.3835,
          0.9568
        ],
        [
          0.3935,
          0.954
        ],
        [
          0.4035,
          0.9527
        ],
        [
          0.4136,
          0.9504
        ],
        [
          0.424,
          0.9477
        ],
        [
          0.434,
          0.944
        ],
        [
          0.444,
          0.9409
        ],
        [
          0.454,
          0.9377
        ],
        [
          0.4641,
          0.9349
        ],
        [
          0.4745,
          0.9331
        ],
        [
          0.4845,
          0.9304
        ],
        [
          0.4945,
          0.9277
        ],
        [
          0.5045,
          0.9254
        ],
        [
          0.5146,
          0.9231
        ],
        [
          0.525,
          0.9204
        ],
        [
          0.535,
          0.9177
        ],
        [
          0.545,
          0.914
        ],
        [
          0.5551,
          0.9095
        ],
        [
          0.5655,
          0.9076
        ],
        [
          0.5755,
          0.9045
        ],
        [
          0.5855,
          0.9031
        ],
        [
          0.5955,
          0.8999
        ],
        [
          0.6056,
          0.8958
        ],
        [
          0.616,
          0.8899
        ],
        [
          0.626,
          0.8858
        ],
        [
          0.636,
          0.8813
        ],
        [
          0.646,
          0.8763
        ],
        [
          0.6561,
          0.8726
        ],
        [
          0.6665,
          0.8694
        ],
        [
          0.6765,
          0.8644
        ],
        [
          0.6865,
          0.8576
        ],
        [
          0.6965,
          0.8508
        ],
        [
          0.7066,
          0.8462
        ],
        [
          0.717,
          0.843
        ],
        [
          0.727,
          0.8376
        ],
        [
          0.737,
          0.8312
        ],
        [
          0.747,
          0.8253
        ],
        [
          0.7571,
          0.8217
        ],
        [
          0.7675,
          0.8167
        ],
        [
          0.7775,
          0.8107
        ],
        [
          0.7875,
          0.8066
        ],
        [
          0.7975,
          0.8003
        ],
        [
          0.8076,
          0.793
        ],
        [
          0.818,
          0.7871
        ],
        [
          0.828,
          0.7825
        ],
        [
          0.838,
          0.7766
        ],
        [
          0.848,
          0.7712
        ],
        [
          0.8581,
          0.7643
        ],
        [
          0.8685,
          0.7584
        ],
        [
          0.8785,
          0.7525
        ],
        [
          0.8885,
          0.7466
        ],
        [
          0.8985,
          0.7398
        ],
        [
          0.9086,
          0.7338
        ],
        [
          0.919,
          0.7252
        ],
        [
          0.929,
          0.7188
        ],
        [
          0.939,
          0.7129
        ],
        [
          0.949,
          0.7075
        ],
        [
          0.9591,
          0.6988
        ],
        [
          0.9695,
          0.6911
        ],
        [
          0.9795,
          0.6847
        ],
        [
          0.9895,
          0.6793
        ],
        [
          0.9995,
          0.6711
        ],
        [
          1.0,
          0.6711
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.24278516564031838,
        "difference": -0.0014884898578545408,
        "worse": 0.6733333333333333,
        "kept_right": 0.2402993739573398,
        "kept_wrong": 0.0024857916829784488,
        "handed_off": 0.7572148343596817
      },
      "usd_per_1000": 0.7691546814124486
    },
    {
      "decision": 374,
      "frontier": 190,
      "frontier_accuracy": 0.9672429481346679,
      "in_sample": {
        "threshold": 0.964977,
        "answered": 0.26160145586897177,
        "accuracy": 0.9672429481346679
      },
      "curve": [
        [
          0.0,
          0.9672
        ],
        [
          0.01,
          0.9672
        ],
        [
          0.02,
          0.9672
        ],
        [
          0.03,
          0.9672
        ],
        [
          0.04,
          0.9672
        ],
        [
          0.05,
          0.9672
        ],
        [
          0.0601,
          0.9672
        ],
        [
          0.0705,
          0.9672
        ],
        [
          0.0805,
          0.9672
        ],
        [
          0.0905,
          0.9672
        ],
        [
          0.1005,
          0.9672
        ],
        [
          0.1106,
          0.9672
        ],
        [
          0.121,
          0.9672
        ],
        [
          0.131,
          0.9672
        ],
        [
          0.141,
          0.9672
        ],
        [
          0.151,
          0.9672
        ],
        [
          0.1611,
          0.9672
        ],
        [
          0.1715,
          0.9672
        ],
        [
          0.1815,
          0.9672
        ],
        [
          0.1915,
          0.9672
        ],
        [
          0.2015,
          0.9668
        ],
        [
          0.2116,
          0.9668
        ],
        [
          0.222,
          0.9668
        ],
        [
          0.232,
          0.9663
        ],
        [
          0.242,
          0.9663
        ],
        [
          0.252,
          0.9672
        ],
        [
          0.2621,
          0.9668
        ],
        [
          0.2725,
          0.9668
        ],
        [
          0.2825,
          0.9663
        ],
        [
          0.2925,
          0.9659
        ],
        [
          0.3025,
          0.965
        ],
        [
          0.3126,
          0.9636
        ],
        [
          0.323,
          0.9622
        ],
        [
          0.333,
          0.9613
        ],
        [
          0.343,
          0.9609
        ],
        [
          0.353,
          0.9591
        ],
        [
          0.3631,
          0.9591
        ],
        [
          0.3735,
          0.9568
        ],
        [
          0.3835,
          0.954
        ],
        [
          0.3935,
          0.9518
        ],
        [
          0.4035,
          0.9509
        ],
        [
          0.4136,
          0.9486
        ],
        [
          0.424,
          0.9459
        ],
        [
          0.434,
          0.9422
        ],
        [
          0.444,
          0.9386
        ],
        [
          0.454,
          0.9359
        ],
        [
          0.4641,
          0.9331
        ],
        [
          0.4745,
          0.9313
        ],
        [
          0.4845,
          0.9286
        ],
        [
          0.4945,
          0.9263
        ],
        [
          0.5045,
          0.9249
        ],
        [
          0.5146,
          0.9217
        ],
        [
          0.525,
          0.919
        ],
        [
          0.535,
          0.9158
        ],
        [
          0.545,
          0.9131
        ],
        [
          0.5551,
          0.9081
        ],
        [
          0.5655,
          0.9063
        ],
        [
          0.5755,
          0.9031
        ],
        [
          0.5855,
          0.9013
        ],
        [
          0.5955,
          0.8981
        ],
        [
          0.6056,
          0.8944
        ],
        [
          0.616,
          0.889
        ],
        [
          0.626,
          0.8849
        ],
        [
          0.636,
          0.8799
        ],
        [
          0.646,
          0.8744
        ],
        [
          0.6561,
          0.8703
        ],
        [
          0.6665,
          0.8667
        ],
        [
          0.6765,
          0.8617
        ],
        [
          0.6865,
          0.8549
        ],
        [
          0.6965,
          0.848
        ],
        [
          0.7066,
          0.8435
        ],
        [
          0.717,
          0.8399
        ],
        [
          0.727,
          0.8339
        ],
        [
          0.737,
          0.8271
        ],
        [
          0.747,
          0.8207
        ],
        [
          0.7571,
          0.8185
        ],
        [
          0.7675,
          0.8139
        ],
        [
          0.7775,
          0.808
        ],
        [
          0.7875,
          0.803
        ],
        [
          0.7975,
          0.7971
        ],
        [
          0.8076,
          0.7903
        ],
        [
          0.818,
          0.7843
        ],
        [
          0.828,
          0.7803
        ],
        [
          0.838,
          0.7743
        ],
        [
          0.848,
          0.7689
        ],
        [
          0.8581,
          0.763
        ],
        [
          0.8685,
          0.7571
        ],
        [
          0.8785,
          0.7511
        ],
        [
          0.8885,
          0.7452
        ],
        [
          0.8985,
          0.7384
        ],
        [
          0.9086,
          0.7325
        ],
        [
          0.919,
          0.7243
        ],
        [
          0.929,
          0.7179
        ],
        [
          0.939,
          0.712
        ],
        [
          0.949,
          0.7066
        ],
        [
          0.9591,
          0.6979
        ],
        [
          0.9695,
          0.6902
        ],
        [
          0.9795,
          0.6847
        ],
        [
          0.9895,
          0.6788
        ],
        [
          0.9995,
          0.6711
        ],
        [
          1.0,
          0.6711
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.2594003696048078,
        "difference": -0.0015217571252040863,
        "worse": 0.5633333333333334,
        "kept_right": 0.25639925200824715,
        "kept_wrong": 0.003001117596560267,
        "handed_off": 0.7405996303951925
      },
      "usd_per_1000": 4.163666874915024
    },
    {
      "decision": 374,
      "frontier": 124,
      "frontier_accuracy": 0.9667879890809827,
      "in_sample": {
        "threshold": 0.982397,
        "answered": 0.19381255686988172,
        "accuracy": 0.9667879890809827
      },
      "curve": [
        [
          0.0,
          0.9668
        ],
        [
          0.01,
          0.9668
        ],
        [
          0.02,
          0.9668
        ],
        [
          0.03,
          0.9668
        ],
        [
          0.04,
          0.9668
        ],
        [
          0.05,
          0.9668
        ],
        [
          0.0601,
          0.9668
        ],
        [
          0.0705,
          0.9668
        ],
        [
          0.0805,
          0.9668
        ],
        [
          0.0905,
          0.9668
        ],
        [
          0.1005,
          0.9668
        ],
        [
          0.1106,
          0.9668
        ],
        [
          0.121,
          0.9668
        ],
        [
          0.131,
          0.9668
        ],
        [
          0.141,
          0.9668
        ],
        [
          0.151,
          0.9668
        ],
        [
          0.1611,
          0.9668
        ],
        [
          0.1715,
          0.9668
        ],
        [
          0.1815,
          0.9668
        ],
        [
          0.1915,
          0.9668
        ],
        [
          0.2015,
          0.9659
        ],
        [
          0.2116,
          0.9659
        ],
        [
          0.222,
          0.9659
        ],
        [
          0.232,
          0.9654
        ],
        [
          0.242,
          0.9654
        ],
        [
          0.252,
          0.9663
        ],
        [
          0.2621,
          0.9659
        ],
        [
          0.2725,
          0.9659
        ],
        [
          0.2825,
          0.9654
        ],
        [
          0.2925,
          0.965
        ],
        [
          0.3025,
          0.9641
        ],
        [
          0.3126,
          0.9622
        ],
        [
          0.323,
          0.9609
        ],
        [
          0.333,
          0.9604
        ],
        [
          0.343,
          0.96
        ],
        [
          0.353,
          0.9586
        ],
        [
          0.3631,
          0.9591
        ],
        [
          0.3735,
          0.9568
        ],
        [
          0.3835,
          0.9536
        ],
        [
          0.3935,
          0.9513
        ],
        [
          0.4035,
          0.9504
        ],
        [
          0.4136,
          0.9481
        ],
        [
          0.424,
          0.9454
        ],
        [
          0.434,
          0.9418
        ],
        [
          0.444,
          0.9381
        ],
        [
          0.454,
          0.9349
        ],
        [
          0.4641,
          0.9327
        ],
        [
          0.4745,
          0.9308
        ],
        [
          0.4845,
          0.9281
        ],
        [
          0.4945,
          0.9249
        ],
        [
          0.5045,
          0.9227
        ],
        [
          0.5146,
          0.919
        ],
        [
          0.525,
          0.9163
        ],
        [
          0.535,
          0.9136
        ],
        [
          0.545,
          0.9104
        ],
        [
          0.5551,
          0.9054
        ],
        [
          0.5655,
          0.9035
        ],
        [
          0.5755,
          0.9004
        ],
        [
          0.5855,
          0.8985
        ],
        [
          0.5955,
          0.8954
        ],
        [
          0.6056,
          0.8917
        ],
        [
          0.616,
          0.8858
        ],
        [
          0.626,
          0.8822
        ],
        [
          0.636,
          0.8772
        ],
        [
          0.646,
          0.8722
        ],
        [
          0.6561,
          0.8685
        ],
        [
          0.6665,
          0.8649
        ],
        [
          0.6765,
          0.8599
        ],
        [
          0.6865,
          0.8535
        ],
        [
          0.6965,
          0.8467
        ],
        [
          0.7066,
          0.843
        ],
        [
          0.717,
          0.8399
        ],
        [
          0.727,
          0.8335
        ],
        [
          0.737,
          0.8267
        ],
        [
          0.747,
          0.8203
        ],
        [
          0.7571,
          0.818
        ],
        [
          0.7675,
          0.8139
        ],
        [
          0.7775,
          0.808
        ],
        [
          0.7875,
          0.803
        ],
        [
          0.7975,
          0.7975
        ],
        [
          0.8076,
          0.7903
        ],
        [
          0.818,
          0.7843
        ],
        [
          0.828,
          0.7803
        ],
        [
          0.838,
          0.7748
        ],
        [
          0.848,
          0.7689
        ],
        [
          0.8581,
          0.7625
        ],
        [
          0.8685,
          0.7566
        ],
        [
          0.8785,
          0.7507
        ],
        [
          0.8885,
          0.7448
        ],
        [
          0.8985,
          0.7379
        ],
        [
          0.9086,
          0.7325
        ],
        [
          0.919,
          0.7247
        ],
        [
          0.929,
          0.7184
        ],
        [
          0.939,
          0.7125
        ],
        [
          0.949,
          0.7066
        ],
        [
          0.9591,
          0.6984
        ],
        [
          0.9695,
          0.6911
        ],
        [
          0.9795,
          0.6852
        ],
        [
          0.9895,
          0.6797
        ],
        [
          0.9995,
          0.6711
        ],
        [
          1.0,
          0.6711
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.23880335006516412,
        "difference": -0.0014762447059636038,
        "worse": 0.6566666666666666,
        "kept_right": 0.23652377067670846,
        "kept_wrong": 0.002279579388455626,
        "handed_off": 0.7611966499348367
      },
      "usd_per_1000": 0.9000599402011356
    },
    {
      "decision": 374,
      "frontier": 278,
      "frontier_accuracy": 0.9499545040946314,
      "in_sample": {
        "threshold": 0.981145,
        "answered": 0.19927206551410373,
        "accuracy": 0.9499545040946314
      },
      "curve": [
        [
          0.0,
          0.95
        ],
        [
          0.01,
          0.95
        ],
        [
          0.02,
          0.95
        ],
        [
          0.03,
          0.95
        ],
        [
          0.04,
          0.95
        ],
        [
          0.05,
          0.95
        ],
        [
          0.0601,
          0.95
        ],
        [
          0.0705,
          0.95
        ],
        [
          0.0805,
          0.95
        ],
        [
          0.0905,
          0.95
        ],
        [
          0.1005,
          0.95
        ],
        [
          0.1106,
          0.95
        ],
        [
          0.121,
          0.95
        ],
        [
          0.131,
          0.95
        ],
        [
          0.141,
          0.95
        ],
        [
          0.151,
          0.95
        ],
        [
          0.1611,
          0.95
        ],
        [
          0.1715,
          0.9504
        ],
        [
          0.1815,
          0.9504
        ],
        [
          0.1915,
          0.9504
        ],
        [
          0.2015,
          0.9495
        ],
        [
          0.2116,
          0.9495
        ],
        [
          0.222,
          0.9495
        ],
        [
          0.232,
          0.949
        ],
        [
          0.242,
          0.949
        ],
        [
          0.252,
          0.9495
        ],
        [
          0.2621,
          0.949
        ],
        [
          0.2725,
          0.949
        ],
        [
          0.2825,
          0.949
        ],
        [
          0.2925,
          0.9486
        ],
        [
          0.3025,
          0.9481
        ],
        [
          0.3126,
          0.9468
        ],
        [
          0.323,
          0.9463
        ],
        [
          0.333,
          0.9459
        ],
        [
          0.343,
          0.9454
        ],
        [
          0.353,
          0.9436
        ],
        [
          0.3631,
          0.944
        ],
        [
          0.3735,
          0.9422
        ],
        [
          0.3835,
          0.939
        ],
        [
          0.3935,
          0.9363
        ],
        [
          0.4035,
          0.9354
        ],
        [
          0.4136,
          0.9336
        ],
        [
          0.424,
          0.9308
        ],
        [
          0.434,
          0.9281
        ],
        [
          0.444,
          0.9258
        ],
        [
          0.454,
          0.924
        ],
        [
          0.4641,
          0.9217
        ],
        [
          0.4745,
          0.9199
        ],
        [
          0.4845,
          0.9181
        ],
        [
          0.4945,
          0.9149
        ],
        [
          0.5045,
          0.9126
        ],
        [
          0.5146,
          0.9095
        ],
        [
          0.525,
          0.9067
        ],
        [
          0.535,
          0.904
        ],
        [
          0.545,
          0.9004
        ],
        [
          0.5551,
          0.8958
        ],
        [
          0.5655,
          0.894
        ],
        [
          0.5755,
          0.8904
        ],
        [
          0.5855,
          0.889
        ],
        [
          0.5955,
          0.8858
        ],
        [
          0.6056,
          0.8817
        ],
        [
          0.616,
          0.8763
        ],
        [
          0.626,
          0.8726
        ],
        [
          0.636,
          0.8676
        ],
        [
          0.646,
          0.8626
        ],
        [
          0.6561,
          0.859
        ],
        [
          0.6665,
          0.8562
        ],
        [
          0.6765,
          0.8512
        ],
        [
          0.6865,
          0.8453
        ],
        [
          0.6965,
          0.8389
        ],
        [
          0.7066,
          0.8344
        ],
        [
          0.717,
          0.8312
        ],
        [
          0.727,
          0.8262
        ],
        [
          0.737,
          0.8194
        ],
        [
          0.747,
          0.813
        ],
        [
          0.7571,
          0.8103
        ],
        [
          0.7675,
          0.8053
        ],
        [
          0.7775,
          0.7998
        ],
        [
          0.7875,
          0.7957
        ],
        [
          0.7975,
          0.7907
        ],
        [
          0.8076,
          0.7843
        ],
        [
          0.818,
          0.7784
        ],
        [
          0.828,
          0.7743
        ],
        [
          0.838,
          0.7689
        ],
        [
          0.848,
          0.7634
        ],
        [
          0.8581,
          0.7575
        ],
        [
          0.8685,
          0.752
        ],
        [
          0.8785,
          0.7466
        ],
        [
          0.8885,
          0.7416
        ],
        [
          0.8985,
          0.7348
        ],
        [
          0.9086,
          0.7293
        ],
        [
          0.919,
          0.7225
        ],
        [
          0.929,
          0.7161
        ],
        [
          0.939,
          0.7106
        ],
        [
          0.949,
          0.7052
        ],
        [
          0.9591,
          0.6979
        ],
        [
          0.9695,
          0.6911
        ],
        [
          0.9795,
          0.6852
        ],
        [
          0.9895,
          0.6797
        ],
        [
          0.9995,
          0.6711
        ],
        [
          1.0,
          0.6711
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.24755991185682005,
        "difference": -0.001285360215652581,
        "worse": 0.6366666666666667,
        "kept_right": 0.24478303688972558,
        "kept_wrong": 0.002776874967094349,
        "handed_off": 0.75244008814318
      },
      "usd_per_1000": 4.9371255812100125
    },
    {
      "decision": 374,
      "frontier": 275,
      "frontier_accuracy": 0.9426751592356688,
      "in_sample": {
        "threshold": 0.982397,
        "answered": 0.19381255686988172,
        "accuracy": 0.9426751592356688
      },
      "curve": [
        [
          0.0,
          0.9427
        ],
        [
          0.01,
          0.9427
        ],
        [
          0.02,
          0.9427
        ],
        [
          0.03,
          0.9427
        ],
        [
          0.04,
          0.9427
        ],
        [
          0.05,
          0.9427
        ],
        [
          0.0601,
          0.9427
        ],
        [
          0.0705,
          0.9427
        ],
        [
          0.0805,
          0.9427
        ],
        [
          0.0905,
          0.9427
        ],
        [
          0.1005,
          0.9427
        ],
        [
          0.1106,
          0.9427
        ],
        [
          0.121,
          0.9427
        ],
        [
          0.131,
          0.9427
        ],
        [
          0.141,
          0.9427
        ],
        [
          0.151,
          0.9427
        ],
        [
          0.1611,
          0.9427
        ],
        [
          0.1715,
          0.9427
        ],
        [
          0.1815,
          0.9427
        ],
        [
          0.1915,
          0.9427
        ],
        [
          0.2015,
          0.9418
        ],
        [
          0.2116,
          0.9418
        ],
        [
          0.222,
          0.9418
        ],
        [
          0.232,
          0.9413
        ],
        [
          0.242,
          0.9413
        ],
        [
          0.252,
          0.9422
        ],
        [
          0.2621,
          0.9418
        ],
        [
          0.2725,
          0.9422
        ],
        [
          0.2825,
          0.9422
        ],
        [
          0.2925,
          0.9418
        ],
        [
          0.3025,
          0.9418
        ],
        [
          0.3126,
          0.9404
        ],
        [
          0.323,
          0.939
        ],
        [
          0.333,
          0.9386
        ],
        [
          0.343,
          0.9381
        ],
        [
          0.353,
          0.9372
        ],
        [
          0.3631,
          0.9372
        ],
        [
          0.3735,
          0.9354
        ],
        [
          0.3835,
          0.9327
        ],
        [
          0.3935,
          0.9308
        ],
        [
          0.4035,
          0.9308
        ],
        [
          0.4136,
          0.929
        ],
        [
          0.424,
          0.9268
        ],
        [
          0.434,
          0.9236
        ],
        [
          0.444,
          0.9208
        ],
        [
          0.454,
          0.9181
        ],
        [
          0.4641,
          0.9158
        ],
        [
          0.4745,
          0.914
        ],
        [
          0.4845,
          0.9117
        ],
        [
          0.4945,
          0.9086
        ],
        [
          0.5045,
          0.9063
        ],
        [
          0.5146,
          0.904
        ],
        [
          0.525,
          0.9013
        ],
        [
          0.535,
          0.8985
        ],
        [
          0.545,
          0.8963
        ],
        [
          0.5551,
          0.8922
        ],
        [
          0.5655,
          0.8904
        ],
        [
          0.5755,
          0.8863
        ],
        [
          0.5855,
          0.8849
        ],
        [
          0.5955,
          0.8822
        ],
        [
          0.6056,
          0.8785
        ],
        [
          0.616,
          0.8726
        ],
        [
          0.626,
          0.869
        ],
        [
          0.636,
          0.864
        ],
        [
          0.646,
          0.8603
        ],
        [
          0.6561,
          0.8576
        ],
        [
          0.6665,
          0.8544
        ],
        [
          0.6765,
          0.8499
        ],
        [
          0.6865,
          0.8439
        ],
        [
          0.6965,
          0.8371
        ],
        [
          0.7066,
          0.833
        ],
        [
          0.717,
          0.8303
        ],
        [
          0.727,
          0.8248
        ],
        [
          0.737,
          0.8185
        ],
        [
          0.747,
          0.8121
        ],
        [
          0.7571,
          0.8094
        ],
        [
          0.7675,
          0.8053
        ],
        [
          0.7775,
          0.7998
        ],
        [
          0.7875,
          0.7957
        ],
        [
          0.7975,
          0.7907
        ],
        [
          0.8076,
          0.7839
        ],
        [
          0.818,
          0.778
        ],
        [
          0.828,
          0.7743
        ],
        [
          0.838,
          0.7684
        ],
        [
          0.848,
          0.7634
        ],
        [
          0.8581,
          0.7575
        ],
        [
          0.8685,
          0.752
        ],
        [
          0.8785,
          0.747
        ],
        [
          0.8885,
          0.7411
        ],
        [
          0.8985,
          0.7348
        ],
        [
          0.9086,
          0.7293
        ],
        [
          0.919,
          0.7216
        ],
        [
          0.929,
          0.7166
        ],
        [
          0.939,
          0.7106
        ],
        [
          0.949,
          0.7052
        ],
        [
          0.9591,
          0.697
        ],
        [
          0.9695,
          0.6897
        ],
        [
          0.9795,
          0.6843
        ],
        [
          0.9895,
          0.6788
        ],
        [
          0.9995,
          0.6711
        ],
        [
          1.0,
          0.6711
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.25461483156699144,
        "difference": -0.0016764546747316056,
        "worse": 0.7,
        "kept_right": 0.2512044596413931,
        "kept_wrong": 0.0034103719255982813,
        "handed_off": 0.7453851684330087
      },
      "usd_per_1000": 0.4231342524935803
    },
    {
      "decision": 374,
      "frontier": 192,
      "frontier_accuracy": 0.9331210191082803,
      "in_sample": {
        "threshold": 0.982397,
        "answered": 0.19381255686988172,
        "accuracy": 0.9331210191082803
      },
      "curve": [
        [
          0.0,
          0.9331
        ],
        [
          0.01,
          0.9331
        ],
        [
          0.02,
          0.9331
        ],
        [
          0.03,
          0.9331
        ],
        [
          0.04,
          0.9331
        ],
        [
          0.05,
          0.9331
        ],
        [
          0.0601,
          0.9331
        ],
        [
          0.0705,
          0.9331
        ],
        [
          0.0805,
          0.9331
        ],
        [
          0.0905,
          0.9331
        ],
        [
          0.1005,
          0.9331
        ],
        [
          0.1106,
          0.9331
        ],
        [
          0.121,
          0.9331
        ],
        [
          0.131,
          0.9331
        ],
        [
          0.141,
          0.9331
        ],
        [
          0.151,
          0.9331
        ],
        [
          0.1611,
          0.9331
        ],
        [
          0.1715,
          0.9331
        ],
        [
          0.1815,
          0.9331
        ],
        [
          0.1915,
          0.9331
        ],
        [
          0.2015,
          0.9322
        ],
        [
          0.2116,
          0.9322
        ],
        [
          0.222,
          0.9322
        ],
        [
          0.232,
          0.9318
        ],
        [
          0.242,
          0.9318
        ],
        [
          0.252,
          0.9327
        ],
        [
          0.2621,
          0.9322
        ],
        [
          0.2725,
          0.9322
        ],
        [
          0.2825,
          0.9318
        ],
        [
          0.2925,
          0.9313
        ],
        [
          0.3025,
          0.9308
        ],
        [
          0.3126,
          0.9304
        ],
        [
          0.323,
          0.9304
        ],
        [
          0.333,
          0.9295
        ],
        [
          0.343,
          0.9295
        ],
        [
          0.353,
          0.9286
        ],
        [
          0.3631,
          0.929
        ],
        [
          0.3735,
          0.9268
        ],
        [
          0.3835,
          0.924
        ],
        [
          0.3935,
          0.9222
        ],
        [
          0.4035,
          0.9208
        ],
        [
          0.4136,
          0.919
        ],
        [
          0.424,
          0.9163
        ],
        [
          0.434,
          0.914
        ],
        [
          0.444,
          0.9108
        ],
        [
          0.454,
          0.9081
        ],
        [
          0.4641,
          0.9058
        ],
        [
          0.4745,
          0.904
        ],
        [
          0.4845,
          0.9013
        ],
        [
          0.4945,
          0.899
        ],
        [
          0.5045,
          0.8967
        ],
        [
          0.5146,
          0.894
        ],
        [
          0.525,
          0.8922
        ],
        [
          0.535,
          0.889
        ],
        [
          0.545,
          0.8867
        ],
        [
          0.5551,
          0.8817
        ],
        [
          0.5655,
          0.8808
        ],
        [
          0.5755,
          0.8772
        ],
        [
          0.5855,
          0.8758
        ],
        [
          0.5955,
          0.8735
        ],
        [
          0.6056,
          0.8712
        ],
        [
          0.616,
          0.8658
        ],
        [
          0.626,
          0.8621
        ],
        [
          0.636,
          0.8571
        ],
        [
          0.646,
          0.8521
        ],
        [
          0.6561,
          0.8485
        ],
        [
          0.6665,
          0.8453
        ],
        [
          0.6765,
          0.8412
        ],
        [
          0.6865,
          0.8353
        ],
        [
          0.6965,
          0.8289
        ],
        [
          0.7066,
          0.8244
        ],
        [
          0.717,
          0.8217
        ],
        [
          0.727,
          0.8176
        ],
        [
          0.737,
          0.8112
        ],
        [
          0.747,
          0.8048
        ],
        [
          0.7571,
          0.8021
        ],
        [
          0.7675,
          0.7985
        ],
        [
          0.7775,
          0.793
        ],
        [
          0.7875,
          0.7884
        ],
        [
          0.7975,
          0.7843
        ],
        [
          0.8076,
          0.7775
        ],
        [
          0.818,
          0.7721
        ],
        [
          0.828,
          0.768
        ],
        [
          0.838,
          0.7625
        ],
        [
          0.848,
          0.7571
        ],
        [
          0.8581,
          0.7525
        ],
        [
          0.8685,
          0.747
        ],
        [
          0.8785,
          0.7416
        ],
        [
          0.8885,
          0.7361
        ],
        [
          0.8985,
          0.7302
        ],
        [
          0.9086,
          0.7252
        ],
        [
          0.919,
          0.7184
        ],
        [
          0.929,
          0.7129
        ],
        [
          0.939,
          0.7079
        ],
        [
          0.949,
          0.7025
        ],
        [
          0.9591,
          0.6956
        ],
        [
          0.9695,
          0.6884
        ],
        [
          0.9795,
          0.6829
        ],
        [
          0.9895,
          0.6779
        ],
        [
          0.9995,
          0.6706
        ],
        [
          1.0,
          0.6711
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.24768609276575831,
        "difference": -0.0019368266015574753,
        "worse": 0.67,
        "kept_right": 0.2443185642184816,
        "kept_wrong": 0.0033675285472766353,
        "handed_off": 0.7523139072342423
      },
      "usd_per_1000": 8.77366120516108
    },
    {
      "decision": 374,
      "frontier": 123,
      "frontier_accuracy": 0.9326660600545951,
      "in_sample": {
        "threshold": 0.982397,
        "answered": 0.19381255686988172,
        "accuracy": 0.9326660600545951
      },
      "curve": [
        [
          0.0,
          0.9327
        ],
        [
          0.01,
          0.9327
        ],
        [
          0.02,
          0.9327
        ],
        [
          0.03,
          0.9327
        ],
        [
          0.04,
          0.9327
        ],
        [
          0.05,
          0.9327
        ],
        [
          0.0601,
          0.9327
        ],
        [
          0.0705,
          0.9327
        ],
        [
          0.0805,
          0.9327
        ],
        [
          0.0905,
          0.9327
        ],
        [
          0.1005,
          0.9327
        ],
        [
          0.1106,
          0.9327
        ],
        [
          0.121,
          0.9327
        ],
        [
          0.131,
          0.9327
        ],
        [
          0.141,
          0.9327
        ],
        [
          0.151,
          0.9327
        ],
        [
          0.1611,
          0.9327
        ],
        [
          0.1715,
          0.9327
        ],
        [
          0.1815,
          0.9327
        ],
        [
          0.1915,
          0.9327
        ],
        [
          0.2015,
          0.9318
        ],
        [
          0.2116,
          0.9318
        ],
        [
          0.222,
          0.9318
        ],
        [
          0.232,
          0.9313
        ],
        [
          0.242,
          0.9313
        ],
        [
          0.252,
          0.9318
        ],
        [
          0.2621,
          0.9318
        ],
        [
          0.2725,
          0.9322
        ],
        [
          0.2825,
          0.9322
        ],
        [
          0.2925,
          0.9318
        ],
        [
          0.3025,
          0.9308
        ],
        [
          0.3126,
          0.929
        ],
        [
          0.323,
          0.9277
        ],
        [
          0.333,
          0.9272
        ],
        [
          0.343,
          0.9263
        ],
        [
          0.353,
          0.9249
        ],
        [
          0.3631,
          0.9249
        ],
        [
          0.3735,
          0.9236
        ],
        [
          0.3835,
          0.9213
        ],
        [
          0.3935,
          0.9186
        ],
        [
          0.4035,
          0.9172
        ],
        [
          0.4136,
          0.9149
        ],
        [
          0.424,
          0.9131
        ],
        [
          0.434,
          0.9099
        ],
        [
          0.444,
          0.9086
        ],
        [
          0.454,
          0.9063
        ],
        [
          0.4641,
          0.9035
        ],
        [
          0.4745,
          0.9013
        ],
        [
          0.4845,
          0.8985
        ],
        [
          0.4945,
          0.8963
        ],
        [
          0.5045,
          0.894
        ],
        [
          0.5146,
          0.8922
        ],
        [
          0.525,
          0.8899
        ],
        [
          0.535,
          0.8872
        ],
        [
          0.545,
          0.884
        ],
        [
          0.5551,
          0.8794
        ],
        [
          0.5655,
          0.8781
        ],
        [
          0.5755,
          0.8744
        ],
        [
          0.5855,
          0.8735
        ],
        [
          0.5955,
          0.8717
        ],
        [
          0.6056,
          0.8681
        ],
        [
          0.616,
          0.8626
        ],
        [
          0.626,
          0.8594
        ],
        [
          0.636,
          0.8549
        ],
        [
          0.646,
          0.8508
        ],
        [
          0.6561,
          0.848
        ],
        [
          0.6665,
          0.8458
        ],
        [
          0.6765,
          0.8421
        ],
        [
          0.6865,
          0.8362
        ],
        [
          0.6965,
          0.8294
        ],
        [
          0.7066,
          0.8258
        ],
        [
          0.717,
          0.8226
        ],
        [
          0.727,
          0.8176
        ],
        [
          0.737,
          0.8116
        ],
        [
          0.747,
          0.8066
        ],
        [
          0.7571,
          0.8039
        ],
        [
          0.7675,
          0.7989
        ],
        [
          0.7775,
          0.793
        ],
        [
          0.7875,
          0.7903
        ],
        [
          0.7975,
          0.7834
        ],
        [
          0.8076,
          0.7784
        ],
        [
          0.818,
          0.7725
        ],
        [
          0.828,
          0.7689
        ],
        [
          0.838,
          0.7639
        ],
        [
          0.848,
          0.7584
        ],
        [
          0.8581,
          0.7516
        ],
        [
          0.8685,
          0.7466
        ],
        [
          0.8785,
          0.7425
        ],
        [
          0.8885,
          0.7379
        ],
        [
          0.8985,
          0.7325
        ],
        [
          0.9086,
          0.7279
        ],
        [
          0.919,
          0.7202
        ],
        [
          0.929,
          0.7143
        ],
        [
          0.939,
          0.7084
        ],
        [
          0.949,
          0.7029
        ],
        [
          0.9591,
          0.6956
        ],
        [
          0.9695,
          0.6888
        ],
        [
          0.9795,
          0.6833
        ],
        [
          0.9895,
          0.6788
        ],
        [
          0.9995,
          0.6711
        ],
        [
          1.0,
          0.6711
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.2522342831057701,
        "difference": -0.001712809973709709,
        "worse": 0.7,
        "kept_right": 0.24924527000658814,
        "kept_wrong": 0.0029890130991819357,
        "handed_off": 0.7477657168942297
      },
      "usd_per_1000": 0.2404616112621549
    },
    {
      "decision": 374,
      "frontier": 274,
      "frontier_accuracy": 0.9085532302092811,
      "in_sample": {
        "threshold": 0.949144,
        "answered": 0.30482256596906276,
        "accuracy": 0.9085532302092811
      },
      "curve": [
        [
          0.0,
          0.9086
        ],
        [
          0.01,
          0.9086
        ],
        [
          0.02,
          0.9086
        ],
        [
          0.03,
          0.9086
        ],
        [
          0.04,
          0.9086
        ],
        [
          0.05,
          0.9086
        ],
        [
          0.0601,
          0.9086
        ],
        [
          0.0705,
          0.9086
        ],
        [
          0.0805,
          0.9086
        ],
        [
          0.0905,
          0.9086
        ],
        [
          0.1005,
          0.9086
        ],
        [
          0.1106,
          0.9086
        ],
        [
          0.121,
          0.9086
        ],
        [
          0.131,
          0.9086
        ],
        [
          0.141,
          0.9086
        ],
        [
          0.151,
          0.9086
        ],
        [
          0.1611,
          0.9086
        ],
        [
          0.1715,
          0.9086
        ],
        [
          0.1815,
          0.9086
        ],
        [
          0.1915,
          0.9086
        ],
        [
          0.2015,
          0.9081
        ],
        [
          0.2116,
          0.9081
        ],
        [
          0.222,
          0.9081
        ],
        [
          0.232,
          0.9076
        ],
        [
          0.242,
          0.9076
        ],
        [
          0.252,
          0.9086
        ],
        [
          0.2621,
          0.9086
        ],
        [
          0.2725,
          0.909
        ],
        [
          0.2825,
          0.909
        ],
        [
          0.2925,
          0.9086
        ],
        [
          0.3025,
          0.9086
        ],
        [
          0.3126,
          0.9072
        ],
        [
          0.323,
          0.9063
        ],
        [
          0.333,
          0.9058
        ],
        [
          0.343,
          0.9058
        ],
        [
          0.353,
          0.9049
        ],
        [
          0.3631,
          0.9054
        ],
        [
          0.3735,
          0.9045
        ],
        [
          0.3835,
          0.9017
        ],
        [
          0.3935,
          0.8995
        ],
        [
          0.4035,
          0.8985
        ],
        [
          0.4136,
          0.8967
        ],
        [
          0.424,
          0.8949
        ],
        [
          0.434,
          0.8926
        ],
        [
          0.444,
          0.8917
        ],
        [
          0.454,
          0.8894
        ],
        [
          0.4641,
          0.8867
        ],
        [
          0.4745,
          0.8854
        ],
        [
          0.4845,
          0.8831
        ],
        [
          0.4945,
          0.8799
        ],
        [
          0.5045,
          0.8776
        ],
        [
          0.5146,
          0.8753
        ],
        [
          0.525,
          0.8726
        ],
        [
          0.535,
          0.8694
        ],
        [
          0.545,
          0.8685
        ],
        [
          0.5551,
          0.8644
        ],
        [
          0.5655,
          0.8626
        ],
        [
          0.5755,
          0.8594
        ],
        [
          0.5855,
          0.8581
        ],
        [
          0.5955,
          0.8553
        ],
        [
          0.6056,
          0.8521
        ],
        [
          0.616,
          0.8462
        ],
        [
          0.626,
          0.8426
        ],
        [
          0.636,
          0.8385
        ],
        [
          0.646,
          0.8348
        ],
        [
          0.6561,
          0.8326
        ],
        [
          0.6665,
          0.8294
        ],
        [
          0.6765,
          0.8253
        ],
        [
          0.6865,
          0.8207
        ],
        [
          0.6965,
          0.8144
        ],
        [
          0.7066,
          0.8112
        ],
        [
          0.717,
          0.8085
        ],
        [
          0.727,
          0.8035
        ],
        [
          0.737,
          0.7975
        ],
        [
          0.747,
          0.7925
        ],
        [
          0.7571,
          0.7903
        ],
        [
          0.7675,
          0.7862
        ],
        [
          0.7775,
          0.7812
        ],
        [
          0.7875,
          0.7775
        ],
        [
          0.7975,
          0.773
        ],
        [
          0.8076,
          0.7666
        ],
        [
          0.818,
          0.7607
        ],
        [
          0.828,
          0.7584
        ],
        [
          0.838,
          0.7525
        ],
        [
          0.848,
          0.7493
        ],
        [
          0.8581,
          0.7434
        ],
        [
          0.8685,
          0.7389
        ],
        [
          0.8785,
          0.7348
        ],
        [
          0.8885,
          0.7288
        ],
        [
          0.8985,
          0.7229
        ],
        [
          0.9086,
          0.7179
        ],
        [
          0.919,
          0.7106
        ],
        [
          0.929,
          0.7066
        ],
        [
          0.939,
          0.702
        ],
        [
          0.949,
          0.6965
        ],
        [
          0.9591,
          0.6906
        ],
        [
          0.9695,
          0.6856
        ],
        [
          0.9795,
          0.6811
        ],
        [
          0.9895,
          0.677
        ],
        [
          0.9995,
          0.6706
        ],
        [
          1.0,
          0.6711
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.2946618628241936,
        "difference": -0.002085918200006906,
        "worse": 0.54,
        "kept_right": 0.2883075603597612,
        "kept_wrong": 0.006354302464432434,
        "handed_off": 0.7053381371758062
      },
      "usd_per_1000": 0.34027108128406136
    },
    {
      "decision": 374,
      "frontier": 47,
      "frontier_accuracy": 0.9044585987261147,
      "in_sample": {
        "threshold": 0.954274,
        "answered": 0.2916287534121929,
        "accuracy": 0.9044585987261147
      },
      "curve": [
        [
          0.0,
          0.9045
        ],
        [
          0.01,
          0.9045
        ],
        [
          0.02,
          0.9045
        ],
        [
          0.03,
          0.9045
        ],
        [
          0.04,
          0.9045
        ],
        [
          0.05,
          0.9045
        ],
        [
          0.0601,
          0.9045
        ],
        [
          0.0705,
          0.9045
        ],
        [
          0.0805,
          0.9045
        ],
        [
          0.0905,
          0.9045
        ],
        [
          0.1005,
          0.9045
        ],
        [
          0.1106,
          0.9045
        ],
        [
          0.121,
          0.9045
        ],
        [
          0.131,
          0.9045
        ],
        [
          0.141,
          0.9045
        ],
        [
          0.151,
          0.9045
        ],
        [
          0.1611,
          0.9045
        ],
        [
          0.1715,
          0.9049
        ],
        [
          0.1815,
          0.9049
        ],
        [
          0.1915,
          0.9049
        ],
        [
          0.2015,
          0.904
        ],
        [
          0.2116,
          0.904
        ],
        [
          0.222,
          0.904
        ],
        [
          0.232,
          0.9035
        ],
        [
          0.242,
          0.904
        ],
        [
          0.252,
          0.9049
        ],
        [
          0.2621,
          0.9045
        ],
        [
          0.2725,
          0.9049
        ],
        [
          0.2825,
          0.9045
        ],
        [
          0.2925,
          0.904
        ],
        [
          0.3025,
          0.9035
        ],
        [
          0.3126,
          0.9031
        ],
        [
          0.323,
          0.9026
        ],
        [
          0.333,
          0.9017
        ],
        [
          0.343,
          0.9022
        ],
        [
          0.353,
          0.9013
        ],
        [
          0.3631,
          0.9013
        ],
        [
          0.3735,
          0.8995
        ],
        [
          0.3835,
          0.8967
        ],
        [
          0.3935,
          0.8954
        ],
        [
          0.4035,
          0.8944
        ],
        [
          0.4136,
          0.894
        ],
        [
          0.424,
          0.8926
        ],
        [
          0.434,
          0.8904
        ],
        [
          0.444,
          0.8872
        ],
        [
          0.454,
          0.8849
        ],
        [
          0.4641,
          0.8826
        ],
        [
          0.4745,
          0.8808
        ],
        [
          0.4845,
          0.8785
        ],
        [
          0.4945,
          0.8763
        ],
        [
          0.5045,
          0.8744
        ],
        [
          0.5146,
          0.8717
        ],
        [
          0.525,
          0.8699
        ],
        [
          0.535,
          0.8676
        ],
        [
          0.545,
          0.8653
        ],
        [
          0.5551,
          0.8621
        ],
        [
          0.5655,
          0.8612
        ],
        [
          0.5755,
          0.8581
        ],
        [
          0.5855,
          0.8567
        ],
        [
          0.5955,
          0.8535
        ],
        [
          0.6056,
          0.8508
        ],
        [
          0.616,
          0.8453
        ],
        [
          0.626,
          0.8412
        ],
        [
          0.636,
          0.8371
        ],
        [
          0.646,
          0.8339
        ],
        [
          0.6561,
          0.8303
        ],
        [
          0.6665,
          0.8276
        ],
        [
          0.6765,
          0.8235
        ],
        [
          0.6865,
          0.818
        ],
        [
          0.6965,
          0.8116
        ],
        [
          0.7066,
          0.8089
        ],
        [
          0.717,
          0.8062
        ],
        [
          0.727,
          0.8021
        ],
        [
          0.737,
          0.7962
        ],
        [
          0.747,
          0.7916
        ],
        [
          0.7571,
          0.7898
        ],
        [
          0.7675,
          0.7848
        ],
        [
          0.7775,
          0.7793
        ],
        [
          0.7875,
          0.7753
        ],
        [
          0.7975,
          0.7702
        ],
        [
          0.8076,
          0.7648
        ],
        [
          0.818,
          0.7589
        ],
        [
          0.828,
          0.7548
        ],
        [
          0.838,
          0.7502
        ],
        [
          0.848,
          0.7452
        ],
        [
          0.8581,
          0.7402
        ],
        [
          0.8685,
          0.7357
        ],
        [
          0.8785,
          0.7316
        ],
        [
          0.8885,
          0.727
        ],
        [
          0.8985,
          0.7211
        ],
        [
          0.9086,
          0.7161
        ],
        [
          0.919,
          0.7111
        ],
        [
          0.929,
          0.7052
        ],
        [
          0.939,
          0.6997
        ],
        [
          0.949,
          0.6961
        ],
        [
          0.9591,
          0.6897
        ],
        [
          0.9695,
          0.6843
        ],
        [
          0.9795,
          0.6802
        ],
        [
          0.9895,
          0.6765
        ],
        [
          0.9995,
          0.6706
        ],
        [
          1.0,
          0.6711
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.29587988056157405,
        "difference": -0.0020368732962349396,
        "worse": 0.6233333333333333,
        "kept_right": 0.2896507145001396,
        "kept_wrong": 0.006229166061434605,
        "handed_off": 0.7041201194384257
      },
      "usd_per_1000": 3.511348532967649
    },
    {
      "decision": 374,
      "frontier": 193,
      "frontier_accuracy": 0.89171974522293,
      "in_sample": {
        "threshold": 0.933729,
        "answered": 0.327115559599636,
        "accuracy": 0.89171974522293
      },
      "curve": [
        [
          0.0,
          0.8917
        ],
        [
          0.01,
          0.8917
        ],
        [
          0.02,
          0.8917
        ],
        [
          0.03,
          0.8917
        ],
        [
          0.04,
          0.8917
        ],
        [
          0.05,
          0.8917
        ],
        [
          0.0601,
          0.8917
        ],
        [
          0.0705,
          0.8917
        ],
        [
          0.0805,
          0.8917
        ],
        [
          0.0905,
          0.8917
        ],
        [
          0.1005,
          0.8917
        ],
        [
          0.1106,
          0.8917
        ],
        [
          0.121,
          0.8917
        ],
        [
          0.131,
          0.8917
        ],
        [
          0.141,
          0.8917
        ],
        [
          0.151,
          0.8917
        ],
        [
          0.1611,
          0.8917
        ],
        [
          0.1715,
          0.8922
        ],
        [
          0.1815,
          0.8922
        ],
        [
          0.1915,
          0.8922
        ],
        [
          0.2015,
          0.8917
        ],
        [
          0.2116,
          0.8917
        ],
        [
          0.222,
          0.8917
        ],
        [
          0.232,
          0.8913
        ],
        [
          0.242,
          0.8913
        ],
        [
          0.252,
          0.8926
        ],
        [
          0.2621,
          0.8922
        ],
        [
          0.2725,
          0.8931
        ],
        [
          0.2825,
          0.8931
        ],
        [
          0.2925,
          0.8926
        ],
        [
          0.3025,
          0.8926
        ],
        [
          0.3126,
          0.8917
        ],
        [
          0.323,
          0.8917
        ],
        [
          0.333,
          0.8904
        ],
        [
          0.343,
          0.8899
        ],
        [
          0.353,
          0.889
        ],
        [
          0.3631,
          0.889
        ],
        [
          0.3735,
          0.8872
        ],
        [
          0.3835,
          0.8844
        ],
        [
          0.3935,
          0.8835
        ],
        [
          0.4035,
          0.8835
        ],
        [
          0.4136,
          0.8817
        ],
        [
          0.424,
          0.8799
        ],
        [
          0.434,
          0.8767
        ],
        [
          0.444,
          0.8749
        ],
        [
          0.454,
          0.8722
        ],
        [
          0.4641,
          0.8708
        ],
        [
          0.4745,
          0.869
        ],
        [
          0.4845,
          0.8667
        ],
        [
          0.4945,
          0.8649
        ],
        [
          0.5045,
          0.8626
        ],
        [
          0.5146,
          0.8599
        ],
        [
          0.525,
          0.8585
        ],
        [
          0.535,
          0.8567
        ],
        [
          0.545,
          0.854
        ],
        [
          0.5551,
          0.8503
        ],
        [
          0.5655,
          0.8494
        ],
        [
          0.5755,
          0.8476
        ],
        [
          0.5855,
          0.8471
        ],
        [
          0.5955,
          0.8453
        ],
        [
          0.6056,
          0.8426
        ],
        [
          0.616,
          0.8376
        ],
        [
          0.626,
          0.8344
        ],
        [
          0.636,
          0.8294
        ],
        [
          0.646,
          0.8258
        ],
        [
          0.6561,
          0.823
        ],
        [
          0.6665,
          0.8198
        ],
        [
          0.6765,
          0.8167
        ],
        [
          0.6865,
          0.8116
        ],
        [
          0.6965,
          0.8062
        ],
        [
          0.7066,
          0.8035
        ],
        [
          0.717,
          0.8016
        ],
        [
          0.727,
          0.7975
        ],
        [
          0.737,
          0.7925
        ],
        [
          0.747,
          0.7862
        ],
        [
          0.7571,
          0.7848
        ],
        [
          0.7675,
          0.7803
        ],
        [
          0.7775,
          0.7753
        ],
        [
          0.7875,
          0.7712
        ],
        [
          0.7975,
          0.7671
        ],
        [
          0.8076,
          0.7621
        ],
        [
          0.818,
          0.7575
        ],
        [
          0.828,
          0.7534
        ],
        [
          0.838,
          0.7489
        ],
        [
          0.848,
          0.7452
        ],
        [
          0.8581,
          0.7398
        ],
        [
          0.8685,
          0.7366
        ],
        [
          0.8785,
          0.732
        ],
        [
          0.8885,
          0.7284
        ],
        [
          0.8985,
          0.7225
        ],
        [
          0.9086,
          0.7179
        ],
        [
          0.919,
          0.7111
        ],
        [
          0.929,
          0.707
        ],
        [
          0.939,
          0.7038
        ],
        [
          0.949,
          0.6984
        ],
        [
          0.9591,
          0.6911
        ],
        [
          0.9695,
          0.6861
        ],
        [
          0.9795,
          0.6806
        ],
        [
          0.9895,
          0.6774
        ],
        [
          0.9995,
          0.6706
        ],
        [
          1.0,
          0.6711
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.3227230714308194,
        "difference": -0.001673330922516047,
        "worse": 0.52,
        "kept_right": 0.3142166802762596,
        "kept_wrong": 0.008506391154559525,
        "handed_off": 0.677276928569181
      },
      "usd_per_1000": 5.286887701713235
    },
    {
      "decision": 374,
      "frontier": 55,
      "frontier_accuracy": 0.8503184713375797,
      "in_sample": {
        "threshold": 0.810749,
        "answered": 0.47042766151046406,
        "accuracy": 0.8503184713375797
      },
      "curve": [
        [
          0.0,
          0.8503
        ],
        [
          0.01,
          0.8503
        ],
        [
          0.02,
          0.8503
        ],
        [
          0.03,
          0.8503
        ],
        [
          0.04,
          0.8512
        ],
        [
          0.05,
          0.8512
        ],
        [
          0.0601,
          0.8512
        ],
        [
          0.0705,
          0.8517
        ],
        [
          0.0805,
          0.8517
        ],
        [
          0.0905,
          0.8517
        ],
        [
          0.1005,
          0.8526
        ],
        [
          0.1106,
          0.853
        ],
        [
          0.121,
          0.8535
        ],
        [
          0.131,
          0.8535
        ],
        [
          0.141,
          0.8535
        ],
        [
          0.151,
          0.854
        ],
        [
          0.1611,
          0.854
        ],
        [
          0.1715,
          0.8544
        ],
        [
          0.1815,
          0.8549
        ],
        [
          0.1915,
          0.8549
        ],
        [
          0.2015,
          0.8553
        ],
        [
          0.2116,
          0.8558
        ],
        [
          0.222,
          0.8567
        ],
        [
          0.232,
          0.8571
        ],
        [
          0.242,
          0.8585
        ],
        [
          0.252,
          0.8594
        ],
        [
          0.2621,
          0.8594
        ],
        [
          0.2725,
          0.8612
        ],
        [
          0.2825,
          0.8612
        ],
        [
          0.2925,
          0.8608
        ],
        [
          0.3025,
          0.8612
        ],
        [
          0.3126,
          0.8608
        ],
        [
          0.323,
          0.8617
        ],
        [
          0.333,
          0.8621
        ],
        [
          0.343,
          0.8612
        ],
        [
          0.353,
          0.8603
        ],
        [
          0.3631,
          0.8603
        ],
        [
          0.3735,
          0.8599
        ],
        [
          0.3835,
          0.8585
        ],
        [
          0.3935,
          0.8581
        ],
        [
          0.4035,
          0.8576
        ],
        [
          0.4136,
          0.8558
        ],
        [
          0.424,
          0.8544
        ],
        [
          0.434,
          0.853
        ],
        [
          0.444,
          0.853
        ],
        [
          0.454,
          0.8508
        ],
        [
          0.4641,
          0.8503
        ],
        [
          0.4745,
          0.849
        ],
        [
          0.4845,
          0.8471
        ],
        [
          0.4945,
          0.8467
        ],
        [
          0.5045,
          0.8462
        ],
        [
          0.5146,
          0.843
        ],
        [
          0.525,
          0.843
        ],
        [
          0.535,
          0.8421
        ],
        [
          0.545,
          0.8394
        ],
        [
          0.5551,
          0.8371
        ],
        [
          0.5655,
          0.8367
        ],
        [
          0.5755,
          0.8353
        ],
        [
          0.5855,
          0.8353
        ],
        [
          0.5955,
          0.833
        ],
        [
          0.6056,
          0.8317
        ],
        [
          0.616,
          0.8267
        ],
        [
          0.626,
          0.823
        ],
        [
          0.636,
          0.8189
        ],
        [
          0.646,
          0.8167
        ],
        [
          0.6561,
          0.8162
        ],
        [
          0.6665,
          0.8148
        ],
        [
          0.6765,
          0.8116
        ],
        [
          0.6865,
          0.8066
        ],
        [
          0.6965,
          0.8016
        ],
        [
          0.7066,
          0.7985
        ],
        [
          0.717,
          0.7962
        ],
        [
          0.727,
          0.7916
        ],
        [
          0.737,
          0.7884
        ],
        [
          0.747,
          0.7825
        ],
        [
          0.7571,
          0.7803
        ],
        [
          0.7675,
          0.7766
        ],
        [
          0.7775,
          0.7716
        ],
        [
          0.7875,
          0.768
        ],
        [
          0.7975,
          0.7639
        ],
        [
          0.8076,
          0.7598
        ],
        [
          0.818,
          0.7552
        ],
        [
          0.828,
          0.7516
        ],
        [
          0.838,
          0.747
        ],
        [
          0.848,
          0.7425
        ],
        [
          0.8581,
          0.7384
        ],
        [
          0.8685,
          0.7343
        ],
        [
          0.8785,
          0.7302
        ],
        [
          0.8885,
          0.7261
        ],
        [
          0.8985,
          0.7207
        ],
        [
          0.9086,
          0.7175
        ],
        [
          0.919,
          0.7106
        ],
        [
          0.929,
          0.7047
        ],
        [
          0.939,
          0.7015
        ],
        [
          0.949,
          0.6975
        ],
        [
          0.9591,
          0.6915
        ],
        [
          0.9695,
          0.687
        ],
        [
          0.9795,
          0.682
        ],
        [
          0.9895,
          0.677
        ],
        [
          0.9995,
          0.6706
        ],
        [
          1.0,
          0.6711
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.4744988063008167,
        "difference": -0.002471240283529272,
        "worse": 0.5166666666666667,
        "kept_right": 0.42944505597941457,
        "kept_wrong": 0.04505375032140226,
        "handed_off": 0.5255011936991831
      },
      "usd_per_1000": 1.9599804283001037
    },
    {
      "decision": 217,
      "frontier": 140,
      "frontier_accuracy": 0.9808917197452229,
      "in_sample": {
        "threshold": 0.9864659363390037,
        "answered": 0.181073703366697,
        "accuracy": 0.9808917197452229
      },
      "curve": [
        [
          0.0,
          0.9809
        ],
        [
          0.01,
          0.9809
        ],
        [
          0.02,
          0.9809
        ],
        [
          0.03,
          0.9809
        ],
        [
          0.04,
          0.9809
        ],
        [
          0.05,
          0.9809
        ],
        [
          0.0601,
          0.9809
        ],
        [
          0.0705,
          0.9809
        ],
        [
          0.0805,
          0.9809
        ],
        [
          0.0905,
          0.9809
        ],
        [
          0.1005,
          0.9809
        ],
        [
          0.1106,
          0.9809
        ],
        [
          0.121,
          0.9809
        ],
        [
          0.131,
          0.9809
        ],
        [
          0.141,
          0.9809
        ],
        [
          0.151,
          0.9809
        ],
        [
          0.1611,
          0.9809
        ],
        [
          0.1715,
          0.9809
        ],
        [
          0.1815,
          0.9804
        ],
        [
          0.1915,
          0.98
        ],
        [
          0.2015,
          0.9795
        ],
        [
          0.2116,
          0.9791
        ],
        [
          0.222,
          0.9786
        ],
        [
          0.232,
          0.9782
        ],
        [
          0.242,
          0.9782
        ],
        [
          0.252,
          0.9782
        ],
        [
          0.2621,
          0.9782
        ],
        [
          0.2725,
          0.9768
        ],
        [
          0.2825,
          0.9768
        ],
        [
          0.2925,
          0.9768
        ],
        [
          0.3025,
          0.9763
        ],
        [
          0.3126,
          0.9759
        ],
        [
          0.323,
          0.9754
        ],
        [
          0.333,
          0.975
        ],
        [
          0.343,
          0.9745
        ],
        [
          0.353,
          0.9736
        ],
        [
          0.3631,
          0.9732
        ],
        [
          0.3735,
          0.9722
        ],
        [
          0.3835,
          0.97
        ],
        [
          0.3935,
          0.9663
        ],
        [
          0.4035,
          0.9645
        ],
        [
          0.4136,
          0.9613
        ],
        [
          0.424,
          0.96
        ],
        [
          0.434,
          0.9568
        ],
        [
          0.444,
          0.9554
        ],
        [
          0.454,
          0.9527
        ],
        [
          0.4641,
          0.9504
        ],
        [
          0.4745,
          0.9481
        ],
        [
          0.4845,
          0.9463
        ],
        [
          0.4945,
          0.9427
        ],
        [
          0.5045,
          0.9399
        ],
        [
          0.5146,
          0.9359
        ],
        [
          0.525,
          0.9327
        ],
        [
          0.535,
          0.9286
        ],
        [
          0.545,
          0.9263
        ],
        [
          0.5551,
          0.9231
        ],
        [
          0.5655,
          0.9172
        ],
        [
          0.5755,
          0.9145
        ],
        [
          0.5855,
          0.9104
        ],
        [
          0.5955,
          0.9076
        ],
        [
          0.6056,
          0.9035
        ],
        [
          0.616,
          0.8985
        ],
        [
          0.626,
          0.8935
        ],
        [
          0.636,
          0.8881
        ],
        [
          0.646,
          0.8844
        ],
        [
          0.6561,
          0.8776
        ],
        [
          0.6665,
          0.8735
        ],
        [
          0.6765,
          0.8699
        ],
        [
          0.6865,
          0.8635
        ],
        [
          0.6965,
          0.8576
        ],
        [
          0.7066,
          0.8517
        ],
        [
          0.717,
          0.8449
        ],
        [
          0.727,
          0.8394
        ],
        [
          0.737,
          0.8339
        ],
        [
          0.747,
          0.8285
        ],
        [
          0.7571,
          0.8226
        ],
        [
          0.7675,
          0.8157
        ],
        [
          0.7775,
          0.8107
        ],
        [
          0.7875,
          0.8044
        ],
        [
          0.7975,
          0.7989
        ],
        [
          0.8076,
          0.7944
        ],
        [
          0.818,
          0.7884
        ],
        [
          0.828,
          0.783
        ],
        [
          0.838,
          0.778
        ],
        [
          0.848,
          0.7725
        ],
        [
          0.8581,
          0.7666
        ],
        [
          0.8685,
          0.7611
        ],
        [
          0.8785,
          0.7552
        ],
        [
          0.8885,
          0.748
        ],
        [
          0.8985,
          0.7425
        ],
        [
          0.9086,
          0.7357
        ],
        [
          0.919,
          0.7298
        ],
        [
          0.929,
          0.7229
        ],
        [
          0.939,
          0.7161
        ],
        [
          0.949,
          0.7097
        ],
        [
          0.9591,
          0.7025
        ],
        [
          0.9695,
          0.6952
        ],
        [
          0.9795,
          0.6879
        ],
        [
          0.9895,
          0.6806
        ],
        [
          0.9995,
          0.6711
        ],
        [
          1.0,
          0.6706
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.19089183260438078,
        "difference": -0.0009368145718758521,
        "worse": 0.43333333333333335,
        "kept_right": 0.18993984442352038,
        "kept_wrong": 0.0009519881808603252,
        "handed_off": 0.809108167395619
      },
      "usd_per_1000": 1.9650808758913112
    },
    {
      "decision": 217,
      "frontier": 277,
      "frontier_accuracy": 0.9795268425841674,
      "in_sample": {
        "threshold": 0.9864659363390037,
        "answered": 0.181073703366697,
        "accuracy": 0.9795268425841674
      },
      "curve": [
        [
          0.0,
          0.9795
        ],
        [
          0.01,
          0.9795
        ],
        [
          0.02,
          0.9795
        ],
        [
          0.03,
          0.9795
        ],
        [
          0.04,
          0.9795
        ],
        [
          0.05,
          0.9795
        ],
        [
          0.0601,
          0.9795
        ],
        [
          0.0705,
          0.9795
        ],
        [
          0.0805,
          0.9795
        ],
        [
          0.0905,
          0.9795
        ],
        [
          0.1005,
          0.9795
        ],
        [
          0.1106,
          0.9795
        ],
        [
          0.121,
          0.9795
        ],
        [
          0.131,
          0.9795
        ],
        [
          0.141,
          0.9795
        ],
        [
          0.151,
          0.9795
        ],
        [
          0.1611,
          0.9795
        ],
        [
          0.1715,
          0.9795
        ],
        [
          0.1815,
          0.9791
        ],
        [
          0.1915,
          0.9786
        ],
        [
          0.2015,
          0.9782
        ],
        [
          0.2116,
          0.9777
        ],
        [
          0.222,
          0.9773
        ],
        [
          0.232,
          0.9768
        ],
        [
          0.242,
          0.9768
        ],
        [
          0.252,
          0.9773
        ],
        [
          0.2621,
          0.9773
        ],
        [
          0.2725,
          0.9759
        ],
        [
          0.2825,
          0.9759
        ],
        [
          0.2925,
          0.9759
        ],
        [
          0.3025,
          0.9754
        ],
        [
          0.3126,
          0.975
        ],
        [
          0.323,
          0.9741
        ],
        [
          0.333,
          0.9736
        ],
        [
          0.343,
          0.9732
        ],
        [
          0.353,
          0.9722
        ],
        [
          0.3631,
          0.9713
        ],
        [
          0.3735,
          0.9704
        ],
        [
          0.3835,
          0.9686
        ],
        [
          0.3935,
          0.9654
        ],
        [
          0.4035,
          0.9636
        ],
        [
          0.4136,
          0.9604
        ],
        [
          0.424,
          0.9591
        ],
        [
          0.434,
          0.9554
        ],
        [
          0.444,
          0.954
        ],
        [
          0.454,
          0.9509
        ],
        [
          0.4641,
          0.9481
        ],
        [
          0.4745,
          0.9463
        ],
        [
          0.4845,
          0.9445
        ],
        [
          0.4945,
          0.9413
        ],
        [
          0.5045,
          0.939
        ],
        [
          0.5146,
          0.9354
        ],
        [
          0.525,
          0.9322
        ],
        [
          0.535,
          0.9281
        ],
        [
          0.545,
          0.9258
        ],
        [
          0.5551,
          0.9227
        ],
        [
          0.5655,
          0.9167
        ],
        [
          0.5755,
          0.9136
        ],
        [
          0.5855,
          0.9095
        ],
        [
          0.5955,
          0.9072
        ],
        [
          0.6056,
          0.9031
        ],
        [
          0.616,
          0.8981
        ],
        [
          0.626,
          0.8931
        ],
        [
          0.636,
          0.8881
        ],
        [
          0.646,
          0.8844
        ],
        [
          0.6561,
          0.8776
        ],
        [
          0.6665,
          0.8735
        ],
        [
          0.6765,
          0.8694
        ],
        [
          0.6865,
          0.8631
        ],
        [
          0.6965,
          0.8571
        ],
        [
          0.7066,
          0.8512
        ],
        [
          0.717,
          0.8444
        ],
        [
          0.727,
          0.838
        ],
        [
          0.737,
          0.8326
        ],
        [
          0.747,
          0.8271
        ],
        [
          0.7571,
          0.8212
        ],
        [
          0.7675,
          0.8144
        ],
        [
          0.7775,
          0.8094
        ],
        [
          0.7875,
          0.803
        ],
        [
          0.7975,
          0.7975
        ],
        [
          0.8076,
          0.793
        ],
        [
          0.818,
          0.7875
        ],
        [
          0.828,
          0.7825
        ],
        [
          0.838,
          0.7775
        ],
        [
          0.848,
          0.7721
        ],
        [
          0.8581,
          0.7662
        ],
        [
          0.8685,
          0.7607
        ],
        [
          0.8785,
          0.7548
        ],
        [
          0.8885,
          0.7475
        ],
        [
          0.8985,
          0.7416
        ],
        [
          0.9086,
          0.7348
        ],
        [
          0.919,
          0.7284
        ],
        [
          0.929,
          0.7216
        ],
        [
          0.939,
          0.7152
        ],
        [
          0.949,
          0.7093
        ],
        [
          0.9591,
          0.702
        ],
        [
          0.9695,
          0.6943
        ],
        [
          0.9795,
          0.687
        ],
        [
          0.9895,
          0.6802
        ],
        [
          0.9995,
          0.6711
        ],
        [
          1.0,
          0.6706
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1940281553707288,
        "difference": -0.0010884870268350916,
        "worse": 0.45,
        "kept_right": 0.19288508422142855,
        "kept_wrong": 0.001143071149300185,
        "handed_off": 0.8059718446292712
      },
      "usd_per_1000": 4.896431342158079
    },
    {
      "decision": 217,
      "frontier": 276,
      "frontier_accuracy": 0.9777070063694268,
      "in_sample": {
        "threshold": 0.9864659363390037,
        "answered": 0.181073703366697,
        "accuracy": 0.9777070063694268
      },
      "curve": [
        [
          0.0,
          0.9777
        ],
        [
          0.01,
          0.9777
        ],
        [
          0.02,
          0.9777
        ],
        [
          0.03,
          0.9777
        ],
        [
          0.04,
          0.9777
        ],
        [
          0.05,
          0.9777
        ],
        [
          0.0601,
          0.9777
        ],
        [
          0.0705,
          0.9777
        ],
        [
          0.0805,
          0.9777
        ],
        [
          0.0905,
          0.9777
        ],
        [
          0.1005,
          0.9777
        ],
        [
          0.1106,
          0.9777
        ],
        [
          0.121,
          0.9777
        ],
        [
          0.131,
          0.9777
        ],
        [
          0.141,
          0.9777
        ],
        [
          0.151,
          0.9777
        ],
        [
          0.1611,
          0.9777
        ],
        [
          0.1715,
          0.9777
        ],
        [
          0.1815,
          0.9773
        ],
        [
          0.1915,
          0.9768
        ],
        [
          0.2015,
          0.9763
        ],
        [
          0.2116,
          0.9759
        ],
        [
          0.222,
          0.9754
        ],
        [
          0.232,
          0.975
        ],
        [
          0.242,
          0.975
        ],
        [
          0.252,
          0.9754
        ],
        [
          0.2621,
          0.9754
        ],
        [
          0.2725,
          0.9745
        ],
        [
          0.2825,
          0.9745
        ],
        [
          0.2925,
          0.9745
        ],
        [
          0.3025,
          0.9741
        ],
        [
          0.3126,
          0.9741
        ],
        [
          0.323,
          0.9732
        ],
        [
          0.333,
          0.9727
        ],
        [
          0.343,
          0.9722
        ],
        [
          0.353,
          0.9718
        ],
        [
          0.3631,
          0.9709
        ],
        [
          0.3735,
          0.97
        ],
        [
          0.3835,
          0.9682
        ],
        [
          0.3935,
          0.965
        ],
        [
          0.4035,
          0.9631
        ],
        [
          0.4136,
          0.9595
        ],
        [
          0.424,
          0.9581
        ],
        [
          0.434,
          0.9545
        ],
        [
          0.444,
          0.9531
        ],
        [
          0.454,
          0.95
        ],
        [
          0.4641,
          0.9477
        ],
        [
          0.4745,
          0.9459
        ],
        [
          0.4845,
          0.944
        ],
        [
          0.4945,
          0.9409
        ],
        [
          0.5045,
          0.9386
        ],
        [
          0.5146,
          0.9349
        ],
        [
          0.525,
          0.9318
        ],
        [
          0.535,
          0.9277
        ],
        [
          0.545,
          0.9254
        ],
        [
          0.5551,
          0.9222
        ],
        [
          0.5655,
          0.9167
        ],
        [
          0.5755,
          0.914
        ],
        [
          0.5855,
          0.9099
        ],
        [
          0.5955,
          0.9072
        ],
        [
          0.6056,
          0.9031
        ],
        [
          0.616,
          0.8981
        ],
        [
          0.626,
          0.8931
        ],
        [
          0.636,
          0.8876
        ],
        [
          0.646,
          0.884
        ],
        [
          0.6561,
          0.8776
        ],
        [
          0.6665,
          0.8735
        ],
        [
          0.6765,
          0.8694
        ],
        [
          0.6865,
          0.8631
        ],
        [
          0.6965,
          0.8571
        ],
        [
          0.7066,
          0.8512
        ],
        [
          0.717,
          0.8444
        ],
        [
          0.727,
          0.838
        ],
        [
          0.737,
          0.8326
        ],
        [
          0.747,
          0.8271
        ],
        [
          0.7571,
          0.8212
        ],
        [
          0.7675,
          0.8144
        ],
        [
          0.7775,
          0.8094
        ],
        [
          0.7875,
          0.803
        ],
        [
          0.7975,
          0.7975
        ],
        [
          0.8076,
          0.793
        ],
        [
          0.818,
          0.7871
        ],
        [
          0.828,
          0.7821
        ],
        [
          0.838,
          0.7771
        ],
        [
          0.848,
          0.7716
        ],
        [
          0.8581,
          0.7662
        ],
        [
          0.8685,
          0.7607
        ],
        [
          0.8785,
          0.7548
        ],
        [
          0.8885,
          0.7475
        ],
        [
          0.8985,
          0.742
        ],
        [
          0.9086,
          0.7352
        ],
        [
          0.919,
          0.7288
        ],
        [
          0.929,
          0.722
        ],
        [
          0.939,
          0.7157
        ],
        [
          0.949,
          0.7093
        ],
        [
          0.9591,
          0.702
        ],
        [
          0.9695,
          0.6943
        ],
        [
          0.9795,
          0.687
        ],
        [
          0.9895,
          0.6802
        ],
        [
          0.9995,
          0.6711
        ],
        [
          1.0,
          0.6706
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.19654247858245774,
        "difference": -0.0012098122385896465,
        "worse": 0.4533333333333333,
        "kept_right": 0.19522347602015858,
        "kept_wrong": 0.001319002562299112,
        "handed_off": 0.8034575214175422
      },
      "usd_per_1000": 1.311125645346147
    },
    {
      "decision": 217,
      "frontier": 48,
      "frontier_accuracy": 0.9758871701546861,
      "in_sample": {
        "threshold": 0.9864659363390037,
        "answered": 0.181073703366697,
        "accuracy": 0.9758871701546861
      },
      "curve": [
        [
          0.0,
          0.9759
        ],
        [
          0.01,
          0.9759
        ],
        [
          0.02,
          0.9759
        ],
        [
          0.03,
          0.9759
        ],
        [
          0.04,
          0.9759
        ],
        [
          0.05,
          0.9759
        ],
        [
          0.0601,
          0.9759
        ],
        [
          0.0705,
          0.9759
        ],
        [
          0.0805,
          0.9759
        ],
        [
          0.0905,
          0.9759
        ],
        [
          0.1005,
          0.9759
        ],
        [
          0.1106,
          0.9759
        ],
        [
          0.121,
          0.9759
        ],
        [
          0.131,
          0.9759
        ],
        [
          0.141,
          0.9759
        ],
        [
          0.151,
          0.9759
        ],
        [
          0.1611,
          0.9759
        ],
        [
          0.1715,
          0.9759
        ],
        [
          0.1815,
          0.9754
        ],
        [
          0.1915,
          0.975
        ],
        [
          0.2015,
          0.9745
        ],
        [
          0.2116,
          0.9741
        ],
        [
          0.222,
          0.9736
        ],
        [
          0.232,
          0.9732
        ],
        [
          0.242,
          0.9732
        ],
        [
          0.252,
          0.9736
        ],
        [
          0.2621,
          0.9736
        ],
        [
          0.2725,
          0.9722
        ],
        [
          0.2825,
          0.9722
        ],
        [
          0.2925,
          0.9722
        ],
        [
          0.3025,
          0.9718
        ],
        [
          0.3126,
          0.9718
        ],
        [
          0.323,
          0.9709
        ],
        [
          0.333,
          0.9704
        ],
        [
          0.343,
          0.97
        ],
        [
          0.353,
          0.9691
        ],
        [
          0.3631,
          0.9686
        ],
        [
          0.3735,
          0.9677
        ],
        [
          0.3835,
          0.9663
        ],
        [
          0.3935,
          0.9631
        ],
        [
          0.4035,
          0.9613
        ],
        [
          0.4136,
          0.9577
        ],
        [
          0.424,
          0.9563
        ],
        [
          0.434,
          0.9527
        ],
        [
          0.444,
          0.9518
        ],
        [
          0.454,
          0.9486
        ],
        [
          0.4641,
          0.9463
        ],
        [
          0.4745,
          0.9445
        ],
        [
          0.4845,
          0.9427
        ],
        [
          0.4945,
          0.9395
        ],
        [
          0.5045,
          0.9372
        ],
        [
          0.5146,
          0.9336
        ],
        [
          0.525,
          0.9304
        ],
        [
          0.535,
          0.9263
        ],
        [
          0.545,
          0.924
        ],
        [
          0.5551,
          0.9208
        ],
        [
          0.5655,
          0.9149
        ],
        [
          0.5755,
          0.9117
        ],
        [
          0.5855,
          0.9076
        ],
        [
          0.5955,
          0.9054
        ],
        [
          0.6056,
          0.9013
        ],
        [
          0.616,
          0.8963
        ],
        [
          0.626,
          0.8913
        ],
        [
          0.636,
          0.8863
        ],
        [
          0.646,
          0.8822
        ],
        [
          0.6561,
          0.8753
        ],
        [
          0.6665,
          0.8712
        ],
        [
          0.6765,
          0.8672
        ],
        [
          0.6865,
          0.8608
        ],
        [
          0.6965,
          0.8549
        ],
        [
          0.7066,
          0.849
        ],
        [
          0.717,
          0.8421
        ],
        [
          0.727,
          0.8367
        ],
        [
          0.737,
          0.8312
        ],
        [
          0.747,
          0.8258
        ],
        [
          0.7571,
          0.8203
        ],
        [
          0.7675,
          0.8135
        ],
        [
          0.7775,
          0.8085
        ],
        [
          0.7875,
          0.8021
        ],
        [
          0.7975,
          0.7966
        ],
        [
          0.8076,
          0.7921
        ],
        [
          0.818,
          0.7866
        ],
        [
          0.828,
          0.7816
        ],
        [
          0.838,
          0.7766
        ],
        [
          0.848,
          0.7712
        ],
        [
          0.8581,
          0.7657
        ],
        [
          0.8685,
          0.7602
        ],
        [
          0.8785,
          0.7539
        ],
        [
          0.8885,
          0.7466
        ],
        [
          0.8985,
          0.7411
        ],
        [
          0.9086,
          0.7343
        ],
        [
          0.919,
          0.7284
        ],
        [
          0.929,
          0.7216
        ],
        [
          0.939,
          0.7152
        ],
        [
          0.949,
          0.7093
        ],
        [
          0.9591,
          0.7025
        ],
        [
          0.9695,
          0.6947
        ],
        [
          0.9795,
          0.6874
        ],
        [
          0.9895,
          0.6802
        ],
        [
          0.9995,
          0.6711
        ],
        [
          1.0,
          0.6706
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1948377649288679,
        "difference": -0.0011248727167832231,
        "worse": 0.45,
        "kept_right": 0.19363404357155783,
        "kept_wrong": 0.0012037213573100132,
        "handed_off": 0.805162235071132
      },
      "usd_per_1000": 4.338109413386668
    },
    {
      "decision": 217,
      "frontier": 191,
      "frontier_accuracy": 0.9740673339399454,
      "in_sample": {
        "threshold": 0.9864659363390037,
        "answered": 0.181073703366697,
        "accuracy": 0.9740673339399454
      },
      "curve": [
        [
          0.0,
          0.9741
        ],
        [
          0.01,
          0.9741
        ],
        [
          0.02,
          0.9741
        ],
        [
          0.03,
          0.9741
        ],
        [
          0.04,
          0.9741
        ],
        [
          0.05,
          0.9741
        ],
        [
          0.0601,
          0.9741
        ],
        [
          0.0705,
          0.9741
        ],
        [
          0.0805,
          0.9741
        ],
        [
          0.0905,
          0.9741
        ],
        [
          0.1005,
          0.9741
        ],
        [
          0.1106,
          0.9741
        ],
        [
          0.121,
          0.9741
        ],
        [
          0.131,
          0.9741
        ],
        [
          0.141,
          0.9741
        ],
        [
          0.151,
          0.9741
        ],
        [
          0.1611,
          0.9741
        ],
        [
          0.1715,
          0.9741
        ],
        [
          0.1815,
          0.9736
        ],
        [
          0.1915,
          0.9732
        ],
        [
          0.2015,
          0.9727
        ],
        [
          0.2116,
          0.9722
        ],
        [
          0.222,
          0.9718
        ],
        [
          0.232,
          0.9713
        ],
        [
          0.242,
          0.9713
        ],
        [
          0.252,
          0.9713
        ],
        [
          0.2621,
          0.9713
        ],
        [
          0.2725,
          0.97
        ],
        [
          0.2825,
          0.97
        ],
        [
          0.2925,
          0.97
        ],
        [
          0.3025,
          0.9695
        ],
        [
          0.3126,
          0.9695
        ],
        [
          0.323,
          0.9686
        ],
        [
          0.333,
          0.9682
        ],
        [
          0.343,
          0.9677
        ],
        [
          0.353,
          0.9668
        ],
        [
          0.3631,
          0.9659
        ],
        [
          0.3735,
          0.965
        ],
        [
          0.3835,
          0.9631
        ],
        [
          0.3935,
          0.96
        ],
        [
          0.4035,
          0.9581
        ],
        [
          0.4136,
          0.955
        ],
        [
          0.424,
          0.9536
        ],
        [
          0.434,
          0.9504
        ],
        [
          0.444,
          0.949
        ],
        [
          0.454,
          0.9459
        ],
        [
          0.4641,
          0.9436
        ],
        [
          0.4745,
          0.9418
        ],
        [
          0.4845,
          0.9399
        ],
        [
          0.4945,
          0.9368
        ],
        [
          0.5045,
          0.9345
        ],
        [
          0.5146,
          0.9308
        ],
        [
          0.525,
          0.9281
        ],
        [
          0.535,
          0.924
        ],
        [
          0.545,
          0.9222
        ],
        [
          0.5551,
          0.919
        ],
        [
          0.5655,
          0.9131
        ],
        [
          0.5755,
          0.9104
        ],
        [
          0.5855,
          0.9063
        ],
        [
          0.5955,
          0.904
        ],
        [
          0.6056,
          0.8999
        ],
        [
          0.616,
          0.8949
        ],
        [
          0.626,
          0.8899
        ],
        [
          0.636,
          0.8849
        ],
        [
          0.646,
          0.8808
        ],
        [
          0.6561,
          0.8744
        ],
        [
          0.6665,
          0.8703
        ],
        [
          0.6765,
          0.8662
        ],
        [
          0.6865,
          0.8594
        ],
        [
          0.6965,
          0.8535
        ],
        [
          0.7066,
          0.848
        ],
        [
          0.717,
          0.8412
        ],
        [
          0.727,
          0.8353
        ],
        [
          0.737,
          0.8298
        ],
        [
          0.747,
          0.8244
        ],
        [
          0.7571,
          0.8185
        ],
        [
          0.7675,
          0.8116
        ],
        [
          0.7775,
          0.8066
        ],
        [
          0.7875,
          0.8003
        ],
        [
          0.7975,
          0.7953
        ],
        [
          0.8076,
          0.7907
        ],
        [
          0.818,
          0.7848
        ],
        [
          0.828,
          0.7798
        ],
        [
          0.838,
          0.7748
        ],
        [
          0.848,
          0.7693
        ],
        [
          0.8581,
          0.7643
        ],
        [
          0.8685,
          0.7589
        ],
        [
          0.8785,
          0.753
        ],
        [
          0.8885,
          0.7461
        ],
        [
          0.8985,
          0.7398
        ],
        [
          0.9086,
          0.7329
        ],
        [
          0.919,
          0.7279
        ],
        [
          0.929,
          0.7211
        ],
        [
          0.939,
          0.7147
        ],
        [
          0.949,
          0.7088
        ],
        [
          0.9591,
          0.7015
        ],
        [
          0.9695,
          0.6938
        ],
        [
          0.9795,
          0.6865
        ],
        [
          0.9895,
          0.6797
        ],
        [
          0.9995,
          0.6711
        ],
        [
          1.0,
          0.6706
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.19072146552357155,
        "difference": -0.0009216076988443476,
        "worse": 0.43333333333333335,
        "kept_right": 0.18977861809502689,
        "kept_wrong": 0.0009428474285446228,
        "handed_off": 0.8092785344764283
      },
      "usd_per_1000": 21.64600825462099
    },
    {
      "decision": 217,
      "frontier": 188,
      "frontier_accuracy": 0.9713375796178344,
      "in_sample": {
        "threshold": 0.9864659363390037,
        "answered": 0.181073703366697,
        "accuracy": 0.9713375796178344
      },
      "curve": [
        [
          0.0,
          0.9713
        ],
        [
          0.01,
          0.9713
        ],
        [
          0.02,
          0.9713
        ],
        [
          0.03,
          0.9713
        ],
        [
          0.04,
          0.9713
        ],
        [
          0.05,
          0.9713
        ],
        [
          0.0601,
          0.9713
        ],
        [
          0.0705,
          0.9713
        ],
        [
          0.0805,
          0.9713
        ],
        [
          0.0905,
          0.9713
        ],
        [
          0.1005,
          0.9713
        ],
        [
          0.1106,
          0.9713
        ],
        [
          0.121,
          0.9713
        ],
        [
          0.131,
          0.9713
        ],
        [
          0.141,
          0.9713
        ],
        [
          0.151,
          0.9713
        ],
        [
          0.1611,
          0.9713
        ],
        [
          0.1715,
          0.9713
        ],
        [
          0.1815,
          0.9709
        ],
        [
          0.1915,
          0.9704
        ],
        [
          0.2015,
          0.97
        ],
        [
          0.2116,
          0.9695
        ],
        [
          0.222,
          0.9691
        ],
        [
          0.232,
          0.9686
        ],
        [
          0.242,
          0.9686
        ],
        [
          0.252,
          0.9686
        ],
        [
          0.2621,
          0.9686
        ],
        [
          0.2725,
          0.9672
        ],
        [
          0.2825,
          0.9672
        ],
        [
          0.2925,
          0.9672
        ],
        [
          0.3025,
          0.9668
        ],
        [
          0.3126,
          0.9668
        ],
        [
          0.323,
          0.9663
        ],
        [
          0.333,
          0.9659
        ],
        [
          0.343,
          0.9654
        ],
        [
          0.353,
          0.9645
        ],
        [
          0.3631,
          0.9641
        ],
        [
          0.3735,
          0.9631
        ],
        [
          0.3835,
          0.9609
        ],
        [
          0.3935,
          0.9577
        ],
        [
          0.4035,
          0.9559
        ],
        [
          0.4136,
          0.9527
        ],
        [
          0.424,
          0.9513
        ],
        [
          0.434,
          0.9481
        ],
        [
          0.444,
          0.9468
        ],
        [
          0.454,
          0.944
        ],
        [
          0.4641,
          0.9418
        ],
        [
          0.4745,
          0.9404
        ],
        [
          0.4845,
          0.9381
        ],
        [
          0.4945,
          0.9349
        ],
        [
          0.5045,
          0.9322
        ],
        [
          0.5146,
          0.9286
        ],
        [
          0.525,
          0.9258
        ],
        [
          0.535,
          0.9217
        ],
        [
          0.545,
          0.9195
        ],
        [
          0.5551,
          0.9163
        ],
        [
          0.5655,
          0.9108
        ],
        [
          0.5755,
          0.9086
        ],
        [
          0.5855,
          0.9045
        ],
        [
          0.5955,
          0.9017
        ],
        [
          0.6056,
          0.8976
        ],
        [
          0.616,
          0.8926
        ],
        [
          0.626,
          0.8876
        ],
        [
          0.636,
          0.8826
        ],
        [
          0.646,
          0.879
        ],
        [
          0.6561,
          0.8722
        ],
        [
          0.6665,
          0.8685
        ],
        [
          0.6765,
          0.8644
        ],
        [
          0.6865,
          0.8581
        ],
        [
          0.6965,
          0.8521
        ],
        [
          0.7066,
          0.8462
        ],
        [
          0.717,
          0.8399
        ],
        [
          0.727,
          0.8344
        ],
        [
          0.737,
          0.8289
        ],
        [
          0.747,
          0.8235
        ],
        [
          0.7571,
          0.8176
        ],
        [
          0.7675,
          0.8107
        ],
        [
          0.7775,
          0.8057
        ],
        [
          0.7875,
          0.7994
        ],
        [
          0.7975,
          0.7939
        ],
        [
          0.8076,
          0.7894
        ],
        [
          0.818,
          0.7834
        ],
        [
          0.828,
          0.7784
        ],
        [
          0.838,
          0.7734
        ],
        [
          0.848,
          0.7684
        ],
        [
          0.8581,
          0.763
        ],
        [
          0.8685,
          0.758
        ],
        [
          0.8785,
          0.7516
        ],
        [
          0.8885,
          0.7448
        ],
        [
          0.8985,
          0.7402
        ],
        [
          0.9086,
          0.7343
        ],
        [
          0.919,
          0.7288
        ],
        [
          0.929,
          0.7225
        ],
        [
          0.939,
          0.7157
        ],
        [
          0.949,
          0.7097
        ],
        [
          0.9591,
          0.7025
        ],
        [
          0.9695,
          0.6947
        ],
        [
          0.9795,
          0.6874
        ],
        [
          0.9895,
          0.6802
        ],
        [
          0.9995,
          0.6711
        ],
        [
          1.0,
          0.6706
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.19181702706069947,
        "difference": -0.0009914041436353562,
        "worse": 0.43333333333333335,
        "kept_right": 0.19078921234072937,
        "kept_wrong": 0.0010278147199700512,
        "handed_off": 0.8081829729393003
      },
      "usd_per_1000": 0.7864004171758008
    },
    {
      "decision": 217,
      "frontier": 190,
      "frontier_accuracy": 0.9672429481346679,
      "in_sample": {
        "threshold": 0.9864659363390037,
        "answered": 0.181073703366697,
        "accuracy": 0.9672429481346679
      },
      "curve": [
        [
          0.0,
          0.9672
        ],
        [
          0.01,
          0.9672
        ],
        [
          0.02,
          0.9672
        ],
        [
          0.03,
          0.9672
        ],
        [
          0.04,
          0.9672
        ],
        [
          0.05,
          0.9672
        ],
        [
          0.0601,
          0.9672
        ],
        [
          0.0705,
          0.9672
        ],
        [
          0.0805,
          0.9672
        ],
        [
          0.0905,
          0.9672
        ],
        [
          0.1005,
          0.9672
        ],
        [
          0.1106,
          0.9672
        ],
        [
          0.121,
          0.9672
        ],
        [
          0.131,
          0.9672
        ],
        [
          0.141,
          0.9672
        ],
        [
          0.151,
          0.9672
        ],
        [
          0.1611,
          0.9672
        ],
        [
          0.1715,
          0.9672
        ],
        [
          0.1815,
          0.9668
        ],
        [
          0.1915,
          0.9663
        ],
        [
          0.2015,
          0.9659
        ],
        [
          0.2116,
          0.9654
        ],
        [
          0.222,
          0.965
        ],
        [
          0.232,
          0.9645
        ],
        [
          0.242,
          0.9645
        ],
        [
          0.252,
          0.965
        ],
        [
          0.2621,
          0.965
        ],
        [
          0.2725,
          0.9636
        ],
        [
          0.2825,
          0.9636
        ],
        [
          0.2925,
          0.9636
        ],
        [
          0.3025,
          0.9636
        ],
        [
          0.3126,
          0.9631
        ],
        [
          0.323,
          0.9622
        ],
        [
          0.333,
          0.9622
        ],
        [
          0.343,
          0.9622
        ],
        [
          0.353,
          0.9618
        ],
        [
          0.3631,
          0.9609
        ],
        [
          0.3735,
          0.96
        ],
        [
          0.3835,
          0.9577
        ],
        [
          0.3935,
          0.9545
        ],
        [
          0.4035,
          0.9527
        ],
        [
          0.4136,
          0.9495
        ],
        [
          0.424,
          0.9481
        ],
        [
          0.434,
          0.9445
        ],
        [
          0.444,
          0.9436
        ],
        [
          0.454,
          0.9404
        ],
        [
          0.4641,
          0.9381
        ],
        [
          0.4745,
          0.9368
        ],
        [
          0.4845,
          0.9354
        ],
        [
          0.4945,
          0.9322
        ],
        [
          0.5045,
          0.9299
        ],
        [
          0.5146,
          0.9263
        ],
        [
          0.525,
          0.9231
        ],
        [
          0.535,
          0.919
        ],
        [
          0.545,
          0.9167
        ],
        [
          0.5551,
          0.9136
        ],
        [
          0.5655,
          0.9076
        ],
        [
          0.5755,
          0.9049
        ],
        [
          0.5855,
          0.9008
        ],
        [
          0.5955,
          0.8985
        ],
        [
          0.6056,
          0.8944
        ],
        [
          0.616,
          0.8894
        ],
        [
          0.626,
          0.8844
        ],
        [
          0.636,
          0.8799
        ],
        [
          0.646,
          0.8763
        ],
        [
          0.6561,
          0.8699
        ],
        [
          0.6665,
          0.8658
        ],
        [
          0.6765,
          0.8617
        ],
        [
          0.6865,
          0.8549
        ],
        [
          0.6965,
          0.849
        ],
        [
          0.7066,
          0.843
        ],
        [
          0.717,
          0.8367
        ],
        [
          0.727,
          0.8312
        ],
        [
          0.737,
          0.8258
        ],
        [
          0.747,
          0.8203
        ],
        [
          0.7571,
          0.8144
        ],
        [
          0.7675,
          0.8085
        ],
        [
          0.7775,
          0.8039
        ],
        [
          0.7875,
          0.7975
        ],
        [
          0.7975,
          0.7921
        ],
        [
          0.8076,
          0.7875
        ],
        [
          0.818,
          0.7821
        ],
        [
          0.828,
          0.7775
        ],
        [
          0.838,
          0.7725
        ],
        [
          0.848,
          0.7671
        ],
        [
          0.8581,
          0.7621
        ],
        [
          0.8685,
          0.7566
        ],
        [
          0.8785,
          0.7507
        ],
        [
          0.8885,
          0.7439
        ],
        [
          0.8985,
          0.7384
        ],
        [
          0.9086,
          0.732
        ],
        [
          0.919,
          0.727
        ],
        [
          0.929,
          0.7202
        ],
        [
          0.939,
          0.7134
        ],
        [
          0.949,
          0.7075
        ],
        [
          0.9591,
          0.7006
        ],
        [
          0.9695,
          0.6934
        ],
        [
          0.9795,
          0.6865
        ],
        [
          0.9895,
          0.6797
        ],
        [
          0.9995,
          0.6711
        ],
        [
          1.0,
          0.6706
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.19617701436418256,
        "difference": -0.0012096798915860192,
        "worse": 0.4533333333333333,
        "kept_right": 0.1948460118571895,
        "kept_wrong": 0.0013310025069929912,
        "handed_off": 0.8038229856358176
      },
      "usd_per_1000": 4.483868331819147
    },
    {
      "decision": 217,
      "frontier": 124,
      "frontier_accuracy": 0.9667879890809827,
      "in_sample": {
        "threshold": 0.9864659363390037,
        "answered": 0.181073703366697,
        "accuracy": 0.9667879890809827
      },
      "curve": [
        [
          0.0,
          0.9668
        ],
        [
          0.01,
          0.9668
        ],
        [
          0.02,
          0.9668
        ],
        [
          0.03,
          0.9668
        ],
        [
          0.04,
          0.9668
        ],
        [
          0.05,
          0.9668
        ],
        [
          0.0601,
          0.9668
        ],
        [
          0.0705,
          0.9668
        ],
        [
          0.0805,
          0.9668
        ],
        [
          0.0905,
          0.9668
        ],
        [
          0.1005,
          0.9668
        ],
        [
          0.1106,
          0.9668
        ],
        [
          0.121,
          0.9668
        ],
        [
          0.131,
          0.9668
        ],
        [
          0.141,
          0.9668
        ],
        [
          0.151,
          0.9668
        ],
        [
          0.1611,
          0.9668
        ],
        [
          0.1715,
          0.9668
        ],
        [
          0.1815,
          0.9663
        ],
        [
          0.1915,
          0.9659
        ],
        [
          0.2015,
          0.9654
        ],
        [
          0.2116,
          0.965
        ],
        [
          0.222,
          0.9645
        ],
        [
          0.232,
          0.9641
        ],
        [
          0.242,
          0.9641
        ],
        [
          0.252,
          0.9645
        ],
        [
          0.2621,
          0.9645
        ],
        [
          0.2725,
          0.9631
        ],
        [
          0.2825,
          0.9631
        ],
        [
          0.2925,
          0.9631
        ],
        [
          0.3025,
          0.9627
        ],
        [
          0.3126,
          0.9627
        ],
        [
          0.323,
          0.9618
        ],
        [
          0.333,
          0.9613
        ],
        [
          0.343,
          0.9609
        ],
        [
          0.353,
          0.9604
        ],
        [
          0.3631,
          0.96
        ],
        [
          0.3735,
          0.9591
        ],
        [
          0.3835,
          0.9572
        ],
        [
          0.3935,
          0.954
        ],
        [
          0.4035,
          0.9522
        ],
        [
          0.4136,
          0.949
        ],
        [
          0.424,
          0.9477
        ],
        [
          0.434,
          0.944
        ],
        [
          0.444,
          0.9427
        ],
        [
          0.454,
          0.9395
        ],
        [
          0.4641,
          0.9372
        ],
        [
          0.4745,
          0.9354
        ],
        [
          0.4845,
          0.9336
        ],
        [
          0.4945,
          0.9304
        ],
        [
          0.5045,
          0.9281
        ],
        [
          0.5146,
          0.9245
        ],
        [
          0.525,
          0.9217
        ],
        [
          0.535,
          0.9177
        ],
        [
          0.545,
          0.9154
        ],
        [
          0.5551,
          0.9122
        ],
        [
          0.5655,
          0.9067
        ],
        [
          0.5755,
          0.9035
        ],
        [
          0.5855,
          0.8995
        ],
        [
          0.5955,
          0.8981
        ],
        [
          0.6056,
          0.894
        ],
        [
          0.616,
          0.889
        ],
        [
          0.626,
          0.884
        ],
        [
          0.636,
          0.8794
        ],
        [
          0.646,
          0.8758
        ],
        [
          0.6561,
          0.8703
        ],
        [
          0.6665,
          0.8662
        ],
        [
          0.6765,
          0.8626
        ],
        [
          0.6865,
          0.8562
        ],
        [
          0.6965,
          0.8503
        ],
        [
          0.7066,
          0.8444
        ],
        [
          0.717,
          0.8376
        ],
        [
          0.727,
          0.8321
        ],
        [
          0.737,
          0.8267
        ],
        [
          0.747,
          0.8212
        ],
        [
          0.7571,
          0.8157
        ],
        [
          0.7675,
          0.8089
        ],
        [
          0.7775,
          0.8039
        ],
        [
          0.7875,
          0.7975
        ],
        [
          0.7975,
          0.7925
        ],
        [
          0.8076,
          0.788
        ],
        [
          0.818,
          0.7825
        ],
        [
          0.828,
          0.7775
        ],
        [
          0.838,
          0.773
        ],
        [
          0.848,
          0.7675
        ],
        [
          0.8581,
          0.7625
        ],
        [
          0.8685,
          0.7571
        ],
        [
          0.8785,
          0.7511
        ],
        [
          0.8885,
          0.7443
        ],
        [
          0.8985,
          0.7389
        ],
        [
          0.9086,
          0.7325
        ],
        [
          0.919,
          0.7275
        ],
        [
          0.929,
          0.722
        ],
        [
          0.939,
          0.7152
        ],
        [
          0.949,
          0.7088
        ],
        [
          0.9591,
          0.7015
        ],
        [
          0.9695,
          0.6938
        ],
        [
          0.9795,
          0.687
        ],
        [
          0.9895,
          0.6802
        ],
        [
          0.9995,
          0.6711
        ],
        [
          1.0,
          0.6706
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1948377649288679,
        "difference": -0.0011248727167832231,
        "worse": 0.45,
        "kept_right": 0.19363404357155783,
        "kept_wrong": 0.0012037213573100132,
        "handed_off": 0.805162235071132
      },
      "usd_per_1000": 0.9178984543986952
    },
    {
      "decision": 217,
      "frontier": 278,
      "frontier_accuracy": 0.9499545040946314,
      "in_sample": {
        "threshold": 0.9864659363390037,
        "answered": 0.181073703366697,
        "accuracy": 0.9499545040946314
      },
      "curve": [
        [
          0.0,
          0.95
        ],
        [
          0.01,
          0.95
        ],
        [
          0.02,
          0.95
        ],
        [
          0.03,
          0.95
        ],
        [
          0.04,
          0.95
        ],
        [
          0.05,
          0.95
        ],
        [
          0.0601,
          0.95
        ],
        [
          0.0705,
          0.95
        ],
        [
          0.0805,
          0.95
        ],
        [
          0.0905,
          0.95
        ],
        [
          0.1005,
          0.95
        ],
        [
          0.1106,
          0.95
        ],
        [
          0.121,
          0.95
        ],
        [
          0.131,
          0.95
        ],
        [
          0.141,
          0.95
        ],
        [
          0.151,
          0.95
        ],
        [
          0.1611,
          0.95
        ],
        [
          0.1715,
          0.95
        ],
        [
          0.1815,
          0.9495
        ],
        [
          0.1915,
          0.949
        ],
        [
          0.2015,
          0.9486
        ],
        [
          0.2116,
          0.9481
        ],
        [
          0.222,
          0.9477
        ],
        [
          0.232,
          0.9477
        ],
        [
          0.242,
          0.9477
        ],
        [
          0.252,
          0.9477
        ],
        [
          0.2621,
          0.9477
        ],
        [
          0.2725,
          0.9463
        ],
        [
          0.2825,
          0.9463
        ],
        [
          0.2925,
          0.9463
        ],
        [
          0.3025,
          0.9463
        ],
        [
          0.3126,
          0.9463
        ],
        [
          0.323,
          0.9454
        ],
        [
          0.333,
          0.9449
        ],
        [
          0.343,
          0.9445
        ],
        [
          0.353,
          0.9436
        ],
        [
          0.3631,
          0.9431
        ],
        [
          0.3735,
          0.9422
        ],
        [
          0.3835,
          0.9404
        ],
        [
          0.3935,
          0.9377
        ],
        [
          0.4035,
          0.9363
        ],
        [
          0.4136,
          0.9331
        ],
        [
          0.424,
          0.9322
        ],
        [
          0.434,
          0.9286
        ],
        [
          0.444,
          0.9277
        ],
        [
          0.454,
          0.9249
        ],
        [
          0.4641,
          0.9227
        ],
        [
          0.4745,
          0.9217
        ],
        [
          0.4845,
          0.9199
        ],
        [
          0.4945,
          0.9181
        ],
        [
          0.5045,
          0.9163
        ],
        [
          0.5146,
          0.9126
        ],
        [
          0.525,
          0.9104
        ],
        [
          0.535,
          0.9063
        ],
        [
          0.545,
          0.904
        ],
        [
          0.5551,
          0.9013
        ],
        [
          0.5655,
          0.8958
        ],
        [
          0.5755,
          0.8931
        ],
        [
          0.5855,
          0.889
        ],
        [
          0.5955,
          0.8872
        ],
        [
          0.6056,
          0.8826
        ],
        [
          0.616,
          0.8781
        ],
        [
          0.626,
          0.8731
        ],
        [
          0.636,
          0.8685
        ],
        [
          0.646,
          0.8649
        ],
        [
          0.6561,
          0.8599
        ],
        [
          0.6665,
          0.8558
        ],
        [
          0.6765,
          0.8521
        ],
        [
          0.6865,
          0.8467
        ],
        [
          0.6965,
          0.8408
        ],
        [
          0.7066,
          0.8358
        ],
        [
          0.717,
          0.8298
        ],
        [
          0.727,
          0.8244
        ],
        [
          0.737,
          0.8189
        ],
        [
          0.747,
          0.8139
        ],
        [
          0.7571,
          0.808
        ],
        [
          0.7675,
          0.8021
        ],
        [
          0.7775,
          0.7971
        ],
        [
          0.7875,
          0.7907
        ],
        [
          0.7975,
          0.7857
        ],
        [
          0.8076,
          0.7812
        ],
        [
          0.818,
          0.7762
        ],
        [
          0.828,
          0.7712
        ],
        [
          0.838,
          0.7666
        ],
        [
          0.848,
          0.7616
        ],
        [
          0.8581,
          0.7571
        ],
        [
          0.8685,
          0.752
        ],
        [
          0.8785,
          0.7466
        ],
        [
          0.8885,
          0.7402
        ],
        [
          0.8985,
          0.7352
        ],
        [
          0.9086,
          0.7288
        ],
        [
          0.919,
          0.7238
        ],
        [
          0.929,
          0.7179
        ],
        [
          0.939,
          0.7116
        ],
        [
          0.949,
          0.7061
        ],
        [
          0.9591,
          0.6997
        ],
        [
          0.9695,
          0.692
        ],
        [
          0.9795,
          0.6865
        ],
        [
          0.9895,
          0.6793
        ],
        [
          0.9995,
          0.6711
        ],
        [
          1.0,
          0.6706
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1970916774027131,
        "difference": -0.0012551458174869007,
        "worse": 0.46,
        "kept_right": 0.19572433846667558,
        "kept_wrong": 0.0013673389360374703,
        "handed_off": 0.8029083225972868
      },
      "usd_per_1000": 5.233755553679475
    },
    {
      "decision": 217,
      "frontier": 275,
      "frontier_accuracy": 0.9426751592356688,
      "in_sample": {
        "threshold": 0.9864659363390037,
        "answered": 0.181073703366697,
        "accuracy": 0.9426751592356688
      },
      "curve": [
        [
          0.0,
          0.9427
        ],
        [
          0.01,
          0.9427
        ],
        [
          0.02,
          0.9427
        ],
        [
          0.03,
          0.9427
        ],
        [
          0.04,
          0.9427
        ],
        [
          0.05,
          0.9427
        ],
        [
          0.0601,
          0.9427
        ],
        [
          0.0705,
          0.9427
        ],
        [
          0.0805,
          0.9427
        ],
        [
          0.0905,
          0.9427
        ],
        [
          0.1005,
          0.9427
        ],
        [
          0.1106,
          0.9427
        ],
        [
          0.121,
          0.9427
        ],
        [
          0.131,
          0.9427
        ],
        [
          0.141,
          0.9427
        ],
        [
          0.151,
          0.9427
        ],
        [
          0.1611,
          0.9427
        ],
        [
          0.1715,
          0.9427
        ],
        [
          0.1815,
          0.9422
        ],
        [
          0.1915,
          0.9418
        ],
        [
          0.2015,
          0.9413
        ],
        [
          0.2116,
          0.9409
        ],
        [
          0.222,
          0.9404
        ],
        [
          0.232,
          0.9404
        ],
        [
          0.242,
          0.9404
        ],
        [
          0.252,
          0.9409
        ],
        [
          0.2621,
          0.9409
        ],
        [
          0.2725,
          0.9395
        ],
        [
          0.2825,
          0.9395
        ],
        [
          0.2925,
          0.9395
        ],
        [
          0.3025,
          0.9395
        ],
        [
          0.3126,
          0.9395
        ],
        [
          0.323,
          0.9386
        ],
        [
          0.333,
          0.9381
        ],
        [
          0.343,
          0.9377
        ],
        [
          0.353,
          0.9368
        ],
        [
          0.3631,
          0.9359
        ],
        [
          0.3735,
          0.9354
        ],
        [
          0.3835,
          0.9336
        ],
        [
          0.3935,
          0.9304
        ],
        [
          0.4035,
          0.9299
        ],
        [
          0.4136,
          0.9268
        ],
        [
          0.424,
          0.9258
        ],
        [
          0.434,
          0.9222
        ],
        [
          0.444,
          0.9213
        ],
        [
          0.454,
          0.9186
        ],
        [
          0.4641,
          0.9172
        ],
        [
          0.4745,
          0.9158
        ],
        [
          0.4845,
          0.914
        ],
        [
          0.4945,
          0.9126
        ],
        [
          0.5045,
          0.9108
        ],
        [
          0.5146,
          0.9076
        ],
        [
          0.525,
          0.9049
        ],
        [
          0.535,
          0.9008
        ],
        [
          0.545,
          0.8995
        ],
        [
          0.5551,
          0.8967
        ],
        [
          0.5655,
          0.8908
        ],
        [
          0.5755,
          0.8876
        ],
        [
          0.5855,
          0.884
        ],
        [
          0.5955,
          0.8826
        ],
        [
          0.6056,
          0.879
        ],
        [
          0.616,
          0.874
        ],
        [
          0.626,
          0.8699
        ],
        [
          0.636,
          0.8658
        ],
        [
          0.646,
          0.8621
        ],
        [
          0.6561,
          0.8562
        ],
        [
          0.6665,
          0.8521
        ],
        [
          0.6765,
          0.849
        ],
        [
          0.6865,
          0.8435
        ],
        [
          0.6965,
          0.8376
        ],
        [
          0.7066,
          0.8321
        ],
        [
          0.717,
          0.8262
        ],
        [
          0.727,
          0.8203
        ],
        [
          0.737,
          0.8148
        ],
        [
          0.747,
          0.8098
        ],
        [
          0.7571,
          0.8044
        ],
        [
          0.7675,
          0.7975
        ],
        [
          0.7775,
          0.793
        ],
        [
          0.7875,
          0.7871
        ],
        [
          0.7975,
          0.7825
        ],
        [
          0.8076,
          0.7784
        ],
        [
          0.818,
          0.773
        ],
        [
          0.828,
          0.7698
        ],
        [
          0.838,
          0.7652
        ],
        [
          0.848,
          0.7602
        ],
        [
          0.8581,
          0.7561
        ],
        [
          0.8685,
          0.7507
        ],
        [
          0.8785,
          0.7461
        ],
        [
          0.8885,
          0.7393
        ],
        [
          0.8985,
          0.7348
        ],
        [
          0.9086,
          0.7279
        ],
        [
          0.919,
          0.7229
        ],
        [
          0.929,
          0.717
        ],
        [
          0.939,
          0.7106
        ],
        [
          0.949,
          0.7052
        ],
        [
          0.9591,
          0.6997
        ],
        [
          0.9695,
          0.6924
        ],
        [
          0.9795,
          0.6856
        ],
        [
          0.9895,
          0.6793
        ],
        [
          0.9995,
          0.6711
        ],
        [
          1.0,
          0.6706
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.20093871271569824,
        "difference": -0.0013492477438741867,
        "worse": 0.47333333333333333,
        "kept_right": 0.1993468445772814,
        "kept_wrong": 0.001591868138416831,
        "handed_off": 0.7990612872843018
      },
      "usd_per_1000": 0.4188923247451396
    },
    {
      "decision": 217,
      "frontier": 192,
      "frontier_accuracy": 0.9331210191082803,
      "in_sample": {
        "threshold": 0.9864659363390037,
        "answered": 0.181073703366697,
        "accuracy": 0.9331210191082803
      },
      "curve": [
        [
          0.0,
          0.9331
        ],
        [
          0.01,
          0.9331
        ],
        [
          0.02,
          0.9331
        ],
        [
          0.03,
          0.9331
        ],
        [
          0.04,
          0.9331
        ],
        [
          0.05,
          0.9331
        ],
        [
          0.0601,
          0.9331
        ],
        [
          0.0705,
          0.9331
        ],
        [
          0.0805,
          0.9331
        ],
        [
          0.0905,
          0.9331
        ],
        [
          0.1005,
          0.9331
        ],
        [
          0.1106,
          0.9331
        ],
        [
          0.121,
          0.9331
        ],
        [
          0.131,
          0.9331
        ],
        [
          0.141,
          0.9331
        ],
        [
          0.151,
          0.9331
        ],
        [
          0.1611,
          0.9331
        ],
        [
          0.1715,
          0.9331
        ],
        [
          0.1815,
          0.9327
        ],
        [
          0.1915,
          0.9322
        ],
        [
          0.2015,
          0.9318
        ],
        [
          0.2116,
          0.9313
        ],
        [
          0.222,
          0.9308
        ],
        [
          0.232,
          0.9304
        ],
        [
          0.242,
          0.9304
        ],
        [
          0.252,
          0.9308
        ],
        [
          0.2621,
          0.9308
        ],
        [
          0.2725,
          0.9295
        ],
        [
          0.2825,
          0.9295
        ],
        [
          0.2925,
          0.9295
        ],
        [
          0.3025,
          0.9299
        ],
        [
          0.3126,
          0.9295
        ],
        [
          0.323,
          0.9286
        ],
        [
          0.333,
          0.9281
        ],
        [
          0.343,
          0.9277
        ],
        [
          0.353,
          0.9277
        ],
        [
          0.3631,
          0.9272
        ],
        [
          0.3735,
          0.9263
        ],
        [
          0.3835,
          0.9249
        ],
        [
          0.3935,
          0.9227
        ],
        [
          0.4035,
          0.9213
        ],
        [
          0.4136,
          0.9181
        ],
        [
          0.424,
          0.9172
        ],
        [
          0.434,
          0.914
        ],
        [
          0.444,
          0.914
        ],
        [
          0.454,
          0.9108
        ],
        [
          0.4641,
          0.909
        ],
        [
          0.4745,
          0.9072
        ],
        [
          0.4845,
          0.9049
        ],
        [
          0.4945,
          0.9022
        ],
        [
          0.5045,
          0.9008
        ],
        [
          0.5146,
          0.8967
        ],
        [
          0.525,
          0.8944
        ],
        [
          0.535,
          0.8908
        ],
        [
          0.545,
          0.889
        ],
        [
          0.5551,
          0.8867
        ],
        [
          0.5655,
          0.8808
        ],
        [
          0.5755,
          0.8781
        ],
        [
          0.5855,
          0.874
        ],
        [
          0.5955,
          0.8735
        ],
        [
          0.6056,
          0.8694
        ],
        [
          0.616,
          0.8653
        ],
        [
          0.626,
          0.8603
        ],
        [
          0.636,
          0.8558
        ],
        [
          0.646,
          0.8521
        ],
        [
          0.6561,
          0.8471
        ],
        [
          0.6665,
          0.843
        ],
        [
          0.6765,
          0.8399
        ],
        [
          0.6865,
          0.8344
        ],
        [
          0.6965,
          0.8285
        ],
        [
          0.7066,
          0.8235
        ],
        [
          0.717,
          0.818
        ],
        [
          0.727,
          0.8121
        ],
        [
          0.737,
          0.8066
        ],
        [
          0.747,
          0.8016
        ],
        [
          0.7571,
          0.7957
        ],
        [
          0.7675,
          0.7916
        ],
        [
          0.7775,
          0.7884
        ],
        [
          0.7875,
          0.7825
        ],
        [
          0.7975,
          0.778
        ],
        [
          0.8076,
          0.7739
        ],
        [
          0.818,
          0.7689
        ],
        [
          0.828,
          0.7643
        ],
        [
          0.838,
          0.7607
        ],
        [
          0.848,
          0.7557
        ],
        [
          0.8581,
          0.752
        ],
        [
          0.8685,
          0.748
        ],
        [
          0.8785,
          0.7429
        ],
        [
          0.8885,
          0.7366
        ],
        [
          0.8985,
          0.7316
        ],
        [
          0.9086,
          0.7252
        ],
        [
          0.919,
          0.7202
        ],
        [
          0.929,
          0.7138
        ],
        [
          0.939,
          0.7079
        ],
        [
          0.949,
          0.7038
        ],
        [
          0.9591,
          0.6979
        ],
        [
          0.9695,
          0.6906
        ],
        [
          0.9795,
          0.6838
        ],
        [
          0.9895,
          0.6779
        ],
        [
          0.9995,
          0.6711
        ],
        [
          1.0,
          0.6706
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.19749277209439026,
        "difference": -0.0012672888727269496,
        "worse": 0.45666666666666667,
        "kept_right": 0.19608900075378868,
        "kept_wrong": 0.0014037713406014963,
        "handed_off": 0.8025072279056098
      },
      "usd_per_1000": 9.324524841641301
    },
    {
      "decision": 217,
      "frontier": 123,
      "frontier_accuracy": 0.9326660600545951,
      "in_sample": {
        "threshold": 0.9864659363390037,
        "answered": 0.181073703366697,
        "accuracy": 0.9326660600545951
      },
      "curve": [
        [
          0.0,
          0.9327
        ],
        [
          0.01,
          0.9327
        ],
        [
          0.02,
          0.9327
        ],
        [
          0.03,
          0.9327
        ],
        [
          0.04,
          0.9327
        ],
        [
          0.05,
          0.9327
        ],
        [
          0.0601,
          0.9327
        ],
        [
          0.0705,
          0.9327
        ],
        [
          0.0805,
          0.9327
        ],
        [
          0.0905,
          0.9327
        ],
        [
          0.1005,
          0.9327
        ],
        [
          0.1106,
          0.9327
        ],
        [
          0.121,
          0.9327
        ],
        [
          0.131,
          0.9327
        ],
        [
          0.141,
          0.9327
        ],
        [
          0.151,
          0.9327
        ],
        [
          0.1611,
          0.9327
        ],
        [
          0.1715,
          0.9327
        ],
        [
          0.1815,
          0.9322
        ],
        [
          0.1915,
          0.9318
        ],
        [
          0.2015,
          0.9313
        ],
        [
          0.2116,
          0.9308
        ],
        [
          0.222,
          0.9304
        ],
        [
          0.232,
          0.9299
        ],
        [
          0.242,
          0.9299
        ],
        [
          0.252,
          0.9299
        ],
        [
          0.2621,
          0.9299
        ],
        [
          0.2725,
          0.9286
        ],
        [
          0.2825,
          0.9286
        ],
        [
          0.2925,
          0.9286
        ],
        [
          0.3025,
          0.9286
        ],
        [
          0.3126,
          0.9286
        ],
        [
          0.323,
          0.9286
        ],
        [
          0.333,
          0.9281
        ],
        [
          0.343,
          0.9277
        ],
        [
          0.353,
          0.9268
        ],
        [
          0.3631,
          0.9268
        ],
        [
          0.3735,
          0.9272
        ],
        [
          0.3835,
          0.9254
        ],
        [
          0.3935,
          0.9217
        ],
        [
          0.4035,
          0.9199
        ],
        [
          0.4136,
          0.9167
        ],
        [
          0.424,
          0.9154
        ],
        [
          0.434,
          0.9122
        ],
        [
          0.444,
          0.9113
        ],
        [
          0.454,
          0.9095
        ],
        [
          0.4641,
          0.9072
        ],
        [
          0.4745,
          0.9049
        ],
        [
          0.4845,
          0.9035
        ],
        [
          0.4945,
          0.9004
        ],
        [
          0.5045,
          0.8985
        ],
        [
          0.5146,
          0.8954
        ],
        [
          0.525,
          0.8931
        ],
        [
          0.535,
          0.889
        ],
        [
          0.545,
          0.8872
        ],
        [
          0.5551,
          0.8854
        ],
        [
          0.5655,
          0.8803
        ],
        [
          0.5755,
          0.8781
        ],
        [
          0.5855,
          0.874
        ],
        [
          0.5955,
          0.8722
        ],
        [
          0.6056,
          0.8676
        ],
        [
          0.616,
          0.8626
        ],
        [
          0.626,
          0.859
        ],
        [
          0.636,
          0.8553
        ],
        [
          0.646,
          0.8512
        ],
        [
          0.6561,
          0.8458
        ],
        [
          0.6665,
          0.8435
        ],
        [
          0.6765,
          0.8399
        ],
        [
          0.6865,
          0.8339
        ],
        [
          0.6965,
          0.8285
        ],
        [
          0.7066,
          0.823
        ],
        [
          0.717,
          0.8176
        ],
        [
          0.727,
          0.813
        ],
        [
          0.737,
          0.8085
        ],
        [
          0.747,
          0.8048
        ],
        [
          0.7571,
          0.7989
        ],
        [
          0.7675,
          0.7916
        ],
        [
          0.7775,
          0.7875
        ],
        [
          0.7875,
          0.7812
        ],
        [
          0.7975,
          0.7766
        ],
        [
          0.8076,
          0.7725
        ],
        [
          0.818,
          0.7671
        ],
        [
          0.828,
          0.7634
        ],
        [
          0.838,
          0.7589
        ],
        [
          0.848,
          0.7539
        ],
        [
          0.8581,
          0.7493
        ],
        [
          0.8685,
          0.7457
        ],
        [
          0.8785,
          0.7398
        ],
        [
          0.8885,
          0.7325
        ],
        [
          0.8985,
          0.7298
        ],
        [
          0.9086,
          0.7238
        ],
        [
          0.919,
          0.7184
        ],
        [
          0.929,
          0.7134
        ],
        [
          0.939,
          0.7066
        ],
        [
          0.949,
          0.7015
        ],
        [
          0.9591,
          0.6947
        ],
        [
          0.9695,
          0.6897
        ],
        [
          0.9795,
          0.6843
        ],
        [
          0.9895,
          0.6779
        ],
        [
          0.9995,
          0.6706
        ],
        [
          1.0,
          0.6706
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.19684076598926023,
        "difference": -0.0012248232504085978,
        "worse": 0.4533333333333333,
        "kept_right": 0.19542789815446085,
        "kept_wrong": 0.001412867834799367,
        "handed_off": 0.8031592340107396
      },
      "usd_per_1000": 0.22348042124665846
    },
    {
      "decision": 217,
      "frontier": 274,
      "frontier_accuracy": 0.9085532302092811,
      "in_sample": {
        "threshold": 0.9864659363390037,
        "answered": 0.181073703366697,
        "accuracy": 0.9085532302092811
      },
      "curve": [
        [
          0.0,
          0.9086
        ],
        [
          0.01,
          0.9086
        ],
        [
          0.02,
          0.9086
        ],
        [
          0.03,
          0.9086
        ],
        [
          0.04,
          0.9086
        ],
        [
          0.05,
          0.9086
        ],
        [
          0.0601,
          0.9086
        ],
        [
          0.0705,
          0.9086
        ],
        [
          0.0805,
          0.9086
        ],
        [
          0.0905,
          0.9086
        ],
        [
          0.1005,
          0.9086
        ],
        [
          0.1106,
          0.9086
        ],
        [
          0.121,
          0.9086
        ],
        [
          0.131,
          0.9086
        ],
        [
          0.141,
          0.9086
        ],
        [
          0.151,
          0.9086
        ],
        [
          0.1611,
          0.9086
        ],
        [
          0.1715,
          0.9086
        ],
        [
          0.1815,
          0.9081
        ],
        [
          0.1915,
          0.9081
        ],
        [
          0.2015,
          0.9076
        ],
        [
          0.2116,
          0.9072
        ],
        [
          0.222,
          0.9067
        ],
        [
          0.232,
          0.9067
        ],
        [
          0.242,
          0.9067
        ],
        [
          0.252,
          0.9076
        ],
        [
          0.2621,
          0.9076
        ],
        [
          0.2725,
          0.9072
        ],
        [
          0.2825,
          0.9072
        ],
        [
          0.2925,
          0.9072
        ],
        [
          0.3025,
          0.9072
        ],
        [
          0.3126,
          0.9067
        ],
        [
          0.323,
          0.9058
        ],
        [
          0.333,
          0.9054
        ],
        [
          0.343,
          0.9054
        ],
        [
          0.353,
          0.9054
        ],
        [
          0.3631,
          0.9049
        ],
        [
          0.3735,
          0.9045
        ],
        [
          0.3835,
          0.9026
        ],
        [
          0.3935,
          0.8999
        ],
        [
          0.4035,
          0.899
        ],
        [
          0.4136,
          0.8963
        ],
        [
          0.424,
          0.8954
        ],
        [
          0.434,
          0.8922
        ],
        [
          0.444,
          0.8917
        ],
        [
          0.454,
          0.889
        ],
        [
          0.4641,
          0.8881
        ],
        [
          0.4745,
          0.8867
        ],
        [
          0.4845,
          0.8854
        ],
        [
          0.4945,
          0.8831
        ],
        [
          0.5045,
          0.8817
        ],
        [
          0.5146,
          0.8781
        ],
        [
          0.525,
          0.8758
        ],
        [
          0.535,
          0.8717
        ],
        [
          0.545,
          0.8708
        ],
        [
          0.5551,
          0.8685
        ],
        [
          0.5655,
          0.8626
        ],
        [
          0.5755,
          0.8603
        ],
        [
          0.5855,
          0.8571
        ],
        [
          0.5955,
          0.8567
        ],
        [
          0.6056,
          0.8526
        ],
        [
          0.616,
          0.848
        ],
        [
          0.626,
          0.8449
        ],
        [
          0.636,
          0.8408
        ],
        [
          0.646,
          0.8371
        ],
        [
          0.6561,
          0.8321
        ],
        [
          0.6665,
          0.8289
        ],
        [
          0.6765,
          0.8253
        ],
        [
          0.6865,
          0.8194
        ],
        [
          0.6965,
          0.8135
        ],
        [
          0.7066,
          0.8085
        ],
        [
          0.717,
          0.8025
        ],
        [
          0.727,
          0.7989
        ],
        [
          0.737,
          0.7939
        ],
        [
          0.747,
          0.7894
        ],
        [
          0.7571,
          0.7839
        ],
        [
          0.7675,
          0.7771
        ],
        [
          0.7775,
          0.7725
        ],
        [
          0.7875,
          0.7666
        ],
        [
          0.7975,
          0.7625
        ],
        [
          0.8076,
          0.7589
        ],
        [
          0.818,
          0.7552
        ],
        [
          0.828,
          0.7525
        ],
        [
          0.838,
          0.7493
        ],
        [
          0.848,
          0.7448
        ],
        [
          0.8581,
          0.7411
        ],
        [
          0.8685,
          0.737
        ],
        [
          0.8785,
          0.732
        ],
        [
          0.8885,
          0.7257
        ],
        [
          0.8985,
          0.7216
        ],
        [
          0.9086,
          0.7161
        ],
        [
          0.919,
          0.7125
        ],
        [
          0.929,
          0.7075
        ],
        [
          0.939,
          0.702
        ],
        [
          0.949,
          0.6975
        ],
        [
          0.9591,
          0.6929
        ],
        [
          0.9695,
          0.6884
        ],
        [
          0.9795,
          0.6829
        ],
        [
          0.9895,
          0.6783
        ],
        [
          0.9995,
          0.6711
        ],
        [
          1.0,
          0.6706
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.24048988143977543,
        "difference": -0.002204254675183799,
        "worse": 0.6166666666666667,
        "kept_right": 0.23660904251975315,
        "kept_wrong": 0.0038808389200223987,
        "handed_off": 0.7595101185602245
      },
      "usd_per_1000": 0.3315026551217705
    },
    {
      "decision": 217,
      "frontier": 47,
      "frontier_accuracy": 0.9044585987261147,
      "in_sample": {
        "threshold": 0.9864659363390037,
        "answered": 0.181073703366697,
        "accuracy": 0.9044585987261147
      },
      "curve": [
        [
          0.0,
          0.9045
        ],
        [
          0.01,
          0.9045
        ],
        [
          0.02,
          0.9045
        ],
        [
          0.03,
          0.9045
        ],
        [
          0.04,
          0.9045
        ],
        [
          0.05,
          0.9045
        ],
        [
          0.0601,
          0.9045
        ],
        [
          0.0705,
          0.9045
        ],
        [
          0.0805,
          0.9045
        ],
        [
          0.0905,
          0.9045
        ],
        [
          0.1005,
          0.9045
        ],
        [
          0.1106,
          0.9045
        ],
        [
          0.121,
          0.9045
        ],
        [
          0.131,
          0.9045
        ],
        [
          0.141,
          0.9045
        ],
        [
          0.151,
          0.9045
        ],
        [
          0.1611,
          0.9045
        ],
        [
          0.1715,
          0.9045
        ],
        [
          0.1815,
          0.904
        ],
        [
          0.1915,
          0.9035
        ],
        [
          0.2015,
          0.9031
        ],
        [
          0.2116,
          0.9035
        ],
        [
          0.222,
          0.9031
        ],
        [
          0.232,
          0.9031
        ],
        [
          0.242,
          0.9035
        ],
        [
          0.252,
          0.904
        ],
        [
          0.2621,
          0.904
        ],
        [
          0.2725,
          0.9026
        ],
        [
          0.2825,
          0.9026
        ],
        [
          0.2925,
          0.9031
        ],
        [
          0.3025,
          0.9026
        ],
        [
          0.3126,
          0.9026
        ],
        [
          0.323,
          0.9017
        ],
        [
          0.333,
          0.9013
        ],
        [
          0.343,
          0.9013
        ],
        [
          0.353,
          0.9004
        ],
        [
          0.3631,
          0.8999
        ],
        [
          0.3735,
          0.8999
        ],
        [
          0.3835,
          0.8985
        ],
        [
          0.3935,
          0.8963
        ],
        [
          0.4035,
          0.8949
        ],
        [
          0.4136,
          0.8922
        ],
        [
          0.424,
          0.8908
        ],
        [
          0.434,
          0.8876
        ],
        [
          0.444,
          0.8876
        ],
        [
          0.454,
          0.8849
        ],
        [
          0.4641,
          0.8831
        ],
        [
          0.4745,
          0.8831
        ],
        [
          0.4845,
          0.8813
        ],
        [
          0.4945,
          0.8794
        ],
        [
          0.5045,
          0.8794
        ],
        [
          0.5146,
          0.8763
        ],
        [
          0.525,
          0.8731
        ],
        [
          0.535,
          0.8694
        ],
        [
          0.545,
          0.8676
        ],
        [
          0.5551,
          0.8653
        ],
        [
          0.5655,
          0.8612
        ],
        [
          0.5755,
          0.8585
        ],
        [
          0.5855,
          0.8553
        ],
        [
          0.5955,
          0.854
        ],
        [
          0.6056,
          0.8503
        ],
        [
          0.616,
          0.8462
        ],
        [
          0.626,
          0.8417
        ],
        [
          0.636,
          0.8385
        ],
        [
          0.646,
          0.8353
        ],
        [
          0.6561,
          0.8317
        ],
        [
          0.6665,
          0.8285
        ],
        [
          0.6765,
          0.8244
        ],
        [
          0.6865,
          0.8185
        ],
        [
          0.6965,
          0.8144
        ],
        [
          0.7066,
          0.8089
        ],
        [
          0.717,
          0.8039
        ],
        [
          0.727,
          0.7985
        ],
        [
          0.737,
          0.7939
        ],
        [
          0.747,
          0.7903
        ],
        [
          0.7571,
          0.7848
        ],
        [
          0.7675,
          0.7803
        ],
        [
          0.7775,
          0.7762
        ],
        [
          0.7875,
          0.7698
        ],
        [
          0.7975,
          0.7657
        ],
        [
          0.8076,
          0.7616
        ],
        [
          0.818,
          0.7571
        ],
        [
          0.828,
          0.7539
        ],
        [
          0.838,
          0.7493
        ],
        [
          0.848,
          0.7448
        ],
        [
          0.8581,
          0.7402
        ],
        [
          0.8685,
          0.737
        ],
        [
          0.8785,
          0.7329
        ],
        [
          0.8885,
          0.7279
        ],
        [
          0.8985,
          0.7238
        ],
        [
          0.9086,
          0.7175
        ],
        [
          0.919,
          0.7129
        ],
        [
          0.929,
          0.7088
        ],
        [
          0.939,
          0.7034
        ],
        [
          0.949,
          0.6988
        ],
        [
          0.9591,
          0.6938
        ],
        [
          0.9695,
          0.6874
        ],
        [
          0.9795,
          0.6829
        ],
        [
          0.9895,
          0.677
        ],
        [
          0.9995,
          0.6706
        ],
        [
          1.0,
          0.6706
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.24623292046176265,
        "difference": -0.002273659412467876,
        "worse": 0.6866666666666666,
        "kept_right": 0.24211913390932907,
        "kept_wrong": 0.004113786552433795,
        "handed_off": 0.7537670795382377
      },
      "usd_per_1000": 3.724278138940375
    },
    {
      "decision": 217,
      "frontier": 193,
      "frontier_accuracy": 0.89171974522293,
      "in_sample": {
        "threshold": 0.9732139370119502,
        "answered": 0.22747952684258416,
        "accuracy": 0.89171974522293
      },
      "curve": [
        [
          0.0,
          0.8917
        ],
        [
          0.01,
          0.8917
        ],
        [
          0.02,
          0.8917
        ],
        [
          0.03,
          0.8917
        ],
        [
          0.04,
          0.8917
        ],
        [
          0.05,
          0.8917
        ],
        [
          0.0601,
          0.8917
        ],
        [
          0.0705,
          0.8917
        ],
        [
          0.0805,
          0.8917
        ],
        [
          0.0905,
          0.8917
        ],
        [
          0.1005,
          0.8917
        ],
        [
          0.1106,
          0.8917
        ],
        [
          0.121,
          0.8917
        ],
        [
          0.131,
          0.8917
        ],
        [
          0.141,
          0.8917
        ],
        [
          0.151,
          0.8917
        ],
        [
          0.1611,
          0.8917
        ],
        [
          0.1715,
          0.8917
        ],
        [
          0.1815,
          0.8913
        ],
        [
          0.1915,
          0.8913
        ],
        [
          0.2015,
          0.8908
        ],
        [
          0.2116,
          0.8908
        ],
        [
          0.222,
          0.8913
        ],
        [
          0.232,
          0.8913
        ],
        [
          0.242,
          0.8908
        ],
        [
          0.252,
          0.8913
        ],
        [
          0.2621,
          0.8913
        ],
        [
          0.2725,
          0.8904
        ],
        [
          0.2825,
          0.8904
        ],
        [
          0.2925,
          0.8904
        ],
        [
          0.3025,
          0.8904
        ],
        [
          0.3126,
          0.8904
        ],
        [
          0.323,
          0.8899
        ],
        [
          0.333,
          0.8894
        ],
        [
          0.343,
          0.8894
        ],
        [
          0.353,
          0.8894
        ],
        [
          0.3631,
          0.8894
        ],
        [
          0.3735,
          0.8885
        ],
        [
          0.3835,
          0.8881
        ],
        [
          0.3935,
          0.8849
        ],
        [
          0.4035,
          0.884
        ],
        [
          0.4136,
          0.8803
        ],
        [
          0.424,
          0.8799
        ],
        [
          0.434,
          0.8767
        ],
        [
          0.444,
          0.8763
        ],
        [
          0.454,
          0.8735
        ],
        [
          0.4641,
          0.8731
        ],
        [
          0.4745,
          0.8712
        ],
        [
          0.4845,
          0.8712
        ],
        [
          0.4945,
          0.8699
        ],
        [
          0.5045,
          0.8685
        ],
        [
          0.5146,
          0.8658
        ],
        [
          0.525,
          0.864
        ],
        [
          0.535,
          0.8608
        ],
        [
          0.545,
          0.8603
        ],
        [
          0.5551,
          0.8585
        ],
        [
          0.5655,
          0.8535
        ],
        [
          0.5755,
          0.8512
        ],
        [
          0.5855,
          0.8485
        ],
        [
          0.5955,
          0.8485
        ],
        [
          0.6056,
          0.8453
        ],
        [
          0.616,
          0.8417
        ],
        [
          0.626,
          0.838
        ],
        [
          0.636,
          0.8348
        ],
        [
          0.646,
          0.8312
        ],
        [
          0.6561,
          0.8267
        ],
        [
          0.6665,
          0.823
        ],
        [
          0.6765,
          0.8198
        ],
        [
          0.6865,
          0.8157
        ],
        [
          0.6965,
          0.8107
        ],
        [
          0.7066,
          0.8062
        ],
        [
          0.717,
          0.8007
        ],
        [
          0.727,
          0.7953
        ],
        [
          0.737,
          0.7907
        ],
        [
          0.747,
          0.7871
        ],
        [
          0.7571,
          0.7821
        ],
        [
          0.7675,
          0.7771
        ],
        [
          0.7775,
          0.7748
        ],
        [
          0.7875,
          0.7698
        ],
        [
          0.7975,
          0.7657
        ],
        [
          0.8076,
          0.7621
        ],
        [
          0.818,
          0.7575
        ],
        [
          0.828,
          0.7534
        ],
        [
          0.838,
          0.7498
        ],
        [
          0.848,
          0.7448
        ],
        [
          0.8581,
          0.742
        ],
        [
          0.8685,
          0.7375
        ],
        [
          0.8785,
          0.732
        ],
        [
          0.8885,
          0.7257
        ],
        [
          0.8985,
          0.7216
        ],
        [
          0.9086,
          0.7166
        ],
        [
          0.919,
          0.712
        ],
        [
          0.929,
          0.7075
        ],
        [
          0.939,
          0.7015
        ],
        [
          0.949,
          0.6984
        ],
        [
          0.9591,
          0.6924
        ],
        [
          0.9695,
          0.6865
        ],
        [
          0.9795,
          0.682
        ],
        [
          0.9895,
          0.6765
        ],
        [
          0.9995,
          0.6706
        ],
        [
          1.0,
          0.6706
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.2714037961112533,
        "difference": -0.0023100368201691385,
        "worse": 0.7133333333333334,
        "kept_right": 0.2655650290221995,
        "kept_wrong": 0.005838767089053649,
        "handed_off": 0.7285962038887471
      },
      "usd_per_1000": 5.652629406429306
    },
    {
      "decision": 217,
      "frontier": 55,
      "frontier_accuracy": 0.8503184713375797,
      "in_sample": {
        "threshold": 0.7551592075018005,
        "answered": 0.49545040946314833,
        "accuracy": 0.8503184713375797
      },
      "curve": [
        [
          0.0,
          0.8503
        ],
        [
          0.01,
          0.8503
        ],
        [
          0.02,
          0.8503
        ],
        [
          0.03,
          0.8508
        ],
        [
          0.04,
          0.8508
        ],
        [
          0.05,
          0.8508
        ],
        [
          0.0601,
          0.8508
        ],
        [
          0.0705,
          0.8517
        ],
        [
          0.0805,
          0.8517
        ],
        [
          0.0905,
          0.8517
        ],
        [
          0.1005,
          0.8517
        ],
        [
          0.1106,
          0.8517
        ],
        [
          0.121,
          0.8517
        ],
        [
          0.131,
          0.8517
        ],
        [
          0.141,
          0.8526
        ],
        [
          0.151,
          0.853
        ],
        [
          0.1611,
          0.853
        ],
        [
          0.1715,
          0.8535
        ],
        [
          0.1815,
          0.854
        ],
        [
          0.1915,
          0.8544
        ],
        [
          0.2015,
          0.8553
        ],
        [
          0.2116,
          0.8558
        ],
        [
          0.222,
          0.8567
        ],
        [
          0.232,
          0.8562
        ],
        [
          0.242,
          0.8562
        ],
        [
          0.252,
          0.8558
        ],
        [
          0.2621,
          0.8558
        ],
        [
          0.2725,
          0.8562
        ],
        [
          0.2825,
          0.8567
        ],
        [
          0.2925,
          0.8571
        ],
        [
          0.3025,
          0.8571
        ],
        [
          0.3126,
          0.8581
        ],
        [
          0.323,
          0.8581
        ],
        [
          0.333,
          0.8585
        ],
        [
          0.343,
          0.859
        ],
        [
          0.353,
          0.8594
        ],
        [
          0.3631,
          0.8599
        ],
        [
          0.3735,
          0.8603
        ],
        [
          0.3835,
          0.8612
        ],
        [
          0.3935,
          0.8585
        ],
        [
          0.4035,
          0.8585
        ],
        [
          0.4136,
          0.8558
        ],
        [
          0.424,
          0.8567
        ],
        [
          0.434,
          0.8544
        ],
        [
          0.444,
          0.8553
        ],
        [
          0.454,
          0.854
        ],
        [
          0.4641,
          0.8535
        ],
        [
          0.4745,
          0.8521
        ],
        [
          0.4845,
          0.8517
        ],
        [
          0.4945,
          0.8503
        ],
        [
          0.5045,
          0.8485
        ],
        [
          0.5146,
          0.8458
        ],
        [
          0.525,
          0.8449
        ],
        [
          0.535,
          0.8426
        ],
        [
          0.545,
          0.8426
        ],
        [
          0.5551,
          0.8408
        ],
        [
          0.5655,
          0.8371
        ],
        [
          0.5755,
          0.8353
        ],
        [
          0.5855,
          0.8326
        ],
        [
          0.5955,
          0.8317
        ],
        [
          0.6056,
          0.8294
        ],
        [
          0.616,
          0.8262
        ],
        [
          0.626,
          0.823
        ],
        [
          0.636,
          0.8212
        ],
        [
          0.646,
          0.818
        ],
        [
          0.6561,
          0.8139
        ],
        [
          0.6665,
          0.8103
        ],
        [
          0.6765,
          0.8085
        ],
        [
          0.6865,
          0.8053
        ],
        [
          0.6965,
          0.8012
        ],
        [
          0.7066,
          0.7966
        ],
        [
          0.717,
          0.7912
        ],
        [
          0.727,
          0.7862
        ],
        [
          0.737,
          0.7816
        ],
        [
          0.747,
          0.7784
        ],
        [
          0.7571,
          0.7734
        ],
        [
          0.7675,
          0.7693
        ],
        [
          0.7775,
          0.7671
        ],
        [
          0.7875,
          0.7621
        ],
        [
          0.7975,
          0.7589
        ],
        [
          0.8076,
          0.7561
        ],
        [
          0.818,
          0.7525
        ],
        [
          0.828,
          0.748
        ],
        [
          0.838,
          0.7439
        ],
        [
          0.848,
          0.7398
        ],
        [
          0.8581,
          0.7352
        ],
        [
          0.8685,
          0.7307
        ],
        [
          0.8785,
          0.727
        ],
        [
          0.8885,
          0.7211
        ],
        [
          0.8985,
          0.7179
        ],
        [
          0.9086,
          0.7125
        ],
        [
          0.919,
          0.7093
        ],
        [
          0.929,
          0.7061
        ],
        [
          0.939,
          0.7011
        ],
        [
          0.949,
          0.6975
        ],
        [
          0.9591,
          0.6924
        ],
        [
          0.9695,
          0.6888
        ],
        [
          0.9795,
          0.6843
        ],
        [
          0.9895,
          0.6779
        ],
        [
          0.9995,
          0.6706
        ],
        [
          1.0,
          0.6706
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.4941834318807408,
        "difference": -0.001490078491470799,
        "worse": 0.4766666666666667,
        "kept_right": 0.44802497883908055,
        "kept_wrong": 0.04615845304166024,
        "handed_off": 0.5058165681192592
      },
      "usd_per_1000": 1.8561872401579744
    },
    {
      "decision": 145,
      "frontier": 140,
      "frontier_accuracy": 0.9808917197452229,
      "in_sample": {
        "threshold": 0.9851364969800199,
        "answered": 0.18289353958143767,
        "accuracy": 0.9808917197452229
      },
      "curve": [
        [
          0.0,
          0.9809
        ],
        [
          0.01,
          0.9809
        ],
        [
          0.02,
          0.9809
        ],
        [
          0.03,
          0.9809
        ],
        [
          0.04,
          0.9809
        ],
        [
          0.05,
          0.9809
        ],
        [
          0.0601,
          0.9809
        ],
        [
          0.0705,
          0.9809
        ],
        [
          0.0805,
          0.9809
        ],
        [
          0.0905,
          0.9809
        ],
        [
          0.1005,
          0.9809
        ],
        [
          0.1106,
          0.9809
        ],
        [
          0.121,
          0.9809
        ],
        [
          0.131,
          0.9809
        ],
        [
          0.141,
          0.9809
        ],
        [
          0.151,
          0.9809
        ],
        [
          0.1611,
          0.9809
        ],
        [
          0.1715,
          0.9809
        ],
        [
          0.1815,
          0.9809
        ],
        [
          0.1915,
          0.9804
        ],
        [
          0.2015,
          0.9795
        ],
        [
          0.2116,
          0.9795
        ],
        [
          0.222,
          0.9791
        ],
        [
          0.232,
          0.9791
        ],
        [
          0.242,
          0.9791
        ],
        [
          0.252,
          0.9791
        ],
        [
          0.2621,
          0.9782
        ],
        [
          0.2725,
          0.9782
        ],
        [
          0.2825,
          0.9782
        ],
        [
          0.2925,
          0.9782
        ],
        [
          0.3025,
          0.9768
        ],
        [
          0.3126,
          0.9773
        ],
        [
          0.323,
          0.9768
        ],
        [
          0.333,
          0.975
        ],
        [
          0.343,
          0.9732
        ],
        [
          0.353,
          0.9732
        ],
        [
          0.3631,
          0.9713
        ],
        [
          0.3735,
          0.9704
        ],
        [
          0.3835,
          0.9686
        ],
        [
          0.3935,
          0.9672
        ],
        [
          0.4035,
          0.9645
        ],
        [
          0.4136,
          0.9636
        ],
        [
          0.424,
          0.9622
        ],
        [
          0.434,
          0.9591
        ],
        [
          0.444,
          0.9568
        ],
        [
          0.454,
          0.9559
        ],
        [
          0.4641,
          0.9518
        ],
        [
          0.4745,
          0.95
        ],
        [
          0.4845,
          0.9481
        ],
        [
          0.4945,
          0.9445
        ],
        [
          0.5045,
          0.9399
        ],
        [
          0.5146,
          0.9381
        ],
        [
          0.525,
          0.9345
        ],
        [
          0.535,
          0.9304
        ],
        [
          0.545,
          0.924
        ],
        [
          0.5551,
          0.9195
        ],
        [
          0.5655,
          0.9167
        ],
        [
          0.5755,
          0.9126
        ],
        [
          0.5855,
          0.9095
        ],
        [
          0.5955,
          0.904
        ],
        [
          0.6056,
          0.899
        ],
        [
          0.616,
          0.8944
        ],
        [
          0.626,
          0.8899
        ],
        [
          0.636,
          0.8858
        ],
        [
          0.646,
          0.8803
        ],
        [
          0.6561,
          0.8758
        ],
        [
          0.6665,
          0.8708
        ],
        [
          0.6765,
          0.8685
        ],
        [
          0.6865,
          0.8626
        ],
        [
          0.6965,
          0.8567
        ],
        [
          0.7066,
          0.853
        ],
        [
          0.717,
          0.849
        ],
        [
          0.727,
          0.8426
        ],
        [
          0.737,
          0.8376
        ],
        [
          0.747,
          0.833
        ],
        [
          0.7571,
          0.8271
        ],
        [
          0.7675,
          0.8207
        ],
        [
          0.7775,
          0.8148
        ],
        [
          0.7875,
          0.8085
        ],
        [
          0.7975,
          0.8016
        ],
        [
          0.8076,
          0.7966
        ],
        [
          0.818,
          0.7903
        ],
        [
          0.828,
          0.7834
        ],
        [
          0.838,
          0.7784
        ],
        [
          0.848,
          0.7712
        ],
        [
          0.8581,
          0.7666
        ],
        [
          0.8685,
          0.7607
        ],
        [
          0.8785,
          0.7539
        ],
        [
          0.8885,
          0.7475
        ],
        [
          0.8985,
          0.7407
        ],
        [
          0.9086,
          0.7343
        ],
        [
          0.919,
          0.727
        ],
        [
          0.929,
          0.7211
        ],
        [
          0.939,
          0.7147
        ],
        [
          0.949,
          0.7084
        ],
        [
          0.9591,
          0.7015
        ],
        [
          0.9695,
          0.6929
        ],
        [
          0.9795,
          0.6856
        ],
        [
          0.9895,
          0.6774
        ],
        [
          0.9995,
          0.6692
        ],
        [
          1.0,
          0.6688
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.19410589913823859,
        "difference": -0.0008823570603721185,
        "worse": 0.5033333333333333,
        "kept_right": 0.19319623071785885,
        "kept_wrong": 0.0009096684203796821,
        "handed_off": 0.8058941008617615
      },
      "usd_per_1000": 1.9573057173764885
    },
    {
      "decision": 145,
      "frontier": 277,
      "frontier_accuracy": 0.9795268425841674,
      "in_sample": {
        "threshold": 0.9851364969800199,
        "answered": 0.18289353958143767,
        "accuracy": 0.9795268425841674
      },
      "curve": [
        [
          0.0,
          0.9795
        ],
        [
          0.01,
          0.9795
        ],
        [
          0.02,
          0.9795
        ],
        [
          0.03,
          0.9795
        ],
        [
          0.04,
          0.9795
        ],
        [
          0.05,
          0.9795
        ],
        [
          0.0601,
          0.9795
        ],
        [
          0.0705,
          0.9795
        ],
        [
          0.0805,
          0.9795
        ],
        [
          0.0905,
          0.9795
        ],
        [
          0.1005,
          0.9795
        ],
        [
          0.1106,
          0.9795
        ],
        [
          0.121,
          0.9795
        ],
        [
          0.131,
          0.9795
        ],
        [
          0.141,
          0.9795
        ],
        [
          0.151,
          0.9795
        ],
        [
          0.1611,
          0.9795
        ],
        [
          0.1715,
          0.9795
        ],
        [
          0.1815,
          0.9795
        ],
        [
          0.1915,
          0.9791
        ],
        [
          0.2015,
          0.9782
        ],
        [
          0.2116,
          0.9782
        ],
        [
          0.222,
          0.9777
        ],
        [
          0.232,
          0.9782
        ],
        [
          0.242,
          0.9782
        ],
        [
          0.252,
          0.9782
        ],
        [
          0.2621,
          0.9773
        ],
        [
          0.2725,
          0.9773
        ],
        [
          0.2825,
          0.9773
        ],
        [
          0.2925,
          0.9773
        ],
        [
          0.3025,
          0.9759
        ],
        [
          0.3126,
          0.9759
        ],
        [
          0.323,
          0.9754
        ],
        [
          0.333,
          0.9736
        ],
        [
          0.343,
          0.9713
        ],
        [
          0.353,
          0.9713
        ],
        [
          0.3631,
          0.9695
        ],
        [
          0.3735,
          0.9686
        ],
        [
          0.3835,
          0.9672
        ],
        [
          0.3935,
          0.9659
        ],
        [
          0.4035,
          0.9631
        ],
        [
          0.4136,
          0.9622
        ],
        [
          0.424,
          0.9613
        ],
        [
          0.434,
          0.9581
        ],
        [
          0.444,
          0.9559
        ],
        [
          0.454,
          0.955
        ],
        [
          0.4641,
          0.9509
        ],
        [
          0.4745,
          0.949
        ],
        [
          0.4845,
          0.9472
        ],
        [
          0.4945,
          0.9436
        ],
        [
          0.5045,
          0.939
        ],
        [
          0.5146,
          0.9377
        ],
        [
          0.525,
          0.934
        ],
        [
          0.535,
          0.9299
        ],
        [
          0.545,
          0.9236
        ],
        [
          0.5551,
          0.919
        ],
        [
          0.5655,
          0.9163
        ],
        [
          0.5755,
          0.9117
        ],
        [
          0.5855,
          0.9086
        ],
        [
          0.5955,
          0.9035
        ],
        [
          0.6056,
          0.899
        ],
        [
          0.616,
          0.894
        ],
        [
          0.626,
          0.8894
        ],
        [
          0.636,
          0.8854
        ],
        [
          0.646,
          0.8799
        ],
        [
          0.6561,
          0.8753
        ],
        [
          0.6665,
          0.8703
        ],
        [
          0.6765,
          0.8681
        ],
        [
          0.6865,
          0.8621
        ],
        [
          0.6965,
          0.8562
        ],
        [
          0.7066,
          0.8521
        ],
        [
          0.717,
          0.848
        ],
        [
          0.727,
          0.8421
        ],
        [
          0.737,
          0.8371
        ],
        [
          0.747,
          0.8326
        ],
        [
          0.7571,
          0.8267
        ],
        [
          0.7675,
          0.8207
        ],
        [
          0.7775,
          0.8148
        ],
        [
          0.7875,
          0.808
        ],
        [
          0.7975,
          0.8012
        ],
        [
          0.8076,
          0.7962
        ],
        [
          0.818,
          0.7898
        ],
        [
          0.828,
          0.783
        ],
        [
          0.838,
          0.778
        ],
        [
          0.848,
          0.7707
        ],
        [
          0.8581,
          0.7657
        ],
        [
          0.8685,
          0.7598
        ],
        [
          0.8785,
          0.753
        ],
        [
          0.8885,
          0.747
        ],
        [
          0.8985,
          0.7402
        ],
        [
          0.9086,
          0.7343
        ],
        [
          0.919,
          0.727
        ],
        [
          0.929,
          0.7216
        ],
        [
          0.939,
          0.7147
        ],
        [
          0.949,
          0.7088
        ],
        [
          0.9591,
          0.702
        ],
        [
          0.9695,
          0.6934
        ],
        [
          0.9795,
          0.6856
        ],
        [
          0.9895,
          0.6774
        ],
        [
          0.9995,
          0.6692
        ],
        [
          1.0,
          0.6688
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.20173701764314966,
        "difference": -0.0011006384334868136,
        "worse": 0.53,
        "kept_right": 0.2005392797273189,
        "kept_wrong": 0.001197737915830763,
        "handed_off": 0.7982629823568503
      },
      "usd_per_1000": 4.849672813671492
    },
    {
      "decision": 145,
      "frontier": 276,
      "frontier_accuracy": 0.9777070063694268,
      "in_sample": {
        "threshold": 0.9851364969800199,
        "answered": 0.18289353958143767,
        "accuracy": 0.9777070063694268
      },
      "curve": [
        [
          0.0,
          0.9777
        ],
        [
          0.01,
          0.9777
        ],
        [
          0.02,
          0.9777
        ],
        [
          0.03,
          0.9777
        ],
        [
          0.04,
          0.9777
        ],
        [
          0.05,
          0.9777
        ],
        [
          0.0601,
          0.9777
        ],
        [
          0.0705,
          0.9777
        ],
        [
          0.0805,
          0.9777
        ],
        [
          0.0905,
          0.9777
        ],
        [
          0.1005,
          0.9777
        ],
        [
          0.1106,
          0.9777
        ],
        [
          0.121,
          0.9777
        ],
        [
          0.131,
          0.9777
        ],
        [
          0.141,
          0.9777
        ],
        [
          0.151,
          0.9777
        ],
        [
          0.1611,
          0.9777
        ],
        [
          0.1715,
          0.9777
        ],
        [
          0.1815,
          0.9777
        ],
        [
          0.1915,
          0.9773
        ],
        [
          0.2015,
          0.9763
        ],
        [
          0.2116,
          0.9763
        ],
        [
          0.222,
          0.9759
        ],
        [
          0.232,
          0.9763
        ],
        [
          0.242,
          0.9768
        ],
        [
          0.252,
          0.9768
        ],
        [
          0.2621,
          0.9759
        ],
        [
          0.2725,
          0.9759
        ],
        [
          0.2825,
          0.9759
        ],
        [
          0.2925,
          0.9759
        ],
        [
          0.3025,
          0.975
        ],
        [
          0.3126,
          0.975
        ],
        [
          0.323,
          0.9745
        ],
        [
          0.333,
          0.9727
        ],
        [
          0.343,
          0.9704
        ],
        [
          0.353,
          0.9704
        ],
        [
          0.3631,
          0.9691
        ],
        [
          0.3735,
          0.9686
        ],
        [
          0.3835,
          0.9672
        ],
        [
          0.3935,
          0.9659
        ],
        [
          0.4035,
          0.9631
        ],
        [
          0.4136,
          0.9622
        ],
        [
          0.424,
          0.9613
        ],
        [
          0.434,
          0.9581
        ],
        [
          0.444,
          0.9559
        ],
        [
          0.454,
          0.955
        ],
        [
          0.4641,
          0.9509
        ],
        [
          0.4745,
          0.949
        ],
        [
          0.4845,
          0.9472
        ],
        [
          0.4945,
          0.9436
        ],
        [
          0.5045,
          0.939
        ],
        [
          0.5146,
          0.9377
        ],
        [
          0.525,
          0.934
        ],
        [
          0.535,
          0.9299
        ],
        [
          0.545,
          0.9236
        ],
        [
          0.5551,
          0.919
        ],
        [
          0.5655,
          0.9163
        ],
        [
          0.5755,
          0.9117
        ],
        [
          0.5855,
          0.9086
        ],
        [
          0.5955,
          0.9031
        ],
        [
          0.6056,
          0.8985
        ],
        [
          0.616,
          0.8935
        ],
        [
          0.626,
          0.8885
        ],
        [
          0.636,
          0.8844
        ],
        [
          0.646,
          0.8794
        ],
        [
          0.6561,
          0.8749
        ],
        [
          0.6665,
          0.8699
        ],
        [
          0.6765,
          0.8676
        ],
        [
          0.6865,
          0.8621
        ],
        [
          0.6965,
          0.8562
        ],
        [
          0.7066,
          0.8521
        ],
        [
          0.717,
          0.848
        ],
        [
          0.727,
          0.8421
        ],
        [
          0.737,
          0.8371
        ],
        [
          0.747,
          0.8321
        ],
        [
          0.7571,
          0.8262
        ],
        [
          0.7675,
          0.8207
        ],
        [
          0.7775,
          0.8148
        ],
        [
          0.7875,
          0.808
        ],
        [
          0.7975,
          0.8012
        ],
        [
          0.8076,
          0.7962
        ],
        [
          0.818,
          0.7898
        ],
        [
          0.828,
          0.783
        ],
        [
          0.838,
          0.778
        ],
        [
          0.848,
          0.7707
        ],
        [
          0.8581,
          0.7662
        ],
        [
          0.8685,
          0.7602
        ],
        [
          0.8785,
          0.7534
        ],
        [
          0.8885,
          0.7475
        ],
        [
          0.8985,
          0.7407
        ],
        [
          0.9086,
          0.7348
        ],
        [
          0.919,
          0.727
        ],
        [
          0.929,
          0.7211
        ],
        [
          0.939,
          0.7143
        ],
        [
          0.949,
          0.7088
        ],
        [
          0.9591,
          0.702
        ],
        [
          0.9695,
          0.6934
        ],
        [
          0.9795,
          0.6856
        ],
        [
          0.9895,
          0.6774
        ],
        [
          0.9995,
          0.6692
        ],
        [
          1.0,
          0.6688
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.2190466207715356,
        "difference": -0.0015008897816964142,
        "worse": 0.6033333333333334,
        "kept_right": 0.21715144355999036,
        "kept_wrong": 0.001895177211545097,
        "handed_off": 0.7809533792284643
      },
      "usd_per_1000": 1.2746196470378308
    },
    {
      "decision": 145,
      "frontier": 48,
      "frontier_accuracy": 0.9758871701546861,
      "in_sample": {
        "threshold": 0.9851364969800199,
        "answered": 0.18289353958143767,
        "accuracy": 0.9758871701546861
      },
      "curve": [
        [
          0.0,
          0.9759
        ],
        [
          0.01,
          0.9759
        ],
        [
          0.02,
          0.9759
        ],
        [
          0.03,
          0.9759
        ],
        [
          0.04,
          0.9759
        ],
        [
          0.05,
          0.9759
        ],
        [
          0.0601,
          0.9759
        ],
        [
          0.0705,
          0.9759
        ],
        [
          0.0805,
          0.9759
        ],
        [
          0.0905,
          0.9759
        ],
        [
          0.1005,
          0.9759
        ],
        [
          0.1106,
          0.9759
        ],
        [
          0.121,
          0.9759
        ],
        [
          0.131,
          0.9759
        ],
        [
          0.141,
          0.9759
        ],
        [
          0.151,
          0.9759
        ],
        [
          0.1611,
          0.9759
        ],
        [
          0.1715,
          0.9759
        ],
        [
          0.1815,
          0.9759
        ],
        [
          0.1915,
          0.9754
        ],
        [
          0.2015,
          0.9745
        ],
        [
          0.2116,
          0.9745
        ],
        [
          0.222,
          0.9741
        ],
        [
          0.232,
          0.9745
        ],
        [
          0.242,
          0.9745
        ],
        [
          0.252,
          0.9745
        ],
        [
          0.2621,
          0.9736
        ],
        [
          0.2725,
          0.9736
        ],
        [
          0.2825,
          0.9741
        ],
        [
          0.2925,
          0.9741
        ],
        [
          0.3025,
          0.9732
        ],
        [
          0.3126,
          0.9732
        ],
        [
          0.323,
          0.9727
        ],
        [
          0.333,
          0.9709
        ],
        [
          0.343,
          0.9686
        ],
        [
          0.353,
          0.9686
        ],
        [
          0.3631,
          0.9668
        ],
        [
          0.3735,
          0.9659
        ],
        [
          0.3835,
          0.9641
        ],
        [
          0.3935,
          0.9631
        ],
        [
          0.4035,
          0.9604
        ],
        [
          0.4136,
          0.9595
        ],
        [
          0.424,
          0.9586
        ],
        [
          0.434,
          0.9554
        ],
        [
          0.444,
          0.9531
        ],
        [
          0.454,
          0.9522
        ],
        [
          0.4641,
          0.9481
        ],
        [
          0.4745,
          0.9463
        ],
        [
          0.4845,
          0.9445
        ],
        [
          0.4945,
          0.9409
        ],
        [
          0.5045,
          0.9363
        ],
        [
          0.5146,
          0.9349
        ],
        [
          0.525,
          0.9313
        ],
        [
          0.535,
          0.9272
        ],
        [
          0.545,
          0.9208
        ],
        [
          0.5551,
          0.9163
        ],
        [
          0.5655,
          0.9136
        ],
        [
          0.5755,
          0.909
        ],
        [
          0.5855,
          0.9058
        ],
        [
          0.5955,
          0.9008
        ],
        [
          0.6056,
          0.8963
        ],
        [
          0.616,
          0.8913
        ],
        [
          0.626,
          0.8867
        ],
        [
          0.636,
          0.8826
        ],
        [
          0.646,
          0.8776
        ],
        [
          0.6561,
          0.8731
        ],
        [
          0.6665,
          0.8681
        ],
        [
          0.6765,
          0.8658
        ],
        [
          0.6865,
          0.8603
        ],
        [
          0.6965,
          0.8544
        ],
        [
          0.7066,
          0.8508
        ],
        [
          0.717,
          0.8467
        ],
        [
          0.727,
          0.8412
        ],
        [
          0.737,
          0.8362
        ],
        [
          0.747,
          0.8317
        ],
        [
          0.7571,
          0.8258
        ],
        [
          0.7675,
          0.8198
        ],
        [
          0.7775,
          0.8144
        ],
        [
          0.7875,
          0.8076
        ],
        [
          0.7975,
          0.8007
        ],
        [
          0.8076,
          0.7957
        ],
        [
          0.818,
          0.7894
        ],
        [
          0.828,
          0.7825
        ],
        [
          0.838,
          0.7775
        ],
        [
          0.848,
          0.7702
        ],
        [
          0.8581,
          0.7652
        ],
        [
          0.8685,
          0.7593
        ],
        [
          0.8785,
          0.7525
        ],
        [
          0.8885,
          0.7466
        ],
        [
          0.8985,
          0.7393
        ],
        [
          0.9086,
          0.7329
        ],
        [
          0.919,
          0.7257
        ],
        [
          0.929,
          0.7202
        ],
        [
          0.939,
          0.7138
        ],
        [
          0.949,
          0.7084
        ],
        [
          0.9591,
          0.7015
        ],
        [
          0.9695,
          0.6934
        ],
        [
          0.9795,
          0.6856
        ],
        [
          0.9895,
          0.6774
        ],
        [
          0.9995,
          0.6692
        ],
        [
          1.0,
          0.6688
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.20896818179845023,
        "difference": -0.0012946523904456822,
        "worse": 0.54,
        "kept_right": 0.20737022022810184,
        "kept_wrong": 0.0015979615703483388,
        "handed_off": 0.7910318182015496
      },
      "usd_per_1000": 4.262112834535161
    },
    {
      "decision": 145,
      "frontier": 191,
      "frontier_accuracy": 0.9740673339399454,
      "in_sample": {
        "threshold": 0.9851364969800199,
        "answered": 0.18289353958143767,
        "accuracy": 0.9740673339399454
      },
      "curve": [
        [
          0.0,
          0.9741
        ],
        [
          0.01,
          0.9741
        ],
        [
          0.02,
          0.9741
        ],
        [
          0.03,
          0.9741
        ],
        [
          0.04,
          0.9741
        ],
        [
          0.05,
          0.9741
        ],
        [
          0.0601,
          0.9741
        ],
        [
          0.0705,
          0.9741
        ],
        [
          0.0805,
          0.9741
        ],
        [
          0.0905,
          0.9741
        ],
        [
          0.1005,
          0.9741
        ],
        [
          0.1106,
          0.9741
        ],
        [
          0.121,
          0.9741
        ],
        [
          0.131,
          0.9741
        ],
        [
          0.141,
          0.9741
        ],
        [
          0.151,
          0.9741
        ],
        [
          0.1611,
          0.9741
        ],
        [
          0.1715,
          0.9741
        ],
        [
          0.1815,
          0.9741
        ],
        [
          0.1915,
          0.9736
        ],
        [
          0.2015,
          0.9727
        ],
        [
          0.2116,
          0.9727
        ],
        [
          0.222,
          0.9722
        ],
        [
          0.232,
          0.9722
        ],
        [
          0.242,
          0.9722
        ],
        [
          0.252,
          0.9722
        ],
        [
          0.2621,
          0.9713
        ],
        [
          0.2725,
          0.9713
        ],
        [
          0.2825,
          0.9713
        ],
        [
          0.2925,
          0.9713
        ],
        [
          0.3025,
          0.9704
        ],
        [
          0.3126,
          0.9704
        ],
        [
          0.323,
          0.97
        ],
        [
          0.333,
          0.9682
        ],
        [
          0.343,
          0.9659
        ],
        [
          0.353,
          0.9659
        ],
        [
          0.3631,
          0.9641
        ],
        [
          0.3735,
          0.9631
        ],
        [
          0.3835,
          0.9613
        ],
        [
          0.3935,
          0.9604
        ],
        [
          0.4035,
          0.9577
        ],
        [
          0.4136,
          0.9568
        ],
        [
          0.424,
          0.9559
        ],
        [
          0.434,
          0.9531
        ],
        [
          0.444,
          0.9509
        ],
        [
          0.454,
          0.95
        ],
        [
          0.4641,
          0.9463
        ],
        [
          0.4745,
          0.9445
        ],
        [
          0.4845,
          0.9427
        ],
        [
          0.4945,
          0.939
        ],
        [
          0.5045,
          0.9345
        ],
        [
          0.5146,
          0.9327
        ],
        [
          0.525,
          0.929
        ],
        [
          0.535,
          0.9249
        ],
        [
          0.545,
          0.9186
        ],
        [
          0.5551,
          0.914
        ],
        [
          0.5655,
          0.9108
        ],
        [
          0.5755,
          0.9063
        ],
        [
          0.5855,
          0.9031
        ],
        [
          0.5955,
          0.8985
        ],
        [
          0.6056,
          0.894
        ],
        [
          0.616,
          0.889
        ],
        [
          0.626,
          0.884
        ],
        [
          0.636,
          0.8803
        ],
        [
          0.646,
          0.8753
        ],
        [
          0.6561,
          0.8708
        ],
        [
          0.6665,
          0.8658
        ],
        [
          0.6765,
          0.8631
        ],
        [
          0.6865,
          0.8581
        ],
        [
          0.6965,
          0.8526
        ],
        [
          0.7066,
          0.849
        ],
        [
          0.717,
          0.8449
        ],
        [
          0.727,
          0.8389
        ],
        [
          0.737,
          0.8339
        ],
        [
          0.747,
          0.8294
        ],
        [
          0.7571,
          0.8235
        ],
        [
          0.7675,
          0.818
        ],
        [
          0.7775,
          0.8126
        ],
        [
          0.7875,
          0.8071
        ],
        [
          0.7975,
          0.8007
        ],
        [
          0.8076,
          0.7957
        ],
        [
          0.818,
          0.7894
        ],
        [
          0.828,
          0.7825
        ],
        [
          0.838,
          0.7775
        ],
        [
          0.848,
          0.7702
        ],
        [
          0.8581,
          0.7657
        ],
        [
          0.8685,
          0.7598
        ],
        [
          0.8785,
          0.753
        ],
        [
          0.8885,
          0.747
        ],
        [
          0.8985,
          0.7398
        ],
        [
          0.9086,
          0.7338
        ],
        [
          0.919,
          0.7266
        ],
        [
          0.929,
          0.7207
        ],
        [
          0.939,
          0.7138
        ],
        [
          0.949,
          0.7079
        ],
        [
          0.9591,
          0.7011
        ],
        [
          0.9695,
          0.6924
        ],
        [
          0.9795,
          0.6856
        ],
        [
          0.9895,
          0.6774
        ],
        [
          0.9995,
          0.6692
        ],
        [
          1.0,
          0.6688
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1957244312676092,
        "difference": -0.0009641945535283963,
        "worse": 0.5033333333333333,
        "kept_right": 0.19470563331546234,
        "kept_wrong": 0.0010187979521467832,
        "handed_off": 0.8042755687323908
      },
      "usd_per_1000": 21.512240477021482
    },
    {
      "decision": 145,
      "frontier": 188,
      "frontier_accuracy": 0.9713375796178344,
      "in_sample": {
        "threshold": 0.9825243141806727,
        "answered": 0.1924476797088262,
        "accuracy": 0.9713375796178344
      },
      "curve": [
        [
          0.0,
          0.9713
        ],
        [
          0.01,
          0.9713
        ],
        [
          0.02,
          0.9713
        ],
        [
          0.03,
          0.9713
        ],
        [
          0.04,
          0.9713
        ],
        [
          0.05,
          0.9713
        ],
        [
          0.0601,
          0.9713
        ],
        [
          0.0705,
          0.9713
        ],
        [
          0.0805,
          0.9713
        ],
        [
          0.0905,
          0.9713
        ],
        [
          0.1005,
          0.9713
        ],
        [
          0.1106,
          0.9713
        ],
        [
          0.121,
          0.9713
        ],
        [
          0.131,
          0.9713
        ],
        [
          0.141,
          0.9718
        ],
        [
          0.151,
          0.9718
        ],
        [
          0.1611,
          0.9718
        ],
        [
          0.1715,
          0.9718
        ],
        [
          0.1815,
          0.9718
        ],
        [
          0.1915,
          0.9713
        ],
        [
          0.2015,
          0.9704
        ],
        [
          0.2116,
          0.9704
        ],
        [
          0.222,
          0.97
        ],
        [
          0.232,
          0.97
        ],
        [
          0.242,
          0.97
        ],
        [
          0.252,
          0.97
        ],
        [
          0.2621,
          0.9691
        ],
        [
          0.2725,
          0.9691
        ],
        [
          0.2825,
          0.9691
        ],
        [
          0.2925,
          0.9691
        ],
        [
          0.3025,
          0.9682
        ],
        [
          0.3126,
          0.9686
        ],
        [
          0.323,
          0.9682
        ],
        [
          0.333,
          0.9663
        ],
        [
          0.343,
          0.9645
        ],
        [
          0.353,
          0.9645
        ],
        [
          0.3631,
          0.9622
        ],
        [
          0.3735,
          0.9618
        ],
        [
          0.3835,
          0.9609
        ],
        [
          0.3935,
          0.96
        ],
        [
          0.4035,
          0.9572
        ],
        [
          0.4136,
          0.9563
        ],
        [
          0.424,
          0.955
        ],
        [
          0.434,
          0.9518
        ],
        [
          0.444,
          0.9495
        ],
        [
          0.454,
          0.9486
        ],
        [
          0.4641,
          0.9449
        ],
        [
          0.4745,
          0.9431
        ],
        [
          0.4845,
          0.9413
        ],
        [
          0.4945,
          0.9377
        ],
        [
          0.5045,
          0.9331
        ],
        [
          0.5146,
          0.9318
        ],
        [
          0.525,
          0.9286
        ],
        [
          0.535,
          0.9245
        ],
        [
          0.545,
          0.9181
        ],
        [
          0.5551,
          0.9136
        ],
        [
          0.5655,
          0.9108
        ],
        [
          0.5755,
          0.9067
        ],
        [
          0.5855,
          0.904
        ],
        [
          0.5955,
          0.8985
        ],
        [
          0.6056,
          0.894
        ],
        [
          0.616,
          0.8894
        ],
        [
          0.626,
          0.8854
        ],
        [
          0.636,
          0.8813
        ],
        [
          0.646,
          0.8758
        ],
        [
          0.6561,
          0.8712
        ],
        [
          0.6665,
          0.8662
        ],
        [
          0.6765,
          0.8635
        ],
        [
          0.6865,
          0.8576
        ],
        [
          0.6965,
          0.8521
        ],
        [
          0.7066,
          0.848
        ],
        [
          0.717,
          0.8439
        ],
        [
          0.727,
          0.838
        ],
        [
          0.737,
          0.833
        ],
        [
          0.747,
          0.8289
        ],
        [
          0.7571,
          0.823
        ],
        [
          0.7675,
          0.8176
        ],
        [
          0.7775,
          0.8116
        ],
        [
          0.7875,
          0.8048
        ],
        [
          0.7975,
          0.7985
        ],
        [
          0.8076,
          0.7934
        ],
        [
          0.818,
          0.7875
        ],
        [
          0.828,
          0.7807
        ],
        [
          0.838,
          0.7757
        ],
        [
          0.848,
          0.7684
        ],
        [
          0.8581,
          0.7643
        ],
        [
          0.8685,
          0.7584
        ],
        [
          0.8785,
          0.7516
        ],
        [
          0.8885,
          0.7461
        ],
        [
          0.8985,
          0.7393
        ],
        [
          0.9086,
          0.7329
        ],
        [
          0.919,
          0.7261
        ],
        [
          0.929,
          0.7197
        ],
        [
          0.939,
          0.7129
        ],
        [
          0.949,
          0.7066
        ],
        [
          0.9591,
          0.7006
        ],
        [
          0.9695,
          0.692
        ],
        [
          0.9795,
          0.6856
        ],
        [
          0.9895,
          0.6774
        ],
        [
          0.9995,
          0.6692
        ],
        [
          1.0,
          0.6688
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.20915224027233026,
        "difference": -0.0009006270582004801,
        "worse": 0.44,
        "kept_right": 0.20763314108872016,
        "kept_wrong": 0.0015190991836099789,
        "handed_off": 0.7908477597276696
      },
      "usd_per_1000": 0.7696989822773077
    },
    {
      "decision": 145,
      "frontier": 190,
      "frontier_accuracy": 0.9672429481346679,
      "in_sample": {
        "threshold": 0.9851364969800199,
        "answered": 0.18289353958143767,
        "accuracy": 0.9672429481346679
      },
      "curve": [
        [
          0.0,
          0.9672
        ],
        [
          0.01,
          0.9672
        ],
        [
          0.02,
          0.9672
        ],
        [
          0.03,
          0.9672
        ],
        [
          0.04,
          0.9672
        ],
        [
          0.05,
          0.9672
        ],
        [
          0.0601,
          0.9672
        ],
        [
          0.0705,
          0.9672
        ],
        [
          0.0805,
          0.9672
        ],
        [
          0.0905,
          0.9672
        ],
        [
          0.1005,
          0.9672
        ],
        [
          0.1106,
          0.9672
        ],
        [
          0.121,
          0.9672
        ],
        [
          0.131,
          0.9672
        ],
        [
          0.141,
          0.9672
        ],
        [
          0.151,
          0.9672
        ],
        [
          0.1611,
          0.9672
        ],
        [
          0.1715,
          0.9672
        ],
        [
          0.1815,
          0.9672
        ],
        [
          0.1915,
          0.9668
        ],
        [
          0.2015,
          0.9659
        ],
        [
          0.2116,
          0.9659
        ],
        [
          0.222,
          0.9654
        ],
        [
          0.232,
          0.9659
        ],
        [
          0.242,
          0.9659
        ],
        [
          0.252,
          0.9659
        ],
        [
          0.2621,
          0.965
        ],
        [
          0.2725,
          0.9654
        ],
        [
          0.2825,
          0.9654
        ],
        [
          0.2925,
          0.9654
        ],
        [
          0.3025,
          0.9641
        ],
        [
          0.3126,
          0.9641
        ],
        [
          0.323,
          0.9641
        ],
        [
          0.333,
          0.9627
        ],
        [
          0.343,
          0.9604
        ],
        [
          0.353,
          0.9604
        ],
        [
          0.3631,
          0.9591
        ],
        [
          0.3735,
          0.9581
        ],
        [
          0.3835,
          0.9568
        ],
        [
          0.3935,
          0.9554
        ],
        [
          0.4035,
          0.9527
        ],
        [
          0.4136,
          0.9518
        ],
        [
          0.424,
          0.9509
        ],
        [
          0.434,
          0.9477
        ],
        [
          0.444,
          0.9454
        ],
        [
          0.454,
          0.9445
        ],
        [
          0.4641,
          0.9404
        ],
        [
          0.4745,
          0.9386
        ],
        [
          0.4845,
          0.9368
        ],
        [
          0.4945,
          0.9331
        ],
        [
          0.5045,
          0.9286
        ],
        [
          0.5146,
          0.9268
        ],
        [
          0.525,
          0.9236
        ],
        [
          0.535,
          0.9195
        ],
        [
          0.545,
          0.9131
        ],
        [
          0.5551,
          0.9086
        ],
        [
          0.5655,
          0.9058
        ],
        [
          0.5755,
          0.9013
        ],
        [
          0.5855,
          0.8981
        ],
        [
          0.5955,
          0.8935
        ],
        [
          0.6056,
          0.889
        ],
        [
          0.616,
          0.8844
        ],
        [
          0.626,
          0.8794
        ],
        [
          0.636,
          0.8758
        ],
        [
          0.646,
          0.8708
        ],
        [
          0.6561,
          0.8662
        ],
        [
          0.6665,
          0.8617
        ],
        [
          0.6765,
          0.8599
        ],
        [
          0.6865,
          0.8544
        ],
        [
          0.6965,
          0.849
        ],
        [
          0.7066,
          0.8462
        ],
        [
          0.717,
          0.8421
        ],
        [
          0.727,
          0.8367
        ],
        [
          0.737,
          0.8317
        ],
        [
          0.747,
          0.8276
        ],
        [
          0.7571,
          0.8217
        ],
        [
          0.7675,
          0.8167
        ],
        [
          0.7775,
          0.8107
        ],
        [
          0.7875,
          0.8044
        ],
        [
          0.7975,
          0.798
        ],
        [
          0.8076,
          0.793
        ],
        [
          0.818,
          0.7866
        ],
        [
          0.828,
          0.7798
        ],
        [
          0.838,
          0.7748
        ],
        [
          0.848,
          0.7675
        ],
        [
          0.8581,
          0.763
        ],
        [
          0.8685,
          0.7571
        ],
        [
          0.8785,
          0.7502
        ],
        [
          0.8885,
          0.7448
        ],
        [
          0.8985,
          0.7379
        ],
        [
          0.9086,
          0.7325
        ],
        [
          0.919,
          0.7252
        ],
        [
          0.929,
          0.7197
        ],
        [
          0.939,
          0.7129
        ],
        [
          0.949,
          0.707
        ],
        [
          0.9591,
          0.7002
        ],
        [
          0.9695,
          0.6915
        ],
        [
          0.9795,
          0.6847
        ],
        [
          0.9895,
          0.6765
        ],
        [
          0.9995,
          0.6692
        ],
        [
          1.0,
          0.6688
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.21106529217770995,
        "difference": -0.0014008871903844213,
        "worse": 0.5533333333333333,
        "kept_right": 0.20932769608228813,
        "kept_wrong": 0.0017375960954217422,
        "handed_off": 0.7889347078222897
      },
      "usd_per_1000": 4.400962673679845
    },
    {
      "decision": 145,
      "frontier": 124,
      "frontier_accuracy": 0.9667879890809827,
      "in_sample": {
        "threshold": 0.9851364969800199,
        "answered": 0.18289353958143767,
        "accuracy": 0.9667879890809827
      },
      "curve": [
        [
          0.0,
          0.9668
        ],
        [
          0.01,
          0.9668
        ],
        [
          0.02,
          0.9668
        ],
        [
          0.03,
          0.9668
        ],
        [
          0.04,
          0.9668
        ],
        [
          0.05,
          0.9668
        ],
        [
          0.0601,
          0.9668
        ],
        [
          0.0705,
          0.9668
        ],
        [
          0.0805,
          0.9668
        ],
        [
          0.0905,
          0.9668
        ],
        [
          0.1005,
          0.9668
        ],
        [
          0.1106,
          0.9668
        ],
        [
          0.121,
          0.9668
        ],
        [
          0.131,
          0.9668
        ],
        [
          0.141,
          0.9668
        ],
        [
          0.151,
          0.9668
        ],
        [
          0.1611,
          0.9668
        ],
        [
          0.1715,
          0.9668
        ],
        [
          0.1815,
          0.9668
        ],
        [
          0.1915,
          0.9663
        ],
        [
          0.2015,
          0.9654
        ],
        [
          0.2116,
          0.9654
        ],
        [
          0.222,
          0.965
        ],
        [
          0.232,
          0.9654
        ],
        [
          0.242,
          0.9654
        ],
        [
          0.252,
          0.9654
        ],
        [
          0.2621,
          0.9645
        ],
        [
          0.2725,
          0.9645
        ],
        [
          0.2825,
          0.965
        ],
        [
          0.2925,
          0.965
        ],
        [
          0.3025,
          0.9641
        ],
        [
          0.3126,
          0.9641
        ],
        [
          0.323,
          0.9636
        ],
        [
          0.333,
          0.9618
        ],
        [
          0.343,
          0.9595
        ],
        [
          0.353,
          0.9595
        ],
        [
          0.3631,
          0.9581
        ],
        [
          0.3735,
          0.9577
        ],
        [
          0.3835,
          0.9563
        ],
        [
          0.3935,
          0.955
        ],
        [
          0.4035,
          0.9522
        ],
        [
          0.4136,
          0.9513
        ],
        [
          0.424,
          0.9504
        ],
        [
          0.434,
          0.9472
        ],
        [
          0.444,
          0.9449
        ],
        [
          0.454,
          0.944
        ],
        [
          0.4641,
          0.9404
        ],
        [
          0.4745,
          0.9386
        ],
        [
          0.4845,
          0.9372
        ],
        [
          0.4945,
          0.9336
        ],
        [
          0.5045,
          0.929
        ],
        [
          0.5146,
          0.9277
        ],
        [
          0.525,
          0.924
        ],
        [
          0.535,
          0.9199
        ],
        [
          0.545,
          0.9136
        ],
        [
          0.5551,
          0.909
        ],
        [
          0.5655,
          0.9063
        ],
        [
          0.5755,
          0.9017
        ],
        [
          0.5855,
          0.8985
        ],
        [
          0.5955,
          0.8935
        ],
        [
          0.6056,
          0.889
        ],
        [
          0.616,
          0.884
        ],
        [
          0.626,
          0.8799
        ],
        [
          0.636,
          0.8758
        ],
        [
          0.646,
          0.8712
        ],
        [
          0.6561,
          0.8667
        ],
        [
          0.6665,
          0.8621
        ],
        [
          0.6765,
          0.8603
        ],
        [
          0.6865,
          0.8558
        ],
        [
          0.6965,
          0.8499
        ],
        [
          0.7066,
          0.8453
        ],
        [
          0.717,
          0.8412
        ],
        [
          0.727,
          0.8353
        ],
        [
          0.737,
          0.8303
        ],
        [
          0.747,
          0.8262
        ],
        [
          0.7571,
          0.8203
        ],
        [
          0.7675,
          0.8144
        ],
        [
          0.7775,
          0.8089
        ],
        [
          0.7875,
          0.8035
        ],
        [
          0.7975,
          0.7971
        ],
        [
          0.8076,
          0.7921
        ],
        [
          0.818,
          0.7857
        ],
        [
          0.828,
          0.7789
        ],
        [
          0.838,
          0.7739
        ],
        [
          0.848,
          0.7666
        ],
        [
          0.8581,
          0.7621
        ],
        [
          0.8685,
          0.7561
        ],
        [
          0.8785,
          0.7493
        ],
        [
          0.8885,
          0.7448
        ],
        [
          0.8985,
          0.7384
        ],
        [
          0.9086,
          0.7325
        ],
        [
          0.919,
          0.7252
        ],
        [
          0.929,
          0.7197
        ],
        [
          0.939,
          0.7129
        ],
        [
          0.949,
          0.707
        ],
        [
          0.9591,
          0.7006
        ],
        [
          0.9695,
          0.692
        ],
        [
          0.9795,
          0.6852
        ],
        [
          0.9895,
          0.677
        ],
        [
          0.9995,
          0.6692
        ],
        [
          1.0,
          0.6688
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.20896818179845023,
        "difference": -0.0012946523904456822,
        "worse": 0.54,
        "kept_right": 0.20737022022810184,
        "kept_wrong": 0.0015979615703483388,
        "handed_off": 0.7910318182015496
      },
      "usd_per_1000": 0.9019258117519757
    },
    {
      "decision": 145,
      "frontier": 278,
      "frontier_accuracy": 0.9499545040946314,
      "in_sample": {
        "threshold": 0.9851364969800199,
        "answered": 0.18289353958143767,
        "accuracy": 0.9499545040946314
      },
      "curve": [
        [
          0.0,
          0.95
        ],
        [
          0.01,
          0.95
        ],
        [
          0.02,
          0.95
        ],
        [
          0.03,
          0.95
        ],
        [
          0.04,
          0.95
        ],
        [
          0.05,
          0.95
        ],
        [
          0.0601,
          0.95
        ],
        [
          0.0705,
          0.95
        ],
        [
          0.0805,
          0.95
        ],
        [
          0.0905,
          0.95
        ],
        [
          0.1005,
          0.95
        ],
        [
          0.1106,
          0.95
        ],
        [
          0.121,
          0.95
        ],
        [
          0.131,
          0.95
        ],
        [
          0.141,
          0.95
        ],
        [
          0.151,
          0.95
        ],
        [
          0.1611,
          0.95
        ],
        [
          0.1715,
          0.95
        ],
        [
          0.1815,
          0.95
        ],
        [
          0.1915,
          0.9495
        ],
        [
          0.2015,
          0.9486
        ],
        [
          0.2116,
          0.9486
        ],
        [
          0.222,
          0.9481
        ],
        [
          0.232,
          0.9481
        ],
        [
          0.242,
          0.9481
        ],
        [
          0.252,
          0.9481
        ],
        [
          0.2621,
          0.9472
        ],
        [
          0.2725,
          0.9472
        ],
        [
          0.2825,
          0.9477
        ],
        [
          0.2925,
          0.9477
        ],
        [
          0.3025,
          0.9468
        ],
        [
          0.3126,
          0.9472
        ],
        [
          0.323,
          0.9468
        ],
        [
          0.333,
          0.9449
        ],
        [
          0.343,
          0.9427
        ],
        [
          0.353,
          0.9431
        ],
        [
          0.3631,
          0.9413
        ],
        [
          0.3735,
          0.9404
        ],
        [
          0.3835,
          0.9395
        ],
        [
          0.3935,
          0.9386
        ],
        [
          0.4035,
          0.9359
        ],
        [
          0.4136,
          0.9349
        ],
        [
          0.424,
          0.934
        ],
        [
          0.434,
          0.9308
        ],
        [
          0.444,
          0.929
        ],
        [
          0.454,
          0.9277
        ],
        [
          0.4641,
          0.924
        ],
        [
          0.4745,
          0.9222
        ],
        [
          0.4845,
          0.9204
        ],
        [
          0.4945,
          0.9167
        ],
        [
          0.5045,
          0.9126
        ],
        [
          0.5146,
          0.9113
        ],
        [
          0.525,
          0.9076
        ],
        [
          0.535,
          0.904
        ],
        [
          0.545,
          0.8981
        ],
        [
          0.5551,
          0.8935
        ],
        [
          0.5655,
          0.8913
        ],
        [
          0.5755,
          0.8881
        ],
        [
          0.5855,
          0.8858
        ],
        [
          0.5955,
          0.8808
        ],
        [
          0.6056,
          0.8763
        ],
        [
          0.616,
          0.8712
        ],
        [
          0.626,
          0.8676
        ],
        [
          0.636,
          0.864
        ],
        [
          0.646,
          0.8594
        ],
        [
          0.6561,
          0.8553
        ],
        [
          0.6665,
          0.8508
        ],
        [
          0.6765,
          0.849
        ],
        [
          0.6865,
          0.8439
        ],
        [
          0.6965,
          0.8385
        ],
        [
          0.7066,
          0.8353
        ],
        [
          0.717,
          0.8326
        ],
        [
          0.727,
          0.8276
        ],
        [
          0.737,
          0.823
        ],
        [
          0.747,
          0.8189
        ],
        [
          0.7571,
          0.813
        ],
        [
          0.7675,
          0.8076
        ],
        [
          0.7775,
          0.8021
        ],
        [
          0.7875,
          0.7971
        ],
        [
          0.7975,
          0.7912
        ],
        [
          0.8076,
          0.7871
        ],
        [
          0.818,
          0.7807
        ],
        [
          0.828,
          0.7748
        ],
        [
          0.838,
          0.7702
        ],
        [
          0.848,
          0.7625
        ],
        [
          0.8581,
          0.7584
        ],
        [
          0.8685,
          0.753
        ],
        [
          0.8785,
          0.7475
        ],
        [
          0.8885,
          0.7429
        ],
        [
          0.8985,
          0.7361
        ],
        [
          0.9086,
          0.7298
        ],
        [
          0.919,
          0.7225
        ],
        [
          0.929,
          0.717
        ],
        [
          0.939,
          0.7106
        ],
        [
          0.949,
          0.7047
        ],
        [
          0.9591,
          0.6988
        ],
        [
          0.9695,
          0.6902
        ],
        [
          0.9795,
          0.6829
        ],
        [
          0.9895,
          0.6761
        ],
        [
          0.9995,
          0.6692
        ],
        [
          1.0,
          0.6688
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.20025117543291898,
        "difference": -0.0010824481525224675,
        "worse": 0.5066666666666667,
        "kept_right": 0.19898375458531806,
        "kept_wrong": 0.001267420847600941,
        "handed_off": 0.7997488245670809
      },
      "usd_per_1000": 5.21319092981631
    },
    {
      "decision": 145,
      "frontier": 275,
      "frontier_accuracy": 0.9426751592356688,
      "in_sample": {
        "threshold": 0.9851364969800199,
        "answered": 0.18289353958143767,
        "accuracy": 0.9426751592356688
      },
      "curve": [
        [
          0.0,
          0.9427
        ],
        [
          0.01,
          0.9427
        ],
        [
          0.02,
          0.9427
        ],
        [
          0.03,
          0.9427
        ],
        [
          0.04,
          0.9427
        ],
        [
          0.05,
          0.9427
        ],
        [
          0.0601,
          0.9427
        ],
        [
          0.0705,
          0.9427
        ],
        [
          0.0805,
          0.9427
        ],
        [
          0.0905,
          0.9427
        ],
        [
          0.1005,
          0.9427
        ],
        [
          0.1106,
          0.9427
        ],
        [
          0.121,
          0.9427
        ],
        [
          0.131,
          0.9427
        ],
        [
          0.141,
          0.9427
        ],
        [
          0.151,
          0.9427
        ],
        [
          0.1611,
          0.9427
        ],
        [
          0.1715,
          0.9427
        ],
        [
          0.1815,
          0.9427
        ],
        [
          0.1915,
          0.9422
        ],
        [
          0.2015,
          0.9413
        ],
        [
          0.2116,
          0.9413
        ],
        [
          0.222,
          0.9409
        ],
        [
          0.232,
          0.9413
        ],
        [
          0.242,
          0.9413
        ],
        [
          0.252,
          0.9413
        ],
        [
          0.2621,
          0.9404
        ],
        [
          0.2725,
          0.9404
        ],
        [
          0.2825,
          0.9404
        ],
        [
          0.2925,
          0.9404
        ],
        [
          0.3025,
          0.9395
        ],
        [
          0.3126,
          0.9399
        ],
        [
          0.323,
          0.9395
        ],
        [
          0.333,
          0.9381
        ],
        [
          0.343,
          0.9359
        ],
        [
          0.353,
          0.9363
        ],
        [
          0.3631,
          0.9345
        ],
        [
          0.3735,
          0.9336
        ],
        [
          0.3835,
          0.9327
        ],
        [
          0.3935,
          0.9318
        ],
        [
          0.4035,
          0.929
        ],
        [
          0.4136,
          0.9281
        ],
        [
          0.424,
          0.9277
        ],
        [
          0.434,
          0.9249
        ],
        [
          0.444,
          0.9227
        ],
        [
          0.454,
          0.9222
        ],
        [
          0.4641,
          0.9186
        ],
        [
          0.4745,
          0.9167
        ],
        [
          0.4845,
          0.9149
        ],
        [
          0.4945,
          0.9117
        ],
        [
          0.5045,
          0.9081
        ],
        [
          0.5146,
          0.9072
        ],
        [
          0.525,
          0.9035
        ],
        [
          0.535,
          0.8999
        ],
        [
          0.545,
          0.894
        ],
        [
          0.5551,
          0.8894
        ],
        [
          0.5655,
          0.8867
        ],
        [
          0.5755,
          0.8831
        ],
        [
          0.5855,
          0.8813
        ],
        [
          0.5955,
          0.8767
        ],
        [
          0.6056,
          0.8722
        ],
        [
          0.616,
          0.8694
        ],
        [
          0.626,
          0.8653
        ],
        [
          0.636,
          0.8621
        ],
        [
          0.646,
          0.8576
        ],
        [
          0.6561,
          0.8535
        ],
        [
          0.6665,
          0.849
        ],
        [
          0.6765,
          0.8467
        ],
        [
          0.6865,
          0.8426
        ],
        [
          0.6965,
          0.8371
        ],
        [
          0.7066,
          0.8335
        ],
        [
          0.717,
          0.8303
        ],
        [
          0.727,
          0.8248
        ],
        [
          0.737,
          0.8198
        ],
        [
          0.747,
          0.8157
        ],
        [
          0.7571,
          0.8098
        ],
        [
          0.7675,
          0.8048
        ],
        [
          0.7775,
          0.7998
        ],
        [
          0.7875,
          0.7944
        ],
        [
          0.7975,
          0.7889
        ],
        [
          0.8076,
          0.7853
        ],
        [
          0.818,
          0.7789
        ],
        [
          0.828,
          0.7721
        ],
        [
          0.838,
          0.7671
        ],
        [
          0.848,
          0.7602
        ],
        [
          0.8581,
          0.7552
        ],
        [
          0.8685,
          0.7507
        ],
        [
          0.8785,
          0.7448
        ],
        [
          0.8885,
          0.7393
        ],
        [
          0.8985,
          0.7334
        ],
        [
          0.9086,
          0.7288
        ],
        [
          0.919,
          0.7216
        ],
        [
          0.929,
          0.7166
        ],
        [
          0.939,
          0.7102
        ],
        [
          0.949,
          0.7043
        ],
        [
          0.9591,
          0.6979
        ],
        [
          0.9695,
          0.6897
        ],
        [
          0.9795,
          0.6829
        ],
        [
          0.9895,
          0.6765
        ],
        [
          0.9995,
          0.6692
        ],
        [
          1.0,
          0.6688
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.2055989767924637,
        "difference": -0.001243115235694148,
        "worse": 0.5333333333333333,
        "kept_right": 0.2041284038173069,
        "kept_wrong": 0.001470572975156717,
        "handed_off": 0.7944010232075361
      },
      "usd_per_1000": 0.41649455739099306
    },
    {
      "decision": 145,
      "frontier": 192,
      "frontier_accuracy": 0.9331210191082803,
      "in_sample": {
        "threshold": 0.9851364969800199,
        "answered": 0.18289353958143767,
        "accuracy": 0.9331210191082803
      },
      "curve": [
        [
          0.0,
          0.9331
        ],
        [
          0.01,
          0.9331
        ],
        [
          0.02,
          0.9331
        ],
        [
          0.03,
          0.9331
        ],
        [
          0.04,
          0.9331
        ],
        [
          0.05,
          0.9331
        ],
        [
          0.0601,
          0.9331
        ],
        [
          0.0705,
          0.9331
        ],
        [
          0.0805,
          0.9331
        ],
        [
          0.0905,
          0.9331
        ],
        [
          0.1005,
          0.9331
        ],
        [
          0.1106,
          0.9331
        ],
        [
          0.121,
          0.9331
        ],
        [
          0.131,
          0.9331
        ],
        [
          0.141,
          0.9331
        ],
        [
          0.151,
          0.9331
        ],
        [
          0.1611,
          0.9331
        ],
        [
          0.1715,
          0.9331
        ],
        [
          0.1815,
          0.9331
        ],
        [
          0.1915,
          0.9327
        ],
        [
          0.2015,
          0.9318
        ],
        [
          0.2116,
          0.9318
        ],
        [
          0.222,
          0.9313
        ],
        [
          0.232,
          0.9318
        ],
        [
          0.242,
          0.9318
        ],
        [
          0.252,
          0.9318
        ],
        [
          0.2621,
          0.9308
        ],
        [
          0.2725,
          0.9313
        ],
        [
          0.2825,
          0.9318
        ],
        [
          0.2925,
          0.9318
        ],
        [
          0.3025,
          0.9304
        ],
        [
          0.3126,
          0.9304
        ],
        [
          0.323,
          0.9299
        ],
        [
          0.333,
          0.9281
        ],
        [
          0.343,
          0.9263
        ],
        [
          0.353,
          0.9268
        ],
        [
          0.3631,
          0.9245
        ],
        [
          0.3735,
          0.924
        ],
        [
          0.3835,
          0.9231
        ],
        [
          0.3935,
          0.9227
        ],
        [
          0.4035,
          0.9199
        ],
        [
          0.4136,
          0.919
        ],
        [
          0.424,
          0.9177
        ],
        [
          0.434,
          0.9163
        ],
        [
          0.444,
          0.914
        ],
        [
          0.454,
          0.9136
        ],
        [
          0.4641,
          0.9104
        ],
        [
          0.4745,
          0.9086
        ],
        [
          0.4845,
          0.9072
        ],
        [
          0.4945,
          0.9035
        ],
        [
          0.5045,
          0.8995
        ],
        [
          0.5146,
          0.8985
        ],
        [
          0.525,
          0.8954
        ],
        [
          0.535,
          0.8917
        ],
        [
          0.545,
          0.8858
        ],
        [
          0.5551,
          0.8817
        ],
        [
          0.5655,
          0.879
        ],
        [
          0.5755,
          0.8753
        ],
        [
          0.5855,
          0.8726
        ],
        [
          0.5955,
          0.8681
        ],
        [
          0.6056,
          0.8635
        ],
        [
          0.616,
          0.8599
        ],
        [
          0.626,
          0.8567
        ],
        [
          0.636,
          0.854
        ],
        [
          0.646,
          0.849
        ],
        [
          0.6561,
          0.8444
        ],
        [
          0.6665,
          0.8403
        ],
        [
          0.6765,
          0.8385
        ],
        [
          0.6865,
          0.8339
        ],
        [
          0.6965,
          0.8298
        ],
        [
          0.7066,
          0.8271
        ],
        [
          0.717,
          0.8239
        ],
        [
          0.727,
          0.8194
        ],
        [
          0.737,
          0.8153
        ],
        [
          0.747,
          0.8121
        ],
        [
          0.7571,
          0.8071
        ],
        [
          0.7675,
          0.8016
        ],
        [
          0.7775,
          0.7971
        ],
        [
          0.7875,
          0.793
        ],
        [
          0.7975,
          0.7866
        ],
        [
          0.8076,
          0.7825
        ],
        [
          0.818,
          0.7766
        ],
        [
          0.828,
          0.7707
        ],
        [
          0.838,
          0.7657
        ],
        [
          0.848,
          0.7589
        ],
        [
          0.8581,
          0.7548
        ],
        [
          0.8685,
          0.7493
        ],
        [
          0.8785,
          0.7439
        ],
        [
          0.8885,
          0.7398
        ],
        [
          0.8985,
          0.7329
        ],
        [
          0.9086,
          0.727
        ],
        [
          0.919,
          0.7197
        ],
        [
          0.929,
          0.7152
        ],
        [
          0.939,
          0.7093
        ],
        [
          0.949,
          0.7043
        ],
        [
          0.9591,
          0.6975
        ],
        [
          0.9695,
          0.6888
        ],
        [
          0.9795,
          0.6833
        ],
        [
          0.9895,
          0.6765
        ],
        [
          0.9995,
          0.6692
        ],
        [
          1.0,
          0.6688
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.216156560493564,
        "difference": -0.0014341733479642836,
        "worse": 0.5666666666666667,
        "kept_right": 0.21425838617196752,
        "kept_wrong": 0.0018981743215963688,
        "handed_off": 0.7838434395064356
      },
      "usd_per_1000": 9.107846370876226
    },
    {
      "decision": 145,
      "frontier": 123,
      "frontier_accuracy": 0.9326660600545951,
      "in_sample": {
        "threshold": 0.9851364969800199,
        "answered": 0.18289353958143767,
        "accuracy": 0.9326660600545951
      },
      "curve": [
        [
          0.0,
          0.9327
        ],
        [
          0.01,
          0.9327
        ],
        [
          0.02,
          0.9327
        ],
        [
          0.03,
          0.9327
        ],
        [
          0.04,
          0.9327
        ],
        [
          0.05,
          0.9327
        ],
        [
          0.0601,
          0.9327
        ],
        [
          0.0705,
          0.9327
        ],
        [
          0.0805,
          0.9327
        ],
        [
          0.0905,
          0.9327
        ],
        [
          0.1005,
          0.9327
        ],
        [
          0.1106,
          0.9327
        ],
        [
          0.121,
          0.9327
        ],
        [
          0.131,
          0.9327
        ],
        [
          0.141,
          0.9327
        ],
        [
          0.151,
          0.9327
        ],
        [
          0.1611,
          0.9327
        ],
        [
          0.1715,
          0.9327
        ],
        [
          0.1815,
          0.9327
        ],
        [
          0.1915,
          0.9322
        ],
        [
          0.2015,
          0.9313
        ],
        [
          0.2116,
          0.9313
        ],
        [
          0.222,
          0.9308
        ],
        [
          0.232,
          0.9308
        ],
        [
          0.242,
          0.9308
        ],
        [
          0.252,
          0.9308
        ],
        [
          0.2621,
          0.9299
        ],
        [
          0.2725,
          0.9299
        ],
        [
          0.2825,
          0.9304
        ],
        [
          0.2925,
          0.9304
        ],
        [
          0.3025,
          0.9295
        ],
        [
          0.3126,
          0.9299
        ],
        [
          0.323,
          0.9295
        ],
        [
          0.333,
          0.9277
        ],
        [
          0.343,
          0.9263
        ],
        [
          0.353,
          0.9258
        ],
        [
          0.3631,
          0.9254
        ],
        [
          0.3735,
          0.9245
        ],
        [
          0.3835,
          0.9231
        ],
        [
          0.3935,
          0.9222
        ],
        [
          0.4035,
          0.9195
        ],
        [
          0.4136,
          0.9186
        ],
        [
          0.424,
          0.9172
        ],
        [
          0.434,
          0.914
        ],
        [
          0.444,
          0.9131
        ],
        [
          0.454,
          0.9122
        ],
        [
          0.4641,
          0.909
        ],
        [
          0.4745,
          0.9076
        ],
        [
          0.4845,
          0.9063
        ],
        [
          0.4945,
          0.9035
        ],
        [
          0.5045,
          0.8999
        ],
        [
          0.5146,
          0.899
        ],
        [
          0.525,
          0.8963
        ],
        [
          0.535,
          0.8922
        ],
        [
          0.545,
          0.8872
        ],
        [
          0.5551,
          0.8831
        ],
        [
          0.5655,
          0.8808
        ],
        [
          0.5755,
          0.8776
        ],
        [
          0.5855,
          0.8744
        ],
        [
          0.5955,
          0.869
        ],
        [
          0.6056,
          0.8653
        ],
        [
          0.616,
          0.8612
        ],
        [
          0.626,
          0.8585
        ],
        [
          0.636,
          0.8553
        ],
        [
          0.646,
          0.8512
        ],
        [
          0.6561,
          0.8471
        ],
        [
          0.6665,
          0.8421
        ],
        [
          0.6765,
          0.8399
        ],
        [
          0.6865,
          0.8344
        ],
        [
          0.6965,
          0.8289
        ],
        [
          0.7066,
          0.8258
        ],
        [
          0.717,
          0.823
        ],
        [
          0.727,
          0.8176
        ],
        [
          0.737,
          0.8135
        ],
        [
          0.747,
          0.8089
        ],
        [
          0.7571,
          0.8044
        ],
        [
          0.7675,
          0.7989
        ],
        [
          0.7775,
          0.7934
        ],
        [
          0.7875,
          0.788
        ],
        [
          0.7975,
          0.7834
        ],
        [
          0.8076,
          0.7789
        ],
        [
          0.818,
          0.7743
        ],
        [
          0.828,
          0.7675
        ],
        [
          0.838,
          0.763
        ],
        [
          0.848,
          0.7557
        ],
        [
          0.8581,
          0.7516
        ],
        [
          0.8685,
          0.747
        ],
        [
          0.8785,
          0.7402
        ],
        [
          0.8885,
          0.7352
        ],
        [
          0.8985,
          0.7288
        ],
        [
          0.9086,
          0.7238
        ],
        [
          0.919,
          0.7179
        ],
        [
          0.929,
          0.7129
        ],
        [
          0.939,
          0.7075
        ],
        [
          0.949,
          0.7015
        ],
        [
          0.9591,
          0.697
        ],
        [
          0.9695,
          0.6897
        ],
        [
          0.9795,
          0.6847
        ],
        [
          0.9895,
          0.6774
        ],
        [
          0.9995,
          0.6692
        ],
        [
          1.0,
          0.6688
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.20025117543291898,
        "difference": -0.0010824481525224675,
        "worse": 0.5066666666666667,
        "kept_right": 0.19898375458531806,
        "kept_wrong": 0.001267420847600941,
        "handed_off": 0.7997488245670809
      },
      "usd_per_1000": 0.22256444004145357
    },
    {
      "decision": 145,
      "frontier": 274,
      "frontier_accuracy": 0.9085532302092811,
      "in_sample": {
        "threshold": 0.9825243141806727,
        "answered": 0.1924476797088262,
        "accuracy": 0.9085532302092811
      },
      "curve": [
        [
          0.0,
          0.9086
        ],
        [
          0.01,
          0.9086
        ],
        [
          0.02,
          0.9086
        ],
        [
          0.03,
          0.9086
        ],
        [
          0.04,
          0.9086
        ],
        [
          0.05,
          0.9086
        ],
        [
          0.0601,
          0.9086
        ],
        [
          0.0705,
          0.9086
        ],
        [
          0.0805,
          0.9086
        ],
        [
          0.0905,
          0.9086
        ],
        [
          0.1005,
          0.9086
        ],
        [
          0.1106,
          0.9086
        ],
        [
          0.121,
          0.9086
        ],
        [
          0.131,
          0.9086
        ],
        [
          0.141,
          0.9086
        ],
        [
          0.151,
          0.9086
        ],
        [
          0.1611,
          0.9086
        ],
        [
          0.1715,
          0.9086
        ],
        [
          0.1815,
          0.9086
        ],
        [
          0.1915,
          0.9086
        ],
        [
          0.2015,
          0.9076
        ],
        [
          0.2116,
          0.9076
        ],
        [
          0.222,
          0.9072
        ],
        [
          0.232,
          0.9076
        ],
        [
          0.242,
          0.9081
        ],
        [
          0.252,
          0.9081
        ],
        [
          0.2621,
          0.9076
        ],
        [
          0.2725,
          0.9076
        ],
        [
          0.2825,
          0.9076
        ],
        [
          0.2925,
          0.9081
        ],
        [
          0.3025,
          0.9072
        ],
        [
          0.3126,
          0.9076
        ],
        [
          0.323,
          0.9076
        ],
        [
          0.333,
          0.9063
        ],
        [
          0.343,
          0.904
        ],
        [
          0.353,
          0.9045
        ],
        [
          0.3631,
          0.9035
        ],
        [
          0.3735,
          0.904
        ],
        [
          0.3835,
          0.9031
        ],
        [
          0.3935,
          0.9022
        ],
        [
          0.4035,
          0.8999
        ],
        [
          0.4136,
          0.899
        ],
        [
          0.424,
          0.8981
        ],
        [
          0.434,
          0.8958
        ],
        [
          0.444,
          0.8935
        ],
        [
          0.454,
          0.8926
        ],
        [
          0.4641,
          0.889
        ],
        [
          0.4745,
          0.8872
        ],
        [
          0.4845,
          0.8854
        ],
        [
          0.4945,
          0.8822
        ],
        [
          0.5045,
          0.879
        ],
        [
          0.5146,
          0.8785
        ],
        [
          0.525,
          0.8753
        ],
        [
          0.535,
          0.8722
        ],
        [
          0.545,
          0.8676
        ],
        [
          0.5551,
          0.8644
        ],
        [
          0.5655,
          0.8612
        ],
        [
          0.5755,
          0.8576
        ],
        [
          0.5855,
          0.8558
        ],
        [
          0.5955,
          0.8512
        ],
        [
          0.6056,
          0.8476
        ],
        [
          0.616,
          0.8435
        ],
        [
          0.626,
          0.8394
        ],
        [
          0.636,
          0.8367
        ],
        [
          0.646,
          0.833
        ],
        [
          0.6561,
          0.8294
        ],
        [
          0.6665,
          0.8248
        ],
        [
          0.6765,
          0.8226
        ],
        [
          0.6865,
          0.818
        ],
        [
          0.6965,
          0.8135
        ],
        [
          0.7066,
          0.8107
        ],
        [
          0.717,
          0.8089
        ],
        [
          0.727,
          0.8035
        ],
        [
          0.737,
          0.7989
        ],
        [
          0.747,
          0.7953
        ],
        [
          0.7571,
          0.7903
        ],
        [
          0.7675,
          0.7853
        ],
        [
          0.7775,
          0.7807
        ],
        [
          0.7875,
          0.7757
        ],
        [
          0.7975,
          0.7707
        ],
        [
          0.8076,
          0.7662
        ],
        [
          0.818,
          0.7607
        ],
        [
          0.828,
          0.7543
        ],
        [
          0.838,
          0.7502
        ],
        [
          0.848,
          0.7434
        ],
        [
          0.8581,
          0.7402
        ],
        [
          0.8685,
          0.7357
        ],
        [
          0.8785,
          0.7293
        ],
        [
          0.8885,
          0.7252
        ],
        [
          0.8985,
          0.7197
        ],
        [
          0.9086,
          0.7161
        ],
        [
          0.919,
          0.7093
        ],
        [
          0.929,
          0.7056
        ],
        [
          0.939,
          0.7002
        ],
        [
          0.949,
          0.697
        ],
        [
          0.9591,
          0.6924
        ],
        [
          0.9695,
          0.6852
        ],
        [
          0.9795,
          0.6811
        ],
        [
          0.9895,
          0.6756
        ],
        [
          0.9995,
          0.6692
        ],
        [
          1.0,
          0.6688
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.2760165667538391,
        "difference": -0.002240631661133829,
        "worse": 0.7033333333333334,
        "kept_right": 0.27027408874763487,
        "kept_wrong": 0.005742478006204227,
        "handed_off": 0.7239834332461609
      },
      "usd_per_1000": 0.31635955905692864
    },
    {
      "decision": 145,
      "frontier": 47,
      "frontier_accuracy": 0.9044585987261147,
      "in_sample": {
        "threshold": 0.9851364969800199,
        "answered": 0.18289353958143767,
        "accuracy": 0.9044585987261147
      },
      "curve": [
        [
          0.0,
          0.9045
        ],
        [
          0.01,
          0.9045
        ],
        [
          0.02,
          0.9045
        ],
        [
          0.03,
          0.9045
        ],
        [
          0.04,
          0.9045
        ],
        [
          0.05,
          0.9045
        ],
        [
          0.0601,
          0.9045
        ],
        [
          0.0705,
          0.9045
        ],
        [
          0.0805,
          0.9045
        ],
        [
          0.0905,
          0.9045
        ],
        [
          0.1005,
          0.9045
        ],
        [
          0.1106,
          0.9045
        ],
        [
          0.121,
          0.9045
        ],
        [
          0.131,
          0.9045
        ],
        [
          0.141,
          0.9045
        ],
        [
          0.151,
          0.9045
        ],
        [
          0.1611,
          0.9045
        ],
        [
          0.1715,
          0.9045
        ],
        [
          0.1815,
          0.9045
        ],
        [
          0.1915,
          0.904
        ],
        [
          0.2015,
          0.9035
        ],
        [
          0.2116,
          0.9035
        ],
        [
          0.222,
          0.9031
        ],
        [
          0.232,
          0.9035
        ],
        [
          0.242,
          0.9035
        ],
        [
          0.252,
          0.9035
        ],
        [
          0.2621,
          0.9026
        ],
        [
          0.2725,
          0.9026
        ],
        [
          0.2825,
          0.9031
        ],
        [
          0.2925,
          0.9035
        ],
        [
          0.3025,
          0.9026
        ],
        [
          0.3126,
          0.9031
        ],
        [
          0.323,
          0.9026
        ],
        [
          0.333,
          0.9017
        ],
        [
          0.343,
          0.8999
        ],
        [
          0.353,
          0.8995
        ],
        [
          0.3631,
          0.8981
        ],
        [
          0.3735,
          0.8972
        ],
        [
          0.3835,
          0.8963
        ],
        [
          0.3935,
          0.8963
        ],
        [
          0.4035,
          0.8935
        ],
        [
          0.4136,
          0.894
        ],
        [
          0.424,
          0.8931
        ],
        [
          0.434,
          0.8908
        ],
        [
          0.444,
          0.8885
        ],
        [
          0.454,
          0.8876
        ],
        [
          0.4641,
          0.8844
        ],
        [
          0.4745,
          0.8831
        ],
        [
          0.4845,
          0.8826
        ],
        [
          0.4945,
          0.8799
        ],
        [
          0.5045,
          0.8753
        ],
        [
          0.5146,
          0.874
        ],
        [
          0.525,
          0.8712
        ],
        [
          0.535,
          0.8676
        ],
        [
          0.545,
          0.8621
        ],
        [
          0.5551,
          0.8576
        ],
        [
          0.5655,
          0.8549
        ],
        [
          0.5755,
          0.8517
        ],
        [
          0.5855,
          0.8494
        ],
        [
          0.5955,
          0.8444
        ],
        [
          0.6056,
          0.8408
        ],
        [
          0.616,
          0.8376
        ],
        [
          0.626,
          0.8339
        ],
        [
          0.636,
          0.8317
        ],
        [
          0.646,
          0.8267
        ],
        [
          0.6561,
          0.823
        ],
        [
          0.6665,
          0.818
        ],
        [
          0.6765,
          0.8167
        ],
        [
          0.6865,
          0.8135
        ],
        [
          0.6965,
          0.8085
        ],
        [
          0.7066,
          0.8057
        ],
        [
          0.717,
          0.8039
        ],
        [
          0.727,
          0.7998
        ],
        [
          0.737,
          0.7966
        ],
        [
          0.747,
          0.7934
        ],
        [
          0.7571,
          0.7884
        ],
        [
          0.7675,
          0.7848
        ],
        [
          0.7775,
          0.7798
        ],
        [
          0.7875,
          0.7766
        ],
        [
          0.7975,
          0.7721
        ],
        [
          0.8076,
          0.768
        ],
        [
          0.818,
          0.7625
        ],
        [
          0.828,
          0.7566
        ],
        [
          0.838,
          0.7525
        ],
        [
          0.848,
          0.747
        ],
        [
          0.8581,
          0.7434
        ],
        [
          0.8685,
          0.7384
        ],
        [
          0.8785,
          0.7329
        ],
        [
          0.8885,
          0.7284
        ],
        [
          0.8985,
          0.722
        ],
        [
          0.9086,
          0.7166
        ],
        [
          0.919,
          0.7102
        ],
        [
          0.929,
          0.707
        ],
        [
          0.939,
          0.7025
        ],
        [
          0.949,
          0.6979
        ],
        [
          0.9591,
          0.6924
        ],
        [
          0.9695,
          0.6861
        ],
        [
          0.9795,
          0.6802
        ],
        [
          0.9895,
          0.6752
        ],
        [
          0.9995,
          0.6692
        ],
        [
          1.0,
          0.6688
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.25199988684521957,
        "difference": -0.0023193230780552206,
        "worse": 0.6733333333333333,
        "kept_right": 0.24786756657096157,
        "kept_wrong": 0.004132320274257803,
        "handed_off": 0.7480001131547805
      },
      "usd_per_1000": 3.695843615384201
    },
    {
      "decision": 145,
      "frontier": 193,
      "frontier_accuracy": 0.89171974522293,
      "in_sample": {
        "threshold": 0.9553897068461282,
        "answered": 0.25796178343949044,
        "accuracy": 0.89171974522293
      },
      "curve": [
        [
          0.0,
          0.8917
        ],
        [
          0.01,
          0.8917
        ],
        [
          0.02,
          0.8917
        ],
        [
          0.03,
          0.8917
        ],
        [
          0.04,
          0.8917
        ],
        [
          0.05,
          0.8917
        ],
        [
          0.0601,
          0.8917
        ],
        [
          0.0705,
          0.8917
        ],
        [
          0.0805,
          0.8917
        ],
        [
          0.0905,
          0.8917
        ],
        [
          0.1005,
          0.8917
        ],
        [
          0.1106,
          0.8917
        ],
        [
          0.121,
          0.8917
        ],
        [
          0.131,
          0.8917
        ],
        [
          0.141,
          0.8917
        ],
        [
          0.151,
          0.8917
        ],
        [
          0.1611,
          0.8917
        ],
        [
          0.1715,
          0.8917
        ],
        [
          0.1815,
          0.8917
        ],
        [
          0.1915,
          0.8917
        ],
        [
          0.2015,
          0.8922
        ],
        [
          0.2116,
          0.8922
        ],
        [
          0.222,
          0.8917
        ],
        [
          0.232,
          0.8922
        ],
        [
          0.242,
          0.8922
        ],
        [
          0.252,
          0.8922
        ],
        [
          0.2621,
          0.8913
        ],
        [
          0.2725,
          0.8913
        ],
        [
          0.2825,
          0.8913
        ],
        [
          0.2925,
          0.8913
        ],
        [
          0.3025,
          0.8904
        ],
        [
          0.3126,
          0.8908
        ],
        [
          0.323,
          0.8904
        ],
        [
          0.333,
          0.889
        ],
        [
          0.343,
          0.8872
        ],
        [
          0.353,
          0.8867
        ],
        [
          0.3631,
          0.8854
        ],
        [
          0.3735,
          0.8844
        ],
        [
          0.3835,
          0.8844
        ],
        [
          0.3935,
          0.8835
        ],
        [
          0.4035,
          0.8808
        ],
        [
          0.4136,
          0.8803
        ],
        [
          0.424,
          0.8803
        ],
        [
          0.434,
          0.8794
        ],
        [
          0.444,
          0.8772
        ],
        [
          0.454,
          0.8763
        ],
        [
          0.4641,
          0.8731
        ],
        [
          0.4745,
          0.8717
        ],
        [
          0.4845,
          0.8703
        ],
        [
          0.4945,
          0.8672
        ],
        [
          0.5045,
          0.8635
        ],
        [
          0.5146,
          0.8631
        ],
        [
          0.525,
          0.8608
        ],
        [
          0.535,
          0.8576
        ],
        [
          0.545,
          0.8517
        ],
        [
          0.5551,
          0.8476
        ],
        [
          0.5655,
          0.8458
        ],
        [
          0.5755,
          0.8426
        ],
        [
          0.5855,
          0.8403
        ],
        [
          0.5955,
          0.8362
        ],
        [
          0.6056,
          0.833
        ],
        [
          0.616,
          0.8298
        ],
        [
          0.626,
          0.8267
        ],
        [
          0.636,
          0.8239
        ],
        [
          0.646,
          0.8203
        ],
        [
          0.6561,
          0.8167
        ],
        [
          0.6665,
          0.8139
        ],
        [
          0.6765,
          0.813
        ],
        [
          0.6865,
          0.8094
        ],
        [
          0.6965,
          0.8048
        ],
        [
          0.7066,
          0.8021
        ],
        [
          0.717,
          0.7994
        ],
        [
          0.727,
          0.7957
        ],
        [
          0.737,
          0.7925
        ],
        [
          0.747,
          0.7894
        ],
        [
          0.7571,
          0.7853
        ],
        [
          0.7675,
          0.7816
        ],
        [
          0.7775,
          0.7766
        ],
        [
          0.7875,
          0.7725
        ],
        [
          0.7975,
          0.7675
        ],
        [
          0.8076,
          0.7648
        ],
        [
          0.818,
          0.7602
        ],
        [
          0.828,
          0.7561
        ],
        [
          0.838,
          0.7516
        ],
        [
          0.848,
          0.7457
        ],
        [
          0.8581,
          0.742
        ],
        [
          0.8685,
          0.7361
        ],
        [
          0.8785,
          0.7316
        ],
        [
          0.8885,
          0.7284
        ],
        [
          0.8985,
          0.7225
        ],
        [
          0.9086,
          0.7184
        ],
        [
          0.919,
          0.7116
        ],
        [
          0.929,
          0.7079
        ],
        [
          0.939,
          0.7034
        ],
        [
          0.949,
          0.6997
        ],
        [
          0.9591,
          0.6943
        ],
        [
          0.9695,
          0.6861
        ],
        [
          0.9795,
          0.6797
        ],
        [
          0.9895,
          0.6738
        ],
        [
          0.9995,
          0.6688
        ],
        [
          1.0,
          0.6688
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.286416091970103,
        "difference": -0.0016038700607607344,
        "worse": 0.5833333333333334,
        "kept_right": 0.28145015855797484,
        "kept_wrong": 0.00496593341212821,
        "handed_off": 0.7135839080298975
      },
      "usd_per_1000": 5.536320293292841
    },
    {
      "decision": 145,
      "frontier": 55,
      "frontier_accuracy": 0.8503184713375797,
      "in_sample": {
        "threshold": 0.7446587945265318,
        "answered": 0.48498635122838946,
        "accuracy": 0.8503184713375797
      },
      "curve": [
        [
          0.0,
          0.8503
        ],
        [
          0.01,
          0.8503
        ],
        [
          0.02,
          0.8503
        ],
        [
          0.03,
          0.8508
        ],
        [
          0.04,
          0.8508
        ],
        [
          0.05,
          0.8508
        ],
        [
          0.0601,
          0.8512
        ],
        [
          0.0705,
          0.8521
        ],
        [
          0.0805,
          0.8526
        ],
        [
          0.0905,
          0.8526
        ],
        [
          0.1005,
          0.8526
        ],
        [
          0.1106,
          0.8526
        ],
        [
          0.121,
          0.8526
        ],
        [
          0.131,
          0.853
        ],
        [
          0.141,
          0.853
        ],
        [
          0.151,
          0.853
        ],
        [
          0.1611,
          0.853
        ],
        [
          0.1715,
          0.8535
        ],
        [
          0.1815,
          0.854
        ],
        [
          0.1915,
          0.8544
        ],
        [
          0.2015,
          0.8544
        ],
        [
          0.2116,
          0.8553
        ],
        [
          0.222,
          0.8558
        ],
        [
          0.232,
          0.8558
        ],
        [
          0.242,
          0.8558
        ],
        [
          0.252,
          0.8567
        ],
        [
          0.2621,
          0.8571
        ],
        [
          0.2725,
          0.8581
        ],
        [
          0.2825,
          0.8599
        ],
        [
          0.2925,
          0.8603
        ],
        [
          0.3025,
          0.8603
        ],
        [
          0.3126,
          0.8608
        ],
        [
          0.323,
          0.8608
        ],
        [
          0.333,
          0.859
        ],
        [
          0.343,
          0.8571
        ],
        [
          0.353,
          0.8585
        ],
        [
          0.3631,
          0.8571
        ],
        [
          0.3735,
          0.8581
        ],
        [
          0.3835,
          0.8581
        ],
        [
          0.3935,
          0.8581
        ],
        [
          0.4035,
          0.8558
        ],
        [
          0.4136,
          0.8558
        ],
        [
          0.424,
          0.8562
        ],
        [
          0.434,
          0.8562
        ],
        [
          0.444,
          0.8553
        ],
        [
          0.454,
          0.8553
        ],
        [
          0.4641,
          0.8517
        ],
        [
          0.4745,
          0.8517
        ],
        [
          0.4845,
          0.8508
        ],
        [
          0.4945,
          0.848
        ],
        [
          0.5045,
          0.8444
        ],
        [
          0.5146,
          0.8444
        ],
        [
          0.525,
          0.8417
        ],
        [
          0.535,
          0.8389
        ],
        [
          0.545,
          0.8348
        ],
        [
          0.5551,
          0.8321
        ],
        [
          0.5655,
          0.8308
        ],
        [
          0.5755,
          0.8285
        ],
        [
          0.5855,
          0.8271
        ],
        [
          0.5955,
          0.823
        ],
        [
          0.6056,
          0.8194
        ],
        [
          0.616,
          0.8171
        ],
        [
          0.626,
          0.8144
        ],
        [
          0.636,
          0.8112
        ],
        [
          0.646,
          0.8066
        ],
        [
          0.6561,
          0.803
        ],
        [
          0.6665,
          0.7998
        ],
        [
          0.6765,
          0.7985
        ],
        [
          0.6865,
          0.7962
        ],
        [
          0.6965,
          0.7921
        ],
        [
          0.7066,
          0.7903
        ],
        [
          0.717,
          0.788
        ],
        [
          0.727,
          0.7843
        ],
        [
          0.737,
          0.7812
        ],
        [
          0.747,
          0.778
        ],
        [
          0.7571,
          0.7739
        ],
        [
          0.7675,
          0.7702
        ],
        [
          0.7775,
          0.7666
        ],
        [
          0.7875,
          0.7625
        ],
        [
          0.7975,
          0.7593
        ],
        [
          0.8076,
          0.7557
        ],
        [
          0.818,
          0.7502
        ],
        [
          0.828,
          0.7461
        ],
        [
          0.838,
          0.7429
        ],
        [
          0.848,
          0.737
        ],
        [
          0.8581,
          0.7343
        ],
        [
          0.8685,
          0.7293
        ],
        [
          0.8785,
          0.7261
        ],
        [
          0.8885,
          0.7225
        ],
        [
          0.8985,
          0.7161
        ],
        [
          0.9086,
          0.7143
        ],
        [
          0.919,
          0.7088
        ],
        [
          0.929,
          0.7056
        ],
        [
          0.939,
          0.7015
        ],
        [
          0.949,
          0.6975
        ],
        [
          0.9591,
          0.6934
        ],
        [
          0.9695,
          0.6861
        ],
        [
          0.9795,
          0.6797
        ],
        [
          0.9895,
          0.6747
        ],
        [
          0.9995,
          0.6692
        ],
        [
          1.0,
          0.6688
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.48314591113670324,
        "difference": -0.0009956317531289965,
        "worse": 0.4533333333333333,
        "kept_right": 0.4426845553172104,
        "kept_wrong": 0.04046135581949293,
        "handed_off": 0.5168540888632966
      },
      "usd_per_1000": 1.8965220232933593
    },
    {
      "decision": 177,
      "frontier": 140,
      "frontier_accuracy": 0.9808917197452229,
      "in_sample": {
        "threshold": 0.9537515044212341,
        "answered": 0.14240218380345768,
        "accuracy": 0.9808917197452229
      },
      "curve": [
        [
          0.0,
          0.9809
        ],
        [
          0.01,
          0.9809
        ],
        [
          0.02,
          0.9809
        ],
        [
          0.03,
          0.9809
        ],
        [
          0.04,
          0.9809
        ],
        [
          0.05,
          0.9809
        ],
        [
          0.0601,
          0.9809
        ],
        [
          0.0705,
          0.9809
        ],
        [
          0.0805,
          0.9809
        ],
        [
          0.0905,
          0.9809
        ],
        [
          0.1005,
          0.9809
        ],
        [
          0.1106,
          0.9809
        ],
        [
          0.121,
          0.9809
        ],
        [
          0.131,
          0.9809
        ],
        [
          0.141,
          0.9809
        ],
        [
          0.151,
          0.9804
        ],
        [
          0.1611,
          0.9804
        ],
        [
          0.1715,
          0.98
        ],
        [
          0.1815,
          0.9795
        ],
        [
          0.1915,
          0.9786
        ],
        [
          0.2015,
          0.9782
        ],
        [
          0.2116,
          0.9777
        ],
        [
          0.222,
          0.9777
        ],
        [
          0.232,
          0.9763
        ],
        [
          0.242,
          0.9763
        ],
        [
          0.252,
          0.9754
        ],
        [
          0.2621,
          0.9745
        ],
        [
          0.2725,
          0.9736
        ],
        [
          0.2825,
          0.9722
        ],
        [
          0.2925,
          0.9709
        ],
        [
          0.3025,
          0.9713
        ],
        [
          0.3126,
          0.9704
        ],
        [
          0.323,
          0.9677
        ],
        [
          0.333,
          0.9672
        ],
        [
          0.343,
          0.9654
        ],
        [
          0.353,
          0.965
        ],
        [
          0.3631,
          0.9636
        ],
        [
          0.3735,
          0.9618
        ],
        [
          0.3835,
          0.9613
        ],
        [
          0.3935,
          0.96
        ],
        [
          0.4035,
          0.9591
        ],
        [
          0.4136,
          0.9572
        ],
        [
          0.424,
          0.954
        ],
        [
          0.434,
          0.9527
        ],
        [
          0.444,
          0.9495
        ],
        [
          0.454,
          0.9468
        ],
        [
          0.4641,
          0.944
        ],
        [
          0.4745,
          0.9409
        ],
        [
          0.4845,
          0.9359
        ],
        [
          0.4945,
          0.9331
        ],
        [
          0.5045,
          0.9304
        ],
        [
          0.5146,
          0.9254
        ],
        [
          0.525,
          0.9227
        ],
        [
          0.535,
          0.919
        ],
        [
          0.545,
          0.9167
        ],
        [
          0.5551,
          0.9126
        ],
        [
          0.5655,
          0.9072
        ],
        [
          0.5755,
          0.9035
        ],
        [
          0.5855,
          0.8981
        ],
        [
          0.5955,
          0.8926
        ],
        [
          0.6056,
          0.8894
        ],
        [
          0.616,
          0.8849
        ],
        [
          0.626,
          0.8785
        ],
        [
          0.636,
          0.8726
        ],
        [
          0.646,
          0.8681
        ],
        [
          0.6561,
          0.8621
        ],
        [
          0.6665,
          0.8571
        ],
        [
          0.6765,
          0.8512
        ],
        [
          0.6865,
          0.8453
        ],
        [
          0.6965,
          0.8399
        ],
        [
          0.7066,
          0.8376
        ],
        [
          0.717,
          0.8298
        ],
        [
          0.727,
          0.8262
        ],
        [
          0.737,
          0.818
        ],
        [
          0.747,
          0.8135
        ],
        [
          0.7571,
          0.8066
        ],
        [
          0.7675,
          0.798
        ],
        [
          0.7775,
          0.793
        ],
        [
          0.7875,
          0.7898
        ],
        [
          0.7975,
          0.7834
        ],
        [
          0.8076,
          0.7789
        ],
        [
          0.818,
          0.7748
        ],
        [
          0.828,
          0.768
        ],
        [
          0.838,
          0.7621
        ],
        [
          0.848,
          0.7543
        ],
        [
          0.8581,
          0.7489
        ],
        [
          0.8685,
          0.7407
        ],
        [
          0.8785,
          0.7343
        ],
        [
          0.8885,
          0.7298
        ],
        [
          0.8985,
          0.722
        ],
        [
          0.9086,
          0.7152
        ],
        [
          0.919,
          0.7093
        ],
        [
          0.929,
          0.7029
        ],
        [
          0.939,
          0.6956
        ],
        [
          0.949,
          0.6888
        ],
        [
          0.9591,
          0.6838
        ],
        [
          0.9695,
          0.6765
        ],
        [
          0.9795,
          0.6688
        ],
        [
          0.9895,
          0.6611
        ],
        [
          0.9995,
          0.6529
        ],
        [
          1.0,
          0.6524
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.15830064751271303,
        "difference": -0.0008699930722628275,
        "worse": 0.5066666666666667,
        "kept_right": 0.1574306544404503,
        "kept_wrong": 0.0008699930722628275,
        "handed_off": 0.8416993524872872
      },
      "usd_per_1000": 2.0555694394575457
    },
    {
      "decision": 177,
      "frontier": 277,
      "frontier_accuracy": 0.9795268425841674,
      "in_sample": {
        "threshold": 0.9537515044212341,
        "answered": 0.14240218380345768,
        "accuracy": 0.9795268425841674
      },
      "curve": [
        [
          0.0,
          0.9795
        ],
        [
          0.01,
          0.9795
        ],
        [
          0.02,
          0.9795
        ],
        [
          0.03,
          0.9795
        ],
        [
          0.04,
          0.9795
        ],
        [
          0.05,
          0.9795
        ],
        [
          0.0601,
          0.9795
        ],
        [
          0.0705,
          0.9795
        ],
        [
          0.0805,
          0.9795
        ],
        [
          0.0905,
          0.9795
        ],
        [
          0.1005,
          0.9795
        ],
        [
          0.1106,
          0.9795
        ],
        [
          0.121,
          0.9795
        ],
        [
          0.131,
          0.9795
        ],
        [
          0.141,
          0.9795
        ],
        [
          0.151,
          0.9791
        ],
        [
          0.1611,
          0.9791
        ],
        [
          0.1715,
          0.9786
        ],
        [
          0.1815,
          0.9782
        ],
        [
          0.1915,
          0.9773
        ],
        [
          0.2015,
          0.9768
        ],
        [
          0.2116,
          0.9763
        ],
        [
          0.222,
          0.9763
        ],
        [
          0.232,
          0.975
        ],
        [
          0.242,
          0.975
        ],
        [
          0.252,
          0.9741
        ],
        [
          0.2621,
          0.9732
        ],
        [
          0.2725,
          0.9727
        ],
        [
          0.2825,
          0.9713
        ],
        [
          0.2925,
          0.97
        ],
        [
          0.3025,
          0.9709
        ],
        [
          0.3126,
          0.9695
        ],
        [
          0.323,
          0.9668
        ],
        [
          0.333,
          0.9668
        ],
        [
          0.343,
          0.965
        ],
        [
          0.353,
          0.9645
        ],
        [
          0.3631,
          0.9627
        ],
        [
          0.3735,
          0.9609
        ],
        [
          0.3835,
          0.9604
        ],
        [
          0.3935,
          0.9591
        ],
        [
          0.4035,
          0.9581
        ],
        [
          0.4136,
          0.9563
        ],
        [
          0.424,
          0.9531
        ],
        [
          0.434,
          0.9518
        ],
        [
          0.444,
          0.9486
        ],
        [
          0.454,
          0.9459
        ],
        [
          0.4641,
          0.9436
        ],
        [
          0.4745,
          0.9395
        ],
        [
          0.4845,
          0.9345
        ],
        [
          0.4945,
          0.9318
        ],
        [
          0.5045,
          0.9286
        ],
        [
          0.5146,
          0.924
        ],
        [
          0.525,
          0.9217
        ],
        [
          0.535,
          0.9181
        ],
        [
          0.545,
          0.9163
        ],
        [
          0.5551,
          0.9117
        ],
        [
          0.5655,
          0.9063
        ],
        [
          0.5755,
          0.9026
        ],
        [
          0.5855,
          0.8972
        ],
        [
          0.5955,
          0.8922
        ],
        [
          0.6056,
          0.889
        ],
        [
          0.616,
          0.8844
        ],
        [
          0.626,
          0.8781
        ],
        [
          0.636,
          0.8722
        ],
        [
          0.646,
          0.8676
        ],
        [
          0.6561,
          0.8617
        ],
        [
          0.6665,
          0.8567
        ],
        [
          0.6765,
          0.8512
        ],
        [
          0.6865,
          0.8453
        ],
        [
          0.6965,
          0.8399
        ],
        [
          0.7066,
          0.8376
        ],
        [
          0.717,
          0.8303
        ],
        [
          0.727,
          0.8262
        ],
        [
          0.737,
          0.818
        ],
        [
          0.747,
          0.8135
        ],
        [
          0.7571,
          0.8062
        ],
        [
          0.7675,
          0.7971
        ],
        [
          0.7775,
          0.7912
        ],
        [
          0.7875,
          0.7884
        ],
        [
          0.7975,
          0.7821
        ],
        [
          0.8076,
          0.7775
        ],
        [
          0.818,
          0.7734
        ],
        [
          0.828,
          0.7666
        ],
        [
          0.838,
          0.7607
        ],
        [
          0.848,
          0.753
        ],
        [
          0.8581,
          0.748
        ],
        [
          0.8685,
          0.7393
        ],
        [
          0.8785,
          0.7334
        ],
        [
          0.8885,
          0.7288
        ],
        [
          0.8985,
          0.7216
        ],
        [
          0.9086,
          0.7147
        ],
        [
          0.919,
          0.7093
        ],
        [
          0.929,
          0.7029
        ],
        [
          0.939,
          0.6956
        ],
        [
          0.949,
          0.6888
        ],
        [
          0.9591,
          0.6838
        ],
        [
          0.9695,
          0.6765
        ],
        [
          0.9795,
          0.6688
        ],
        [
          0.9895,
          0.6611
        ],
        [
          0.9995,
          0.6529
        ],
        [
          1.0,
          0.6524
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.15830064751271303,
        "difference": -0.0008699930722628275,
        "worse": 0.5066666666666667,
        "kept_right": 0.1574306544404503,
        "kept_wrong": 0.0008699930722628275,
        "handed_off": 0.8416993524872872
      },
      "usd_per_1000": 5.124785632530589
    },
    {
      "decision": 177,
      "frontier": 276,
      "frontier_accuracy": 0.9777070063694268,
      "in_sample": {
        "threshold": 0.9537515044212341,
        "answered": 0.14240218380345768,
        "accuracy": 0.9777070063694268
      },
      "curve": [
        [
          0.0,
          0.9777
        ],
        [
          0.01,
          0.9777
        ],
        [
          0.02,
          0.9777
        ],
        [
          0.03,
          0.9777
        ],
        [
          0.04,
          0.9777
        ],
        [
          0.05,
          0.9777
        ],
        [
          0.0601,
          0.9777
        ],
        [
          0.0705,
          0.9777
        ],
        [
          0.0805,
          0.9777
        ],
        [
          0.0905,
          0.9777
        ],
        [
          0.1005,
          0.9777
        ],
        [
          0.1106,
          0.9777
        ],
        [
          0.121,
          0.9777
        ],
        [
          0.131,
          0.9777
        ],
        [
          0.141,
          0.9777
        ],
        [
          0.151,
          0.9773
        ],
        [
          0.1611,
          0.9773
        ],
        [
          0.1715,
          0.9768
        ],
        [
          0.1815,
          0.9763
        ],
        [
          0.1915,
          0.9759
        ],
        [
          0.2015,
          0.9754
        ],
        [
          0.2116,
          0.975
        ],
        [
          0.222,
          0.975
        ],
        [
          0.232,
          0.9736
        ],
        [
          0.242,
          0.9736
        ],
        [
          0.252,
          0.9727
        ],
        [
          0.2621,
          0.9722
        ],
        [
          0.2725,
          0.9718
        ],
        [
          0.2825,
          0.9704
        ],
        [
          0.2925,
          0.9691
        ],
        [
          0.3025,
          0.97
        ],
        [
          0.3126,
          0.9686
        ],
        [
          0.323,
          0.9659
        ],
        [
          0.333,
          0.9663
        ],
        [
          0.343,
          0.965
        ],
        [
          0.353,
          0.9645
        ],
        [
          0.3631,
          0.9627
        ],
        [
          0.3735,
          0.9609
        ],
        [
          0.3835,
          0.9604
        ],
        [
          0.3935,
          0.9591
        ],
        [
          0.4035,
          0.9581
        ],
        [
          0.4136,
          0.9563
        ],
        [
          0.424,
          0.9531
        ],
        [
          0.434,
          0.9518
        ],
        [
          0.444,
          0.9486
        ],
        [
          0.454,
          0.9459
        ],
        [
          0.4641,
          0.9436
        ],
        [
          0.4745,
          0.9395
        ],
        [
          0.4845,
          0.9345
        ],
        [
          0.4945,
          0.9318
        ],
        [
          0.5045,
          0.9286
        ],
        [
          0.5146,
          0.924
        ],
        [
          0.525,
          0.9217
        ],
        [
          0.535,
          0.9181
        ],
        [
          0.545,
          0.9163
        ],
        [
          0.5551,
          0.9117
        ],
        [
          0.5655,
          0.9067
        ],
        [
          0.5755,
          0.9031
        ],
        [
          0.5855,
          0.8976
        ],
        [
          0.5955,
          0.8926
        ],
        [
          0.6056,
          0.8899
        ],
        [
          0.616,
          0.8854
        ],
        [
          0.626,
          0.879
        ],
        [
          0.636,
          0.8726
        ],
        [
          0.646,
          0.8681
        ],
        [
          0.6561,
          0.8621
        ],
        [
          0.6665,
          0.8571
        ],
        [
          0.6765,
          0.8517
        ],
        [
          0.6865,
          0.8458
        ],
        [
          0.6965,
          0.8403
        ],
        [
          0.7066,
          0.8385
        ],
        [
          0.717,
          0.8312
        ],
        [
          0.727,
          0.8271
        ],
        [
          0.737,
          0.8189
        ],
        [
          0.747,
          0.8144
        ],
        [
          0.7571,
          0.8071
        ],
        [
          0.7675,
          0.798
        ],
        [
          0.7775,
          0.7921
        ],
        [
          0.7875,
          0.7894
        ],
        [
          0.7975,
          0.783
        ],
        [
          0.8076,
          0.7784
        ],
        [
          0.818,
          0.7743
        ],
        [
          0.828,
          0.7675
        ],
        [
          0.838,
          0.7616
        ],
        [
          0.848,
          0.7539
        ],
        [
          0.8581,
          0.7484
        ],
        [
          0.8685,
          0.7402
        ],
        [
          0.8785,
          0.7338
        ],
        [
          0.8885,
          0.7293
        ],
        [
          0.8985,
          0.722
        ],
        [
          0.9086,
          0.7152
        ],
        [
          0.919,
          0.7093
        ],
        [
          0.929,
          0.7029
        ],
        [
          0.939,
          0.6956
        ],
        [
          0.949,
          0.6888
        ],
        [
          0.9591,
          0.6838
        ],
        [
          0.9695,
          0.6765
        ],
        [
          0.9795,
          0.6688
        ],
        [
          0.9895,
          0.6611
        ],
        [
          0.9995,
          0.6529
        ],
        [
          1.0,
          0.6524
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.15884638632934817,
        "difference": -0.0008609104100971802,
        "worse": 0.5066666666666667,
        "kept_right": 0.15795517286389044,
        "kept_wrong": 0.0008912134654577428,
        "handed_off": 0.8411536136706521
      },
      "usd_per_1000": 1.3839230004140675
    },
    {
      "decision": 177,
      "frontier": 48,
      "frontier_accuracy": 0.9758871701546861,
      "in_sample": {
        "threshold": 0.9537515044212341,
        "answered": 0.14240218380345768,
        "accuracy": 0.9758871701546861
      },
      "curve": [
        [
          0.0,
          0.9759
        ],
        [
          0.01,
          0.9759
        ],
        [
          0.02,
          0.9759
        ],
        [
          0.03,
          0.9759
        ],
        [
          0.04,
          0.9759
        ],
        [
          0.05,
          0.9759
        ],
        [
          0.0601,
          0.9759
        ],
        [
          0.0705,
          0.9759
        ],
        [
          0.0805,
          0.9759
        ],
        [
          0.0905,
          0.9759
        ],
        [
          0.1005,
          0.9759
        ],
        [
          0.1106,
          0.9759
        ],
        [
          0.121,
          0.9759
        ],
        [
          0.131,
          0.9759
        ],
        [
          0.141,
          0.9759
        ],
        [
          0.151,
          0.9754
        ],
        [
          0.1611,
          0.9754
        ],
        [
          0.1715,
          0.975
        ],
        [
          0.1815,
          0.9745
        ],
        [
          0.1915,
          0.9736
        ],
        [
          0.2015,
          0.9732
        ],
        [
          0.2116,
          0.9727
        ],
        [
          0.222,
          0.9727
        ],
        [
          0.232,
          0.9713
        ],
        [
          0.242,
          0.9713
        ],
        [
          0.252,
          0.9704
        ],
        [
          0.2621,
          0.9695
        ],
        [
          0.2725,
          0.9686
        ],
        [
          0.2825,
          0.9672
        ],
        [
          0.2925,
          0.9659
        ],
        [
          0.3025,
          0.9668
        ],
        [
          0.3126,
          0.9654
        ],
        [
          0.323,
          0.9627
        ],
        [
          0.333,
          0.9631
        ],
        [
          0.343,
          0.9613
        ],
        [
          0.353,
          0.9609
        ],
        [
          0.3631,
          0.9591
        ],
        [
          0.3735,
          0.9572
        ],
        [
          0.3835,
          0.9568
        ],
        [
          0.3935,
          0.9554
        ],
        [
          0.4035,
          0.955
        ],
        [
          0.4136,
          0.9531
        ],
        [
          0.424,
          0.95
        ],
        [
          0.434,
          0.9486
        ],
        [
          0.444,
          0.9454
        ],
        [
          0.454,
          0.9427
        ],
        [
          0.4641,
          0.9404
        ],
        [
          0.4745,
          0.9363
        ],
        [
          0.4845,
          0.9313
        ],
        [
          0.4945,
          0.9286
        ],
        [
          0.5045,
          0.9254
        ],
        [
          0.5146,
          0.9208
        ],
        [
          0.525,
          0.9186
        ],
        [
          0.535,
          0.9149
        ],
        [
          0.545,
          0.9136
        ],
        [
          0.5551,
          0.909
        ],
        [
          0.5655,
          0.904
        ],
        [
          0.5755,
          0.9004
        ],
        [
          0.5855,
          0.8954
        ],
        [
          0.5955,
          0.8908
        ],
        [
          0.6056,
          0.8881
        ],
        [
          0.616,
          0.8835
        ],
        [
          0.626,
          0.8772
        ],
        [
          0.636,
          0.8708
        ],
        [
          0.646,
          0.8662
        ],
        [
          0.6561,
          0.8608
        ],
        [
          0.6665,
          0.8558
        ],
        [
          0.6765,
          0.8503
        ],
        [
          0.6865,
          0.8444
        ],
        [
          0.6965,
          0.8389
        ],
        [
          0.7066,
          0.8367
        ],
        [
          0.717,
          0.8294
        ],
        [
          0.727,
          0.8248
        ],
        [
          0.737,
          0.8167
        ],
        [
          0.747,
          0.8126
        ],
        [
          0.7571,
          0.8057
        ],
        [
          0.7675,
          0.7971
        ],
        [
          0.7775,
          0.7916
        ],
        [
          0.7875,
          0.7884
        ],
        [
          0.7975,
          0.7816
        ],
        [
          0.8076,
          0.7771
        ],
        [
          0.818,
          0.773
        ],
        [
          0.828,
          0.7662
        ],
        [
          0.838,
          0.7602
        ],
        [
          0.848,
          0.7525
        ],
        [
          0.8581,
          0.7475
        ],
        [
          0.8685,
          0.7393
        ],
        [
          0.8785,
          0.7329
        ],
        [
          0.8885,
          0.7284
        ],
        [
          0.8985,
          0.7211
        ],
        [
          0.9086,
          0.7143
        ],
        [
          0.919,
          0.7088
        ],
        [
          0.929,
          0.7029
        ],
        [
          0.939,
          0.6956
        ],
        [
          0.949,
          0.6888
        ],
        [
          0.9591,
          0.6838
        ],
        [
          0.9695,
          0.6765
        ],
        [
          0.9795,
          0.6688
        ],
        [
          0.9895,
          0.6611
        ],
        [
          0.9995,
          0.6529
        ],
        [
          1.0,
          0.6524
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.15830064751271303,
        "difference": -0.0008699930722628275,
        "worse": 0.5066666666666667,
        "kept_right": 0.1574306544404503,
        "kept_wrong": 0.0008699930722628275,
        "handed_off": 0.8416993524872872
      },
      "usd_per_1000": 4.5462614218256885
    },
    {
      "decision": 177,
      "frontier": 191,
      "frontier_accuracy": 0.9740673339399454,
      "in_sample": {
        "threshold": 0.9537515044212341,
        "answered": 0.14240218380345768,
        "accuracy": 0.9740673339399454
      },
      "curve": [
        [
          0.0,
          0.9741
        ],
        [
          0.01,
          0.9741
        ],
        [
          0.02,
          0.9741
        ],
        [
          0.03,
          0.9741
        ],
        [
          0.04,
          0.9741
        ],
        [
          0.05,
          0.9741
        ],
        [
          0.0601,
          0.9741
        ],
        [
          0.0705,
          0.9741
        ],
        [
          0.0805,
          0.9741
        ],
        [
          0.0905,
          0.9741
        ],
        [
          0.1005,
          0.9741
        ],
        [
          0.1106,
          0.9741
        ],
        [
          0.121,
          0.9741
        ],
        [
          0.131,
          0.9741
        ],
        [
          0.141,
          0.9741
        ],
        [
          0.151,
          0.9736
        ],
        [
          0.1611,
          0.9736
        ],
        [
          0.1715,
          0.9732
        ],
        [
          0.1815,
          0.9727
        ],
        [
          0.1915,
          0.9718
        ],
        [
          0.2015,
          0.9713
        ],
        [
          0.2116,
          0.9709
        ],
        [
          0.222,
          0.9709
        ],
        [
          0.232,
          0.97
        ],
        [
          0.242,
          0.97
        ],
        [
          0.252,
          0.9691
        ],
        [
          0.2621,
          0.9682
        ],
        [
          0.2725,
          0.9672
        ],
        [
          0.2825,
          0.9659
        ],
        [
          0.2925,
          0.9645
        ],
        [
          0.3025,
          0.965
        ],
        [
          0.3126,
          0.9641
        ],
        [
          0.323,
          0.9618
        ],
        [
          0.333,
          0.9618
        ],
        [
          0.343,
          0.96
        ],
        [
          0.353,
          0.9595
        ],
        [
          0.3631,
          0.9577
        ],
        [
          0.3735,
          0.9559
        ],
        [
          0.3835,
          0.9554
        ],
        [
          0.3935,
          0.954
        ],
        [
          0.4035,
          0.9536
        ],
        [
          0.4136,
          0.9518
        ],
        [
          0.424,
          0.9486
        ],
        [
          0.434,
          0.9472
        ],
        [
          0.444,
          0.944
        ],
        [
          0.454,
          0.9418
        ],
        [
          0.4641,
          0.9399
        ],
        [
          0.4745,
          0.9359
        ],
        [
          0.4845,
          0.9308
        ],
        [
          0.4945,
          0.9281
        ],
        [
          0.5045,
          0.9249
        ],
        [
          0.5146,
          0.9204
        ],
        [
          0.525,
          0.9186
        ],
        [
          0.535,
          0.9149
        ],
        [
          0.545,
          0.9131
        ],
        [
          0.5551,
          0.9086
        ],
        [
          0.5655,
          0.9035
        ],
        [
          0.5755,
          0.8999
        ],
        [
          0.5855,
          0.8949
        ],
        [
          0.5955,
          0.8904
        ],
        [
          0.6056,
          0.8876
        ],
        [
          0.616,
          0.8831
        ],
        [
          0.626,
          0.8767
        ],
        [
          0.636,
          0.8703
        ],
        [
          0.646,
          0.8658
        ],
        [
          0.6561,
          0.8608
        ],
        [
          0.6665,
          0.8553
        ],
        [
          0.6765,
          0.8494
        ],
        [
          0.6865,
          0.8435
        ],
        [
          0.6965,
          0.838
        ],
        [
          0.7066,
          0.8367
        ],
        [
          0.717,
          0.8294
        ],
        [
          0.727,
          0.8248
        ],
        [
          0.737,
          0.8167
        ],
        [
          0.747,
          0.8121
        ],
        [
          0.7571,
          0.8057
        ],
        [
          0.7675,
          0.7966
        ],
        [
          0.7775,
          0.7912
        ],
        [
          0.7875,
          0.7884
        ],
        [
          0.7975,
          0.7821
        ],
        [
          0.8076,
          0.7775
        ],
        [
          0.818,
          0.7734
        ],
        [
          0.828,
          0.7666
        ],
        [
          0.838,
          0.7611
        ],
        [
          0.848,
          0.7539
        ],
        [
          0.8581,
          0.7489
        ],
        [
          0.8685,
          0.7402
        ],
        [
          0.8785,
          0.7338
        ],
        [
          0.8885,
          0.7288
        ],
        [
          0.8985,
          0.722
        ],
        [
          0.9086,
          0.7152
        ],
        [
          0.919,
          0.7093
        ],
        [
          0.929,
          0.7029
        ],
        [
          0.939,
          0.6956
        ],
        [
          0.949,
          0.6888
        ],
        [
          0.9591,
          0.6838
        ],
        [
          0.9695,
          0.6765
        ],
        [
          0.9795,
          0.6688
        ],
        [
          0.9895,
          0.6611
        ],
        [
          0.9995,
          0.6529
        ],
        [
          1.0,
          0.6524
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1583218596339252,
        "difference": -0.0008699930722628275,
        "worse": 0.5066666666666667,
        "kept_right": 0.15745186656166243,
        "kept_wrong": 0.0008699930722628275,
        "handed_off": 0.8416781403660749
      },
      "usd_per_1000": 22.523946194907126
    },
    {
      "decision": 177,
      "frontier": 188,
      "frontier_accuracy": 0.9713375796178344,
      "in_sample": {
        "threshold": 0.9537515044212341,
        "answered": 0.14240218380345768,
        "accuracy": 0.9713375796178344
      },
      "curve": [
        [
          0.0,
          0.9713
        ],
        [
          0.01,
          0.9713
        ],
        [
          0.02,
          0.9713
        ],
        [
          0.03,
          0.9713
        ],
        [
          0.04,
          0.9713
        ],
        [
          0.05,
          0.9713
        ],
        [
          0.0601,
          0.9713
        ],
        [
          0.0705,
          0.9713
        ],
        [
          0.0805,
          0.9713
        ],
        [
          0.0905,
          0.9713
        ],
        [
          0.1005,
          0.9713
        ],
        [
          0.1106,
          0.9713
        ],
        [
          0.121,
          0.9713
        ],
        [
          0.131,
          0.9713
        ],
        [
          0.141,
          0.9713
        ],
        [
          0.151,
          0.9709
        ],
        [
          0.1611,
          0.9709
        ],
        [
          0.1715,
          0.9709
        ],
        [
          0.1815,
          0.9704
        ],
        [
          0.1915,
          0.9695
        ],
        [
          0.2015,
          0.9691
        ],
        [
          0.2116,
          0.9686
        ],
        [
          0.222,
          0.9686
        ],
        [
          0.232,
          0.9672
        ],
        [
          0.242,
          0.9672
        ],
        [
          0.252,
          0.9668
        ],
        [
          0.2621,
          0.9663
        ],
        [
          0.2725,
          0.9659
        ],
        [
          0.2825,
          0.9645
        ],
        [
          0.2925,
          0.9631
        ],
        [
          0.3025,
          0.9636
        ],
        [
          0.3126,
          0.9627
        ],
        [
          0.323,
          0.96
        ],
        [
          0.333,
          0.96
        ],
        [
          0.343,
          0.9581
        ],
        [
          0.353,
          0.9577
        ],
        [
          0.3631,
          0.9563
        ],
        [
          0.3735,
          0.9545
        ],
        [
          0.3835,
          0.954
        ],
        [
          0.3935,
          0.9531
        ],
        [
          0.4035,
          0.9527
        ],
        [
          0.4136,
          0.9509
        ],
        [
          0.424,
          0.9472
        ],
        [
          0.434,
          0.9459
        ],
        [
          0.444,
          0.9427
        ],
        [
          0.454,
          0.9399
        ],
        [
          0.4641,
          0.9377
        ],
        [
          0.4745,
          0.9349
        ],
        [
          0.4845,
          0.9299
        ],
        [
          0.4945,
          0.9272
        ],
        [
          0.5045,
          0.9245
        ],
        [
          0.5146,
          0.9195
        ],
        [
          0.525,
          0.9167
        ],
        [
          0.535,
          0.9131
        ],
        [
          0.545,
          0.9113
        ],
        [
          0.5551,
          0.9072
        ],
        [
          0.5655,
          0.9022
        ],
        [
          0.5755,
          0.8985
        ],
        [
          0.5855,
          0.8935
        ],
        [
          0.5955,
          0.8885
        ],
        [
          0.6056,
          0.8854
        ],
        [
          0.616,
          0.8808
        ],
        [
          0.626,
          0.8744
        ],
        [
          0.636,
          0.8685
        ],
        [
          0.646,
          0.864
        ],
        [
          0.6561,
          0.8581
        ],
        [
          0.6665,
          0.8526
        ],
        [
          0.6765,
          0.8471
        ],
        [
          0.6865,
          0.8412
        ],
        [
          0.6965,
          0.8362
        ],
        [
          0.7066,
          0.8344
        ],
        [
          0.717,
          0.8271
        ],
        [
          0.727,
          0.8235
        ],
        [
          0.737,
          0.8153
        ],
        [
          0.747,
          0.8098
        ],
        [
          0.7571,
          0.803
        ],
        [
          0.7675,
          0.7944
        ],
        [
          0.7775,
          0.7894
        ],
        [
          0.7875,
          0.7862
        ],
        [
          0.7975,
          0.7798
        ],
        [
          0.8076,
          0.7753
        ],
        [
          0.818,
          0.7712
        ],
        [
          0.828,
          0.7648
        ],
        [
          0.838,
          0.7589
        ],
        [
          0.848,
          0.752
        ],
        [
          0.8581,
          0.747
        ],
        [
          0.8685,
          0.7389
        ],
        [
          0.8785,
          0.7329
        ],
        [
          0.8885,
          0.7284
        ],
        [
          0.8985,
          0.7207
        ],
        [
          0.9086,
          0.7143
        ],
        [
          0.919,
          0.7084
        ],
        [
          0.929,
          0.7025
        ],
        [
          0.939,
          0.6956
        ],
        [
          0.949,
          0.6888
        ],
        [
          0.9591,
          0.6838
        ],
        [
          0.9695,
          0.6765
        ],
        [
          0.9795,
          0.6688
        ],
        [
          0.9895,
          0.6611
        ],
        [
          0.9995,
          0.6529
        ],
        [
          1.0,
          0.6524
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1649493333596704,
        "difference": -0.0011519797469792096,
        "worse": 0.5966666666666667,
        "kept_right": 0.16367308065720648,
        "kept_wrong": 0.0012762527024639788,
        "handed_off": 0.8350506666403297
      },
      "usd_per_1000": 0.8239329481311317
    },
    {
      "decision": 177,
      "frontier": 190,
      "frontier_accuracy": 0.9672429481346679,
      "in_sample": {
        "threshold": 0.9537515044212341,
        "answered": 0.14240218380345768,
        "accuracy": 0.9672429481346679
      },
      "curve": [
        [
          0.0,
          0.9672
        ],
        [
          0.01,
          0.9672
        ],
        [
          0.02,
          0.9672
        ],
        [
          0.03,
          0.9672
        ],
        [
          0.04,
          0.9672
        ],
        [
          0.05,
          0.9672
        ],
        [
          0.0601,
          0.9672
        ],
        [
          0.0705,
          0.9672
        ],
        [
          0.0805,
          0.9672
        ],
        [
          0.0905,
          0.9672
        ],
        [
          0.1005,
          0.9672
        ],
        [
          0.1106,
          0.9672
        ],
        [
          0.121,
          0.9672
        ],
        [
          0.131,
          0.9672
        ],
        [
          0.141,
          0.9672
        ],
        [
          0.151,
          0.9668
        ],
        [
          0.1611,
          0.9668
        ],
        [
          0.1715,
          0.9663
        ],
        [
          0.1815,
          0.9659
        ],
        [
          0.1915,
          0.9654
        ],
        [
          0.2015,
          0.965
        ],
        [
          0.2116,
          0.9645
        ],
        [
          0.222,
          0.9645
        ],
        [
          0.232,
          0.9636
        ],
        [
          0.242,
          0.9636
        ],
        [
          0.252,
          0.9627
        ],
        [
          0.2621,
          0.9622
        ],
        [
          0.2725,
          0.9622
        ],
        [
          0.2825,
          0.9609
        ],
        [
          0.2925,
          0.9595
        ],
        [
          0.3025,
          0.96
        ],
        [
          0.3126,
          0.9586
        ],
        [
          0.323,
          0.9559
        ],
        [
          0.333,
          0.9559
        ],
        [
          0.343,
          0.9545
        ],
        [
          0.353,
          0.954
        ],
        [
          0.3631,
          0.9522
        ],
        [
          0.3735,
          0.9504
        ],
        [
          0.3835,
          0.9504
        ],
        [
          0.3935,
          0.949
        ],
        [
          0.4035,
          0.9481
        ],
        [
          0.4136,
          0.9463
        ],
        [
          0.424,
          0.9431
        ],
        [
          0.434,
          0.9418
        ],
        [
          0.444,
          0.9386
        ],
        [
          0.454,
          0.9363
        ],
        [
          0.4641,
          0.934
        ],
        [
          0.4745,
          0.9299
        ],
        [
          0.4845,
          0.9249
        ],
        [
          0.4945,
          0.9222
        ],
        [
          0.5045,
          0.919
        ],
        [
          0.5146,
          0.9145
        ],
        [
          0.525,
          0.9122
        ],
        [
          0.535,
          0.9086
        ],
        [
          0.545,
          0.9067
        ],
        [
          0.5551,
          0.9022
        ],
        [
          0.5655,
          0.8972
        ],
        [
          0.5755,
          0.8935
        ],
        [
          0.5855,
          0.8885
        ],
        [
          0.5955,
          0.884
        ],
        [
          0.6056,
          0.8813
        ],
        [
          0.616,
          0.8767
        ],
        [
          0.626,
          0.8703
        ],
        [
          0.636,
          0.864
        ],
        [
          0.646,
          0.8594
        ],
        [
          0.6561,
          0.8549
        ],
        [
          0.6665,
          0.8499
        ],
        [
          0.6765,
          0.8444
        ],
        [
          0.6865,
          0.8385
        ],
        [
          0.6965,
          0.833
        ],
        [
          0.7066,
          0.8312
        ],
        [
          0.717,
          0.8239
        ],
        [
          0.727,
          0.8203
        ],
        [
          0.737,
          0.8121
        ],
        [
          0.747,
          0.808
        ],
        [
          0.7571,
          0.8021
        ],
        [
          0.7675,
          0.7934
        ],
        [
          0.7775,
          0.788
        ],
        [
          0.7875,
          0.7853
        ],
        [
          0.7975,
          0.7789
        ],
        [
          0.8076,
          0.7743
        ],
        [
          0.818,
          0.7702
        ],
        [
          0.828,
          0.7634
        ],
        [
          0.838,
          0.758
        ],
        [
          0.848,
          0.7507
        ],
        [
          0.8581,
          0.7457
        ],
        [
          0.8685,
          0.7375
        ],
        [
          0.8785,
          0.7316
        ],
        [
          0.8885,
          0.7266
        ],
        [
          0.8985,
          0.7193
        ],
        [
          0.9086,
          0.7134
        ],
        [
          0.919,
          0.7079
        ],
        [
          0.929,
          0.7015
        ],
        [
          0.939,
          0.6943
        ],
        [
          0.949,
          0.6874
        ],
        [
          0.9591,
          0.6829
        ],
        [
          0.9695,
          0.6756
        ],
        [
          0.9795,
          0.6683
        ],
        [
          0.9895,
          0.6606
        ],
        [
          0.9995,
          0.6529
        ],
        [
          1.0,
          0.6524
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.16291049466975038,
        "difference": -0.0011518502278240709,
        "worse": 0.5366666666666666,
        "kept_right": 0.1616980355387065,
        "kept_wrong": 0.001212459131043962,
        "handed_off": 0.8370895053302497
      },
      "usd_per_1000": 4.680760704035373
    },
    {
      "decision": 177,
      "frontier": 124,
      "frontier_accuracy": 0.9667879890809827,
      "in_sample": {
        "threshold": 0.9537515044212341,
        "answered": 0.14240218380345768,
        "accuracy": 0.9667879890809827
      },
      "curve": [
        [
          0.0,
          0.9668
        ],
        [
          0.01,
          0.9668
        ],
        [
          0.02,
          0.9668
        ],
        [
          0.03,
          0.9668
        ],
        [
          0.04,
          0.9668
        ],
        [
          0.05,
          0.9668
        ],
        [
          0.0601,
          0.9668
        ],
        [
          0.0705,
          0.9668
        ],
        [
          0.0805,
          0.9668
        ],
        [
          0.0905,
          0.9668
        ],
        [
          0.1005,
          0.9668
        ],
        [
          0.1106,
          0.9668
        ],
        [
          0.121,
          0.9668
        ],
        [
          0.131,
          0.9668
        ],
        [
          0.141,
          0.9668
        ],
        [
          0.151,
          0.9663
        ],
        [
          0.1611,
          0.9663
        ],
        [
          0.1715,
          0.9659
        ],
        [
          0.1815,
          0.9654
        ],
        [
          0.1915,
          0.965
        ],
        [
          0.2015,
          0.9645
        ],
        [
          0.2116,
          0.9641
        ],
        [
          0.222,
          0.9641
        ],
        [
          0.232,
          0.9627
        ],
        [
          0.242,
          0.9627
        ],
        [
          0.252,
          0.9618
        ],
        [
          0.2621,
          0.9609
        ],
        [
          0.2725,
          0.9604
        ],
        [
          0.2825,
          0.9591
        ],
        [
          0.2925,
          0.9577
        ],
        [
          0.3025,
          0.9586
        ],
        [
          0.3126,
          0.9572
        ],
        [
          0.323,
          0.9545
        ],
        [
          0.333,
          0.955
        ],
        [
          0.343,
          0.9536
        ],
        [
          0.353,
          0.9531
        ],
        [
          0.3631,
          0.9513
        ],
        [
          0.3735,
          0.9495
        ],
        [
          0.3835,
          0.949
        ],
        [
          0.3935,
          0.9477
        ],
        [
          0.4035,
          0.9468
        ],
        [
          0.4136,
          0.9449
        ],
        [
          0.424,
          0.9418
        ],
        [
          0.434,
          0.9404
        ],
        [
          0.444,
          0.9372
        ],
        [
          0.454,
          0.9345
        ],
        [
          0.4641,
          0.9331
        ],
        [
          0.4745,
          0.9295
        ],
        [
          0.4845,
          0.9245
        ],
        [
          0.4945,
          0.9217
        ],
        [
          0.5045,
          0.9186
        ],
        [
          0.5146,
          0.914
        ],
        [
          0.525,
          0.9122
        ],
        [
          0.535,
          0.9086
        ],
        [
          0.545,
          0.9072
        ],
        [
          0.5551,
          0.9026
        ],
        [
          0.5655,
          0.8976
        ],
        [
          0.5755,
          0.894
        ],
        [
          0.5855,
          0.889
        ],
        [
          0.5955,
          0.8844
        ],
        [
          0.6056,
          0.8822
        ],
        [
          0.616,
          0.8776
        ],
        [
          0.626,
          0.8717
        ],
        [
          0.636,
          0.8658
        ],
        [
          0.646,
          0.8612
        ],
        [
          0.6561,
          0.8562
        ],
        [
          0.6665,
          0.8512
        ],
        [
          0.6765,
          0.8453
        ],
        [
          0.6865,
          0.8394
        ],
        [
          0.6965,
          0.8339
        ],
        [
          0.7066,
          0.8321
        ],
        [
          0.717,
          0.8248
        ],
        [
          0.727,
          0.8212
        ],
        [
          0.737,
          0.813
        ],
        [
          0.747,
          0.8085
        ],
        [
          0.7571,
          0.8016
        ],
        [
          0.7675,
          0.7925
        ],
        [
          0.7775,
          0.7871
        ],
        [
          0.7875,
          0.7843
        ],
        [
          0.7975,
          0.7775
        ],
        [
          0.8076,
          0.773
        ],
        [
          0.818,
          0.7689
        ],
        [
          0.828,
          0.7621
        ],
        [
          0.838,
          0.7561
        ],
        [
          0.848,
          0.7493
        ],
        [
          0.8581,
          0.7439
        ],
        [
          0.8685,
          0.7361
        ],
        [
          0.8785,
          0.7302
        ],
        [
          0.8885,
          0.7261
        ],
        [
          0.8985,
          0.7197
        ],
        [
          0.9086,
          0.7129
        ],
        [
          0.919,
          0.7075
        ],
        [
          0.929,
          0.7015
        ],
        [
          0.939,
          0.6947
        ],
        [
          0.949,
          0.6879
        ],
        [
          0.9591,
          0.6833
        ],
        [
          0.9695,
          0.6765
        ],
        [
          0.9795,
          0.6688
        ],
        [
          0.9895,
          0.6611
        ],
        [
          0.9995,
          0.6529
        ],
        [
          1.0,
          0.6524
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.15884638632934817,
        "difference": -0.0008609104100971802,
        "worse": 0.5066666666666667,
        "kept_right": 0.15795517286389044,
        "kept_wrong": 0.0008912134654577428,
        "handed_off": 0.8411536136706521
      },
      "usd_per_1000": 0.9702292640498431
    },
    {
      "decision": 177,
      "frontier": 278,
      "frontier_accuracy": 0.9499545040946314,
      "in_sample": {
        "threshold": 0.9537515044212341,
        "answered": 0.14240218380345768,
        "accuracy": 0.9499545040946314
      },
      "curve": [
        [
          0.0,
          0.95
        ],
        [
          0.01,
          0.95
        ],
        [
          0.02,
          0.95
        ],
        [
          0.03,
          0.95
        ],
        [
          0.04,
          0.95
        ],
        [
          0.05,
          0.95
        ],
        [
          0.0601,
          0.95
        ],
        [
          0.0705,
          0.95
        ],
        [
          0.0805,
          0.95
        ],
        [
          0.0905,
          0.95
        ],
        [
          0.1005,
          0.95
        ],
        [
          0.1106,
          0.95
        ],
        [
          0.121,
          0.95
        ],
        [
          0.131,
          0.95
        ],
        [
          0.141,
          0.95
        ],
        [
          0.151,
          0.9495
        ],
        [
          0.1611,
          0.9495
        ],
        [
          0.1715,
          0.949
        ],
        [
          0.1815,
          0.9486
        ],
        [
          0.1915,
          0.9477
        ],
        [
          0.2015,
          0.9472
        ],
        [
          0.2116,
          0.9468
        ],
        [
          0.222,
          0.9468
        ],
        [
          0.232,
          0.9454
        ],
        [
          0.242,
          0.9454
        ],
        [
          0.252,
          0.9445
        ],
        [
          0.2621,
          0.9436
        ],
        [
          0.2725,
          0.9431
        ],
        [
          0.2825,
          0.9418
        ],
        [
          0.2925,
          0.9409
        ],
        [
          0.3025,
          0.9413
        ],
        [
          0.3126,
          0.9399
        ],
        [
          0.323,
          0.9372
        ],
        [
          0.333,
          0.9372
        ],
        [
          0.343,
          0.9354
        ],
        [
          0.353,
          0.9349
        ],
        [
          0.3631,
          0.934
        ],
        [
          0.3735,
          0.9322
        ],
        [
          0.3835,
          0.9318
        ],
        [
          0.3935,
          0.9304
        ],
        [
          0.4035,
          0.9304
        ],
        [
          0.4136,
          0.9295
        ],
        [
          0.424,
          0.9268
        ],
        [
          0.434,
          0.9258
        ],
        [
          0.444,
          0.9227
        ],
        [
          0.454,
          0.9199
        ],
        [
          0.4641,
          0.9181
        ],
        [
          0.4745,
          0.9149
        ],
        [
          0.4845,
          0.9099
        ],
        [
          0.4945,
          0.9086
        ],
        [
          0.5045,
          0.9058
        ],
        [
          0.5146,
          0.9017
        ],
        [
          0.525,
          0.9008
        ],
        [
          0.535,
          0.8972
        ],
        [
          0.545,
          0.8958
        ],
        [
          0.5551,
          0.8922
        ],
        [
          0.5655,
          0.8876
        ],
        [
          0.5755,
          0.8844
        ],
        [
          0.5855,
          0.8803
        ],
        [
          0.5955,
          0.8758
        ],
        [
          0.6056,
          0.8735
        ],
        [
          0.616,
          0.869
        ],
        [
          0.626,
          0.8631
        ],
        [
          0.636,
          0.8571
        ],
        [
          0.646,
          0.8526
        ],
        [
          0.6561,
          0.848
        ],
        [
          0.6665,
          0.843
        ],
        [
          0.6765,
          0.838
        ],
        [
          0.6865,
          0.8321
        ],
        [
          0.6965,
          0.8271
        ],
        [
          0.7066,
          0.8253
        ],
        [
          0.717,
          0.818
        ],
        [
          0.727,
          0.8144
        ],
        [
          0.737,
          0.8062
        ],
        [
          0.747,
          0.8016
        ],
        [
          0.7571,
          0.7953
        ],
        [
          0.7675,
          0.7871
        ],
        [
          0.7775,
          0.7821
        ],
        [
          0.7875,
          0.7789
        ],
        [
          0.7975,
          0.773
        ],
        [
          0.8076,
          0.768
        ],
        [
          0.818,
          0.7639
        ],
        [
          0.828,
          0.7575
        ],
        [
          0.838,
          0.7516
        ],
        [
          0.848,
          0.7448
        ],
        [
          0.8581,
          0.7398
        ],
        [
          0.8685,
          0.732
        ],
        [
          0.8785,
          0.7266
        ],
        [
          0.8885,
          0.722
        ],
        [
          0.8985,
          0.7161
        ],
        [
          0.9086,
          0.7093
        ],
        [
          0.919,
          0.7038
        ],
        [
          0.929,
          0.6979
        ],
        [
          0.939,
          0.6915
        ],
        [
          0.949,
          0.6852
        ],
        [
          0.9591,
          0.6815
        ],
        [
          0.9695,
          0.6742
        ],
        [
          0.9795,
          0.6679
        ],
        [
          0.9895,
          0.6611
        ],
        [
          0.9995,
          0.6529
        ],
        [
          1.0,
          0.6524
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.15830064751271303,
        "difference": -0.0008699930722628275,
        "worse": 0.5066666666666667,
        "kept_right": 0.1574306544404503,
        "kept_wrong": 0.0008699930722628275,
        "handed_off": 0.8416993524872872
      },
      "usd_per_1000": 5.4978867537597775
    },
    {
      "decision": 177,
      "frontier": 275,
      "frontier_accuracy": 0.9426751592356688,
      "in_sample": {
        "threshold": 0.9537515044212341,
        "answered": 0.14240218380345768,
        "accuracy": 0.9426751592356688
      },
      "curve": [
        [
          0.0,
          0.9427
        ],
        [
          0.01,
          0.9427
        ],
        [
          0.02,
          0.9427
        ],
        [
          0.03,
          0.9427
        ],
        [
          0.04,
          0.9427
        ],
        [
          0.05,
          0.9427
        ],
        [
          0.0601,
          0.9427
        ],
        [
          0.0705,
          0.9427
        ],
        [
          0.0805,
          0.9427
        ],
        [
          0.0905,
          0.9427
        ],
        [
          0.1005,
          0.9427
        ],
        [
          0.1106,
          0.9427
        ],
        [
          0.121,
          0.9427
        ],
        [
          0.131,
          0.9427
        ],
        [
          0.141,
          0.9427
        ],
        [
          0.151,
          0.9422
        ],
        [
          0.1611,
          0.9422
        ],
        [
          0.1715,
          0.9418
        ],
        [
          0.1815,
          0.9413
        ],
        [
          0.1915,
          0.9404
        ],
        [
          0.2015,
          0.9399
        ],
        [
          0.2116,
          0.9395
        ],
        [
          0.222,
          0.9395
        ],
        [
          0.232,
          0.9381
        ],
        [
          0.242,
          0.9381
        ],
        [
          0.252,
          0.9372
        ],
        [
          0.2621,
          0.9363
        ],
        [
          0.2725,
          0.9359
        ],
        [
          0.2825,
          0.9345
        ],
        [
          0.2925,
          0.9336
        ],
        [
          0.3025,
          0.934
        ],
        [
          0.3126,
          0.9336
        ],
        [
          0.323,
          0.9308
        ],
        [
          0.333,
          0.9313
        ],
        [
          0.343,
          0.9295
        ],
        [
          0.353,
          0.929
        ],
        [
          0.3631,
          0.9277
        ],
        [
          0.3735,
          0.9258
        ],
        [
          0.3835,
          0.9254
        ],
        [
          0.3935,
          0.924
        ],
        [
          0.4035,
          0.9236
        ],
        [
          0.4136,
          0.9217
        ],
        [
          0.424,
          0.9195
        ],
        [
          0.434,
          0.9181
        ],
        [
          0.444,
          0.9149
        ],
        [
          0.454,
          0.9136
        ],
        [
          0.4641,
          0.9126
        ],
        [
          0.4745,
          0.9104
        ],
        [
          0.4845,
          0.9054
        ],
        [
          0.4945,
          0.9035
        ],
        [
          0.5045,
          0.9013
        ],
        [
          0.5146,
          0.8967
        ],
        [
          0.525,
          0.8958
        ],
        [
          0.535,
          0.8922
        ],
        [
          0.545,
          0.8908
        ],
        [
          0.5551,
          0.8876
        ],
        [
          0.5655,
          0.8826
        ],
        [
          0.5755,
          0.879
        ],
        [
          0.5855,
          0.8744
        ],
        [
          0.5955,
          0.8708
        ],
        [
          0.6056,
          0.8685
        ],
        [
          0.616,
          0.864
        ],
        [
          0.626,
          0.8585
        ],
        [
          0.636,
          0.8521
        ],
        [
          0.646,
          0.848
        ],
        [
          0.6561,
          0.8435
        ],
        [
          0.6665,
          0.8385
        ],
        [
          0.6765,
          0.833
        ],
        [
          0.6865,
          0.8276
        ],
        [
          0.6965,
          0.8226
        ],
        [
          0.7066,
          0.8207
        ],
        [
          0.717,
          0.8135
        ],
        [
          0.727,
          0.8103
        ],
        [
          0.737,
          0.8035
        ],
        [
          0.747,
          0.7989
        ],
        [
          0.7571,
          0.7934
        ],
        [
          0.7675,
          0.7848
        ],
        [
          0.7775,
          0.7793
        ],
        [
          0.7875,
          0.7766
        ],
        [
          0.7975,
          0.7712
        ],
        [
          0.8076,
          0.7671
        ],
        [
          0.818,
          0.763
        ],
        [
          0.828,
          0.7561
        ],
        [
          0.838,
          0.7502
        ],
        [
          0.848,
          0.7429
        ],
        [
          0.8581,
          0.7384
        ],
        [
          0.8685,
          0.7307
        ],
        [
          0.8785,
          0.7252
        ],
        [
          0.8885,
          0.7207
        ],
        [
          0.8985,
          0.7143
        ],
        [
          0.9086,
          0.7079
        ],
        [
          0.919,
          0.702
        ],
        [
          0.929,
          0.6965
        ],
        [
          0.939,
          0.6902
        ],
        [
          0.949,
          0.6838
        ],
        [
          0.9591,
          0.6797
        ],
        [
          0.9695,
          0.6729
        ],
        [
          0.9795,
          0.6665
        ],
        [
          0.9895,
          0.6601
        ],
        [
          0.9995,
          0.6529
        ],
        [
          1.0,
          0.6524
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.15830064751271303,
        "difference": -0.0008699930722628275,
        "worse": 0.5066666666666667,
        "kept_right": 0.1574306544404503,
        "kept_wrong": 0.0008699930722628275,
        "handed_off": 0.8416993524872872
      },
      "usd_per_1000": 0.4524772965094963
    },
    {
      "decision": 177,
      "frontier": 192,
      "frontier_accuracy": 0.9331210191082803,
      "in_sample": {
        "threshold": 0.9537515044212341,
        "answered": 0.14240218380345768,
        "accuracy": 0.9331210191082803
      },
      "curve": [
        [
          0.0,
          0.9331
        ],
        [
          0.01,
          0.9331
        ],
        [
          0.02,
          0.9331
        ],
        [
          0.03,
          0.9331
        ],
        [
          0.04,
          0.9331
        ],
        [
          0.05,
          0.9331
        ],
        [
          0.0601,
          0.9331
        ],
        [
          0.0705,
          0.9331
        ],
        [
          0.0805,
          0.9331
        ],
        [
          0.0905,
          0.9331
        ],
        [
          0.1005,
          0.9331
        ],
        [
          0.1106,
          0.9331
        ],
        [
          0.121,
          0.9331
        ],
        [
          0.131,
          0.9331
        ],
        [
          0.141,
          0.9331
        ],
        [
          0.151,
          0.9327
        ],
        [
          0.1611,
          0.9327
        ],
        [
          0.1715,
          0.9322
        ],
        [
          0.1815,
          0.9318
        ],
        [
          0.1915,
          0.9313
        ],
        [
          0.2015,
          0.9308
        ],
        [
          0.2116,
          0.9304
        ],
        [
          0.222,
          0.9304
        ],
        [
          0.232,
          0.929
        ],
        [
          0.242,
          0.929
        ],
        [
          0.252,
          0.9281
        ],
        [
          0.2621,
          0.9277
        ],
        [
          0.2725,
          0.9272
        ],
        [
          0.2825,
          0.9258
        ],
        [
          0.2925,
          0.9245
        ],
        [
          0.3025,
          0.9249
        ],
        [
          0.3126,
          0.9236
        ],
        [
          0.323,
          0.9213
        ],
        [
          0.333,
          0.9217
        ],
        [
          0.343,
          0.9204
        ],
        [
          0.353,
          0.9204
        ],
        [
          0.3631,
          0.919
        ],
        [
          0.3735,
          0.9172
        ],
        [
          0.3835,
          0.9172
        ],
        [
          0.3935,
          0.9163
        ],
        [
          0.4035,
          0.9158
        ],
        [
          0.4136,
          0.9149
        ],
        [
          0.424,
          0.9122
        ],
        [
          0.434,
          0.9113
        ],
        [
          0.444,
          0.9081
        ],
        [
          0.454,
          0.9058
        ],
        [
          0.4641,
          0.9045
        ],
        [
          0.4745,
          0.9013
        ],
        [
          0.4845,
          0.8972
        ],
        [
          0.4945,
          0.8958
        ],
        [
          0.5045,
          0.8926
        ],
        [
          0.5146,
          0.8885
        ],
        [
          0.525,
          0.8872
        ],
        [
          0.535,
          0.884
        ],
        [
          0.545,
          0.8822
        ],
        [
          0.5551,
          0.8785
        ],
        [
          0.5655,
          0.8749
        ],
        [
          0.5755,
          0.8712
        ],
        [
          0.5855,
          0.8672
        ],
        [
          0.5955,
          0.8626
        ],
        [
          0.6056,
          0.8603
        ],
        [
          0.616,
          0.8562
        ],
        [
          0.626,
          0.8508
        ],
        [
          0.636,
          0.8449
        ],
        [
          0.646,
          0.8403
        ],
        [
          0.6561,
          0.8362
        ],
        [
          0.6665,
          0.8308
        ],
        [
          0.6765,
          0.8253
        ],
        [
          0.6865,
          0.8194
        ],
        [
          0.6965,
          0.8144
        ],
        [
          0.7066,
          0.813
        ],
        [
          0.717,
          0.8062
        ],
        [
          0.727,
          0.803
        ],
        [
          0.737,
          0.7966
        ],
        [
          0.747,
          0.793
        ],
        [
          0.7571,
          0.7875
        ],
        [
          0.7675,
          0.7793
        ],
        [
          0.7775,
          0.7739
        ],
        [
          0.7875,
          0.7712
        ],
        [
          0.7975,
          0.7657
        ],
        [
          0.8076,
          0.7616
        ],
        [
          0.818,
          0.7575
        ],
        [
          0.828,
          0.7511
        ],
        [
          0.838,
          0.7461
        ],
        [
          0.848,
          0.7389
        ],
        [
          0.8581,
          0.7348
        ],
        [
          0.8685,
          0.7261
        ],
        [
          0.8785,
          0.7216
        ],
        [
          0.8885,
          0.717
        ],
        [
          0.8985,
          0.7111
        ],
        [
          0.9086,
          0.7056
        ],
        [
          0.919,
          0.7006
        ],
        [
          0.929,
          0.6943
        ],
        [
          0.939,
          0.6874
        ],
        [
          0.949,
          0.6811
        ],
        [
          0.9591,
          0.6774
        ],
        [
          0.9695,
          0.6715
        ],
        [
          0.9795,
          0.6656
        ],
        [
          0.9895,
          0.6606
        ],
        [
          0.9995,
          0.6529
        ],
        [
          1.0,
          0.6524
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.161895722854291,
        "difference": -0.0010882414023886662,
        "worse": 0.5333333333333333,
        "kept_right": 0.16078019488775316,
        "kept_wrong": 0.0011155279665378652,
        "handed_off": 0.8381042771457091
      },
      "usd_per_1000": 9.749438235083709
    },
    {
      "decision": 177,
      "frontier": 123,
      "frontier_accuracy": 0.9326660600545951,
      "in_sample": {
        "threshold": 0.9537515044212341,
        "answered": 0.14240218380345768,
        "accuracy": 0.9326660600545951
      },
      "curve": [
        [
          0.0,
          0.9327
        ],
        [
          0.01,
          0.9327
        ],
        [
          0.02,
          0.9327
        ],
        [
          0.03,
          0.9327
        ],
        [
          0.04,
          0.9327
        ],
        [
          0.05,
          0.9327
        ],
        [
          0.0601,
          0.9327
        ],
        [
          0.0705,
          0.9327
        ],
        [
          0.0805,
          0.9327
        ],
        [
          0.0905,
          0.9327
        ],
        [
          0.1005,
          0.9327
        ],
        [
          0.1106,
          0.9327
        ],
        [
          0.121,
          0.9327
        ],
        [
          0.131,
          0.9327
        ],
        [
          0.141,
          0.9327
        ],
        [
          0.151,
          0.9322
        ],
        [
          0.1611,
          0.9322
        ],
        [
          0.1715,
          0.9318
        ],
        [
          0.1815,
          0.9313
        ],
        [
          0.1915,
          0.9304
        ],
        [
          0.2015,
          0.9299
        ],
        [
          0.2116,
          0.9295
        ],
        [
          0.222,
          0.9295
        ],
        [
          0.232,
          0.9281
        ],
        [
          0.242,
          0.9281
        ],
        [
          0.252,
          0.9277
        ],
        [
          0.2621,
          0.9268
        ],
        [
          0.2725,
          0.9263
        ],
        [
          0.2825,
          0.9254
        ],
        [
          0.2925,
          0.924
        ],
        [
          0.3025,
          0.924
        ],
        [
          0.3126,
          0.924
        ],
        [
          0.323,
          0.9213
        ],
        [
          0.333,
          0.9213
        ],
        [
          0.343,
          0.9195
        ],
        [
          0.353,
          0.919
        ],
        [
          0.3631,
          0.9177
        ],
        [
          0.3735,
          0.9158
        ],
        [
          0.3835,
          0.9163
        ],
        [
          0.3935,
          0.9158
        ],
        [
          0.4035,
          0.9154
        ],
        [
          0.4136,
          0.9149
        ],
        [
          0.424,
          0.9117
        ],
        [
          0.434,
          0.9108
        ],
        [
          0.444,
          0.9076
        ],
        [
          0.454,
          0.9054
        ],
        [
          0.4641,
          0.9031
        ],
        [
          0.4745,
          0.8999
        ],
        [
          0.4845,
          0.8949
        ],
        [
          0.4945,
          0.8931
        ],
        [
          0.5045,
          0.8904
        ],
        [
          0.5146,
          0.8854
        ],
        [
          0.525,
          0.8831
        ],
        [
          0.535,
          0.8799
        ],
        [
          0.545,
          0.8785
        ],
        [
          0.5551,
          0.8749
        ],
        [
          0.5655,
          0.8712
        ],
        [
          0.5755,
          0.8681
        ],
        [
          0.5855,
          0.8635
        ],
        [
          0.5955,
          0.8599
        ],
        [
          0.6056,
          0.8571
        ],
        [
          0.616,
          0.8526
        ],
        [
          0.626,
          0.8467
        ],
        [
          0.636,
          0.8403
        ],
        [
          0.646,
          0.8367
        ],
        [
          0.6561,
          0.8321
        ],
        [
          0.6665,
          0.828
        ],
        [
          0.6765,
          0.823
        ],
        [
          0.6865,
          0.818
        ],
        [
          0.6965,
          0.813
        ],
        [
          0.7066,
          0.8116
        ],
        [
          0.717,
          0.8044
        ],
        [
          0.727,
          0.8007
        ],
        [
          0.737,
          0.7944
        ],
        [
          0.747,
          0.7903
        ],
        [
          0.7571,
          0.7839
        ],
        [
          0.7675,
          0.7762
        ],
        [
          0.7775,
          0.7712
        ],
        [
          0.7875,
          0.768
        ],
        [
          0.7975,
          0.7625
        ],
        [
          0.8076,
          0.7589
        ],
        [
          0.818,
          0.7552
        ],
        [
          0.828,
          0.7498
        ],
        [
          0.838,
          0.7452
        ],
        [
          0.848,
          0.7389
        ],
        [
          0.8581,
          0.7343
        ],
        [
          0.8685,
          0.7279
        ],
        [
          0.8785,
          0.722
        ],
        [
          0.8885,
          0.7184
        ],
        [
          0.8985,
          0.7116
        ],
        [
          0.9086,
          0.7061
        ],
        [
          0.919,
          0.7002
        ],
        [
          0.929,
          0.6952
        ],
        [
          0.939,
          0.6884
        ],
        [
          0.949,
          0.6833
        ],
        [
          0.9591,
          0.6793
        ],
        [
          0.9695,
          0.6729
        ],
        [
          0.9795,
          0.6665
        ],
        [
          0.9895,
          0.6592
        ],
        [
          0.9995,
          0.6529
        ],
        [
          1.0,
          0.6524
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.15830064751271303,
        "difference": -0.0008699930722628275,
        "worse": 0.5066666666666667,
        "kept_right": 0.1574306544404503,
        "kept_wrong": 0.0008699930722628275,
        "handed_off": 0.8416993524872872
      },
      "usd_per_1000": 0.24547880277183293
    },
    {
      "decision": 177,
      "frontier": 274,
      "frontier_accuracy": 0.9085532302092811,
      "in_sample": {
        "threshold": 0.9537515044212341,
        "answered": 0.14240218380345768,
        "accuracy": 0.9085532302092811
      },
      "curve": [
        [
          0.0,
          0.9086
        ],
        [
          0.01,
          0.9086
        ],
        [
          0.02,
          0.9086
        ],
        [
          0.03,
          0.9086
        ],
        [
          0.04,
          0.9086
        ],
        [
          0.05,
          0.9086
        ],
        [
          0.0601,
          0.9086
        ],
        [
          0.0705,
          0.9086
        ],
        [
          0.0805,
          0.9086
        ],
        [
          0.0905,
          0.9086
        ],
        [
          0.1005,
          0.9086
        ],
        [
          0.1106,
          0.9086
        ],
        [
          0.121,
          0.9086
        ],
        [
          0.131,
          0.9086
        ],
        [
          0.141,
          0.9086
        ],
        [
          0.151,
          0.9081
        ],
        [
          0.1611,
          0.9081
        ],
        [
          0.1715,
          0.9076
        ],
        [
          0.1815,
          0.9072
        ],
        [
          0.1915,
          0.9067
        ],
        [
          0.2015,
          0.9063
        ],
        [
          0.2116,
          0.9058
        ],
        [
          0.222,
          0.9058
        ],
        [
          0.232,
          0.9045
        ],
        [
          0.242,
          0.9045
        ],
        [
          0.252,
          0.9035
        ],
        [
          0.2621,
          0.9031
        ],
        [
          0.2725,
          0.9026
        ],
        [
          0.2825,
          0.9013
        ],
        [
          0.2925,
          0.9004
        ],
        [
          0.3025,
          0.9013
        ],
        [
          0.3126,
          0.9013
        ],
        [
          0.323,
          0.899
        ],
        [
          0.333,
          0.899
        ],
        [
          0.343,
          0.8981
        ],
        [
          0.353,
          0.8976
        ],
        [
          0.3631,
          0.8963
        ],
        [
          0.3735,
          0.8954
        ],
        [
          0.3835,
          0.8954
        ],
        [
          0.3935,
          0.8944
        ],
        [
          0.4035,
          0.894
        ],
        [
          0.4136,
          0.8931
        ],
        [
          0.424,
          0.8908
        ],
        [
          0.434,
          0.8894
        ],
        [
          0.444,
          0.8872
        ],
        [
          0.454,
          0.8854
        ],
        [
          0.4641,
          0.8844
        ],
        [
          0.4745,
          0.8822
        ],
        [
          0.4845,
          0.8776
        ],
        [
          0.4945,
          0.8758
        ],
        [
          0.5045,
          0.8735
        ],
        [
          0.5146,
          0.8694
        ],
        [
          0.525,
          0.8681
        ],
        [
          0.535,
          0.8644
        ],
        [
          0.545,
          0.8631
        ],
        [
          0.5551,
          0.8594
        ],
        [
          0.5655,
          0.8549
        ],
        [
          0.5755,
          0.8521
        ],
        [
          0.5855,
          0.848
        ],
        [
          0.5955,
          0.8439
        ],
        [
          0.6056,
          0.8421
        ],
        [
          0.616,
          0.838
        ],
        [
          0.626,
          0.8335
        ],
        [
          0.636,
          0.8271
        ],
        [
          0.646,
          0.823
        ],
        [
          0.6561,
          0.8189
        ],
        [
          0.6665,
          0.8153
        ],
        [
          0.6765,
          0.8103
        ],
        [
          0.6865,
          0.8048
        ],
        [
          0.6965,
          0.7998
        ],
        [
          0.7066,
          0.7998
        ],
        [
          0.717,
          0.7939
        ],
        [
          0.727,
          0.7912
        ],
        [
          0.737,
          0.7834
        ],
        [
          0.747,
          0.7793
        ],
        [
          0.7571,
          0.7748
        ],
        [
          0.7675,
          0.7662
        ],
        [
          0.7775,
          0.7616
        ],
        [
          0.7875,
          0.7584
        ],
        [
          0.7975,
          0.7534
        ],
        [
          0.8076,
          0.7493
        ],
        [
          0.818,
          0.7461
        ],
        [
          0.828,
          0.7402
        ],
        [
          0.838,
          0.7357
        ],
        [
          0.848,
          0.7307
        ],
        [
          0.8581,
          0.7275
        ],
        [
          0.8685,
          0.7202
        ],
        [
          0.8785,
          0.7152
        ],
        [
          0.8885,
          0.7106
        ],
        [
          0.8985,
          0.7047
        ],
        [
          0.9086,
          0.6988
        ],
        [
          0.919,
          0.6943
        ],
        [
          0.929,
          0.6893
        ],
        [
          0.939,
          0.6843
        ],
        [
          0.949,
          0.6779
        ],
        [
          0.9591,
          0.6752
        ],
        [
          0.9695,
          0.6697
        ],
        [
          0.9795,
          0.6642
        ],
        [
          0.9895,
          0.6592
        ],
        [
          0.9995,
          0.6529
        ],
        [
          1.0,
          0.6524
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1596039435413519,
        "difference": -0.0009184696485867248,
        "worse": 0.5066666666666667,
        "kept_right": 0.1586157547791202,
        "kept_wrong": 0.0009881887622317173,
        "handed_off": 0.8403960564586485
      },
      "usd_per_1000": 0.3776270612117516
    },
    {
      "decision": 177,
      "frontier": 47,
      "frontier_accuracy": 0.9044585987261147,
      "in_sample": {
        "threshold": 0.9460216760635376,
        "answered": 0.1696997270245678,
        "accuracy": 0.9044585987261147
      },
      "curve": [
        [
          0.0,
          0.9045
        ],
        [
          0.01,
          0.9045
        ],
        [
          0.02,
          0.9045
        ],
        [
          0.03,
          0.9045
        ],
        [
          0.04,
          0.9045
        ],
        [
          0.05,
          0.9045
        ],
        [
          0.0601,
          0.9045
        ],
        [
          0.0705,
          0.9045
        ],
        [
          0.0805,
          0.9045
        ],
        [
          0.0905,
          0.9045
        ],
        [
          0.1005,
          0.9045
        ],
        [
          0.1106,
          0.9045
        ],
        [
          0.121,
          0.9045
        ],
        [
          0.131,
          0.9045
        ],
        [
          0.141,
          0.9045
        ],
        [
          0.151,
          0.9045
        ],
        [
          0.1611,
          0.9045
        ],
        [
          0.1715,
          0.904
        ],
        [
          0.1815,
          0.9035
        ],
        [
          0.1915,
          0.9026
        ],
        [
          0.2015,
          0.9026
        ],
        [
          0.2116,
          0.9022
        ],
        [
          0.222,
          0.9022
        ],
        [
          0.232,
          0.9008
        ],
        [
          0.242,
          0.9008
        ],
        [
          0.252,
          0.8999
        ],
        [
          0.2621,
          0.899
        ],
        [
          0.2725,
          0.8985
        ],
        [
          0.2825,
          0.8972
        ],
        [
          0.2925,
          0.8958
        ],
        [
          0.3025,
          0.8958
        ],
        [
          0.3126,
          0.8949
        ],
        [
          0.323,
          0.8922
        ],
        [
          0.333,
          0.8926
        ],
        [
          0.343,
          0.8913
        ],
        [
          0.353,
          0.8913
        ],
        [
          0.3631,
          0.8913
        ],
        [
          0.3735,
          0.8904
        ],
        [
          0.3835,
          0.8908
        ],
        [
          0.3935,
          0.8899
        ],
        [
          0.4035,
          0.889
        ],
        [
          0.4136,
          0.8885
        ],
        [
          0.424,
          0.8867
        ],
        [
          0.434,
          0.8858
        ],
        [
          0.444,
          0.8826
        ],
        [
          0.454,
          0.8803
        ],
        [
          0.4641,
          0.8781
        ],
        [
          0.4745,
          0.8753
        ],
        [
          0.4845,
          0.8712
        ],
        [
          0.4945,
          0.869
        ],
        [
          0.5045,
          0.8662
        ],
        [
          0.5146,
          0.8631
        ],
        [
          0.525,
          0.8617
        ],
        [
          0.535,
          0.8581
        ],
        [
          0.545,
          0.8567
        ],
        [
          0.5551,
          0.8535
        ],
        [
          0.5655,
          0.8499
        ],
        [
          0.5755,
          0.8467
        ],
        [
          0.5855,
          0.8439
        ],
        [
          0.5955,
          0.8403
        ],
        [
          0.6056,
          0.8385
        ],
        [
          0.616,
          0.8344
        ],
        [
          0.626,
          0.8289
        ],
        [
          0.636,
          0.8244
        ],
        [
          0.646,
          0.8198
        ],
        [
          0.6561,
          0.8153
        ],
        [
          0.6665,
          0.8112
        ],
        [
          0.6765,
          0.808
        ],
        [
          0.6865,
          0.8035
        ],
        [
          0.6965,
          0.7989
        ],
        [
          0.7066,
          0.7971
        ],
        [
          0.717,
          0.7907
        ],
        [
          0.727,
          0.7871
        ],
        [
          0.737,
          0.7798
        ],
        [
          0.747,
          0.7766
        ],
        [
          0.7571,
          0.7707
        ],
        [
          0.7675,
          0.763
        ],
        [
          0.7775,
          0.7571
        ],
        [
          0.7875,
          0.7539
        ],
        [
          0.7975,
          0.7484
        ],
        [
          0.8076,
          0.7452
        ],
        [
          0.818,
          0.7429
        ],
        [
          0.828,
          0.7384
        ],
        [
          0.838,
          0.7329
        ],
        [
          0.848,
          0.7266
        ],
        [
          0.8581,
          0.7234
        ],
        [
          0.8685,
          0.717
        ],
        [
          0.8785,
          0.7129
        ],
        [
          0.8885,
          0.7093
        ],
        [
          0.8985,
          0.7034
        ],
        [
          0.9086,
          0.6984
        ],
        [
          0.919,
          0.6929
        ],
        [
          0.929,
          0.6888
        ],
        [
          0.939,
          0.6824
        ],
        [
          0.949,
          0.6779
        ],
        [
          0.9591,
          0.6756
        ],
        [
          0.9695,
          0.6702
        ],
        [
          0.9795,
          0.6642
        ],
        [
          0.9895,
          0.6597
        ],
        [
          0.9995,
          0.6529
        ],
        [
          1.0,
          0.6524
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.177515252593992,
        "difference": -0.0009942798696329377,
        "worse": 0.43666666666666665,
        "kept_right": 0.1760389747494473,
        "kept_wrong": 0.0014762778445447677,
        "handed_off": 0.8224847474060083
      },
      "usd_per_1000": 4.0747436328749975
    },
    {
      "decision": 177,
      "frontier": 193,
      "frontier_accuracy": 0.89171974522293,
      "in_sample": {
        "threshold": 0.9460216760635376,
        "answered": 0.1696997270245678,
        "accuracy": 0.89171974522293
      },
      "curve": [
        [
          0.0,
          0.8917
        ],
        [
          0.01,
          0.8917
        ],
        [
          0.02,
          0.8917
        ],
        [
          0.03,
          0.8917
        ],
        [
          0.04,
          0.8917
        ],
        [
          0.05,
          0.8917
        ],
        [
          0.0601,
          0.8917
        ],
        [
          0.0705,
          0.8917
        ],
        [
          0.0805,
          0.8917
        ],
        [
          0.0905,
          0.8917
        ],
        [
          0.1005,
          0.8917
        ],
        [
          0.1106,
          0.8917
        ],
        [
          0.121,
          0.8917
        ],
        [
          0.131,
          0.8917
        ],
        [
          0.141,
          0.8917
        ],
        [
          0.151,
          0.8917
        ],
        [
          0.1611,
          0.8917
        ],
        [
          0.1715,
          0.8913
        ],
        [
          0.1815,
          0.8913
        ],
        [
          0.1915,
          0.8908
        ],
        [
          0.2015,
          0.8908
        ],
        [
          0.2116,
          0.8904
        ],
        [
          0.222,
          0.8904
        ],
        [
          0.232,
          0.8894
        ],
        [
          0.242,
          0.8894
        ],
        [
          0.252,
          0.8885
        ],
        [
          0.2621,
          0.8881
        ],
        [
          0.2725,
          0.8876
        ],
        [
          0.2825,
          0.8863
        ],
        [
          0.2925,
          0.8849
        ],
        [
          0.3025,
          0.8854
        ],
        [
          0.3126,
          0.8854
        ],
        [
          0.323,
          0.8835
        ],
        [
          0.333,
          0.8844
        ],
        [
          0.343,
          0.8835
        ],
        [
          0.353,
          0.8835
        ],
        [
          0.3631,
          0.8826
        ],
        [
          0.3735,
          0.8803
        ],
        [
          0.3835,
          0.8799
        ],
        [
          0.3935,
          0.8799
        ],
        [
          0.4035,
          0.8785
        ],
        [
          0.4136,
          0.8776
        ],
        [
          0.424,
          0.8753
        ],
        [
          0.434,
          0.8744
        ],
        [
          0.444,
          0.8717
        ],
        [
          0.454,
          0.8708
        ],
        [
          0.4641,
          0.8699
        ],
        [
          0.4745,
          0.869
        ],
        [
          0.4845,
          0.864
        ],
        [
          0.4945,
          0.8617
        ],
        [
          0.5045,
          0.859
        ],
        [
          0.5146,
          0.8558
        ],
        [
          0.525,
          0.854
        ],
        [
          0.535,
          0.8517
        ],
        [
          0.545,
          0.8508
        ],
        [
          0.5551,
          0.848
        ],
        [
          0.5655,
          0.8444
        ],
        [
          0.5755,
          0.8412
        ],
        [
          0.5855,
          0.8367
        ],
        [
          0.5955,
          0.833
        ],
        [
          0.6056,
          0.8317
        ],
        [
          0.616,
          0.828
        ],
        [
          0.626,
          0.823
        ],
        [
          0.636,
          0.8176
        ],
        [
          0.646,
          0.8135
        ],
        [
          0.6561,
          0.8103
        ],
        [
          0.6665,
          0.8076
        ],
        [
          0.6765,
          0.803
        ],
        [
          0.6865,
          0.7971
        ],
        [
          0.6965,
          0.7944
        ],
        [
          0.7066,
          0.7953
        ],
        [
          0.717,
          0.7884
        ],
        [
          0.727,
          0.7857
        ],
        [
          0.737,
          0.7793
        ],
        [
          0.747,
          0.7762
        ],
        [
          0.7571,
          0.7721
        ],
        [
          0.7675,
          0.7652
        ],
        [
          0.7775,
          0.7593
        ],
        [
          0.7875,
          0.7575
        ],
        [
          0.7975,
          0.752
        ],
        [
          0.8076,
          0.7484
        ],
        [
          0.818,
          0.7452
        ],
        [
          0.828,
          0.7393
        ],
        [
          0.838,
          0.7357
        ],
        [
          0.848,
          0.7302
        ],
        [
          0.8581,
          0.7261
        ],
        [
          0.8685,
          0.7188
        ],
        [
          0.8785,
          0.7134
        ],
        [
          0.8885,
          0.7088
        ],
        [
          0.8985,
          0.7038
        ],
        [
          0.9086,
          0.6993
        ],
        [
          0.919,
          0.6947
        ],
        [
          0.929,
          0.6888
        ],
        [
          0.939,
          0.6833
        ],
        [
          0.949,
          0.6774
        ],
        [
          0.9591,
          0.6756
        ],
        [
          0.9695,
          0.6697
        ],
        [
          0.9795,
          0.6642
        ],
        [
          0.9895,
          0.6592
        ],
        [
          0.9995,
          0.6529
        ],
        [
          1.0,
          0.6524
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.18377027239545127,
        "difference": -0.0008397949059274529,
        "worse": 0.4266666666666667,
        "kept_right": 0.18185424148607826,
        "kept_wrong": 0.001916030909373152,
        "handed_off": 0.8162297276045493
      },
      "usd_per_1000": 6.343225142296005
    },
    {
      "decision": 177,
      "frontier": 55,
      "frontier_accuracy": 0.8503184713375797,
      "in_sample": {
        "threshold": 0.7587605118751526,
        "answered": 0.45632393084622386,
        "accuracy": 0.8503184713375797
      },
      "curve": [
        [
          0.0,
          0.8503
        ],
        [
          0.01,
          0.8503
        ],
        [
          0.02,
          0.8503
        ],
        [
          0.03,
          0.8503
        ],
        [
          0.04,
          0.8512
        ],
        [
          0.05,
          0.8512
        ],
        [
          0.0601,
          0.8512
        ],
        [
          0.0705,
          0.8512
        ],
        [
          0.0805,
          0.8512
        ],
        [
          0.0905,
          0.8517
        ],
        [
          0.1005,
          0.8517
        ],
        [
          0.1106,
          0.8517
        ],
        [
          0.121,
          0.8521
        ],
        [
          0.131,
          0.8521
        ],
        [
          0.141,
          0.8526
        ],
        [
          0.151,
          0.8526
        ],
        [
          0.1611,
          0.853
        ],
        [
          0.1715,
          0.8535
        ],
        [
          0.1815,
          0.8535
        ],
        [
          0.1915,
          0.8535
        ],
        [
          0.2015,
          0.854
        ],
        [
          0.2116,
          0.8535
        ],
        [
          0.222,
          0.8535
        ],
        [
          0.232,
          0.8544
        ],
        [
          0.242,
          0.8558
        ],
        [
          0.252,
          0.8549
        ],
        [
          0.2621,
          0.8549
        ],
        [
          0.2725,
          0.8553
        ],
        [
          0.2825,
          0.8549
        ],
        [
          0.2925,
          0.8549
        ],
        [
          0.3025,
          0.8558
        ],
        [
          0.3126,
          0.8553
        ],
        [
          0.323,
          0.8544
        ],
        [
          0.333,
          0.8553
        ],
        [
          0.343,
          0.8549
        ],
        [
          0.353,
          0.8553
        ],
        [
          0.3631,
          0.8549
        ],
        [
          0.3735,
          0.8549
        ],
        [
          0.3835,
          0.8549
        ],
        [
          0.3935,
          0.854
        ],
        [
          0.4035,
          0.854
        ],
        [
          0.4136,
          0.8553
        ],
        [
          0.424,
          0.8535
        ],
        [
          0.434,
          0.853
        ],
        [
          0.444,
          0.8508
        ],
        [
          0.454,
          0.8499
        ],
        [
          0.4641,
          0.8485
        ],
        [
          0.4745,
          0.8467
        ],
        [
          0.4845,
          0.843
        ],
        [
          0.4945,
          0.8412
        ],
        [
          0.5045,
          0.8403
        ],
        [
          0.5146,
          0.8371
        ],
        [
          0.525,
          0.8362
        ],
        [
          0.535,
          0.8335
        ],
        [
          0.545,
          0.8326
        ],
        [
          0.5551,
          0.8303
        ],
        [
          0.5655,
          0.8271
        ],
        [
          0.5755,
          0.8244
        ],
        [
          0.5855,
          0.8212
        ],
        [
          0.5955,
          0.8189
        ],
        [
          0.6056,
          0.8171
        ],
        [
          0.616,
          0.8139
        ],
        [
          0.626,
          0.8094
        ],
        [
          0.636,
          0.8044
        ],
        [
          0.646,
          0.8007
        ],
        [
          0.6561,
          0.7975
        ],
        [
          0.6665,
          0.7953
        ],
        [
          0.6765,
          0.7912
        ],
        [
          0.6865,
          0.7862
        ],
        [
          0.6965,
          0.783
        ],
        [
          0.7066,
          0.7825
        ],
        [
          0.717,
          0.7775
        ],
        [
          0.727,
          0.7743
        ],
        [
          0.737,
          0.768
        ],
        [
          0.747,
          0.7652
        ],
        [
          0.7571,
          0.7616
        ],
        [
          0.7675,
          0.7557
        ],
        [
          0.7775,
          0.7507
        ],
        [
          0.7875,
          0.7484
        ],
        [
          0.7975,
          0.7434
        ],
        [
          0.8076,
          0.7402
        ],
        [
          0.818,
          0.7379
        ],
        [
          0.828,
          0.7325
        ],
        [
          0.838,
          0.7275
        ],
        [
          0.848,
          0.7229
        ],
        [
          0.8581,
          0.7197
        ],
        [
          0.8685,
          0.7138
        ],
        [
          0.8785,
          0.7097
        ],
        [
          0.8885,
          0.7056
        ],
        [
          0.8985,
          0.6988
        ],
        [
          0.9086,
          0.6943
        ],
        [
          0.919,
          0.6902
        ],
        [
          0.929,
          0.6856
        ],
        [
          0.939,
          0.6811
        ],
        [
          0.949,
          0.6761
        ],
        [
          0.9591,
          0.6742
        ],
        [
          0.9695,
          0.6692
        ],
        [
          0.9795,
          0.6642
        ],
        [
          0.9895,
          0.6588
        ],
        [
          0.9995,
          0.6529
        ],
        [
          1.0,
          0.6524
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.4517954562510866,
        "difference": -0.0013351880657828303,
        "worse": 0.4866666666666667,
        "kept_right": 0.4113766002046441,
        "kept_wrong": 0.04041885604644293,
        "handed_off": 0.5482045437489134
      },
      "usd_per_1000": 2.022734169240105
    },
    {
      "decision": 346,
      "frontier": 140,
      "frontier_accuracy": 0.9808917197452229,
      "in_sample": {
        "threshold": 0.9926909804344177,
        "answered": 0.1105550500454959,
        "accuracy": 0.9808917197452229
      },
      "curve": [
        [
          0.0,
          0.9809
        ],
        [
          0.01,
          0.9809
        ],
        [
          0.02,
          0.9809
        ],
        [
          0.03,
          0.9809
        ],
        [
          0.04,
          0.9809
        ],
        [
          0.05,
          0.9809
        ],
        [
          0.0601,
          0.9809
        ],
        [
          0.0705,
          0.9809
        ],
        [
          0.0805,
          0.9809
        ],
        [
          0.0905,
          0.9809
        ],
        [
          0.1005,
          0.9809
        ],
        [
          0.1106,
          0.9809
        ],
        [
          0.121,
          0.9804
        ],
        [
          0.131,
          0.9804
        ],
        [
          0.141,
          0.9804
        ],
        [
          0.151,
          0.9804
        ],
        [
          0.1611,
          0.9804
        ],
        [
          0.1715,
          0.98
        ],
        [
          0.1815,
          0.98
        ],
        [
          0.1915,
          0.98
        ],
        [
          0.2015,
          0.98
        ],
        [
          0.2116,
          0.98
        ],
        [
          0.222,
          0.9786
        ],
        [
          0.232,
          0.9777
        ],
        [
          0.242,
          0.9777
        ],
        [
          0.252,
          0.9773
        ],
        [
          0.2621,
          0.9763
        ],
        [
          0.2725,
          0.9759
        ],
        [
          0.2825,
          0.9754
        ],
        [
          0.2925,
          0.9745
        ],
        [
          0.3025,
          0.9745
        ],
        [
          0.3126,
          0.9718
        ],
        [
          0.323,
          0.9704
        ],
        [
          0.333,
          0.9691
        ],
        [
          0.343,
          0.9677
        ],
        [
          0.353,
          0.9668
        ],
        [
          0.3631,
          0.9659
        ],
        [
          0.3735,
          0.9627
        ],
        [
          0.3835,
          0.96
        ],
        [
          0.3935,
          0.9581
        ],
        [
          0.4035,
          0.9559
        ],
        [
          0.4136,
          0.9527
        ],
        [
          0.424,
          0.9513
        ],
        [
          0.434,
          0.9481
        ],
        [
          0.444,
          0.9449
        ],
        [
          0.454,
          0.9427
        ],
        [
          0.4641,
          0.9404
        ],
        [
          0.4745,
          0.9372
        ],
        [
          0.4845,
          0.9336
        ],
        [
          0.4945,
          0.9304
        ],
        [
          0.5045,
          0.9263
        ],
        [
          0.5146,
          0.9222
        ],
        [
          0.525,
          0.9195
        ],
        [
          0.535,
          0.9163
        ],
        [
          0.545,
          0.9131
        ],
        [
          0.5551,
          0.9086
        ],
        [
          0.5655,
          0.9067
        ],
        [
          0.5755,
          0.9004
        ],
        [
          0.5855,
          0.8958
        ],
        [
          0.5955,
          0.8894
        ],
        [
          0.6056,
          0.884
        ],
        [
          0.616,
          0.8794
        ],
        [
          0.626,
          0.874
        ],
        [
          0.636,
          0.8703
        ],
        [
          0.646,
          0.8644
        ],
        [
          0.6561,
          0.859
        ],
        [
          0.6665,
          0.853
        ],
        [
          0.6765,
          0.8485
        ],
        [
          0.6865,
          0.8435
        ],
        [
          0.6965,
          0.8362
        ],
        [
          0.7066,
          0.8326
        ],
        [
          0.717,
          0.8285
        ],
        [
          0.727,
          0.8217
        ],
        [
          0.737,
          0.8148
        ],
        [
          0.747,
          0.8076
        ],
        [
          0.7571,
          0.8003
        ],
        [
          0.7675,
          0.7948
        ],
        [
          0.7775,
          0.7894
        ],
        [
          0.7875,
          0.7834
        ],
        [
          0.7975,
          0.7771
        ],
        [
          0.8076,
          0.7702
        ],
        [
          0.818,
          0.7657
        ],
        [
          0.828,
          0.7575
        ],
        [
          0.838,
          0.7507
        ],
        [
          0.848,
          0.7439
        ],
        [
          0.8581,
          0.737
        ],
        [
          0.8685,
          0.7307
        ],
        [
          0.8785,
          0.7229
        ],
        [
          0.8885,
          0.7175
        ],
        [
          0.8985,
          0.7116
        ],
        [
          0.9086,
          0.7066
        ],
        [
          0.919,
          0.6988
        ],
        [
          0.929,
          0.6929
        ],
        [
          0.939,
          0.6861
        ],
        [
          0.949,
          0.6793
        ],
        [
          0.9591,
          0.672
        ],
        [
          0.9695,
          0.6647
        ],
        [
          0.9795,
          0.6561
        ],
        [
          0.9895,
          0.6474
        ],
        [
          0.9995,
          0.6392
        ],
        [
          1.0,
          0.6388
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.15150356017456393,
        "difference": -0.0009185468388837493,
        "worse": 0.5066666666666667,
        "kept_right": 0.1505728866089038,
        "kept_wrong": 0.0009306735656601574,
        "handed_off": 0.8484964398254358
      },
      "usd_per_1000": 2.068009920043157
    },
    {
      "decision": 346,
      "frontier": 277,
      "frontier_accuracy": 0.9795268425841674,
      "in_sample": {
        "threshold": 0.9926909804344177,
        "answered": 0.1105550500454959,
        "accuracy": 0.9795268425841674
      },
      "curve": [
        [
          0.0,
          0.9795
        ],
        [
          0.01,
          0.9795
        ],
        [
          0.02,
          0.9795
        ],
        [
          0.03,
          0.9795
        ],
        [
          0.04,
          0.9795
        ],
        [
          0.05,
          0.9795
        ],
        [
          0.0601,
          0.9795
        ],
        [
          0.0705,
          0.9795
        ],
        [
          0.0805,
          0.9795
        ],
        [
          0.0905,
          0.9795
        ],
        [
          0.1005,
          0.9795
        ],
        [
          0.1106,
          0.9795
        ],
        [
          0.121,
          0.9791
        ],
        [
          0.131,
          0.9791
        ],
        [
          0.141,
          0.9791
        ],
        [
          0.151,
          0.9791
        ],
        [
          0.1611,
          0.9791
        ],
        [
          0.1715,
          0.9786
        ],
        [
          0.1815,
          0.9786
        ],
        [
          0.1915,
          0.9786
        ],
        [
          0.2015,
          0.9786
        ],
        [
          0.2116,
          0.9786
        ],
        [
          0.222,
          0.9773
        ],
        [
          0.232,
          0.9763
        ],
        [
          0.242,
          0.9763
        ],
        [
          0.252,
          0.9763
        ],
        [
          0.2621,
          0.9754
        ],
        [
          0.2725,
          0.9745
        ],
        [
          0.2825,
          0.9736
        ],
        [
          0.2925,
          0.9727
        ],
        [
          0.3025,
          0.9727
        ],
        [
          0.3126,
          0.97
        ],
        [
          0.323,
          0.9686
        ],
        [
          0.333,
          0.9668
        ],
        [
          0.343,
          0.9654
        ],
        [
          0.353,
          0.9645
        ],
        [
          0.3631,
          0.9636
        ],
        [
          0.3735,
          0.9604
        ],
        [
          0.3835,
          0.9577
        ],
        [
          0.3935,
          0.9559
        ],
        [
          0.4035,
          0.954
        ],
        [
          0.4136,
          0.9509
        ],
        [
          0.424,
          0.95
        ],
        [
          0.434,
          0.9463
        ],
        [
          0.444,
          0.9436
        ],
        [
          0.454,
          0.9409
        ],
        [
          0.4641,
          0.9386
        ],
        [
          0.4745,
          0.9354
        ],
        [
          0.4845,
          0.9318
        ],
        [
          0.4945,
          0.9286
        ],
        [
          0.5045,
          0.9245
        ],
        [
          0.5146,
          0.9204
        ],
        [
          0.525,
          0.9181
        ],
        [
          0.535,
          0.9149
        ],
        [
          0.545,
          0.9117
        ],
        [
          0.5551,
          0.9067
        ],
        [
          0.5655,
          0.9045
        ],
        [
          0.5755,
          0.8981
        ],
        [
          0.5855,
          0.8935
        ],
        [
          0.5955,
          0.8872
        ],
        [
          0.6056,
          0.8817
        ],
        [
          0.616,
          0.8767
        ],
        [
          0.626,
          0.8712
        ],
        [
          0.636,
          0.8676
        ],
        [
          0.646,
          0.8617
        ],
        [
          0.6561,
          0.8562
        ],
        [
          0.6665,
          0.8508
        ],
        [
          0.6765,
          0.8462
        ],
        [
          0.6865,
          0.8412
        ],
        [
          0.6965,
          0.8339
        ],
        [
          0.7066,
          0.8308
        ],
        [
          0.717,
          0.8267
        ],
        [
          0.727,
          0.8198
        ],
        [
          0.737,
          0.813
        ],
        [
          0.747,
          0.8062
        ],
        [
          0.7571,
          0.7994
        ],
        [
          0.7675,
          0.7939
        ],
        [
          0.7775,
          0.7884
        ],
        [
          0.7875,
          0.783
        ],
        [
          0.7975,
          0.7766
        ],
        [
          0.8076,
          0.7698
        ],
        [
          0.818,
          0.7652
        ],
        [
          0.828,
          0.7571
        ],
        [
          0.838,
          0.7502
        ],
        [
          0.848,
          0.7439
        ],
        [
          0.8581,
          0.737
        ],
        [
          0.8685,
          0.7311
        ],
        [
          0.8785,
          0.7234
        ],
        [
          0.8885,
          0.7179
        ],
        [
          0.8985,
          0.712
        ],
        [
          0.9086,
          0.7061
        ],
        [
          0.919,
          0.6984
        ],
        [
          0.929,
          0.692
        ],
        [
          0.939,
          0.6852
        ],
        [
          0.949,
          0.6793
        ],
        [
          0.9591,
          0.6715
        ],
        [
          0.9695,
          0.6642
        ],
        [
          0.9795,
          0.6556
        ],
        [
          0.9895,
          0.6474
        ],
        [
          0.9995,
          0.6392
        ],
        [
          1.0,
          0.6388
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.15167639498306754,
        "difference": -0.0009306790803153537,
        "worse": 0.5066666666666667,
        "kept_right": 0.1507335891759758,
        "kept_wrong": 0.0009428058070917618,
        "handed_off": 0.8483236050169323
      },
      "usd_per_1000": 5.160963026028296
    },
    {
      "decision": 346,
      "frontier": 276,
      "frontier_accuracy": 0.9777070063694268,
      "in_sample": {
        "threshold": 0.9926909804344177,
        "answered": 0.1105550500454959,
        "accuracy": 0.9777070063694268
      },
      "curve": [
        [
          0.0,
          0.9777
        ],
        [
          0.01,
          0.9777
        ],
        [
          0.02,
          0.9777
        ],
        [
          0.03,
          0.9777
        ],
        [
          0.04,
          0.9777
        ],
        [
          0.05,
          0.9777
        ],
        [
          0.0601,
          0.9777
        ],
        [
          0.0705,
          0.9777
        ],
        [
          0.0805,
          0.9777
        ],
        [
          0.0905,
          0.9777
        ],
        [
          0.1005,
          0.9777
        ],
        [
          0.1106,
          0.9777
        ],
        [
          0.121,
          0.9773
        ],
        [
          0.131,
          0.9773
        ],
        [
          0.141,
          0.9773
        ],
        [
          0.151,
          0.9773
        ],
        [
          0.1611,
          0.9773
        ],
        [
          0.1715,
          0.9768
        ],
        [
          0.1815,
          0.9768
        ],
        [
          0.1915,
          0.9768
        ],
        [
          0.2015,
          0.9768
        ],
        [
          0.2116,
          0.9773
        ],
        [
          0.222,
          0.9759
        ],
        [
          0.232,
          0.975
        ],
        [
          0.242,
          0.975
        ],
        [
          0.252,
          0.975
        ],
        [
          0.2621,
          0.9745
        ],
        [
          0.2725,
          0.9736
        ],
        [
          0.2825,
          0.9727
        ],
        [
          0.2925,
          0.9722
        ],
        [
          0.3025,
          0.9722
        ],
        [
          0.3126,
          0.9695
        ],
        [
          0.323,
          0.9682
        ],
        [
          0.333,
          0.9668
        ],
        [
          0.343,
          0.9654
        ],
        [
          0.353,
          0.9645
        ],
        [
          0.3631,
          0.9636
        ],
        [
          0.3735,
          0.9604
        ],
        [
          0.3835,
          0.9577
        ],
        [
          0.3935,
          0.9563
        ],
        [
          0.4035,
          0.9545
        ],
        [
          0.4136,
          0.9513
        ],
        [
          0.424,
          0.9504
        ],
        [
          0.434,
          0.9472
        ],
        [
          0.444,
          0.9445
        ],
        [
          0.454,
          0.9418
        ],
        [
          0.4641,
          0.9395
        ],
        [
          0.4745,
          0.9363
        ],
        [
          0.4845,
          0.9327
        ],
        [
          0.4945,
          0.9295
        ],
        [
          0.5045,
          0.9254
        ],
        [
          0.5146,
          0.9213
        ],
        [
          0.525,
          0.919
        ],
        [
          0.535,
          0.9154
        ],
        [
          0.545,
          0.9122
        ],
        [
          0.5551,
          0.9067
        ],
        [
          0.5655,
          0.9045
        ],
        [
          0.5755,
          0.8981
        ],
        [
          0.5855,
          0.8935
        ],
        [
          0.5955,
          0.8872
        ],
        [
          0.6056,
          0.8817
        ],
        [
          0.616,
          0.8767
        ],
        [
          0.626,
          0.8712
        ],
        [
          0.636,
          0.8676
        ],
        [
          0.646,
          0.8617
        ],
        [
          0.6561,
          0.8562
        ],
        [
          0.6665,
          0.8508
        ],
        [
          0.6765,
          0.8462
        ],
        [
          0.6865,
          0.8417
        ],
        [
          0.6965,
          0.8344
        ],
        [
          0.7066,
          0.8308
        ],
        [
          0.717,
          0.8267
        ],
        [
          0.727,
          0.8198
        ],
        [
          0.737,
          0.813
        ],
        [
          0.747,
          0.8062
        ],
        [
          0.7571,
          0.7994
        ],
        [
          0.7675,
          0.7939
        ],
        [
          0.7775,
          0.7884
        ],
        [
          0.7875,
          0.783
        ],
        [
          0.7975,
          0.7766
        ],
        [
          0.8076,
          0.7698
        ],
        [
          0.818,
          0.7652
        ],
        [
          0.828,
          0.7571
        ],
        [
          0.838,
          0.7502
        ],
        [
          0.848,
          0.7439
        ],
        [
          0.8581,
          0.7375
        ],
        [
          0.8685,
          0.7316
        ],
        [
          0.8785,
          0.7238
        ],
        [
          0.8885,
          0.7184
        ],
        [
          0.8985,
          0.7125
        ],
        [
          0.9086,
          0.7066
        ],
        [
          0.919,
          0.6988
        ],
        [
          0.929,
          0.6924
        ],
        [
          0.939,
          0.6856
        ],
        [
          0.949,
          0.6793
        ],
        [
          0.9591,
          0.6715
        ],
        [
          0.9695,
          0.6642
        ],
        [
          0.9795,
          0.6556
        ],
        [
          0.9895,
          0.6474
        ],
        [
          0.9995,
          0.6392
        ],
        [
          1.0,
          0.6388
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1717262233726893,
        "difference": -0.0012066025072837553,
        "worse": 0.61,
        "kept_right": 0.17030435332167054,
        "kept_wrong": 0.0014218700510187207,
        "handed_off": 0.8282737766273104
      },
      "usd_per_1000": 1.3590270804477462
    },
    {
      "decision": 346,
      "frontier": 48,
      "frontier_accuracy": 0.9758871701546861,
      "in_sample": {
        "threshold": 0.9926909804344177,
        "answered": 0.1105550500454959,
        "accuracy": 0.9758871701546861
      },
      "curve": [
        [
          0.0,
          0.9759
        ],
        [
          0.01,
          0.9759
        ],
        [
          0.02,
          0.9759
        ],
        [
          0.03,
          0.9759
        ],
        [
          0.04,
          0.9759
        ],
        [
          0.05,
          0.9759
        ],
        [
          0.0601,
          0.9759
        ],
        [
          0.0705,
          0.9759
        ],
        [
          0.0805,
          0.9759
        ],
        [
          0.0905,
          0.9759
        ],
        [
          0.1005,
          0.9759
        ],
        [
          0.1106,
          0.9759
        ],
        [
          0.121,
          0.9754
        ],
        [
          0.131,
          0.9754
        ],
        [
          0.141,
          0.9754
        ],
        [
          0.151,
          0.9754
        ],
        [
          0.1611,
          0.9754
        ],
        [
          0.1715,
          0.975
        ],
        [
          0.1815,
          0.975
        ],
        [
          0.1915,
          0.975
        ],
        [
          0.2015,
          0.975
        ],
        [
          0.2116,
          0.9754
        ],
        [
          0.222,
          0.9741
        ],
        [
          0.232,
          0.9732
        ],
        [
          0.242,
          0.9732
        ],
        [
          0.252,
          0.9732
        ],
        [
          0.2621,
          0.9722
        ],
        [
          0.2725,
          0.9713
        ],
        [
          0.2825,
          0.9704
        ],
        [
          0.2925,
          0.9695
        ],
        [
          0.3025,
          0.9695
        ],
        [
          0.3126,
          0.9668
        ],
        [
          0.323,
          0.9654
        ],
        [
          0.333,
          0.9636
        ],
        [
          0.343,
          0.9622
        ],
        [
          0.353,
          0.9613
        ],
        [
          0.3631,
          0.9604
        ],
        [
          0.3735,
          0.9577
        ],
        [
          0.3835,
          0.9554
        ],
        [
          0.3935,
          0.9536
        ],
        [
          0.4035,
          0.9518
        ],
        [
          0.4136,
          0.9486
        ],
        [
          0.424,
          0.9477
        ],
        [
          0.434,
          0.9445
        ],
        [
          0.444,
          0.9418
        ],
        [
          0.454,
          0.939
        ],
        [
          0.4641,
          0.9368
        ],
        [
          0.4745,
          0.9336
        ],
        [
          0.4845,
          0.9299
        ],
        [
          0.4945,
          0.9268
        ],
        [
          0.5045,
          0.9227
        ],
        [
          0.5146,
          0.9186
        ],
        [
          0.525,
          0.9163
        ],
        [
          0.535,
          0.9126
        ],
        [
          0.545,
          0.9095
        ],
        [
          0.5551,
          0.904
        ],
        [
          0.5655,
          0.9022
        ],
        [
          0.5755,
          0.8958
        ],
        [
          0.5855,
          0.8913
        ],
        [
          0.5955,
          0.8854
        ],
        [
          0.6056,
          0.8799
        ],
        [
          0.616,
          0.8749
        ],
        [
          0.626,
          0.8694
        ],
        [
          0.636,
          0.8658
        ],
        [
          0.646,
          0.8599
        ],
        [
          0.6561,
          0.8544
        ],
        [
          0.6665,
          0.849
        ],
        [
          0.6765,
          0.8444
        ],
        [
          0.6865,
          0.8394
        ],
        [
          0.6965,
          0.8321
        ],
        [
          0.7066,
          0.8294
        ],
        [
          0.717,
          0.8253
        ],
        [
          0.727,
          0.8185
        ],
        [
          0.737,
          0.8116
        ],
        [
          0.747,
          0.8048
        ],
        [
          0.7571,
          0.7975
        ],
        [
          0.7675,
          0.7916
        ],
        [
          0.7775,
          0.7862
        ],
        [
          0.7875,
          0.7807
        ],
        [
          0.7975,
          0.7743
        ],
        [
          0.8076,
          0.7675
        ],
        [
          0.818,
          0.7625
        ],
        [
          0.828,
          0.7543
        ],
        [
          0.838,
          0.7475
        ],
        [
          0.848,
          0.7411
        ],
        [
          0.8581,
          0.7348
        ],
        [
          0.8685,
          0.7288
        ],
        [
          0.8785,
          0.7211
        ],
        [
          0.8885,
          0.7157
        ],
        [
          0.8985,
          0.7102
        ],
        [
          0.9086,
          0.7043
        ],
        [
          0.919,
          0.697
        ],
        [
          0.929,
          0.6906
        ],
        [
          0.939,
          0.6838
        ],
        [
          0.949,
          0.6779
        ],
        [
          0.9591,
          0.6706
        ],
        [
          0.9695,
          0.6633
        ],
        [
          0.9795,
          0.6547
        ],
        [
          0.9895,
          0.647
        ],
        [
          0.9995,
          0.6392
        ],
        [
          1.0,
          0.6388
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.17161708831117217,
        "difference": -0.0012005446585507471,
        "worse": 0.61,
        "kept_right": 0.1702073422296022,
        "kept_wrong": 0.0014097460815699107,
        "handed_off": 0.8283829116888275
      },
      "usd_per_1000": 4.470640215555903
    },
    {
      "decision": 346,
      "frontier": 191,
      "frontier_accuracy": 0.9740673339399454,
      "in_sample": {
        "threshold": 0.9781514406204224,
        "answered": 0.21292083712465878,
        "accuracy": 0.9740673339399454
      },
      "curve": [
        [
          0.0,
          0.9741
        ],
        [
          0.01,
          0.9741
        ],
        [
          0.02,
          0.9741
        ],
        [
          0.03,
          0.9741
        ],
        [
          0.04,
          0.9741
        ],
        [
          0.05,
          0.9741
        ],
        [
          0.0601,
          0.9741
        ],
        [
          0.0705,
          0.9741
        ],
        [
          0.0805,
          0.9741
        ],
        [
          0.0905,
          0.9741
        ],
        [
          0.1005,
          0.9741
        ],
        [
          0.1106,
          0.9741
        ],
        [
          0.121,
          0.9736
        ],
        [
          0.131,
          0.9736
        ],
        [
          0.141,
          0.9736
        ],
        [
          0.151,
          0.9736
        ],
        [
          0.1611,
          0.9736
        ],
        [
          0.1715,
          0.9732
        ],
        [
          0.1815,
          0.9732
        ],
        [
          0.1915,
          0.9732
        ],
        [
          0.2015,
          0.9732
        ],
        [
          0.2116,
          0.9741
        ],
        [
          0.222,
          0.9727
        ],
        [
          0.232,
          0.9718
        ],
        [
          0.242,
          0.9718
        ],
        [
          0.252,
          0.9718
        ],
        [
          0.2621,
          0.9709
        ],
        [
          0.2725,
          0.97
        ],
        [
          0.2825,
          0.9691
        ],
        [
          0.2925,
          0.9682
        ],
        [
          0.3025,
          0.9682
        ],
        [
          0.3126,
          0.9654
        ],
        [
          0.323,
          0.9641
        ],
        [
          0.333,
          0.9622
        ],
        [
          0.343,
          0.9609
        ],
        [
          0.353,
          0.96
        ],
        [
          0.3631,
          0.9591
        ],
        [
          0.3735,
          0.9563
        ],
        [
          0.3835,
          0.954
        ],
        [
          0.3935,
          0.9527
        ],
        [
          0.4035,
          0.9504
        ],
        [
          0.4136,
          0.9477
        ],
        [
          0.424,
          0.9468
        ],
        [
          0.434,
          0.9436
        ],
        [
          0.444,
          0.9409
        ],
        [
          0.454,
          0.9381
        ],
        [
          0.4641,
          0.9354
        ],
        [
          0.4745,
          0.9318
        ],
        [
          0.4845,
          0.9281
        ],
        [
          0.4945,
          0.9249
        ],
        [
          0.5045,
          0.9208
        ],
        [
          0.5146,
          0.9167
        ],
        [
          0.525,
          0.914
        ],
        [
          0.535,
          0.9104
        ],
        [
          0.545,
          0.9076
        ],
        [
          0.5551,
          0.9022
        ],
        [
          0.5655,
          0.8999
        ],
        [
          0.5755,
          0.8935
        ],
        [
          0.5855,
          0.889
        ],
        [
          0.5955,
          0.8826
        ],
        [
          0.6056,
          0.8772
        ],
        [
          0.616,
          0.8722
        ],
        [
          0.626,
          0.8667
        ],
        [
          0.636,
          0.8631
        ],
        [
          0.646,
          0.8571
        ],
        [
          0.6561,
          0.8521
        ],
        [
          0.6665,
          0.8471
        ],
        [
          0.6765,
          0.843
        ],
        [
          0.6865,
          0.8385
        ],
        [
          0.6965,
          0.8312
        ],
        [
          0.7066,
          0.8276
        ],
        [
          0.717,
          0.8235
        ],
        [
          0.727,
          0.8171
        ],
        [
          0.737,
          0.8103
        ],
        [
          0.747,
          0.803
        ],
        [
          0.7571,
          0.7957
        ],
        [
          0.7675,
          0.7898
        ],
        [
          0.7775,
          0.7848
        ],
        [
          0.7875,
          0.7798
        ],
        [
          0.7975,
          0.7734
        ],
        [
          0.8076,
          0.7666
        ],
        [
          0.818,
          0.7621
        ],
        [
          0.828,
          0.7539
        ],
        [
          0.838,
          0.747
        ],
        [
          0.848,
          0.7407
        ],
        [
          0.8581,
          0.7338
        ],
        [
          0.8685,
          0.7279
        ],
        [
          0.8785,
          0.7202
        ],
        [
          0.8885,
          0.7152
        ],
        [
          0.8985,
          0.7093
        ],
        [
          0.9086,
          0.7043
        ],
        [
          0.919,
          0.697
        ],
        [
          0.929,
          0.6906
        ],
        [
          0.939,
          0.6838
        ],
        [
          0.949,
          0.6779
        ],
        [
          0.9591,
          0.6706
        ],
        [
          0.9695,
          0.6633
        ],
        [
          0.9795,
          0.6551
        ],
        [
          0.9895,
          0.6474
        ],
        [
          0.9995,
          0.6392
        ],
        [
          1.0,
          0.6388
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.19158334989771808,
        "difference": -0.0012430157959764554,
        "worse": 0.53,
        "kept_right": 0.18965829172366333,
        "kept_wrong": 0.0019250581740548013,
        "handed_off": 0.8084166501022817
      },
      "usd_per_1000": 21.630608201935146
    },
    {
      "decision": 346,
      "frontier": 188,
      "frontier_accuracy": 0.9713375796178344,
      "in_sample": {
        "threshold": 0.9926909804344177,
        "answered": 0.1105550500454959,
        "accuracy": 0.9713375796178344
      },
      "curve": [
        [
          0.0,
          0.9713
        ],
        [
          0.01,
          0.9713
        ],
        [
          0.02,
          0.9713
        ],
        [
          0.03,
          0.9713
        ],
        [
          0.04,
          0.9713
        ],
        [
          0.05,
          0.9713
        ],
        [
          0.0601,
          0.9713
        ],
        [
          0.0705,
          0.9713
        ],
        [
          0.0805,
          0.9713
        ],
        [
          0.0905,
          0.9713
        ],
        [
          0.1005,
          0.9713
        ],
        [
          0.1106,
          0.9713
        ],
        [
          0.121,
          0.9709
        ],
        [
          0.131,
          0.9709
        ],
        [
          0.141,
          0.9709
        ],
        [
          0.151,
          0.9709
        ],
        [
          0.1611,
          0.9709
        ],
        [
          0.1715,
          0.9704
        ],
        [
          0.1815,
          0.9704
        ],
        [
          0.1915,
          0.9704
        ],
        [
          0.2015,
          0.9704
        ],
        [
          0.2116,
          0.9709
        ],
        [
          0.222,
          0.9695
        ],
        [
          0.232,
          0.9686
        ],
        [
          0.242,
          0.9686
        ],
        [
          0.252,
          0.9682
        ],
        [
          0.2621,
          0.9672
        ],
        [
          0.2725,
          0.9668
        ],
        [
          0.2825,
          0.9663
        ],
        [
          0.2925,
          0.9654
        ],
        [
          0.3025,
          0.9659
        ],
        [
          0.3126,
          0.9631
        ],
        [
          0.323,
          0.9618
        ],
        [
          0.333,
          0.9604
        ],
        [
          0.343,
          0.9591
        ],
        [
          0.353,
          0.9581
        ],
        [
          0.3631,
          0.9577
        ],
        [
          0.3735,
          0.955
        ],
        [
          0.3835,
          0.9527
        ],
        [
          0.3935,
          0.9513
        ],
        [
          0.4035,
          0.949
        ],
        [
          0.4136,
          0.9459
        ],
        [
          0.424,
          0.9449
        ],
        [
          0.434,
          0.9418
        ],
        [
          0.444,
          0.9381
        ],
        [
          0.454,
          0.9359
        ],
        [
          0.4641,
          0.9331
        ],
        [
          0.4745,
          0.9304
        ],
        [
          0.4845,
          0.9268
        ],
        [
          0.4945,
          0.9236
        ],
        [
          0.5045,
          0.9195
        ],
        [
          0.5146,
          0.9154
        ],
        [
          0.525,
          0.9126
        ],
        [
          0.535,
          0.9095
        ],
        [
          0.545,
          0.9063
        ],
        [
          0.5551,
          0.9022
        ],
        [
          0.5655,
          0.9004
        ],
        [
          0.5755,
          0.894
        ],
        [
          0.5855,
          0.8894
        ],
        [
          0.5955,
          0.8831
        ],
        [
          0.6056,
          0.8776
        ],
        [
          0.616,
          0.8731
        ],
        [
          0.626,
          0.8681
        ],
        [
          0.636,
          0.8644
        ],
        [
          0.646,
          0.8585
        ],
        [
          0.6561,
          0.853
        ],
        [
          0.6665,
          0.8471
        ],
        [
          0.6765,
          0.843
        ],
        [
          0.6865,
          0.8385
        ],
        [
          0.6965,
          0.8312
        ],
        [
          0.7066,
          0.8276
        ],
        [
          0.717,
          0.8235
        ],
        [
          0.727,
          0.8176
        ],
        [
          0.737,
          0.8107
        ],
        [
          0.747,
          0.8039
        ],
        [
          0.7571,
          0.7971
        ],
        [
          0.7675,
          0.7912
        ],
        [
          0.7775,
          0.7857
        ],
        [
          0.7875,
          0.7803
        ],
        [
          0.7975,
          0.7739
        ],
        [
          0.8076,
          0.7671
        ],
        [
          0.818,
          0.7621
        ],
        [
          0.828,
          0.7539
        ],
        [
          0.838,
          0.747
        ],
        [
          0.848,
          0.7402
        ],
        [
          0.8581,
          0.7338
        ],
        [
          0.8685,
          0.7279
        ],
        [
          0.8785,
          0.7207
        ],
        [
          0.8885,
          0.7157
        ],
        [
          0.8985,
          0.7102
        ],
        [
          0.9086,
          0.7047
        ],
        [
          0.919,
          0.6975
        ],
        [
          0.929,
          0.6911
        ],
        [
          0.939,
          0.6843
        ],
        [
          0.949,
          0.6779
        ],
        [
          0.9591,
          0.6706
        ],
        [
          0.9695,
          0.6633
        ],
        [
          0.9795,
          0.6547
        ],
        [
          0.9895,
          0.6474
        ],
        [
          0.9995,
          0.6392
        ],
        [
          1.0,
          0.6388
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.17148361703109963,
        "difference": -0.0012005501882870425,
        "worse": 0.61,
        "kept_right": 0.17007690125256,
        "kept_wrong": 0.0014067157785396076,
        "handed_off": 0.8285163829689001
      },
      "usd_per_1000": 0.8136351840844174
    },
    {
      "decision": 346,
      "frontier": 190,
      "frontier_accuracy": 0.9672429481346679,
      "in_sample": {
        "threshold": 0.9867754578590393,
        "answered": 0.1624203821656051,
        "accuracy": 0.9672429481346679
      },
      "curve": [
        [
          0.0,
          0.9672
        ],
        [
          0.01,
          0.9672
        ],
        [
          0.02,
          0.9672
        ],
        [
          0.03,
          0.9672
        ],
        [
          0.04,
          0.9672
        ],
        [
          0.05,
          0.9672
        ],
        [
          0.0601,
          0.9672
        ],
        [
          0.0705,
          0.9672
        ],
        [
          0.0805,
          0.9672
        ],
        [
          0.0905,
          0.9672
        ],
        [
          0.1005,
          0.9672
        ],
        [
          0.1106,
          0.9672
        ],
        [
          0.121,
          0.9668
        ],
        [
          0.131,
          0.9668
        ],
        [
          0.141,
          0.9668
        ],
        [
          0.151,
          0.9668
        ],
        [
          0.1611,
          0.9672
        ],
        [
          0.1715,
          0.9668
        ],
        [
          0.1815,
          0.9668
        ],
        [
          0.1915,
          0.9668
        ],
        [
          0.2015,
          0.9668
        ],
        [
          0.2116,
          0.9668
        ],
        [
          0.222,
          0.9654
        ],
        [
          0.232,
          0.9645
        ],
        [
          0.242,
          0.9645
        ],
        [
          0.252,
          0.9645
        ],
        [
          0.2621,
          0.9641
        ],
        [
          0.2725,
          0.9631
        ],
        [
          0.2825,
          0.9622
        ],
        [
          0.2925,
          0.9618
        ],
        [
          0.3025,
          0.9618
        ],
        [
          0.3126,
          0.9591
        ],
        [
          0.323,
          0.9577
        ],
        [
          0.333,
          0.9559
        ],
        [
          0.343,
          0.9545
        ],
        [
          0.353,
          0.9536
        ],
        [
          0.3631,
          0.9527
        ],
        [
          0.3735,
          0.9504
        ],
        [
          0.3835,
          0.9477
        ],
        [
          0.3935,
          0.9468
        ],
        [
          0.4035,
          0.9445
        ],
        [
          0.4136,
          0.9413
        ],
        [
          0.424,
          0.9404
        ],
        [
          0.434,
          0.9372
        ],
        [
          0.444,
          0.9345
        ],
        [
          0.454,
          0.9318
        ],
        [
          0.4641,
          0.9295
        ],
        [
          0.4745,
          0.9258
        ],
        [
          0.4845,
          0.9222
        ],
        [
          0.4945,
          0.919
        ],
        [
          0.5045,
          0.9154
        ],
        [
          0.5146,
          0.9113
        ],
        [
          0.525,
          0.909
        ],
        [
          0.535,
          0.9054
        ],
        [
          0.545,
          0.9026
        ],
        [
          0.5551,
          0.8981
        ],
        [
          0.5655,
          0.8958
        ],
        [
          0.5755,
          0.8894
        ],
        [
          0.5855,
          0.8849
        ],
        [
          0.5955,
          0.8785
        ],
        [
          0.6056,
          0.8731
        ],
        [
          0.616,
          0.8681
        ],
        [
          0.626,
          0.8626
        ],
        [
          0.636,
          0.859
        ],
        [
          0.646,
          0.853
        ],
        [
          0.6561,
          0.848
        ],
        [
          0.6665,
          0.8426
        ],
        [
          0.6765,
          0.838
        ],
        [
          0.6865,
          0.8335
        ],
        [
          0.6965,
          0.8262
        ],
        [
          0.7066,
          0.8235
        ],
        [
          0.717,
          0.8198
        ],
        [
          0.727,
          0.8135
        ],
        [
          0.737,
          0.8066
        ],
        [
          0.747,
          0.8003
        ],
        [
          0.7571,
          0.7934
        ],
        [
          0.7675,
          0.7884
        ],
        [
          0.7775,
          0.783
        ],
        [
          0.7875,
          0.778
        ],
        [
          0.7975,
          0.7716
        ],
        [
          0.8076,
          0.7648
        ],
        [
          0.818,
          0.7602
        ],
        [
          0.828,
          0.752
        ],
        [
          0.838,
          0.7452
        ],
        [
          0.848,
          0.7389
        ],
        [
          0.8581,
          0.7329
        ],
        [
          0.8685,
          0.727
        ],
        [
          0.8785,
          0.7193
        ],
        [
          0.8885,
          0.7147
        ],
        [
          0.8985,
          0.7093
        ],
        [
          0.9086,
          0.7043
        ],
        [
          0.919,
          0.697
        ],
        [
          0.929,
          0.6906
        ],
        [
          0.939,
          0.6838
        ],
        [
          0.949,
          0.6774
        ],
        [
          0.9591,
          0.6702
        ],
        [
          0.9695,
          0.6629
        ],
        [
          0.9795,
          0.6547
        ],
        [
          0.9895,
          0.6474
        ],
        [
          0.9995,
          0.6392
        ],
        [
          1.0,
          0.6388
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.18109414902446647,
        "difference": -0.001203478414743888,
        "worse": 0.55,
        "kept_right": 0.17960573355945567,
        "kept_wrong": 0.0014884154650108591,
        "handed_off": 0.8189058509755333
      },
      "usd_per_1000": 4.575502304071077
    },
    {
      "decision": 346,
      "frontier": 124,
      "frontier_accuracy": 0.9667879890809827,
      "in_sample": {
        "threshold": 0.9926909804344177,
        "answered": 0.1105550500454959,
        "accuracy": 0.9667879890809827
      },
      "curve": [
        [
          0.0,
          0.9668
        ],
        [
          0.01,
          0.9668
        ],
        [
          0.02,
          0.9668
        ],
        [
          0.03,
          0.9668
        ],
        [
          0.04,
          0.9668
        ],
        [
          0.05,
          0.9668
        ],
        [
          0.0601,
          0.9668
        ],
        [
          0.0705,
          0.9668
        ],
        [
          0.0805,
          0.9668
        ],
        [
          0.0905,
          0.9668
        ],
        [
          0.1005,
          0.9668
        ],
        [
          0.1106,
          0.9668
        ],
        [
          0.121,
          0.9663
        ],
        [
          0.131,
          0.9663
        ],
        [
          0.141,
          0.9663
        ],
        [
          0.151,
          0.9663
        ],
        [
          0.1611,
          0.9663
        ],
        [
          0.1715,
          0.9659
        ],
        [
          0.1815,
          0.9659
        ],
        [
          0.1915,
          0.9659
        ],
        [
          0.2015,
          0.9659
        ],
        [
          0.2116,
          0.9663
        ],
        [
          0.222,
          0.965
        ],
        [
          0.232,
          0.9641
        ],
        [
          0.242,
          0.9641
        ],
        [
          0.252,
          0.9641
        ],
        [
          0.2621,
          0.9636
        ],
        [
          0.2725,
          0.9627
        ],
        [
          0.2825,
          0.9618
        ],
        [
          0.2925,
          0.9613
        ],
        [
          0.3025,
          0.9613
        ],
        [
          0.3126,
          0.9586
        ],
        [
          0.323,
          0.9572
        ],
        [
          0.333,
          0.9554
        ],
        [
          0.343,
          0.954
        ],
        [
          0.353,
          0.9531
        ],
        [
          0.3631,
          0.9522
        ],
        [
          0.3735,
          0.9495
        ],
        [
          0.3835,
          0.9468
        ],
        [
          0.3935,
          0.9449
        ],
        [
          0.4035,
          0.9431
        ],
        [
          0.4136,
          0.9399
        ],
        [
          0.424,
          0.939
        ],
        [
          0.434,
          0.9359
        ],
        [
          0.444,
          0.9331
        ],
        [
          0.454,
          0.9304
        ],
        [
          0.4641,
          0.9281
        ],
        [
          0.4745,
          0.9249
        ],
        [
          0.4845,
          0.9213
        ],
        [
          0.4945,
          0.9181
        ],
        [
          0.5045,
          0.9145
        ],
        [
          0.5146,
          0.9108
        ],
        [
          0.525,
          0.9086
        ],
        [
          0.535,
          0.9054
        ],
        [
          0.545,
          0.9022
        ],
        [
          0.5551,
          0.8972
        ],
        [
          0.5655,
          0.8949
        ],
        [
          0.5755,
          0.8885
        ],
        [
          0.5855,
          0.884
        ],
        [
          0.5955,
          0.8781
        ],
        [
          0.6056,
          0.8726
        ],
        [
          0.616,
          0.8676
        ],
        [
          0.626,
          0.8621
        ],
        [
          0.636,
          0.8585
        ],
        [
          0.646,
          0.8526
        ],
        [
          0.6561,
          0.8471
        ],
        [
          0.6665,
          0.8421
        ],
        [
          0.6765,
          0.8376
        ],
        [
          0.6865,
          0.8326
        ],
        [
          0.6965,
          0.8253
        ],
        [
          0.7066,
          0.8221
        ],
        [
          0.717,
          0.818
        ],
        [
          0.727,
          0.8112
        ],
        [
          0.737,
          0.8044
        ],
        [
          0.747,
          0.7971
        ],
        [
          0.7571,
          0.7907
        ],
        [
          0.7675,
          0.7857
        ],
        [
          0.7775,
          0.7816
        ],
        [
          0.7875,
          0.7766
        ],
        [
          0.7975,
          0.7702
        ],
        [
          0.8076,
          0.7639
        ],
        [
          0.818,
          0.7589
        ],
        [
          0.828,
          0.7507
        ],
        [
          0.838,
          0.7439
        ],
        [
          0.848,
          0.7384
        ],
        [
          0.8581,
          0.732
        ],
        [
          0.8685,
          0.7261
        ],
        [
          0.8785,
          0.7184
        ],
        [
          0.8885,
          0.7138
        ],
        [
          0.8985,
          0.7093
        ],
        [
          0.9086,
          0.7043
        ],
        [
          0.919,
          0.697
        ],
        [
          0.929,
          0.6906
        ],
        [
          0.939,
          0.6843
        ],
        [
          0.949,
          0.6774
        ],
        [
          0.9591,
          0.6702
        ],
        [
          0.9695,
          0.6629
        ],
        [
          0.9795,
          0.6542
        ],
        [
          0.9895,
          0.6474
        ],
        [
          0.9995,
          0.6392
        ],
        [
          1.0,
          0.6388
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1717262233726893,
        "difference": -0.0012066025072837553,
        "worse": 0.61,
        "kept_right": 0.17030435332167054,
        "kept_wrong": 0.0014218700510187207,
        "handed_off": 0.8282737766273104
      },
      "usd_per_1000": 0.9516678682698595
    },
    {
      "decision": 346,
      "frontier": 278,
      "frontier_accuracy": 0.9499545040946314,
      "in_sample": {
        "threshold": 0.9926909804344177,
        "answered": 0.1105550500454959,
        "accuracy": 0.9499545040946314
      },
      "curve": [
        [
          0.0,
          0.95
        ],
        [
          0.01,
          0.95
        ],
        [
          0.02,
          0.95
        ],
        [
          0.03,
          0.95
        ],
        [
          0.04,
          0.95
        ],
        [
          0.05,
          0.95
        ],
        [
          0.0601,
          0.95
        ],
        [
          0.0705,
          0.95
        ],
        [
          0.0805,
          0.95
        ],
        [
          0.0905,
          0.95
        ],
        [
          0.1005,
          0.95
        ],
        [
          0.1106,
          0.95
        ],
        [
          0.121,
          0.9495
        ],
        [
          0.131,
          0.9495
        ],
        [
          0.141,
          0.9495
        ],
        [
          0.151,
          0.9495
        ],
        [
          0.1611,
          0.9495
        ],
        [
          0.1715,
          0.949
        ],
        [
          0.1815,
          0.949
        ],
        [
          0.1915,
          0.949
        ],
        [
          0.2015,
          0.949
        ],
        [
          0.2116,
          0.9495
        ],
        [
          0.222,
          0.9481
        ],
        [
          0.232,
          0.9472
        ],
        [
          0.242,
          0.9472
        ],
        [
          0.252,
          0.9472
        ],
        [
          0.2621,
          0.9472
        ],
        [
          0.2725,
          0.9463
        ],
        [
          0.2825,
          0.9454
        ],
        [
          0.2925,
          0.9445
        ],
        [
          0.3025,
          0.9445
        ],
        [
          0.3126,
          0.9418
        ],
        [
          0.323,
          0.9404
        ],
        [
          0.333,
          0.9386
        ],
        [
          0.343,
          0.9377
        ],
        [
          0.353,
          0.9368
        ],
        [
          0.3631,
          0.9363
        ],
        [
          0.3735,
          0.9336
        ],
        [
          0.3835,
          0.9313
        ],
        [
          0.3935,
          0.9299
        ],
        [
          0.4035,
          0.9277
        ],
        [
          0.4136,
          0.9245
        ],
        [
          0.424,
          0.9236
        ],
        [
          0.434,
          0.9208
        ],
        [
          0.444,
          0.9181
        ],
        [
          0.454,
          0.9163
        ],
        [
          0.4641,
          0.9149
        ],
        [
          0.4745,
          0.9122
        ],
        [
          0.4845,
          0.909
        ],
        [
          0.4945,
          0.9063
        ],
        [
          0.5045,
          0.9022
        ],
        [
          0.5146,
          0.8981
        ],
        [
          0.525,
          0.8954
        ],
        [
          0.535,
          0.8922
        ],
        [
          0.545,
          0.889
        ],
        [
          0.5551,
          0.8844
        ],
        [
          0.5655,
          0.8831
        ],
        [
          0.5755,
          0.8767
        ],
        [
          0.5855,
          0.8722
        ],
        [
          0.5955,
          0.8667
        ],
        [
          0.6056,
          0.8617
        ],
        [
          0.616,
          0.8581
        ],
        [
          0.626,
          0.8526
        ],
        [
          0.636,
          0.8499
        ],
        [
          0.646,
          0.8444
        ],
        [
          0.6561,
          0.8389
        ],
        [
          0.6665,
          0.8339
        ],
        [
          0.6765,
          0.8303
        ],
        [
          0.6865,
          0.8258
        ],
        [
          0.6965,
          0.8185
        ],
        [
          0.7066,
          0.8157
        ],
        [
          0.717,
          0.8121
        ],
        [
          0.727,
          0.8057
        ],
        [
          0.737,
          0.7989
        ],
        [
          0.747,
          0.7921
        ],
        [
          0.7571,
          0.7866
        ],
        [
          0.7675,
          0.7812
        ],
        [
          0.7775,
          0.7762
        ],
        [
          0.7875,
          0.7712
        ],
        [
          0.7975,
          0.7657
        ],
        [
          0.8076,
          0.7593
        ],
        [
          0.818,
          0.7548
        ],
        [
          0.828,
          0.747
        ],
        [
          0.838,
          0.7402
        ],
        [
          0.848,
          0.7334
        ],
        [
          0.8581,
          0.7275
        ],
        [
          0.8685,
          0.7211
        ],
        [
          0.8785,
          0.7138
        ],
        [
          0.8885,
          0.7093
        ],
        [
          0.8985,
          0.7043
        ],
        [
          0.9086,
          0.6988
        ],
        [
          0.919,
          0.692
        ],
        [
          0.929,
          0.687
        ],
        [
          0.939,
          0.6811
        ],
        [
          0.949,
          0.6756
        ],
        [
          0.9591,
          0.6688
        ],
        [
          0.9695,
          0.662
        ],
        [
          0.9795,
          0.6529
        ],
        [
          0.9895,
          0.6456
        ],
        [
          0.9995,
          0.6388
        ],
        [
          1.0,
          0.6388
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.17569444875380844,
        "difference": -0.0014400106348758537,
        "worse": 0.6133333333333333,
        "kept_right": 0.17390274877397746,
        "kept_wrong": 0.001791699979830862,
        "handed_off": 0.8243055512461915
      },
      "usd_per_1000": 5.380671147957062
    },
    {
      "decision": 346,
      "frontier": 275,
      "frontier_accuracy": 0.9426751592356688,
      "in_sample": {
        "threshold": 0.9775926470756531,
        "answered": 0.2156505914467698,
        "accuracy": 0.9426751592356688
      },
      "curve": [
        [
          0.0,
          0.9427
        ],
        [
          0.01,
          0.9427
        ],
        [
          0.02,
          0.9427
        ],
        [
          0.03,
          0.9427
        ],
        [
          0.04,
          0.9427
        ],
        [
          0.05,
          0.9427
        ],
        [
          0.0601,
          0.9427
        ],
        [
          0.0705,
          0.9427
        ],
        [
          0.0805,
          0.9427
        ],
        [
          0.0905,
          0.9427
        ],
        [
          0.1005,
          0.9427
        ],
        [
          0.1106,
          0.9427
        ],
        [
          0.121,
          0.9422
        ],
        [
          0.131,
          0.9422
        ],
        [
          0.141,
          0.9422
        ],
        [
          0.151,
          0.9422
        ],
        [
          0.1611,
          0.9422
        ],
        [
          0.1715,
          0.9422
        ],
        [
          0.1815,
          0.9422
        ],
        [
          0.1915,
          0.9422
        ],
        [
          0.2015,
          0.9422
        ],
        [
          0.2116,
          0.9431
        ],
        [
          0.222,
          0.9418
        ],
        [
          0.232,
          0.9409
        ],
        [
          0.242,
          0.9409
        ],
        [
          0.252,
          0.9409
        ],
        [
          0.2621,
          0.9404
        ],
        [
          0.2725,
          0.9395
        ],
        [
          0.2825,
          0.9386
        ],
        [
          0.2925,
          0.9377
        ],
        [
          0.3025,
          0.9377
        ],
        [
          0.3126,
          0.9349
        ],
        [
          0.323,
          0.9336
        ],
        [
          0.333,
          0.9318
        ],
        [
          0.343,
          0.9308
        ],
        [
          0.353,
          0.9304
        ],
        [
          0.3631,
          0.9299
        ],
        [
          0.3735,
          0.9277
        ],
        [
          0.3835,
          0.9254
        ],
        [
          0.3935,
          0.9236
        ],
        [
          0.4035,
          0.9213
        ],
        [
          0.4136,
          0.9186
        ],
        [
          0.424,
          0.9177
        ],
        [
          0.434,
          0.9149
        ],
        [
          0.444,
          0.9122
        ],
        [
          0.454,
          0.9099
        ],
        [
          0.4641,
          0.9076
        ],
        [
          0.4745,
          0.9058
        ],
        [
          0.4845,
          0.9026
        ],
        [
          0.4945,
          0.9004
        ],
        [
          0.5045,
          0.8963
        ],
        [
          0.5146,
          0.8926
        ],
        [
          0.525,
          0.8904
        ],
        [
          0.535,
          0.8876
        ],
        [
          0.545,
          0.8844
        ],
        [
          0.5551,
          0.8799
        ],
        [
          0.5655,
          0.8781
        ],
        [
          0.5755,
          0.8717
        ],
        [
          0.5855,
          0.8676
        ],
        [
          0.5955,
          0.8612
        ],
        [
          0.6056,
          0.8567
        ],
        [
          0.616,
          0.8521
        ],
        [
          0.626,
          0.8471
        ],
        [
          0.636,
          0.8449
        ],
        [
          0.646,
          0.8389
        ],
        [
          0.6561,
          0.8339
        ],
        [
          0.6665,
          0.8289
        ],
        [
          0.6765,
          0.8244
        ],
        [
          0.6865,
          0.8198
        ],
        [
          0.6965,
          0.8126
        ],
        [
          0.7066,
          0.8094
        ],
        [
          0.717,
          0.8062
        ],
        [
          0.727,
          0.7998
        ],
        [
          0.737,
          0.793
        ],
        [
          0.747,
          0.7871
        ],
        [
          0.7571,
          0.7807
        ],
        [
          0.7675,
          0.7762
        ],
        [
          0.7775,
          0.7725
        ],
        [
          0.7875,
          0.7675
        ],
        [
          0.7975,
          0.7616
        ],
        [
          0.8076,
          0.7552
        ],
        [
          0.818,
          0.7511
        ],
        [
          0.828,
          0.7434
        ],
        [
          0.838,
          0.7375
        ],
        [
          0.848,
          0.7316
        ],
        [
          0.8581,
          0.7257
        ],
        [
          0.8685,
          0.7207
        ],
        [
          0.8785,
          0.7129
        ],
        [
          0.8885,
          0.7079
        ],
        [
          0.8985,
          0.702
        ],
        [
          0.9086,
          0.6975
        ],
        [
          0.919,
          0.6906
        ],
        [
          0.929,
          0.6847
        ],
        [
          0.939,
          0.6793
        ],
        [
          0.949,
          0.6733
        ],
        [
          0.9591,
          0.6661
        ],
        [
          0.9695,
          0.6611
        ],
        [
          0.9795,
          0.6529
        ],
        [
          0.9895,
          0.646
        ],
        [
          0.9995,
          0.6388
        ],
        [
          1.0,
          0.6388
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.21335421245114483,
        "difference": -0.0011642305442330274,
        "worse": 0.5166666666666667,
        "kept_right": 0.21076215878066806,
        "kept_wrong": 0.0025920536704768203,
        "handed_off": 0.7866457875488545
      },
      "usd_per_1000": 0.4201491360959965
    },
    {
      "decision": 346,
      "frontier": 192,
      "frontier_accuracy": 0.9331210191082803,
      "in_sample": {
        "threshold": 0.9867754578590393,
        "answered": 0.1624203821656051,
        "accuracy": 0.9331210191082803
      },
      "curve": [
        [
          0.0,
          0.9331
        ],
        [
          0.01,
          0.9331
        ],
        [
          0.02,
          0.9331
        ],
        [
          0.03,
          0.9331
        ],
        [
          0.04,
          0.9331
        ],
        [
          0.05,
          0.9331
        ],
        [
          0.0601,
          0.9331
        ],
        [
          0.0705,
          0.9331
        ],
        [
          0.0805,
          0.9331
        ],
        [
          0.0905,
          0.9331
        ],
        [
          0.1005,
          0.9331
        ],
        [
          0.1106,
          0.9331
        ],
        [
          0.121,
          0.9327
        ],
        [
          0.131,
          0.9327
        ],
        [
          0.141,
          0.9327
        ],
        [
          0.151,
          0.9327
        ],
        [
          0.1611,
          0.9331
        ],
        [
          0.1715,
          0.9327
        ],
        [
          0.1815,
          0.9327
        ],
        [
          0.1915,
          0.9327
        ],
        [
          0.2015,
          0.9327
        ],
        [
          0.2116,
          0.9327
        ],
        [
          0.222,
          0.9313
        ],
        [
          0.232,
          0.9304
        ],
        [
          0.242,
          0.9308
        ],
        [
          0.252,
          0.9308
        ],
        [
          0.2621,
          0.9304
        ],
        [
          0.2725,
          0.9295
        ],
        [
          0.2825,
          0.9286
        ],
        [
          0.2925,
          0.9277
        ],
        [
          0.3025,
          0.9277
        ],
        [
          0.3126,
          0.9254
        ],
        [
          0.323,
          0.924
        ],
        [
          0.333,
          0.9222
        ],
        [
          0.343,
          0.9208
        ],
        [
          0.353,
          0.9199
        ],
        [
          0.3631,
          0.9195
        ],
        [
          0.3735,
          0.9163
        ],
        [
          0.3835,
          0.914
        ],
        [
          0.3935,
          0.9126
        ],
        [
          0.4035,
          0.9113
        ],
        [
          0.4136,
          0.909
        ],
        [
          0.424,
          0.9076
        ],
        [
          0.434,
          0.9054
        ],
        [
          0.444,
          0.9022
        ],
        [
          0.454,
          0.9004
        ],
        [
          0.4641,
          0.8985
        ],
        [
          0.4745,
          0.8954
        ],
        [
          0.4845,
          0.8922
        ],
        [
          0.4945,
          0.8899
        ],
        [
          0.5045,
          0.8867
        ],
        [
          0.5146,
          0.8826
        ],
        [
          0.525,
          0.8813
        ],
        [
          0.535,
          0.8776
        ],
        [
          0.545,
          0.8744
        ],
        [
          0.5551,
          0.8703
        ],
        [
          0.5655,
          0.8681
        ],
        [
          0.5755,
          0.8626
        ],
        [
          0.5855,
          0.859
        ],
        [
          0.5955,
          0.8535
        ],
        [
          0.6056,
          0.848
        ],
        [
          0.616,
          0.8439
        ],
        [
          0.626,
          0.8389
        ],
        [
          0.636,
          0.8362
        ],
        [
          0.646,
          0.8303
        ],
        [
          0.6561,
          0.8262
        ],
        [
          0.6665,
          0.8212
        ],
        [
          0.6765,
          0.8171
        ],
        [
          0.6865,
          0.8135
        ],
        [
          0.6965,
          0.8062
        ],
        [
          0.7066,
          0.8039
        ],
        [
          0.717,
          0.8007
        ],
        [
          0.727,
          0.7948
        ],
        [
          0.737,
          0.7889
        ],
        [
          0.747,
          0.7825
        ],
        [
          0.7571,
          0.7775
        ],
        [
          0.7675,
          0.773
        ],
        [
          0.7775,
          0.7689
        ],
        [
          0.7875,
          0.7639
        ],
        [
          0.7975,
          0.758
        ],
        [
          0.8076,
          0.7516
        ],
        [
          0.818,
          0.748
        ],
        [
          0.828,
          0.7407
        ],
        [
          0.838,
          0.7343
        ],
        [
          0.848,
          0.7275
        ],
        [
          0.8581,
          0.7211
        ],
        [
          0.8685,
          0.7157
        ],
        [
          0.8785,
          0.7093
        ],
        [
          0.8885,
          0.7052
        ],
        [
          0.8985,
          0.7002
        ],
        [
          0.9086,
          0.6956
        ],
        [
          0.919,
          0.6893
        ],
        [
          0.929,
          0.6856
        ],
        [
          0.939,
          0.6802
        ],
        [
          0.949,
          0.6747
        ],
        [
          0.9591,
          0.6692
        ],
        [
          0.9695,
          0.6624
        ],
        [
          0.9795,
          0.6538
        ],
        [
          0.9895,
          0.647
        ],
        [
          0.9995,
          0.6392
        ],
        [
          1.0,
          0.6388
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.18607078687291362,
        "difference": -0.0014673193431655163,
        "worse": 0.5666666666666667,
        "kept_right": 0.18419116653943618,
        "kept_wrong": 0.0018796203334774822,
        "handed_off": 0.8139292131270861
      },
      "usd_per_1000": 9.464773875065063
    },
    {
      "decision": 346,
      "frontier": 123,
      "frontier_accuracy": 0.9326660600545951,
      "in_sample": {
        "threshold": 0.9781514406204224,
        "answered": 0.21292083712465878,
        "accuracy": 0.9326660600545951
      },
      "curve": [
        [
          0.0,
          0.9327
        ],
        [
          0.01,
          0.9327
        ],
        [
          0.02,
          0.9327
        ],
        [
          0.03,
          0.9327
        ],
        [
          0.04,
          0.9327
        ],
        [
          0.05,
          0.9327
        ],
        [
          0.0601,
          0.9327
        ],
        [
          0.0705,
          0.9327
        ],
        [
          0.0805,
          0.9327
        ],
        [
          0.0905,
          0.9327
        ],
        [
          0.1005,
          0.9327
        ],
        [
          0.1106,
          0.9327
        ],
        [
          0.121,
          0.9322
        ],
        [
          0.131,
          0.9322
        ],
        [
          0.141,
          0.9322
        ],
        [
          0.151,
          0.9322
        ],
        [
          0.1611,
          0.9322
        ],
        [
          0.1715,
          0.9318
        ],
        [
          0.1815,
          0.9318
        ],
        [
          0.1915,
          0.9318
        ],
        [
          0.2015,
          0.9318
        ],
        [
          0.2116,
          0.9327
        ],
        [
          0.222,
          0.9313
        ],
        [
          0.232,
          0.9304
        ],
        [
          0.242,
          0.9304
        ],
        [
          0.252,
          0.9299
        ],
        [
          0.2621,
          0.9299
        ],
        [
          0.2725,
          0.9295
        ],
        [
          0.2825,
          0.929
        ],
        [
          0.2925,
          0.9286
        ],
        [
          0.3025,
          0.9286
        ],
        [
          0.3126,
          0.9258
        ],
        [
          0.323,
          0.9249
        ],
        [
          0.333,
          0.9236
        ],
        [
          0.343,
          0.9222
        ],
        [
          0.353,
          0.9213
        ],
        [
          0.3631,
          0.9208
        ],
        [
          0.3735,
          0.9177
        ],
        [
          0.3835,
          0.9154
        ],
        [
          0.3935,
          0.914
        ],
        [
          0.4035,
          0.9122
        ],
        [
          0.4136,
          0.9095
        ],
        [
          0.424,
          0.9081
        ],
        [
          0.434,
          0.9049
        ],
        [
          0.444,
          0.9022
        ],
        [
          0.454,
          0.8999
        ],
        [
          0.4641,
          0.8972
        ],
        [
          0.4745,
          0.8944
        ],
        [
          0.4845,
          0.8908
        ],
        [
          0.4945,
          0.8881
        ],
        [
          0.5045,
          0.8854
        ],
        [
          0.5146,
          0.8817
        ],
        [
          0.525,
          0.8794
        ],
        [
          0.535,
          0.8772
        ],
        [
          0.545,
          0.874
        ],
        [
          0.5551,
          0.8708
        ],
        [
          0.5655,
          0.8694
        ],
        [
          0.5755,
          0.8631
        ],
        [
          0.5855,
          0.8594
        ],
        [
          0.5955,
          0.8535
        ],
        [
          0.6056,
          0.8485
        ],
        [
          0.616,
          0.8444
        ],
        [
          0.626,
          0.8394
        ],
        [
          0.636,
          0.8367
        ],
        [
          0.646,
          0.8312
        ],
        [
          0.6561,
          0.8262
        ],
        [
          0.6665,
          0.8217
        ],
        [
          0.6765,
          0.8176
        ],
        [
          0.6865,
          0.8135
        ],
        [
          0.6965,
          0.8062
        ],
        [
          0.7066,
          0.8039
        ],
        [
          0.717,
          0.8003
        ],
        [
          0.727,
          0.7953
        ],
        [
          0.737,
          0.7889
        ],
        [
          0.747,
          0.7821
        ],
        [
          0.7571,
          0.7762
        ],
        [
          0.7675,
          0.7702
        ],
        [
          0.7775,
          0.7662
        ],
        [
          0.7875,
          0.7611
        ],
        [
          0.7975,
          0.7561
        ],
        [
          0.8076,
          0.7493
        ],
        [
          0.818,
          0.7448
        ],
        [
          0.828,
          0.7375
        ],
        [
          0.838,
          0.7311
        ],
        [
          0.848,
          0.7247
        ],
        [
          0.8581,
          0.7184
        ],
        [
          0.8685,
          0.712
        ],
        [
          0.8785,
          0.7056
        ],
        [
          0.8885,
          0.702
        ],
        [
          0.8985,
          0.6975
        ],
        [
          0.9086,
          0.6943
        ],
        [
          0.919,
          0.6879
        ],
        [
          0.929,
          0.6829
        ],
        [
          0.939,
          0.677
        ],
        [
          0.949,
          0.6711
        ],
        [
          0.9591,
          0.6652
        ],
        [
          0.9695,
          0.6592
        ],
        [
          0.9795,
          0.6515
        ],
        [
          0.9895,
          0.646
        ],
        [
          0.9995,
          0.6392
        ],
        [
          1.0,
          0.6388
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.19613688675677413,
        "difference": -0.0015461593901470517,
        "worse": 0.5433333333333333,
        "kept_right": 0.19376314203797054,
        "kept_wrong": 0.0023737447188037202,
        "handed_off": 0.8038631132432256
      },
      "usd_per_1000": 0.2313142216766858
    },
    {
      "decision": 346,
      "frontier": 274,
      "frontier_accuracy": 0.9085532302092811,
      "in_sample": {
        "threshold": 0.9781514406204224,
        "answered": 0.21292083712465878,
        "accuracy": 0.9085532302092811
      },
      "curve": [
        [
          0.0,
          0.9086
        ],
        [
          0.01,
          0.9086
        ],
        [
          0.02,
          0.9086
        ],
        [
          0.03,
          0.9086
        ],
        [
          0.04,
          0.9086
        ],
        [
          0.05,
          0.9086
        ],
        [
          0.0601,
          0.9086
        ],
        [
          0.0705,
          0.9086
        ],
        [
          0.0805,
          0.9086
        ],
        [
          0.0905,
          0.9086
        ],
        [
          0.1005,
          0.9086
        ],
        [
          0.1106,
          0.9086
        ],
        [
          0.121,
          0.9081
        ],
        [
          0.131,
          0.9081
        ],
        [
          0.141,
          0.9081
        ],
        [
          0.151,
          0.9081
        ],
        [
          0.1611,
          0.9081
        ],
        [
          0.1715,
          0.9081
        ],
        [
          0.1815,
          0.9081
        ],
        [
          0.1915,
          0.9081
        ],
        [
          0.2015,
          0.9081
        ],
        [
          0.2116,
          0.9086
        ],
        [
          0.222,
          0.9072
        ],
        [
          0.232,
          0.9067
        ],
        [
          0.242,
          0.9072
        ],
        [
          0.252,
          0.9072
        ],
        [
          0.2621,
          0.9072
        ],
        [
          0.2725,
          0.9063
        ],
        [
          0.2825,
          0.9054
        ],
        [
          0.2925,
          0.9054
        ],
        [
          0.3025,
          0.9054
        ],
        [
          0.3126,
          0.9026
        ],
        [
          0.323,
          0.9017
        ],
        [
          0.333,
          0.9004
        ],
        [
          0.343,
          0.899
        ],
        [
          0.353,
          0.8985
        ],
        [
          0.3631,
          0.8981
        ],
        [
          0.3735,
          0.8958
        ],
        [
          0.3835,
          0.8935
        ],
        [
          0.3935,
          0.8931
        ],
        [
          0.4035,
          0.8913
        ],
        [
          0.4136,
          0.8885
        ],
        [
          0.424,
          0.8876
        ],
        [
          0.434,
          0.8849
        ],
        [
          0.444,
          0.8831
        ],
        [
          0.454,
          0.8808
        ],
        [
          0.4641,
          0.8785
        ],
        [
          0.4745,
          0.8763
        ],
        [
          0.4845,
          0.8735
        ],
        [
          0.4945,
          0.8708
        ],
        [
          0.5045,
          0.8672
        ],
        [
          0.5146,
          0.8635
        ],
        [
          0.525,
          0.8617
        ],
        [
          0.535,
          0.859
        ],
        [
          0.545,
          0.8567
        ],
        [
          0.5551,
          0.8526
        ],
        [
          0.5655,
          0.8517
        ],
        [
          0.5755,
          0.8453
        ],
        [
          0.5855,
          0.8412
        ],
        [
          0.5955,
          0.8348
        ],
        [
          0.6056,
          0.8294
        ],
        [
          0.616,
          0.8248
        ],
        [
          0.626,
          0.8198
        ],
        [
          0.636,
          0.8176
        ],
        [
          0.646,
          0.8116
        ],
        [
          0.6561,
          0.8071
        ],
        [
          0.6665,
          0.803
        ],
        [
          0.6765,
          0.7994
        ],
        [
          0.6865,
          0.7966
        ],
        [
          0.6965,
          0.7903
        ],
        [
          0.7066,
          0.788
        ],
        [
          0.717,
          0.7848
        ],
        [
          0.727,
          0.7793
        ],
        [
          0.737,
          0.7739
        ],
        [
          0.747,
          0.7675
        ],
        [
          0.7571,
          0.7625
        ],
        [
          0.7675,
          0.7584
        ],
        [
          0.7775,
          0.7548
        ],
        [
          0.7875,
          0.7507
        ],
        [
          0.7975,
          0.7457
        ],
        [
          0.8076,
          0.7398
        ],
        [
          0.818,
          0.7361
        ],
        [
          0.828,
          0.7293
        ],
        [
          0.838,
          0.7234
        ],
        [
          0.848,
          0.7179
        ],
        [
          0.8581,
          0.712
        ],
        [
          0.8685,
          0.7066
        ],
        [
          0.8785,
          0.6997
        ],
        [
          0.8885,
          0.697
        ],
        [
          0.8985,
          0.6924
        ],
        [
          0.9086,
          0.6888
        ],
        [
          0.919,
          0.6833
        ],
        [
          0.929,
          0.6788
        ],
        [
          0.939,
          0.6733
        ],
        [
          0.949,
          0.6674
        ],
        [
          0.9591,
          0.6633
        ],
        [
          0.9695,
          0.6583
        ],
        [
          0.9795,
          0.6506
        ],
        [
          0.9895,
          0.646
        ],
        [
          0.9995,
          0.6392
        ],
        [
          1.0,
          0.6388
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.212667040874282,
        "difference": -0.0016734628044690965,
        "worse": 0.5533333333333333,
        "kept_right": 0.20945659504606884,
        "kept_wrong": 0.003210445828213329,
        "handed_off": 0.7873329591257174
      },
      "usd_per_1000": 0.3510067704319271
    },
    {
      "decision": 346,
      "frontier": 47,
      "frontier_accuracy": 0.9044585987261147,
      "in_sample": {
        "threshold": 0.9781514406204224,
        "answered": 0.21292083712465878,
        "accuracy": 0.9044585987261147
      },
      "curve": [
        [
          0.0,
          0.9045
        ],
        [
          0.01,
          0.9045
        ],
        [
          0.02,
          0.9045
        ],
        [
          0.03,
          0.9045
        ],
        [
          0.04,
          0.9045
        ],
        [
          0.05,
          0.9045
        ],
        [
          0.0601,
          0.9045
        ],
        [
          0.0705,
          0.9045
        ],
        [
          0.0805,
          0.9045
        ],
        [
          0.0905,
          0.9045
        ],
        [
          0.1005,
          0.9045
        ],
        [
          0.1106,
          0.9045
        ],
        [
          0.121,
          0.9045
        ],
        [
          0.131,
          0.9045
        ],
        [
          0.141,
          0.9045
        ],
        [
          0.151,
          0.9045
        ],
        [
          0.1611,
          0.9045
        ],
        [
          0.1715,
          0.904
        ],
        [
          0.1815,
          0.904
        ],
        [
          0.1915,
          0.904
        ],
        [
          0.2015,
          0.904
        ],
        [
          0.2116,
          0.9045
        ],
        [
          0.222,
          0.9031
        ],
        [
          0.232,
          0.9022
        ],
        [
          0.242,
          0.9022
        ],
        [
          0.252,
          0.9022
        ],
        [
          0.2621,
          0.9022
        ],
        [
          0.2725,
          0.9013
        ],
        [
          0.2825,
          0.9004
        ],
        [
          0.2925,
          0.8995
        ],
        [
          0.3025,
          0.8999
        ],
        [
          0.3126,
          0.8976
        ],
        [
          0.323,
          0.8963
        ],
        [
          0.333,
          0.8954
        ],
        [
          0.343,
          0.8944
        ],
        [
          0.353,
          0.8935
        ],
        [
          0.3631,
          0.8931
        ],
        [
          0.3735,
          0.8904
        ],
        [
          0.3835,
          0.8894
        ],
        [
          0.3935,
          0.8876
        ],
        [
          0.4035,
          0.8854
        ],
        [
          0.4136,
          0.8826
        ],
        [
          0.424,
          0.8817
        ],
        [
          0.434,
          0.8799
        ],
        [
          0.444,
          0.8776
        ],
        [
          0.454,
          0.8753
        ],
        [
          0.4641,
          0.874
        ],
        [
          0.4745,
          0.8717
        ],
        [
          0.4845,
          0.8694
        ],
        [
          0.4945,
          0.8672
        ],
        [
          0.5045,
          0.8631
        ],
        [
          0.5146,
          0.8599
        ],
        [
          0.525,
          0.8576
        ],
        [
          0.535,
          0.8549
        ],
        [
          0.545,
          0.8517
        ],
        [
          0.5551,
          0.848
        ],
        [
          0.5655,
          0.8471
        ],
        [
          0.5755,
          0.8421
        ],
        [
          0.5855,
          0.8389
        ],
        [
          0.5955,
          0.833
        ],
        [
          0.6056,
          0.8289
        ],
        [
          0.616,
          0.8253
        ],
        [
          0.626,
          0.8212
        ],
        [
          0.636,
          0.8194
        ],
        [
          0.646,
          0.8135
        ],
        [
          0.6561,
          0.8094
        ],
        [
          0.6665,
          0.8048
        ],
        [
          0.6765,
          0.8021
        ],
        [
          0.6865,
          0.7989
        ],
        [
          0.6965,
          0.7925
        ],
        [
          0.7066,
          0.7898
        ],
        [
          0.717,
          0.7871
        ],
        [
          0.727,
          0.7816
        ],
        [
          0.737,
          0.7757
        ],
        [
          0.747,
          0.7693
        ],
        [
          0.7571,
          0.7639
        ],
        [
          0.7675,
          0.7589
        ],
        [
          0.7775,
          0.7543
        ],
        [
          0.7875,
          0.7498
        ],
        [
          0.7975,
          0.7448
        ],
        [
          0.8076,
          0.7389
        ],
        [
          0.818,
          0.7338
        ],
        [
          0.828,
          0.7275
        ],
        [
          0.838,
          0.7216
        ],
        [
          0.848,
          0.7161
        ],
        [
          0.8581,
          0.7102
        ],
        [
          0.8685,
          0.7043
        ],
        [
          0.8785,
          0.6979
        ],
        [
          0.8885,
          0.6947
        ],
        [
          0.8985,
          0.6906
        ],
        [
          0.9086,
          0.6865
        ],
        [
          0.919,
          0.6824
        ],
        [
          0.929,
          0.6779
        ],
        [
          0.939,
          0.6738
        ],
        [
          0.949,
          0.6683
        ],
        [
          0.9591,
          0.6615
        ],
        [
          0.9695,
          0.6561
        ],
        [
          0.9795,
          0.6497
        ],
        [
          0.9895,
          0.6451
        ],
        [
          0.9995,
          0.6392
        ],
        [
          1.0,
          0.6388
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.21628564920869792,
        "difference": -0.0014673523962810931,
        "worse": 0.5433333333333333,
        "kept_right": 0.21365115992486247,
        "kept_wrong": 0.002634489283835615,
        "handed_off": 0.7837143507913017
      },
      "usd_per_1000": 3.8795804842784105
    },
    {
      "decision": 346,
      "frontier": 193,
      "frontier_accuracy": 0.89171974522293,
      "in_sample": {
        "threshold": 0.9781514406204224,
        "answered": 0.21292083712465878,
        "accuracy": 0.89171974522293
      },
      "curve": [
        [
          0.0,
          0.8917
        ],
        [
          0.01,
          0.8917
        ],
        [
          0.02,
          0.8917
        ],
        [
          0.03,
          0.8917
        ],
        [
          0.04,
          0.8917
        ],
        [
          0.05,
          0.8917
        ],
        [
          0.0601,
          0.8917
        ],
        [
          0.0705,
          0.8917
        ],
        [
          0.0805,
          0.8917
        ],
        [
          0.0905,
          0.8917
        ],
        [
          0.1005,
          0.8917
        ],
        [
          0.1106,
          0.8917
        ],
        [
          0.121,
          0.8917
        ],
        [
          0.131,
          0.8917
        ],
        [
          0.141,
          0.8917
        ],
        [
          0.151,
          0.8917
        ],
        [
          0.1611,
          0.8917
        ],
        [
          0.1715,
          0.8913
        ],
        [
          0.1815,
          0.8913
        ],
        [
          0.1915,
          0.8913
        ],
        [
          0.2015,
          0.8913
        ],
        [
          0.2116,
          0.8917
        ],
        [
          0.222,
          0.8908
        ],
        [
          0.232,
          0.8904
        ],
        [
          0.242,
          0.8908
        ],
        [
          0.252,
          0.8904
        ],
        [
          0.2621,
          0.8908
        ],
        [
          0.2725,
          0.8899
        ],
        [
          0.2825,
          0.8894
        ],
        [
          0.2925,
          0.8885
        ],
        [
          0.3025,
          0.8885
        ],
        [
          0.3126,
          0.8863
        ],
        [
          0.323,
          0.8849
        ],
        [
          0.333,
          0.8835
        ],
        [
          0.343,
          0.8826
        ],
        [
          0.353,
          0.8822
        ],
        [
          0.3631,
          0.8822
        ],
        [
          0.3735,
          0.879
        ],
        [
          0.3835,
          0.8772
        ],
        [
          0.3935,
          0.8763
        ],
        [
          0.4035,
          0.8763
        ],
        [
          0.4136,
          0.874
        ],
        [
          0.424,
          0.8731
        ],
        [
          0.434,
          0.8703
        ],
        [
          0.444,
          0.869
        ],
        [
          0.454,
          0.8681
        ],
        [
          0.4641,
          0.8667
        ],
        [
          0.4745,
          0.864
        ],
        [
          0.4845,
          0.8608
        ],
        [
          0.4945,
          0.8585
        ],
        [
          0.5045,
          0.8549
        ],
        [
          0.5146,
          0.8512
        ],
        [
          0.525,
          0.8508
        ],
        [
          0.535,
          0.8476
        ],
        [
          0.545,
          0.8458
        ],
        [
          0.5551,
          0.843
        ],
        [
          0.5655,
          0.8417
        ],
        [
          0.5755,
          0.8362
        ],
        [
          0.5855,
          0.8321
        ],
        [
          0.5955,
          0.8267
        ],
        [
          0.6056,
          0.823
        ],
        [
          0.616,
          0.8194
        ],
        [
          0.626,
          0.8153
        ],
        [
          0.636,
          0.8135
        ],
        [
          0.646,
          0.8085
        ],
        [
          0.6561,
          0.8057
        ],
        [
          0.6665,
          0.8016
        ],
        [
          0.6765,
          0.798
        ],
        [
          0.6865,
          0.7948
        ],
        [
          0.6965,
          0.788
        ],
        [
          0.7066,
          0.7857
        ],
        [
          0.717,
          0.7816
        ],
        [
          0.727,
          0.7762
        ],
        [
          0.737,
          0.7693
        ],
        [
          0.747,
          0.7634
        ],
        [
          0.7571,
          0.758
        ],
        [
          0.7675,
          0.7548
        ],
        [
          0.7775,
          0.7516
        ],
        [
          0.7875,
          0.7475
        ],
        [
          0.7975,
          0.7439
        ],
        [
          0.8076,
          0.7379
        ],
        [
          0.818,
          0.7343
        ],
        [
          0.828,
          0.7279
        ],
        [
          0.838,
          0.7234
        ],
        [
          0.848,
          0.7188
        ],
        [
          0.8581,
          0.7129
        ],
        [
          0.8685,
          0.7079
        ],
        [
          0.8785,
          0.702
        ],
        [
          0.8885,
          0.6984
        ],
        [
          0.8985,
          0.6934
        ],
        [
          0.9086,
          0.6893
        ],
        [
          0.919,
          0.6833
        ],
        [
          0.929,
          0.6793
        ],
        [
          0.939,
          0.6747
        ],
        [
          0.949,
          0.6702
        ],
        [
          0.9591,
          0.6652
        ],
        [
          0.9695,
          0.6588
        ],
        [
          0.9795,
          0.6501
        ],
        [
          0.9895,
          0.6429
        ],
        [
          0.9995,
          0.6388
        ],
        [
          1.0,
          0.6388
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.2277570939087019,
        "difference": -0.0014671757557977716,
        "worse": 0.58,
        "kept_right": 0.22419218225310275,
        "kept_wrong": 0.0035649116555992602,
        "handed_off": 0.7722429060912982
      },
      "usd_per_1000": 5.998430909322143
    },
    {
      "decision": 346,
      "frontier": 55,
      "frontier_accuracy": 0.8503184713375797,
      "in_sample": {
        "threshold": 0.8659014105796814,
        "answered": 0.4299363057324841,
        "accuracy": 0.8503184713375797
      },
      "curve": [
        [
          0.0,
          0.8503
        ],
        [
          0.01,
          0.8503
        ],
        [
          0.02,
          0.8503
        ],
        [
          0.03,
          0.8503
        ],
        [
          0.04,
          0.8508
        ],
        [
          0.05,
          0.8508
        ],
        [
          0.0601,
          0.8508
        ],
        [
          0.0705,
          0.8512
        ],
        [
          0.0805,
          0.8517
        ],
        [
          0.0905,
          0.8517
        ],
        [
          0.1005,
          0.8521
        ],
        [
          0.1106,
          0.8521
        ],
        [
          0.121,
          0.8526
        ],
        [
          0.131,
          0.853
        ],
        [
          0.141,
          0.854
        ],
        [
          0.151,
          0.8544
        ],
        [
          0.1611,
          0.8553
        ],
        [
          0.1715,
          0.8553
        ],
        [
          0.1815,
          0.8558
        ],
        [
          0.1915,
          0.8558
        ],
        [
          0.2015,
          0.8562
        ],
        [
          0.2116,
          0.8567
        ],
        [
          0.222,
          0.8567
        ],
        [
          0.232,
          0.8567
        ],
        [
          0.242,
          0.8581
        ],
        [
          0.252,
          0.8576
        ],
        [
          0.2621,
          0.8581
        ],
        [
          0.2725,
          0.8585
        ],
        [
          0.2825,
          0.8581
        ],
        [
          0.2925,
          0.8585
        ],
        [
          0.3025,
          0.859
        ],
        [
          0.3126,
          0.8576
        ],
        [
          0.323,
          0.8571
        ],
        [
          0.333,
          0.8571
        ],
        [
          0.343,
          0.8562
        ],
        [
          0.353,
          0.8562
        ],
        [
          0.3631,
          0.8562
        ],
        [
          0.3735,
          0.8544
        ],
        [
          0.3835,
          0.854
        ],
        [
          0.3935,
          0.8535
        ],
        [
          0.4035,
          0.854
        ],
        [
          0.4136,
          0.8521
        ],
        [
          0.424,
          0.8517
        ],
        [
          0.434,
          0.8485
        ],
        [
          0.444,
          0.8462
        ],
        [
          0.454,
          0.8458
        ],
        [
          0.4641,
          0.8453
        ],
        [
          0.4745,
          0.843
        ],
        [
          0.4845,
          0.8403
        ],
        [
          0.4945,
          0.8376
        ],
        [
          0.5045,
          0.8358
        ],
        [
          0.5146,
          0.8339
        ],
        [
          0.525,
          0.8335
        ],
        [
          0.535,
          0.8312
        ],
        [
          0.545,
          0.8303
        ],
        [
          0.5551,
          0.8276
        ],
        [
          0.5655,
          0.8267
        ],
        [
          0.5755,
          0.8217
        ],
        [
          0.5855,
          0.8185
        ],
        [
          0.5955,
          0.8139
        ],
        [
          0.6056,
          0.8107
        ],
        [
          0.616,
          0.8076
        ],
        [
          0.626,
          0.8035
        ],
        [
          0.636,
          0.803
        ],
        [
          0.646,
          0.798
        ],
        [
          0.6561,
          0.7953
        ],
        [
          0.6665,
          0.7912
        ],
        [
          0.6765,
          0.7889
        ],
        [
          0.6865,
          0.7857
        ],
        [
          0.6965,
          0.7798
        ],
        [
          0.7066,
          0.778
        ],
        [
          0.717,
          0.7753
        ],
        [
          0.727,
          0.7698
        ],
        [
          0.737,
          0.7634
        ],
        [
          0.747,
          0.7584
        ],
        [
          0.7571,
          0.7534
        ],
        [
          0.7675,
          0.7502
        ],
        [
          0.7775,
          0.747
        ],
        [
          0.7875,
          0.7434
        ],
        [
          0.7975,
          0.7398
        ],
        [
          0.8076,
          0.7352
        ],
        [
          0.818,
          0.7311
        ],
        [
          0.828,
          0.7247
        ],
        [
          0.838,
          0.7193
        ],
        [
          0.848,
          0.7143
        ],
        [
          0.8581,
          0.7102
        ],
        [
          0.8685,
          0.7061
        ],
        [
          0.8785,
          0.7011
        ],
        [
          0.8885,
          0.6979
        ],
        [
          0.8985,
          0.6938
        ],
        [
          0.9086,
          0.6897
        ],
        [
          0.919,
          0.6843
        ],
        [
          0.929,
          0.6802
        ],
        [
          0.939,
          0.6747
        ],
        [
          0.949,
          0.6692
        ],
        [
          0.9591,
          0.6633
        ],
        [
          0.9695,
          0.6574
        ],
        [
          0.9795,
          0.6501
        ],
        [
          0.9895,
          0.6442
        ],
        [
          0.9995,
          0.6388
        ],
        [
          1.0,
          0.6388
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.430312239189134,
        "difference": -0.0017223924848111028,
        "worse": 0.49,
        "kept_right": 0.3936716719985229,
        "kept_wrong": 0.03664056719061129,
        "handed_off": 0.5696877608108657
      },
      "usd_per_1000": 2.0972386367271514
    },
    {
      "decision": 143,
      "frontier": 140,
      "frontier_accuracy": 0.9808917197452229,
      "in_sample": {
        "threshold": 1.0,
        "answered": 0.12556869881710647,
        "accuracy": 0.9808917197452229
      },
      "curve": [
        [
          0.0,
          0.9809
        ],
        [
          0.1256,
          0.9809
        ],
        [
          0.1674,
          0.9804
        ],
        [
          0.1961,
          0.9795
        ],
        [
          0.2166,
          0.9791
        ],
        [
          0.2316,
          0.9773
        ],
        [
          0.2466,
          0.9759
        ],
        [
          0.2652,
          0.9745
        ],
        [
          0.2771,
          0.9741
        ],
        [
          0.2939,
          0.9713
        ],
        [
          0.3071,
          0.9691
        ],
        [
          0.3221,
          0.9654
        ],
        [
          0.3362,
          0.9631
        ],
        [
          0.354,
          0.9609
        ],
        [
          0.3649,
          0.9591
        ],
        [
          0.3772,
          0.9545
        ],
        [
          0.3922,
          0.9522
        ],
        [
          0.4054,
          0.95
        ],
        [
          0.4208,
          0.9463
        ],
        [
          0.4345,
          0.9418
        ],
        [
          0.4518,
          0.9368
        ],
        [
          0.4709,
          0.9304
        ],
        [
          0.4818,
          0.9263
        ],
        [
          0.4968,
          0.9231
        ],
        [
          0.5109,
          0.9167
        ],
        [
          0.5218,
          0.9104
        ],
        [
          0.5405,
          0.9026
        ],
        [
          0.5591,
          0.8972
        ],
        [
          0.5755,
          0.8904
        ],
        [
          0.5869,
          0.8849
        ],
        [
          0.6083,
          0.8758
        ],
        [
          0.6192,
          0.8712
        ],
        [
          0.631,
          0.8672
        ],
        [
          0.6447,
          0.8617
        ],
        [
          0.6583,
          0.8549
        ],
        [
          0.6793,
          0.8439
        ],
        [
          0.6943,
          0.8371
        ],
        [
          0.7134,
          0.8285
        ],
        [
          0.727,
          0.8203
        ],
        [
          0.7402,
          0.8135
        ],
        [
          0.7566,
          0.803
        ],
        [
          0.7734,
          0.7939
        ],
        [
          0.793,
          0.7825
        ],
        [
          0.8089,
          0.7698
        ],
        [
          0.8221,
          0.7602
        ],
        [
          0.8394,
          0.7489
        ],
        [
          0.853,
          0.7416
        ],
        [
          0.8667,
          0.7338
        ],
        [
          0.8835,
          0.7193
        ],
        [
          0.9017,
          0.7061
        ],
        [
          0.9245,
          0.6911
        ],
        [
          0.9399,
          0.6802
        ],
        [
          0.9568,
          0.6679
        ],
        [
          0.9704,
          0.6579
        ],
        [
          0.9813,
          0.6492
        ],
        [
          0.9923,
          0.6419
        ],
        [
          1.0,
          0.636
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.15459656086239593,
        "difference": -0.0008306873625348495,
        "worse": 0.5533333333333333,
        "kept_right": 0.15281389104542276,
        "kept_wrong": 0.0017826698169731755,
        "handed_off": 0.8454034391376046
      },
      "usd_per_1000": 2.0855541200153676
    },
    {
      "decision": 143,
      "frontier": 277,
      "frontier_accuracy": 0.9795268425841674,
      "in_sample": {
        "threshold": null,
        "answered": 0.0,
        "accuracy": 0.9795268425841674
      },
      "curve": [
        [
          0.0,
          0.9795
        ],
        [
          0.1256,
          0.9791
        ],
        [
          0.1674,
          0.9786
        ],
        [
          0.1961,
          0.9777
        ],
        [
          0.2166,
          0.9773
        ],
        [
          0.2316,
          0.9759
        ],
        [
          0.2466,
          0.9745
        ],
        [
          0.2652,
          0.9727
        ],
        [
          0.2771,
          0.9722
        ],
        [
          0.2939,
          0.97
        ],
        [
          0.3071,
          0.9677
        ],
        [
          0.3221,
          0.9641
        ],
        [
          0.3362,
          0.9622
        ],
        [
          0.354,
          0.9595
        ],
        [
          0.3649,
          0.9577
        ],
        [
          0.3772,
          0.9527
        ],
        [
          0.3922,
          0.9509
        ],
        [
          0.4054,
          0.9486
        ],
        [
          0.4208,
          0.9449
        ],
        [
          0.4345,
          0.9399
        ],
        [
          0.4518,
          0.9354
        ],
        [
          0.4709,
          0.929
        ],
        [
          0.4818,
          0.9249
        ],
        [
          0.4968,
          0.9213
        ],
        [
          0.5109,
          0.9154
        ],
        [
          0.5218,
          0.909
        ],
        [
          0.5405,
          0.9013
        ],
        [
          0.5591,
          0.8958
        ],
        [
          0.5755,
          0.8885
        ],
        [
          0.5869,
          0.8826
        ],
        [
          0.6083,
          0.8735
        ],
        [
          0.6192,
          0.869
        ],
        [
          0.631,
          0.8649
        ],
        [
          0.6447,
          0.8594
        ],
        [
          0.6583,
          0.8526
        ],
        [
          0.6793,
          0.8421
        ],
        [
          0.6943,
          0.8353
        ],
        [
          0.7134,
          0.8262
        ],
        [
          0.727,
          0.818
        ],
        [
          0.7402,
          0.8112
        ],
        [
          0.7566,
          0.8012
        ],
        [
          0.7734,
          0.7921
        ],
        [
          0.793,
          0.7807
        ],
        [
          0.8089,
          0.768
        ],
        [
          0.8221,
          0.7589
        ],
        [
          0.8394,
          0.7475
        ],
        [
          0.853,
          0.7402
        ],
        [
          0.8667,
          0.7325
        ],
        [
          0.8835,
          0.7179
        ],
        [
          0.9017,
          0.7052
        ],
        [
          0.9245,
          0.6902
        ],
        [
          0.9399,
          0.6797
        ],
        [
          0.9568,
          0.6674
        ],
        [
          0.9704,
          0.6574
        ],
        [
          0.9813,
          0.6488
        ],
        [
          0.9923,
          0.6419
        ],
        [
          1.0,
          0.636
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.08207669542071959,
        "difference": -0.000909533275923709,
        "worse": 0.5333333333333333,
        "kept_right": 0.08091853904355693,
        "kept_wrong": 0.001158156377162703,
        "handed_off": 0.9179233045792807
      },
      "usd_per_1000": 5.60815032290645
    },
    {
      "decision": 143,
      "frontier": 276,
      "frontier_accuracy": 0.9777070063694268,
      "in_sample": {
        "threshold": null,
        "answered": 0.0,
        "accuracy": 0.9777070063694268
      },
      "curve": [
        [
          0.0,
          0.9777
        ],
        [
          0.1256,
          0.9773
        ],
        [
          0.1674,
          0.9773
        ],
        [
          0.1961,
          0.9763
        ],
        [
          0.2166,
          0.9759
        ],
        [
          0.2316,
          0.9745
        ],
        [
          0.2466,
          0.9732
        ],
        [
          0.2652,
          0.9718
        ],
        [
          0.2771,
          0.9713
        ],
        [
          0.2939,
          0.9691
        ],
        [
          0.3071,
          0.9668
        ],
        [
          0.3221,
          0.9636
        ],
        [
          0.3362,
          0.9618
        ],
        [
          0.354,
          0.9586
        ],
        [
          0.3649,
          0.9568
        ],
        [
          0.3772,
          0.9518
        ],
        [
          0.3922,
          0.9504
        ],
        [
          0.4054,
          0.9481
        ],
        [
          0.4208,
          0.9449
        ],
        [
          0.4345,
          0.9399
        ],
        [
          0.4518,
          0.9354
        ],
        [
          0.4709,
          0.929
        ],
        [
          0.4818,
          0.9249
        ],
        [
          0.4968,
          0.9213
        ],
        [
          0.5109,
          0.9154
        ],
        [
          0.5218,
          0.909
        ],
        [
          0.5405,
          0.9013
        ],
        [
          0.5591,
          0.8958
        ],
        [
          0.5755,
          0.8885
        ],
        [
          0.5869,
          0.8835
        ],
        [
          0.6083,
          0.8749
        ],
        [
          0.6192,
          0.8703
        ],
        [
          0.631,
          0.8662
        ],
        [
          0.6447,
          0.8608
        ],
        [
          0.6583,
          0.854
        ],
        [
          0.6793,
          0.843
        ],
        [
          0.6943,
          0.8362
        ],
        [
          0.7134,
          0.8271
        ],
        [
          0.727,
          0.8189
        ],
        [
          0.7402,
          0.8121
        ],
        [
          0.7566,
          0.8021
        ],
        [
          0.7734,
          0.793
        ],
        [
          0.793,
          0.7816
        ],
        [
          0.8089,
          0.7689
        ],
        [
          0.8221,
          0.7593
        ],
        [
          0.8394,
          0.748
        ],
        [
          0.853,
          0.7407
        ],
        [
          0.8667,
          0.7329
        ],
        [
          0.8835,
          0.7184
        ],
        [
          0.9017,
          0.7056
        ],
        [
          0.9245,
          0.6906
        ],
        [
          0.9399,
          0.6802
        ],
        [
          0.9568,
          0.6679
        ],
        [
          0.9704,
          0.6579
        ],
        [
          0.9813,
          0.6492
        ],
        [
          0.9923,
          0.6419
        ],
        [
          1.0,
          0.636
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.12009621521181463,
        "difference": -0.0012429524322371377,
        "worse": 0.69,
        "kept_right": 0.11832268603129731,
        "kept_wrong": 0.0017735291805173559,
        "handed_off": 0.8799037847881859
      },
      "usd_per_1000": 1.467807260028366
    },
    {
      "decision": 143,
      "frontier": 48,
      "frontier_accuracy": 0.9758871701546861,
      "in_sample": {
        "threshold": null,
        "answered": 0.0,
        "accuracy": 0.9758871701546861
      },
      "curve": [
        [
          0.0,
          0.9759
        ],
        [
          0.1256,
          0.9754
        ],
        [
          0.1674,
          0.975
        ],
        [
          0.1961,
          0.9741
        ],
        [
          0.2166,
          0.9736
        ],
        [
          0.2316,
          0.9722
        ],
        [
          0.2466,
          0.9709
        ],
        [
          0.2652,
          0.9691
        ],
        [
          0.2771,
          0.9686
        ],
        [
          0.2939,
          0.9659
        ],
        [
          0.3071,
          0.9636
        ],
        [
          0.3221,
          0.9604
        ],
        [
          0.3362,
          0.9586
        ],
        [
          0.354,
          0.9554
        ],
        [
          0.3649,
          0.9536
        ],
        [
          0.3772,
          0.9486
        ],
        [
          0.3922,
          0.9468
        ],
        [
          0.4054,
          0.9445
        ],
        [
          0.4208,
          0.9409
        ],
        [
          0.4345,
          0.9359
        ],
        [
          0.4518,
          0.9313
        ],
        [
          0.4709,
          0.9254
        ],
        [
          0.4818,
          0.9217
        ],
        [
          0.4968,
          0.9181
        ],
        [
          0.5109,
          0.9122
        ],
        [
          0.5218,
          0.9058
        ],
        [
          0.5405,
          0.8981
        ],
        [
          0.5591,
          0.8926
        ],
        [
          0.5755,
          0.8854
        ],
        [
          0.5869,
          0.8794
        ],
        [
          0.6083,
          0.8712
        ],
        [
          0.6192,
          0.8667
        ],
        [
          0.631,
          0.8626
        ],
        [
          0.6447,
          0.8571
        ],
        [
          0.6583,
          0.8503
        ],
        [
          0.6793,
          0.8399
        ],
        [
          0.6943,
          0.8335
        ],
        [
          0.7134,
          0.8248
        ],
        [
          0.727,
          0.8171
        ],
        [
          0.7402,
          0.8103
        ],
        [
          0.7566,
          0.8003
        ],
        [
          0.7734,
          0.7912
        ],
        [
          0.793,
          0.7798
        ],
        [
          0.8089,
          0.7671
        ],
        [
          0.8221,
          0.7584
        ],
        [
          0.8394,
          0.7466
        ],
        [
          0.853,
          0.7393
        ],
        [
          0.8667,
          0.7316
        ],
        [
          0.8835,
          0.717
        ],
        [
          0.9017,
          0.7038
        ],
        [
          0.9245,
          0.6893
        ],
        [
          0.9399,
          0.6788
        ],
        [
          0.9568,
          0.667
        ],
        [
          0.9704,
          0.657
        ],
        [
          0.9813,
          0.6488
        ],
        [
          0.9923,
          0.6419
        ],
        [
          1.0,
          0.636
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.08207669542071959,
        "difference": -0.000909533275923709,
        "worse": 0.5333333333333333,
        "kept_right": 0.08091853904355693,
        "kept_wrong": 0.001158156377162703,
        "handed_off": 0.9179233045792807
      },
      "usd_per_1000": 4.977235194244304
    },
    {
      "decision": 143,
      "frontier": 191,
      "frontier_accuracy": 0.9740673339399454,
      "in_sample": {
        "threshold": null,
        "answered": 0.0,
        "accuracy": 0.9740673339399454
      },
      "curve": [
        [
          0.0,
          0.9741
        ],
        [
          0.1256,
          0.9736
        ],
        [
          0.1674,
          0.9732
        ],
        [
          0.1961,
          0.9722
        ],
        [
          0.2166,
          0.9718
        ],
        [
          0.2316,
          0.97
        ],
        [
          0.2466,
          0.9686
        ],
        [
          0.2652,
          0.9668
        ],
        [
          0.2771,
          0.9663
        ],
        [
          0.2939,
          0.9636
        ],
        [
          0.3071,
          0.9618
        ],
        [
          0.3221,
          0.9586
        ],
        [
          0.3362,
          0.9568
        ],
        [
          0.354,
          0.9536
        ],
        [
          0.3649,
          0.9518
        ],
        [
          0.3772,
          0.9468
        ],
        [
          0.3922,
          0.9449
        ],
        [
          0.4054,
          0.9427
        ],
        [
          0.4208,
          0.9395
        ],
        [
          0.4345,
          0.934
        ],
        [
          0.4518,
          0.9295
        ],
        [
          0.4709,
          0.9231
        ],
        [
          0.4818,
          0.919
        ],
        [
          0.4968,
          0.9154
        ],
        [
          0.5109,
          0.9095
        ],
        [
          0.5218,
          0.9031
        ],
        [
          0.5405,
          0.8963
        ],
        [
          0.5591,
          0.8904
        ],
        [
          0.5755,
          0.8826
        ],
        [
          0.5869,
          0.8776
        ],
        [
          0.6083,
          0.8694
        ],
        [
          0.6192,
          0.8649
        ],
        [
          0.631,
          0.8608
        ],
        [
          0.6447,
          0.8558
        ],
        [
          0.6583,
          0.849
        ],
        [
          0.6793,
          0.8389
        ],
        [
          0.6943,
          0.8321
        ],
        [
          0.7134,
          0.8235
        ],
        [
          0.727,
          0.8153
        ],
        [
          0.7402,
          0.8085
        ],
        [
          0.7566,
          0.7985
        ],
        [
          0.7734,
          0.7894
        ],
        [
          0.793,
          0.7784
        ],
        [
          0.8089,
          0.7662
        ],
        [
          0.8221,
          0.758
        ],
        [
          0.8394,
          0.7466
        ],
        [
          0.853,
          0.7393
        ],
        [
          0.8667,
          0.7316
        ],
        [
          0.8835,
          0.717
        ],
        [
          0.9017,
          0.7043
        ],
        [
          0.9245,
          0.6897
        ],
        [
          0.9399,
          0.6788
        ],
        [
          0.9568,
          0.6674
        ],
        [
          0.9704,
          0.6574
        ],
        [
          0.9813,
          0.6488
        ],
        [
          0.9923,
          0.6419
        ],
        [
          1.0,
          0.636
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.08105480223959077,
        "difference": -0.0008579960657877728,
        "worse": 0.53,
        "kept_right": 0.07995424919327981,
        "kept_wrong": 0.001100553046310964,
        "handed_off": 0.9189451977604098
      },
      "usd_per_1000": 24.61091340591068
    },
    {
      "decision": 143,
      "frontier": 188,
      "frontier_accuracy": 0.9713375796178344,
      "in_sample": {
        "threshold": 1.0,
        "answered": 0.12556869881710647,
        "accuracy": 0.9713375796178344
      },
      "curve": [
        [
          0.0,
          0.9713
        ],
        [
          0.1256,
          0.9713
        ],
        [
          0.1674,
          0.9709
        ],
        [
          0.1961,
          0.97
        ],
        [
          0.2166,
          0.9695
        ],
        [
          0.2316,
          0.9677
        ],
        [
          0.2466,
          0.9659
        ],
        [
          0.2652,
          0.965
        ],
        [
          0.2771,
          0.9645
        ],
        [
          0.2939,
          0.9627
        ],
        [
          0.3071,
          0.9604
        ],
        [
          0.3221,
          0.9568
        ],
        [
          0.3362,
          0.9554
        ],
        [
          0.354,
          0.9531
        ],
        [
          0.3649,
          0.9513
        ],
        [
          0.3772,
          0.9468
        ],
        [
          0.3922,
          0.9454
        ],
        [
          0.4054,
          0.9431
        ],
        [
          0.4208,
          0.9399
        ],
        [
          0.4345,
          0.9349
        ],
        [
          0.4518,
          0.9299
        ],
        [
          0.4709,
          0.9236
        ],
        [
          0.4818,
          0.9195
        ],
        [
          0.4968,
          0.9167
        ],
        [
          0.5109,
          0.9108
        ],
        [
          0.5218,
          0.9045
        ],
        [
          0.5405,
          0.8967
        ],
        [
          0.5591,
          0.8913
        ],
        [
          0.5755,
          0.8849
        ],
        [
          0.5869,
          0.8799
        ],
        [
          0.6083,
          0.8712
        ],
        [
          0.6192,
          0.8667
        ],
        [
          0.631,
          0.8626
        ],
        [
          0.6447,
          0.8567
        ],
        [
          0.6583,
          0.8503
        ],
        [
          0.6793,
          0.8403
        ],
        [
          0.6943,
          0.8339
        ],
        [
          0.7134,
          0.8258
        ],
        [
          0.727,
          0.8176
        ],
        [
          0.7402,
          0.8107
        ],
        [
          0.7566,
          0.8003
        ],
        [
          0.7734,
          0.7916
        ],
        [
          0.793,
          0.7803
        ],
        [
          0.8089,
          0.768
        ],
        [
          0.8221,
          0.7589
        ],
        [
          0.8394,
          0.7466
        ],
        [
          0.853,
          0.7393
        ],
        [
          0.8667,
          0.7316
        ],
        [
          0.8835,
          0.717
        ],
        [
          0.9017,
          0.7043
        ],
        [
          0.9245,
          0.6893
        ],
        [
          0.9399,
          0.6783
        ],
        [
          0.9568,
          0.6674
        ],
        [
          0.9704,
          0.6574
        ],
        [
          0.9813,
          0.6488
        ],
        [
          0.9923,
          0.6419
        ],
        [
          1.0,
          0.636
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.15468755267313297,
        "difference": -0.000839786543608553,
        "worse": 0.5533333333333333,
        "kept_right": 0.15289578367508608,
        "kept_wrong": 0.0017917689980468787,
        "handed_off": 0.8453124473268676
      },
      "usd_per_1000": 0.8548436692181994
    },
    {
      "decision": 143,
      "frontier": 190,
      "frontier_accuracy": 0.9672429481346679,
      "in_sample": {
        "threshold": null,
        "answered": 0.0,
        "accuracy": 0.9672429481346679
      },
      "curve": [
        [
          0.0,
          0.9672
        ],
        [
          0.1256,
          0.9668
        ],
        [
          0.1674,
          0.9663
        ],
        [
          0.1961,
          0.9654
        ],
        [
          0.2166,
          0.965
        ],
        [
          0.2316,
          0.9631
        ],
        [
          0.2466,
          0.9618
        ],
        [
          0.2652,
          0.9604
        ],
        [
          0.2771,
          0.96
        ],
        [
          0.2939,
          0.9581
        ],
        [
          0.3071,
          0.9559
        ],
        [
          0.3221,
          0.9527
        ],
        [
          0.3362,
          0.9509
        ],
        [
          0.354,
          0.9477
        ],
        [
          0.3649,
          0.9463
        ],
        [
          0.3772,
          0.9413
        ],
        [
          0.3922,
          0.9399
        ],
        [
          0.4054,
          0.9377
        ],
        [
          0.4208,
          0.9345
        ],
        [
          0.4345,
          0.9299
        ],
        [
          0.4518,
          0.9254
        ],
        [
          0.4709,
          0.9195
        ],
        [
          0.4818,
          0.9163
        ],
        [
          0.4968,
          0.9126
        ],
        [
          0.5109,
          0.9072
        ],
        [
          0.5218,
          0.9008
        ],
        [
          0.5405,
          0.8935
        ],
        [
          0.5591,
          0.8885
        ],
        [
          0.5755,
          0.8813
        ],
        [
          0.5869,
          0.8758
        ],
        [
          0.6083,
          0.8667
        ],
        [
          0.6192,
          0.8626
        ],
        [
          0.631,
          0.8585
        ],
        [
          0.6447,
          0.8535
        ],
        [
          0.6583,
          0.8467
        ],
        [
          0.6793,
          0.8362
        ],
        [
          0.6943,
          0.8294
        ],
        [
          0.7134,
          0.8207
        ],
        [
          0.727,
          0.813
        ],
        [
          0.7402,
          0.8062
        ],
        [
          0.7566,
          0.7962
        ],
        [
          0.7734,
          0.7871
        ],
        [
          0.793,
          0.7757
        ],
        [
          0.8089,
          0.7639
        ],
        [
          0.8221,
          0.7561
        ],
        [
          0.8394,
          0.7448
        ],
        [
          0.853,
          0.7375
        ],
        [
          0.8667,
          0.7298
        ],
        [
          0.8835,
          0.7157
        ],
        [
          0.9017,
          0.7029
        ],
        [
          0.9245,
          0.6884
        ],
        [
          0.9399,
          0.6788
        ],
        [
          0.9568,
          0.6674
        ],
        [
          0.9704,
          0.6574
        ],
        [
          0.9813,
          0.6488
        ],
        [
          0.9923,
          0.6419
        ],
        [
          1.0,
          0.636
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.08105480223959077,
        "difference": -0.0008579960657877728,
        "worse": 0.53,
        "kept_right": 0.07995424919327981,
        "kept_wrong": 0.001100553046310964,
        "handed_off": 0.9189451977604098
      },
      "usd_per_1000": 5.157599796835876
    },
    {
      "decision": 143,
      "frontier": 124,
      "frontier_accuracy": 0.9667879890809827,
      "in_sample": {
        "threshold": null,
        "answered": 0.0,
        "accuracy": 0.9667879890809827
      },
      "curve": [
        [
          0.0,
          0.9668
        ],
        [
          0.1256,
          0.9663
        ],
        [
          0.1674,
          0.9659
        ],
        [
          0.1961,
          0.965
        ],
        [
          0.2166,
          0.9645
        ],
        [
          0.2316,
          0.9631
        ],
        [
          0.2466,
          0.9618
        ],
        [
          0.2652,
          0.9604
        ],
        [
          0.2771,
          0.96
        ],
        [
          0.2939,
          0.9581
        ],
        [
          0.3071,
          0.9559
        ],
        [
          0.3221,
          0.9527
        ],
        [
          0.3362,
          0.9509
        ],
        [
          0.354,
          0.9481
        ],
        [
          0.3649,
          0.9468
        ],
        [
          0.3772,
          0.9418
        ],
        [
          0.3922,
          0.9404
        ],
        [
          0.4054,
          0.9381
        ],
        [
          0.4208,
          0.9345
        ],
        [
          0.4345,
          0.9295
        ],
        [
          0.4518,
          0.9249
        ],
        [
          0.4709,
          0.9186
        ],
        [
          0.4818,
          0.9149
        ],
        [
          0.4968,
          0.9117
        ],
        [
          0.5109,
          0.9058
        ],
        [
          0.5218,
          0.8995
        ],
        [
          0.5405,
          0.8922
        ],
        [
          0.5591,
          0.8867
        ],
        [
          0.5755,
          0.8799
        ],
        [
          0.5869,
          0.8749
        ],
        [
          0.6083,
          0.8672
        ],
        [
          0.6192,
          0.8631
        ],
        [
          0.631,
          0.859
        ],
        [
          0.6447,
          0.8535
        ],
        [
          0.6583,
          0.8467
        ],
        [
          0.6793,
          0.8362
        ],
        [
          0.6943,
          0.8298
        ],
        [
          0.7134,
          0.8217
        ],
        [
          0.727,
          0.8148
        ],
        [
          0.7402,
          0.808
        ],
        [
          0.7566,
          0.798
        ],
        [
          0.7734,
          0.7889
        ],
        [
          0.793,
          0.7775
        ],
        [
          0.8089,
          0.7652
        ],
        [
          0.8221,
          0.7561
        ],
        [
          0.8394,
          0.7448
        ],
        [
          0.853,
          0.7379
        ],
        [
          0.8667,
          0.7302
        ],
        [
          0.8835,
          0.7157
        ],
        [
          0.9017,
          0.7029
        ],
        [
          0.9245,
          0.6884
        ],
        [
          0.9399,
          0.6783
        ],
        [
          0.9568,
          0.667
        ],
        [
          0.9704,
          0.6574
        ],
        [
          0.9813,
          0.6488
        ],
        [
          0.9923,
          0.6419
        ],
        [
          1.0,
          0.636
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.08225859632236572,
        "difference": -0.0009216600027001173,
        "worse": 0.5333333333333333,
        "kept_right": 0.08108528015806873,
        "kept_wrong": 0.0011733161642970122,
        "handed_off": 0.9177414036776348
      },
      "usd_per_1000": 1.0778261519205201
    },
    {
      "decision": 143,
      "frontier": 278,
      "frontier_accuracy": 0.9499545040946314,
      "in_sample": {
        "threshold": 0.99,
        "answered": 0.16742493175614195,
        "accuracy": 0.9504094631483166
      },
      "curve": [
        [
          0.0,
          0.95
        ],
        [
          0.1256,
          0.95
        ],
        [
          0.1674,
          0.9504
        ],
        [
          0.1961,
          0.9495
        ],
        [
          0.2166,
          0.949
        ],
        [
          0.2316,
          0.9477
        ],
        [
          0.2466,
          0.9463
        ],
        [
          0.2652,
          0.9449
        ],
        [
          0.2771,
          0.9449
        ],
        [
          0.2939,
          0.9436
        ],
        [
          0.3071,
          0.9418
        ],
        [
          0.3221,
          0.9381
        ],
        [
          0.3362,
          0.9363
        ],
        [
          0.354,
          0.9345
        ],
        [
          0.3649,
          0.9327
        ],
        [
          0.3772,
          0.9286
        ],
        [
          0.3922,
          0.9277
        ],
        [
          0.4054,
          0.9254
        ],
        [
          0.4208,
          0.9227
        ],
        [
          0.4345,
          0.9181
        ],
        [
          0.4518,
          0.9136
        ],
        [
          0.4709,
          0.9072
        ],
        [
          0.4818,
          0.904
        ],
        [
          0.4968,
          0.9008
        ],
        [
          0.5109,
          0.8954
        ],
        [
          0.5218,
          0.889
        ],
        [
          0.5405,
          0.8822
        ],
        [
          0.5591,
          0.8781
        ],
        [
          0.5755,
          0.8717
        ],
        [
          0.5869,
          0.8676
        ],
        [
          0.6083,
          0.8599
        ],
        [
          0.6192,
          0.8553
        ],
        [
          0.631,
          0.8521
        ],
        [
          0.6447,
          0.8467
        ],
        [
          0.6583,
          0.8403
        ],
        [
          0.6793,
          0.8308
        ],
        [
          0.6943,
          0.8248
        ],
        [
          0.7134,
          0.8162
        ],
        [
          0.727,
          0.8085
        ],
        [
          0.7402,
          0.8016
        ],
        [
          0.7566,
          0.7921
        ],
        [
          0.7734,
          0.7843
        ],
        [
          0.793,
          0.7739
        ],
        [
          0.8089,
          0.7616
        ],
        [
          0.8221,
          0.7525
        ],
        [
          0.8394,
          0.7416
        ],
        [
          0.853,
          0.7343
        ],
        [
          0.8667,
          0.7279
        ],
        [
          0.8835,
          0.7134
        ],
        [
          0.9017,
          0.7006
        ],
        [
          0.9245,
          0.6874
        ],
        [
          0.9399,
          0.6774
        ],
        [
          0.9568,
          0.6665
        ],
        [
          0.9704,
          0.657
        ],
        [
          0.9813,
          0.6488
        ],
        [
          0.9923,
          0.6419
        ],
        [
          1.0,
          0.636
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1917237544384297,
        "difference": -0.00069418842122909,
        "worse": 0.55,
        "kept_right": 0.18891950178437936,
        "kept_wrong": 0.0028042526540503596,
        "handed_off": 0.8082762455615705
      },
      "usd_per_1000": 5.301365669316331
    },
    {
      "decision": 143,
      "frontier": 275,
      "frontier_accuracy": 0.9426751592356688,
      "in_sample": {
        "threshold": 0.99,
        "answered": 0.16742493175614195,
        "accuracy": 0.9426751592356688
      },
      "curve": [
        [
          0.0,
          0.9427
        ],
        [
          0.1256,
          0.9427
        ],
        [
          0.1674,
          0.9427
        ],
        [
          0.1961,
          0.9418
        ],
        [
          0.2166,
          0.9413
        ],
        [
          0.2316,
          0.9399
        ],
        [
          0.2466,
          0.9386
        ],
        [
          0.2652,
          0.9368
        ],
        [
          0.2771,
          0.9363
        ],
        [
          0.2939,
          0.934
        ],
        [
          0.3071,
          0.9327
        ],
        [
          0.3221,
          0.9295
        ],
        [
          0.3362,
          0.9277
        ],
        [
          0.354,
          0.9263
        ],
        [
          0.3649,
          0.9245
        ],
        [
          0.3772,
          0.9199
        ],
        [
          0.3922,
          0.919
        ],
        [
          0.4054,
          0.9177
        ],
        [
          0.4208,
          0.9149
        ],
        [
          0.4345,
          0.9095
        ],
        [
          0.4518,
          0.9054
        ],
        [
          0.4709,
          0.8999
        ],
        [
          0.4818,
          0.8963
        ],
        [
          0.4968,
          0.8926
        ],
        [
          0.5109,
          0.8867
        ],
        [
          0.5218,
          0.8803
        ],
        [
          0.5405,
          0.874
        ],
        [
          0.5591,
          0.8694
        ],
        [
          0.5755,
          0.8631
        ],
        [
          0.5869,
          0.8585
        ],
        [
          0.6083,
          0.8503
        ],
        [
          0.6192,
          0.8467
        ],
        [
          0.631,
          0.8426
        ],
        [
          0.6447,
          0.8371
        ],
        [
          0.6583,
          0.8303
        ],
        [
          0.6793,
          0.8207
        ],
        [
          0.6943,
          0.8144
        ],
        [
          0.7134,
          0.8066
        ],
        [
          0.727,
          0.7998
        ],
        [
          0.7402,
          0.793
        ],
        [
          0.7566,
          0.7834
        ],
        [
          0.7734,
          0.7757
        ],
        [
          0.793,
          0.7657
        ],
        [
          0.8089,
          0.7539
        ],
        [
          0.8221,
          0.7461
        ],
        [
          0.8394,
          0.7357
        ],
        [
          0.853,
          0.7298
        ],
        [
          0.8667,
          0.7225
        ],
        [
          0.8835,
          0.7093
        ],
        [
          0.9017,
          0.697
        ],
        [
          0.9245,
          0.6833
        ],
        [
          0.9399,
          0.6747
        ],
        [
          0.9568,
          0.6642
        ],
        [
          0.9704,
          0.6547
        ],
        [
          0.9813,
          0.6479
        ],
        [
          0.9923,
          0.6419
        ],
        [
          1.0,
          0.636
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.17834125560528774,
        "difference": -0.0006486814361197583,
        "worse": 0.26666666666666666,
        "kept_right": 0.17617672858808583,
        "kept_wrong": 0.002164527017201988,
        "handed_off": 0.8216587443947124
      },
      "usd_per_1000": 0.46319025206368525
    },
    {
      "decision": 143,
      "frontier": 192,
      "frontier_accuracy": 0.9331210191082803,
      "in_sample": {
        "threshold": 0.97,
        "answered": 0.21656050955414013,
        "accuracy": 0.9335759781619655
      },
      "curve": [
        [
          0.0,
          0.9331
        ],
        [
          0.1256,
          0.9336
        ],
        [
          0.1674,
          0.9345
        ],
        [
          0.1961,
          0.9336
        ],
        [
          0.2166,
          0.9336
        ],
        [
          0.2316,
          0.9327
        ],
        [
          0.2466,
          0.9308
        ],
        [
          0.2652,
          0.9299
        ],
        [
          0.2771,
          0.9299
        ],
        [
          0.2939,
          0.9286
        ],
        [
          0.3071,
          0.9268
        ],
        [
          0.3221,
          0.924
        ],
        [
          0.3362,
          0.9222
        ],
        [
          0.354,
          0.9199
        ],
        [
          0.3649,
          0.9186
        ],
        [
          0.3772,
          0.9136
        ],
        [
          0.3922,
          0.9131
        ],
        [
          0.4054,
          0.9108
        ],
        [
          0.4208,
          0.9086
        ],
        [
          0.4345,
          0.9035
        ],
        [
          0.4518,
          0.8995
        ],
        [
          0.4709,
          0.8944
        ],
        [
          0.4818,
          0.8913
        ],
        [
          0.4968,
          0.8881
        ],
        [
          0.5109,
          0.8826
        ],
        [
          0.5218,
          0.8763
        ],
        [
          0.5405,
          0.8703
        ],
        [
          0.5591,
          0.8667
        ],
        [
          0.5755,
          0.8608
        ],
        [
          0.5869,
          0.8571
        ],
        [
          0.6083,
          0.8499
        ],
        [
          0.6192,
          0.8458
        ],
        [
          0.631,
          0.8417
        ],
        [
          0.6447,
          0.8362
        ],
        [
          0.6583,
          0.8298
        ],
        [
          0.6793,
          0.8207
        ],
        [
          0.6943,
          0.8148
        ],
        [
          0.7134,
          0.8066
        ],
        [
          0.727,
          0.7994
        ],
        [
          0.7402,
          0.793
        ],
        [
          0.7566,
          0.783
        ],
        [
          0.7734,
          0.7757
        ],
        [
          0.793,
          0.7657
        ],
        [
          0.8089,
          0.7539
        ],
        [
          0.8221,
          0.7466
        ],
        [
          0.8394,
          0.7366
        ],
        [
          0.853,
          0.7302
        ],
        [
          0.8667,
          0.7229
        ],
        [
          0.8835,
          0.7084
        ],
        [
          0.9017,
          0.6956
        ],
        [
          0.9245,
          0.6833
        ],
        [
          0.9399,
          0.6761
        ],
        [
          0.9568,
          0.6652
        ],
        [
          0.9704,
          0.6561
        ],
        [
          0.9813,
          0.6483
        ],
        [
          0.9923,
          0.6415
        ],
        [
          1.0,
          0.636
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.21585687410531962,
        "difference": -0.0011791454697770139,
        "worse": 0.53,
        "kept_right": 0.2107305182550272,
        "kept_wrong": 0.005126355850292347,
        "handed_off": 0.7841431258946799
      },
      "usd_per_1000": 9.143996837068322
    },
    {
      "decision": 143,
      "frontier": 123,
      "frontier_accuracy": 0.9326660600545951,
      "in_sample": {
        "threshold": 1.0,
        "answered": 0.12556869881710647,
        "accuracy": 0.9326660600545951
      },
      "curve": [
        [
          0.0,
          0.9327
        ],
        [
          0.1256,
          0.9327
        ],
        [
          0.1674,
          0.9322
        ],
        [
          0.1961,
          0.9318
        ],
        [
          0.2166,
          0.9313
        ],
        [
          0.2316,
          0.9295
        ],
        [
          0.2466,
          0.9281
        ],
        [
          0.2652,
          0.9268
        ],
        [
          0.2771,
          0.9263
        ],
        [
          0.2939,
          0.9245
        ],
        [
          0.3071,
          0.9227
        ],
        [
          0.3221,
          0.919
        ],
        [
          0.3362,
          0.9167
        ],
        [
          0.354,
          0.9154
        ],
        [
          0.3649,
          0.9136
        ],
        [
          0.3772,
          0.9095
        ],
        [
          0.3922,
          0.9081
        ],
        [
          0.4054,
          0.9058
        ],
        [
          0.4208,
          0.9031
        ],
        [
          0.4345,
          0.8985
        ],
        [
          0.4518,
          0.8949
        ],
        [
          0.4709,
          0.8894
        ],
        [
          0.4818,
          0.8867
        ],
        [
          0.4968,
          0.8844
        ],
        [
          0.5109,
          0.8785
        ],
        [
          0.5218,
          0.8722
        ],
        [
          0.5405,
          0.8658
        ],
        [
          0.5591,
          0.8608
        ],
        [
          0.5755,
          0.8549
        ],
        [
          0.5869,
          0.8494
        ],
        [
          0.6083,
          0.8421
        ],
        [
          0.6192,
          0.8385
        ],
        [
          0.631,
          0.8353
        ],
        [
          0.6447,
          0.8298
        ],
        [
          0.6583,
          0.8235
        ],
        [
          0.6793,
          0.8139
        ],
        [
          0.6943,
          0.8076
        ],
        [
          0.7134,
          0.7998
        ],
        [
          0.727,
          0.7925
        ],
        [
          0.7402,
          0.7871
        ],
        [
          0.7566,
          0.7784
        ],
        [
          0.7734,
          0.7693
        ],
        [
          0.793,
          0.7611
        ],
        [
          0.8089,
          0.7516
        ],
        [
          0.8221,
          0.7443
        ],
        [
          0.8394,
          0.7338
        ],
        [
          0.853,
          0.7279
        ],
        [
          0.8667,
          0.7207
        ],
        [
          0.8835,
          0.7088
        ],
        [
          0.9017,
          0.697
        ],
        [
          0.9245,
          0.6843
        ],
        [
          0.9399,
          0.6742
        ],
        [
          0.9568,
          0.6642
        ],
        [
          0.9704,
          0.6551
        ],
        [
          0.9813,
          0.6479
        ],
        [
          0.9923,
          0.641
        ],
        [
          1.0,
          0.636
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.16720773705178968,
        "difference": -0.0011915089060154538,
        "worse": 0.6233333333333333,
        "kept_right": 0.16490965316165102,
        "kept_wrong": 0.002298083890138487,
        "handed_off": 0.8327922629482103
      },
      "usd_per_1000": 0.2641106143998265
    },
    {
      "decision": 143,
      "frontier": 274,
      "frontier_accuracy": 0.9085532302092811,
      "in_sample": {
        "threshold": 0.98,
        "answered": 0.19608735213830755,
        "accuracy": 0.9085532302092811
      },
      "curve": [
        [
          0.0,
          0.9086
        ],
        [
          0.1256,
          0.9086
        ],
        [
          0.1674,
          0.9095
        ],
        [
          0.1961,
          0.9086
        ],
        [
          0.2166,
          0.9081
        ],
        [
          0.2316,
          0.9067
        ],
        [
          0.2466,
          0.9058
        ],
        [
          0.2652,
          0.904
        ],
        [
          0.2771,
          0.904
        ],
        [
          0.2939,
          0.9026
        ],
        [
          0.3071,
          0.9013
        ],
        [
          0.3221,
          0.899
        ],
        [
          0.3362,
          0.8972
        ],
        [
          0.354,
          0.8963
        ],
        [
          0.3649,
          0.8949
        ],
        [
          0.3772,
          0.8904
        ],
        [
          0.3922,
          0.8894
        ],
        [
          0.4054,
          0.8881
        ],
        [
          0.4208,
          0.8858
        ],
        [
          0.4345,
          0.8803
        ],
        [
          0.4518,
          0.8767
        ],
        [
          0.4709,
          0.8722
        ],
        [
          0.4818,
          0.8685
        ],
        [
          0.4968,
          0.8649
        ],
        [
          0.5109,
          0.859
        ],
        [
          0.5218,
          0.853
        ],
        [
          0.5405,
          0.8467
        ],
        [
          0.5591,
          0.8421
        ],
        [
          0.5755,
          0.8358
        ],
        [
          0.5869,
          0.8312
        ],
        [
          0.6083,
          0.8235
        ],
        [
          0.6192,
          0.8194
        ],
        [
          0.631,
          0.8157
        ],
        [
          0.6447,
          0.8107
        ],
        [
          0.6583,
          0.8048
        ],
        [
          0.6793,
          0.7971
        ],
        [
          0.6943,
          0.7925
        ],
        [
          0.7134,
          0.7862
        ],
        [
          0.727,
          0.7807
        ],
        [
          0.7402,
          0.7743
        ],
        [
          0.7566,
          0.7662
        ],
        [
          0.7734,
          0.7593
        ],
        [
          0.793,
          0.7516
        ],
        [
          0.8089,
          0.742
        ],
        [
          0.8221,
          0.7366
        ],
        [
          0.8394,
          0.727
        ],
        [
          0.853,
          0.7229
        ],
        [
          0.8667,
          0.717
        ],
        [
          0.8835,
          0.7047
        ],
        [
          0.9017,
          0.6943
        ],
        [
          0.9245,
          0.6811
        ],
        [
          0.9399,
          0.6724
        ],
        [
          0.9568,
          0.662
        ],
        [
          0.9704,
          0.6538
        ],
        [
          0.9813,
          0.6465
        ],
        [
          0.9923,
          0.641
        ],
        [
          1.0,
          0.636
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.20387848360246957,
        "difference": -0.0007124940583949395,
        "worse": 0.5333333333333333,
        "kept_right": 0.20043748909654577,
        "kept_wrong": 0.0034409945059238347,
        "handed_off": 0.79612151639753
      },
      "usd_per_1000": 0.3797793427000225
    },
    {
      "decision": 143,
      "frontier": 47,
      "frontier_accuracy": 0.9044585987261147,
      "in_sample": {
        "threshold": 0.97,
        "answered": 0.21656050955414013,
        "accuracy": 0.9049135577797999
      },
      "curve": [
        [
          0.0,
          0.9045
        ],
        [
          0.1256,
          0.9049
        ],
        [
          0.1674,
          0.9054
        ],
        [
          0.1961,
          0.9049
        ],
        [
          0.2166,
          0.9049
        ],
        [
          0.2316,
          0.904
        ],
        [
          0.2466,
          0.9026
        ],
        [
          0.2652,
          0.9008
        ],
        [
          0.2771,
          0.9008
        ],
        [
          0.2939,
          0.899
        ],
        [
          0.3071,
          0.8981
        ],
        [
          0.3221,
          0.8954
        ],
        [
          0.3362,
          0.8935
        ],
        [
          0.354,
          0.8922
        ],
        [
          0.3649,
          0.8917
        ],
        [
          0.3772,
          0.8876
        ],
        [
          0.3922,
          0.8867
        ],
        [
          0.4054,
          0.8849
        ],
        [
          0.4208,
          0.8822
        ],
        [
          0.4345,
          0.8785
        ],
        [
          0.4518,
          0.8749
        ],
        [
          0.4709,
          0.8708
        ],
        [
          0.4818,
          0.8681
        ],
        [
          0.4968,
          0.8662
        ],
        [
          0.5109,
          0.8626
        ],
        [
          0.5218,
          0.8562
        ],
        [
          0.5405,
          0.8512
        ],
        [
          0.5591,
          0.8476
        ],
        [
          0.5755,
          0.8421
        ],
        [
          0.5869,
          0.8371
        ],
        [
          0.6083,
          0.8317
        ],
        [
          0.6192,
          0.8276
        ],
        [
          0.631,
          0.8235
        ],
        [
          0.6447,
          0.8194
        ],
        [
          0.6583,
          0.8139
        ],
        [
          0.6793,
          0.8062
        ],
        [
          0.6943,
          0.8007
        ],
        [
          0.7134,
          0.7921
        ],
        [
          0.727,
          0.7843
        ],
        [
          0.7402,
          0.778
        ],
        [
          0.7566,
          0.7693
        ],
        [
          0.7734,
          0.7625
        ],
        [
          0.793,
          0.7539
        ],
        [
          0.8089,
          0.7429
        ],
        [
          0.8221,
          0.7361
        ],
        [
          0.8394,
          0.7257
        ],
        [
          0.853,
          0.7207
        ],
        [
          0.8667,
          0.7152
        ],
        [
          0.8835,
          0.7025
        ],
        [
          0.9017,
          0.6911
        ],
        [
          0.9245,
          0.6797
        ],
        [
          0.9399,
          0.6729
        ],
        [
          0.9568,
          0.6615
        ],
        [
          0.9704,
          0.6529
        ],
        [
          0.9813,
          0.6465
        ],
        [
          0.9923,
          0.641
        ],
        [
          1.0,
          0.636
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.20886583722030086,
        "difference": -0.0010184832389374273,
        "worse": 0.49666666666666665,
        "kept_right": 0.20403346049538654,
        "kept_wrong": 0.004832376724914143,
        "handed_off": 0.7911341627796995
      },
      "usd_per_1000": 3.941190992905696
    },
    {
      "decision": 143,
      "frontier": 193,
      "frontier_accuracy": 0.89171974522293,
      "in_sample": {
        "threshold": 0.94,
        "answered": 0.2556869881710646,
        "accuracy": 0.8926296633303002
      },
      "curve": [
        [
          0.0,
          0.8917
        ],
        [
          0.1256,
          0.8922
        ],
        [
          0.1674,
          0.8926
        ],
        [
          0.1961,
          0.8926
        ],
        [
          0.2166,
          0.8926
        ],
        [
          0.2316,
          0.8931
        ],
        [
          0.2466,
          0.8926
        ],
        [
          0.2652,
          0.8913
        ],
        [
          0.2771,
          0.8908
        ],
        [
          0.2939,
          0.8899
        ],
        [
          0.3071,
          0.8885
        ],
        [
          0.3221,
          0.8854
        ],
        [
          0.3362,
          0.8831
        ],
        [
          0.354,
          0.8822
        ],
        [
          0.3649,
          0.8813
        ],
        [
          0.3772,
          0.8776
        ],
        [
          0.3922,
          0.8767
        ],
        [
          0.4054,
          0.8753
        ],
        [
          0.4208,
          0.874
        ],
        [
          0.4345,
          0.8708
        ],
        [
          0.4518,
          0.8681
        ],
        [
          0.4709,
          0.8626
        ],
        [
          0.4818,
          0.8608
        ],
        [
          0.4968,
          0.8571
        ],
        [
          0.5109,
          0.8535
        ],
        [
          0.5218,
          0.8471
        ],
        [
          0.5405,
          0.8421
        ],
        [
          0.5591,
          0.8385
        ],
        [
          0.5755,
          0.8335
        ],
        [
          0.5869,
          0.8298
        ],
        [
          0.6083,
          0.8244
        ],
        [
          0.6192,
          0.8203
        ],
        [
          0.631,
          0.8176
        ],
        [
          0.6447,
          0.8135
        ],
        [
          0.6583,
          0.8066
        ],
        [
          0.6793,
          0.7975
        ],
        [
          0.6943,
          0.7925
        ],
        [
          0.7134,
          0.7857
        ],
        [
          0.727,
          0.7803
        ],
        [
          0.7402,
          0.7734
        ],
        [
          0.7566,
          0.7657
        ],
        [
          0.7734,
          0.7593
        ],
        [
          0.793,
          0.7507
        ],
        [
          0.8089,
          0.7393
        ],
        [
          0.8221,
          0.732
        ],
        [
          0.8394,
          0.7229
        ],
        [
          0.853,
          0.7175
        ],
        [
          0.8667,
          0.7111
        ],
        [
          0.8835,
          0.6984
        ],
        [
          0.9017,
          0.687
        ],
        [
          0.9245,
          0.6765
        ],
        [
          0.9399,
          0.6692
        ],
        [
          0.9568,
          0.6606
        ],
        [
          0.9704,
          0.6533
        ],
        [
          0.9813,
          0.6465
        ],
        [
          0.9923,
          0.641
        ],
        [
          1.0,
          0.636
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.2601950914253903,
        "difference": -0.0013793575534735605,
        "worse": 0.5333333333333333,
        "kept_right": 0.2506454507874868,
        "kept_wrong": 0.009549640637903584,
        "handed_off": 0.7398049085746097
      },
      "usd_per_1000": 5.772141092153087
    },
    {
      "decision": 143,
      "frontier": 55,
      "frontier_accuracy": 0.8503184713375797,
      "in_sample": {
        "threshold": 0.84,
        "answered": 0.3362147406733394,
        "accuracy": 0.8503184713375797
      },
      "curve": [
        [
          0.0,
          0.8503
        ],
        [
          0.1256,
          0.8526
        ],
        [
          0.1674,
          0.853
        ],
        [
          0.1961,
          0.8549
        ],
        [
          0.2166,
          0.8553
        ],
        [
          0.2316,
          0.8549
        ],
        [
          0.2466,
          0.8544
        ],
        [
          0.2652,
          0.8544
        ],
        [
          0.2771,
          0.8549
        ],
        [
          0.2939,
          0.8553
        ],
        [
          0.3071,
          0.8544
        ],
        [
          0.3221,
          0.8517
        ],
        [
          0.3362,
          0.8503
        ],
        [
          0.354,
          0.8485
        ],
        [
          0.3649,
          0.8471
        ],
        [
          0.3772,
          0.8439
        ],
        [
          0.3922,
          0.843
        ],
        [
          0.4054,
          0.8421
        ],
        [
          0.4208,
          0.843
        ],
        [
          0.4345,
          0.8417
        ],
        [
          0.4518,
          0.8408
        ],
        [
          0.4709,
          0.8358
        ],
        [
          0.4818,
          0.8339
        ],
        [
          0.4968,
          0.8312
        ],
        [
          0.5109,
          0.8285
        ],
        [
          0.5218,
          0.8239
        ],
        [
          0.5405,
          0.8198
        ],
        [
          0.5591,
          0.8167
        ],
        [
          0.5755,
          0.813
        ],
        [
          0.5869,
          0.8089
        ],
        [
          0.6083,
          0.8035
        ],
        [
          0.6192,
          0.7998
        ],
        [
          0.631,
          0.7989
        ],
        [
          0.6447,
          0.7948
        ],
        [
          0.6583,
          0.7903
        ],
        [
          0.6793,
          0.783
        ],
        [
          0.6943,
          0.7784
        ],
        [
          0.7134,
          0.7739
        ],
        [
          0.727,
          0.7707
        ],
        [
          0.7402,
          0.7662
        ],
        [
          0.7566,
          0.7575
        ],
        [
          0.7734,
          0.7511
        ],
        [
          0.793,
          0.7439
        ],
        [
          0.8089,
          0.7338
        ],
        [
          0.8221,
          0.7284
        ],
        [
          0.8394,
          0.7188
        ],
        [
          0.853,
          0.7147
        ],
        [
          0.8667,
          0.7093
        ],
        [
          0.8835,
          0.6965
        ],
        [
          0.9017,
          0.6847
        ],
        [
          0.9245,
          0.6738
        ],
        [
          0.9399,
          0.6679
        ],
        [
          0.9568,
          0.6601
        ],
        [
          0.9704,
          0.6529
        ],
        [
          0.9813,
          0.6465
        ],
        [
          0.9923,
          0.641
        ],
        [
          1.0,
          0.636
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.3437910171421777,
        "difference": -0.0014950224542893786,
        "worse": 0.4766666666666667,
        "kept_right": 0.3196252921109671,
        "kept_wrong": 0.024165725031210946,
        "handed_off": 0.6562089828578223
      },
      "usd_per_1000": 2.4384425350051053
    },
    {
      "decision": 254,
      "frontier": 140,
      "frontier_accuracy": 0.9808917197452229,
      "in_sample": {
        "threshold": 0.9712557594616492,
        "answered": 0.10145586897179254,
        "accuracy": 0.9808917197452229
      },
      "curve": [
        [
          0.0,
          0.9809
        ],
        [
          0.01,
          0.9809
        ],
        [
          0.02,
          0.9809
        ],
        [
          0.03,
          0.9809
        ],
        [
          0.04,
          0.9809
        ],
        [
          0.05,
          0.9809
        ],
        [
          0.0601,
          0.9809
        ],
        [
          0.0705,
          0.9809
        ],
        [
          0.0805,
          0.9809
        ],
        [
          0.0905,
          0.9809
        ],
        [
          0.1005,
          0.9809
        ],
        [
          0.1106,
          0.9804
        ],
        [
          0.121,
          0.9804
        ],
        [
          0.131,
          0.9804
        ],
        [
          0.141,
          0.9804
        ],
        [
          0.151,
          0.98
        ],
        [
          0.1611,
          0.9791
        ],
        [
          0.1715,
          0.9791
        ],
        [
          0.1815,
          0.9791
        ],
        [
          0.1915,
          0.9786
        ],
        [
          0.2015,
          0.9777
        ],
        [
          0.2116,
          0.9773
        ],
        [
          0.222,
          0.9763
        ],
        [
          0.232,
          0.9759
        ],
        [
          0.242,
          0.9759
        ],
        [
          0.252,
          0.9754
        ],
        [
          0.2621,
          0.9741
        ],
        [
          0.2725,
          0.9736
        ],
        [
          0.2825,
          0.9718
        ],
        [
          0.2925,
          0.9709
        ],
        [
          0.3025,
          0.9686
        ],
        [
          0.3126,
          0.9672
        ],
        [
          0.323,
          0.9645
        ],
        [
          0.333,
          0.9631
        ],
        [
          0.343,
          0.9613
        ],
        [
          0.353,
          0.9581
        ],
        [
          0.3631,
          0.9554
        ],
        [
          0.3735,
          0.9522
        ],
        [
          0.3835,
          0.9472
        ],
        [
          0.3935,
          0.9463
        ],
        [
          0.4035,
          0.9431
        ],
        [
          0.4136,
          0.9399
        ],
        [
          0.424,
          0.9368
        ],
        [
          0.434,
          0.9345
        ],
        [
          0.444,
          0.9304
        ],
        [
          0.454,
          0.9277
        ],
        [
          0.4641,
          0.9254
        ],
        [
          0.4745,
          0.9208
        ],
        [
          0.4845,
          0.9163
        ],
        [
          0.4945,
          0.9126
        ],
        [
          0.5045,
          0.9067
        ],
        [
          0.5146,
          0.9022
        ],
        [
          0.525,
          0.8995
        ],
        [
          0.535,
          0.8954
        ],
        [
          0.545,
          0.8899
        ],
        [
          0.5551,
          0.8854
        ],
        [
          0.5655,
          0.8817
        ],
        [
          0.5755,
          0.8772
        ],
        [
          0.5855,
          0.8722
        ],
        [
          0.5955,
          0.8681
        ],
        [
          0.6056,
          0.8631
        ],
        [
          0.616,
          0.8571
        ],
        [
          0.626,
          0.853
        ],
        [
          0.636,
          0.8476
        ],
        [
          0.646,
          0.8421
        ],
        [
          0.6561,
          0.838
        ],
        [
          0.6665,
          0.8303
        ],
        [
          0.6765,
          0.8248
        ],
        [
          0.6865,
          0.8189
        ],
        [
          0.6965,
          0.8116
        ],
        [
          0.7066,
          0.8057
        ],
        [
          0.717,
          0.7985
        ],
        [
          0.727,
          0.7907
        ],
        [
          0.737,
          0.7839
        ],
        [
          0.747,
          0.7775
        ],
        [
          0.7571,
          0.7702
        ],
        [
          0.7675,
          0.763
        ],
        [
          0.7775,
          0.7566
        ],
        [
          0.7875,
          0.7516
        ],
        [
          0.7975,
          0.7466
        ],
        [
          0.8076,
          0.7398
        ],
        [
          0.818,
          0.7329
        ],
        [
          0.828,
          0.7257
        ],
        [
          0.838,
          0.7179
        ],
        [
          0.848,
          0.7125
        ],
        [
          0.8581,
          0.7075
        ],
        [
          0.8685,
          0.7015
        ],
        [
          0.8785,
          0.6947
        ],
        [
          0.8885,
          0.6874
        ],
        [
          0.8985,
          0.6829
        ],
        [
          0.9086,
          0.6783
        ],
        [
          0.919,
          0.6715
        ],
        [
          0.929,
          0.6661
        ],
        [
          0.939,
          0.6583
        ],
        [
          0.949,
          0.6524
        ],
        [
          0.9591,
          0.6465
        ],
        [
          0.9695,
          0.6383
        ],
        [
          0.9795,
          0.6301
        ],
        [
          0.9895,
          0.6228
        ],
        [
          0.9995,
          0.6156
        ],
        [
          1.0,
          0.6151
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.12874186757219655,
        "difference": -0.0009366487594876324,
        "worse": 0.55,
        "kept_right": 0.12780521881270887,
        "kept_wrong": 0.0009366487594876324,
        "handed_off": 0.8712581324278034
      },
      "usd_per_1000": 2.1144911283399117
    },
    {
      "decision": 254,
      "frontier": 277,
      "frontier_accuracy": 0.9795268425841674,
      "in_sample": {
        "threshold": 0.9712557594616492,
        "answered": 0.10145586897179254,
        "accuracy": 0.9795268425841674
      },
      "curve": [
        [
          0.0,
          0.9795
        ],
        [
          0.01,
          0.9795
        ],
        [
          0.02,
          0.9795
        ],
        [
          0.03,
          0.9795
        ],
        [
          0.04,
          0.9795
        ],
        [
          0.05,
          0.9795
        ],
        [
          0.0601,
          0.9795
        ],
        [
          0.0705,
          0.9795
        ],
        [
          0.0805,
          0.9795
        ],
        [
          0.0905,
          0.9795
        ],
        [
          0.1005,
          0.9795
        ],
        [
          0.1106,
          0.9791
        ],
        [
          0.121,
          0.9791
        ],
        [
          0.131,
          0.9791
        ],
        [
          0.141,
          0.9791
        ],
        [
          0.151,
          0.9786
        ],
        [
          0.1611,
          0.9777
        ],
        [
          0.1715,
          0.9777
        ],
        [
          0.1815,
          0.9777
        ],
        [
          0.1915,
          0.9773
        ],
        [
          0.2015,
          0.9763
        ],
        [
          0.2116,
          0.9763
        ],
        [
          0.222,
          0.9754
        ],
        [
          0.232,
          0.9745
        ],
        [
          0.242,
          0.9745
        ],
        [
          0.252,
          0.9745
        ],
        [
          0.2621,
          0.9732
        ],
        [
          0.2725,
          0.9722
        ],
        [
          0.2825,
          0.9704
        ],
        [
          0.2925,
          0.9695
        ],
        [
          0.3025,
          0.9672
        ],
        [
          0.3126,
          0.9659
        ],
        [
          0.323,
          0.9631
        ],
        [
          0.333,
          0.9618
        ],
        [
          0.343,
          0.96
        ],
        [
          0.353,
          0.9568
        ],
        [
          0.3631,
          0.954
        ],
        [
          0.3735,
          0.9509
        ],
        [
          0.3835,
          0.9459
        ],
        [
          0.3935,
          0.9445
        ],
        [
          0.4035,
          0.9413
        ],
        [
          0.4136,
          0.9381
        ],
        [
          0.424,
          0.9345
        ],
        [
          0.434,
          0.9322
        ],
        [
          0.444,
          0.9281
        ],
        [
          0.454,
          0.9254
        ],
        [
          0.4641,
          0.9231
        ],
        [
          0.4745,
          0.9186
        ],
        [
          0.4845,
          0.914
        ],
        [
          0.4945,
          0.9108
        ],
        [
          0.5045,
          0.9049
        ],
        [
          0.5146,
          0.9008
        ],
        [
          0.525,
          0.8976
        ],
        [
          0.535,
          0.8935
        ],
        [
          0.545,
          0.8885
        ],
        [
          0.5551,
          0.884
        ],
        [
          0.5655,
          0.8803
        ],
        [
          0.5755,
          0.8758
        ],
        [
          0.5855,
          0.8703
        ],
        [
          0.5955,
          0.8662
        ],
        [
          0.6056,
          0.8612
        ],
        [
          0.616,
          0.8549
        ],
        [
          0.626,
          0.8512
        ],
        [
          0.636,
          0.8458
        ],
        [
          0.646,
          0.8403
        ],
        [
          0.6561,
          0.8362
        ],
        [
          0.6665,
          0.8285
        ],
        [
          0.6765,
          0.823
        ],
        [
          0.6865,
          0.8176
        ],
        [
          0.6965,
          0.8103
        ],
        [
          0.7066,
          0.8048
        ],
        [
          0.717,
          0.7975
        ],
        [
          0.727,
          0.7907
        ],
        [
          0.737,
          0.7834
        ],
        [
          0.747,
          0.7762
        ],
        [
          0.7571,
          0.7689
        ],
        [
          0.7675,
          0.7621
        ],
        [
          0.7775,
          0.7561
        ],
        [
          0.7875,
          0.7511
        ],
        [
          0.7975,
          0.7461
        ],
        [
          0.8076,
          0.7393
        ],
        [
          0.818,
          0.7325
        ],
        [
          0.828,
          0.7252
        ],
        [
          0.838,
          0.7179
        ],
        [
          0.848,
          0.7125
        ],
        [
          0.8581,
          0.7075
        ],
        [
          0.8685,
          0.7015
        ],
        [
          0.8785,
          0.6943
        ],
        [
          0.8885,
          0.6874
        ],
        [
          0.8985,
          0.6824
        ],
        [
          0.9086,
          0.6779
        ],
        [
          0.919,
          0.6706
        ],
        [
          0.929,
          0.6652
        ],
        [
          0.939,
          0.6583
        ],
        [
          0.949,
          0.6524
        ],
        [
          0.9591,
          0.6465
        ],
        [
          0.9695,
          0.6383
        ],
        [
          0.9795,
          0.6301
        ],
        [
          0.9895,
          0.6228
        ],
        [
          0.9995,
          0.6156
        ],
        [
          1.0,
          0.6151
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.12874186757219655,
        "difference": -0.0009305909057458549,
        "worse": 0.55,
        "kept_right": 0.12780521881270887,
        "kept_wrong": 0.0009366487594876324,
        "handed_off": 0.8712581324278034
      },
      "usd_per_1000": 5.291491990336394
    },
    {
      "decision": 254,
      "frontier": 276,
      "frontier_accuracy": 0.9777070063694268,
      "in_sample": {
        "threshold": 0.9712557594616492,
        "answered": 0.10145586897179254,
        "accuracy": 0.9777070063694268
      },
      "curve": [
        [
          0.0,
          0.9777
        ],
        [
          0.01,
          0.9777
        ],
        [
          0.02,
          0.9777
        ],
        [
          0.03,
          0.9777
        ],
        [
          0.04,
          0.9777
        ],
        [
          0.05,
          0.9777
        ],
        [
          0.0601,
          0.9777
        ],
        [
          0.0705,
          0.9777
        ],
        [
          0.0805,
          0.9777
        ],
        [
          0.0905,
          0.9777
        ],
        [
          0.1005,
          0.9777
        ],
        [
          0.1106,
          0.9773
        ],
        [
          0.121,
          0.9773
        ],
        [
          0.131,
          0.9773
        ],
        [
          0.141,
          0.9773
        ],
        [
          0.151,
          0.9768
        ],
        [
          0.1611,
          0.9759
        ],
        [
          0.1715,
          0.9759
        ],
        [
          0.1815,
          0.9759
        ],
        [
          0.1915,
          0.9754
        ],
        [
          0.2015,
          0.9745
        ],
        [
          0.2116,
          0.9745
        ],
        [
          0.222,
          0.9736
        ],
        [
          0.232,
          0.9727
        ],
        [
          0.242,
          0.9727
        ],
        [
          0.252,
          0.9727
        ],
        [
          0.2621,
          0.9713
        ],
        [
          0.2725,
          0.9709
        ],
        [
          0.2825,
          0.9691
        ],
        [
          0.2925,
          0.9682
        ],
        [
          0.3025,
          0.9663
        ],
        [
          0.3126,
          0.965
        ],
        [
          0.323,
          0.9627
        ],
        [
          0.333,
          0.9613
        ],
        [
          0.343,
          0.9595
        ],
        [
          0.353,
          0.9572
        ],
        [
          0.3631,
          0.9545
        ],
        [
          0.3735,
          0.9513
        ],
        [
          0.3835,
          0.9463
        ],
        [
          0.3935,
          0.9449
        ],
        [
          0.4035,
          0.9418
        ],
        [
          0.4136,
          0.9386
        ],
        [
          0.424,
          0.9349
        ],
        [
          0.434,
          0.9327
        ],
        [
          0.444,
          0.9286
        ],
        [
          0.454,
          0.9258
        ],
        [
          0.4641,
          0.9236
        ],
        [
          0.4745,
          0.919
        ],
        [
          0.4845,
          0.9145
        ],
        [
          0.4945,
          0.9113
        ],
        [
          0.5045,
          0.9054
        ],
        [
          0.5146,
          0.9017
        ],
        [
          0.525,
          0.8981
        ],
        [
          0.535,
          0.894
        ],
        [
          0.545,
          0.889
        ],
        [
          0.5551,
          0.8844
        ],
        [
          0.5655,
          0.8813
        ],
        [
          0.5755,
          0.8767
        ],
        [
          0.5855,
          0.8712
        ],
        [
          0.5955,
          0.8672
        ],
        [
          0.6056,
          0.8621
        ],
        [
          0.616,
          0.8558
        ],
        [
          0.626,
          0.8521
        ],
        [
          0.636,
          0.8467
        ],
        [
          0.646,
          0.8412
        ],
        [
          0.6561,
          0.8371
        ],
        [
          0.6665,
          0.8294
        ],
        [
          0.6765,
          0.8239
        ],
        [
          0.6865,
          0.818
        ],
        [
          0.6965,
          0.8112
        ],
        [
          0.7066,
          0.8057
        ],
        [
          0.717,
          0.7985
        ],
        [
          0.727,
          0.7916
        ],
        [
          0.737,
          0.7843
        ],
        [
          0.747,
          0.7771
        ],
        [
          0.7571,
          0.7698
        ],
        [
          0.7675,
          0.763
        ],
        [
          0.7775,
          0.7566
        ],
        [
          0.7875,
          0.7516
        ],
        [
          0.7975,
          0.7466
        ],
        [
          0.8076,
          0.7398
        ],
        [
          0.818,
          0.7329
        ],
        [
          0.828,
          0.7257
        ],
        [
          0.838,
          0.7179
        ],
        [
          0.848,
          0.7125
        ],
        [
          0.8581,
          0.7075
        ],
        [
          0.8685,
          0.7015
        ],
        [
          0.8785,
          0.6943
        ],
        [
          0.8885,
          0.6874
        ],
        [
          0.8985,
          0.6829
        ],
        [
          0.9086,
          0.6783
        ],
        [
          0.919,
          0.6711
        ],
        [
          0.929,
          0.6656
        ],
        [
          0.939,
          0.6583
        ],
        [
          0.949,
          0.6524
        ],
        [
          0.9591,
          0.6465
        ],
        [
          0.9695,
          0.6383
        ],
        [
          0.9795,
          0.6301
        ],
        [
          0.9895,
          0.6228
        ],
        [
          0.9995,
          0.6156
        ],
        [
          1.0,
          0.6151
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.12874186757219655,
        "difference": -0.0009305909057458549,
        "worse": 0.55,
        "kept_right": 0.12780521881270887,
        "kept_wrong": 0.0009366487594876324,
        "handed_off": 0.8712581324278034
      },
      "usd_per_1000": 1.4201742057657565
    },
    {
      "decision": 254,
      "frontier": 48,
      "frontier_accuracy": 0.9758871701546861,
      "in_sample": {
        "threshold": 0.9712557594616492,
        "answered": 0.10145586897179254,
        "accuracy": 0.9758871701546861
      },
      "curve": [
        [
          0.0,
          0.9759
        ],
        [
          0.01,
          0.9759
        ],
        [
          0.02,
          0.9759
        ],
        [
          0.03,
          0.9759
        ],
        [
          0.04,
          0.9759
        ],
        [
          0.05,
          0.9759
        ],
        [
          0.0601,
          0.9759
        ],
        [
          0.0705,
          0.9759
        ],
        [
          0.0805,
          0.9759
        ],
        [
          0.0905,
          0.9759
        ],
        [
          0.1005,
          0.9759
        ],
        [
          0.1106,
          0.9754
        ],
        [
          0.121,
          0.9754
        ],
        [
          0.131,
          0.9754
        ],
        [
          0.141,
          0.9754
        ],
        [
          0.151,
          0.975
        ],
        [
          0.1611,
          0.9741
        ],
        [
          0.1715,
          0.9741
        ],
        [
          0.1815,
          0.9741
        ],
        [
          0.1915,
          0.9736
        ],
        [
          0.2015,
          0.9727
        ],
        [
          0.2116,
          0.9722
        ],
        [
          0.222,
          0.9713
        ],
        [
          0.232,
          0.9704
        ],
        [
          0.242,
          0.9704
        ],
        [
          0.252,
          0.9704
        ],
        [
          0.2621,
          0.9695
        ],
        [
          0.2725,
          0.9691
        ],
        [
          0.2825,
          0.9672
        ],
        [
          0.2925,
          0.9663
        ],
        [
          0.3025,
          0.9641
        ],
        [
          0.3126,
          0.9627
        ],
        [
          0.323,
          0.96
        ],
        [
          0.333,
          0.9586
        ],
        [
          0.343,
          0.9568
        ],
        [
          0.353,
          0.9536
        ],
        [
          0.3631,
          0.9509
        ],
        [
          0.3735,
          0.9477
        ],
        [
          0.3835,
          0.9427
        ],
        [
          0.3935,
          0.9413
        ],
        [
          0.4035,
          0.9381
        ],
        [
          0.4136,
          0.9349
        ],
        [
          0.424,
          0.9313
        ],
        [
          0.434,
          0.929
        ],
        [
          0.444,
          0.9249
        ],
        [
          0.454,
          0.9222
        ],
        [
          0.4641,
          0.9199
        ],
        [
          0.4745,
          0.9154
        ],
        [
          0.4845,
          0.9113
        ],
        [
          0.4945,
          0.9081
        ],
        [
          0.5045,
          0.9022
        ],
        [
          0.5146,
          0.8981
        ],
        [
          0.525,
          0.8944
        ],
        [
          0.535,
          0.8904
        ],
        [
          0.545,
          0.8854
        ],
        [
          0.5551,
          0.8808
        ],
        [
          0.5655,
          0.8772
        ],
        [
          0.5755,
          0.8726
        ],
        [
          0.5855,
          0.8672
        ],
        [
          0.5955,
          0.8631
        ],
        [
          0.6056,
          0.8581
        ],
        [
          0.616,
          0.8521
        ],
        [
          0.626,
          0.8485
        ],
        [
          0.636,
          0.843
        ],
        [
          0.646,
          0.8376
        ],
        [
          0.6561,
          0.8335
        ],
        [
          0.6665,
          0.8258
        ],
        [
          0.6765,
          0.8207
        ],
        [
          0.6865,
          0.8153
        ],
        [
          0.6965,
          0.8085
        ],
        [
          0.7066,
          0.803
        ],
        [
          0.717,
          0.7953
        ],
        [
          0.727,
          0.788
        ],
        [
          0.737,
          0.7807
        ],
        [
          0.747,
          0.7739
        ],
        [
          0.7571,
          0.7666
        ],
        [
          0.7675,
          0.7598
        ],
        [
          0.7775,
          0.7543
        ],
        [
          0.7875,
          0.7493
        ],
        [
          0.7975,
          0.7443
        ],
        [
          0.8076,
          0.7375
        ],
        [
          0.818,
          0.7307
        ],
        [
          0.828,
          0.7234
        ],
        [
          0.838,
          0.7157
        ],
        [
          0.848,
          0.7097
        ],
        [
          0.8581,
          0.7052
        ],
        [
          0.8685,
          0.6993
        ],
        [
          0.8785,
          0.6929
        ],
        [
          0.8885,
          0.6861
        ],
        [
          0.8985,
          0.6815
        ],
        [
          0.9086,
          0.6774
        ],
        [
          0.919,
          0.6706
        ],
        [
          0.929,
          0.6652
        ],
        [
          0.939,
          0.6583
        ],
        [
          0.949,
          0.6524
        ],
        [
          0.9591,
          0.6465
        ],
        [
          0.9695,
          0.6383
        ],
        [
          0.9795,
          0.6301
        ],
        [
          0.9895,
          0.6228
        ],
        [
          0.9995,
          0.6156
        ],
        [
          1.0,
          0.6151
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.12874186757219655,
        "difference": -0.0009366487594876324,
        "worse": 0.55,
        "kept_right": 0.12780521881270887,
        "kept_wrong": 0.0009366487594876324,
        "handed_off": 0.8712581324278034
      },
      "usd_per_1000": 4.692651178716982
    },
    {
      "decision": 254,
      "frontier": 191,
      "frontier_accuracy": 0.9740673339399454,
      "in_sample": {
        "threshold": 0.9712557594616492,
        "answered": 0.10145586897179254,
        "accuracy": 0.9740673339399454
      },
      "curve": [
        [
          0.0,
          0.9741
        ],
        [
          0.01,
          0.9741
        ],
        [
          0.02,
          0.9741
        ],
        [
          0.03,
          0.9741
        ],
        [
          0.04,
          0.9741
        ],
        [
          0.05,
          0.9741
        ],
        [
          0.0601,
          0.9741
        ],
        [
          0.0705,
          0.9741
        ],
        [
          0.0805,
          0.9741
        ],
        [
          0.0905,
          0.9741
        ],
        [
          0.1005,
          0.9741
        ],
        [
          0.1106,
          0.9736
        ],
        [
          0.121,
          0.9736
        ],
        [
          0.131,
          0.9736
        ],
        [
          0.141,
          0.9736
        ],
        [
          0.151,
          0.9732
        ],
        [
          0.1611,
          0.9727
        ],
        [
          0.1715,
          0.9727
        ],
        [
          0.1815,
          0.9727
        ],
        [
          0.1915,
          0.9722
        ],
        [
          0.2015,
          0.9713
        ],
        [
          0.2116,
          0.9709
        ],
        [
          0.222,
          0.97
        ],
        [
          0.232,
          0.9691
        ],
        [
          0.242,
          0.9691
        ],
        [
          0.252,
          0.9686
        ],
        [
          0.2621,
          0.9672
        ],
        [
          0.2725,
          0.9668
        ],
        [
          0.2825,
          0.9645
        ],
        [
          0.2925,
          0.9641
        ],
        [
          0.3025,
          0.9622
        ],
        [
          0.3126,
          0.9609
        ],
        [
          0.323,
          0.9581
        ],
        [
          0.333,
          0.9572
        ],
        [
          0.343,
          0.9554
        ],
        [
          0.353,
          0.9522
        ],
        [
          0.3631,
          0.9495
        ],
        [
          0.3735,
          0.9463
        ],
        [
          0.3835,
          0.9413
        ],
        [
          0.3935,
          0.9404
        ],
        [
          0.4035,
          0.9372
        ],
        [
          0.4136,
          0.934
        ],
        [
          0.424,
          0.9304
        ],
        [
          0.434,
          0.9281
        ],
        [
          0.444,
          0.924
        ],
        [
          0.454,
          0.9213
        ],
        [
          0.4641,
          0.919
        ],
        [
          0.4745,
          0.9149
        ],
        [
          0.4845,
          0.9108
        ],
        [
          0.4945,
          0.9076
        ],
        [
          0.5045,
          0.9017
        ],
        [
          0.5146,
          0.8985
        ],
        [
          0.525,
          0.8949
        ],
        [
          0.535,
          0.8904
        ],
        [
          0.545,
          0.8849
        ],
        [
          0.5551,
          0.8803
        ],
        [
          0.5655,
          0.8772
        ],
        [
          0.5755,
          0.8726
        ],
        [
          0.5855,
          0.8672
        ],
        [
          0.5955,
          0.8631
        ],
        [
          0.6056,
          0.8581
        ],
        [
          0.616,
          0.8517
        ],
        [
          0.626,
          0.8485
        ],
        [
          0.636,
          0.843
        ],
        [
          0.646,
          0.8376
        ],
        [
          0.6561,
          0.8335
        ],
        [
          0.6665,
          0.8258
        ],
        [
          0.6765,
          0.8203
        ],
        [
          0.6865,
          0.8153
        ],
        [
          0.6965,
          0.8085
        ],
        [
          0.7066,
          0.803
        ],
        [
          0.717,
          0.7953
        ],
        [
          0.727,
          0.7884
        ],
        [
          0.737,
          0.7812
        ],
        [
          0.747,
          0.7743
        ],
        [
          0.7571,
          0.7671
        ],
        [
          0.7675,
          0.7602
        ],
        [
          0.7775,
          0.7539
        ],
        [
          0.7875,
          0.7489
        ],
        [
          0.7975,
          0.7439
        ],
        [
          0.8076,
          0.737
        ],
        [
          0.818,
          0.7302
        ],
        [
          0.828,
          0.7229
        ],
        [
          0.838,
          0.7152
        ],
        [
          0.848,
          0.7097
        ],
        [
          0.8581,
          0.7052
        ],
        [
          0.8685,
          0.6993
        ],
        [
          0.8785,
          0.6924
        ],
        [
          0.8885,
          0.6856
        ],
        [
          0.8985,
          0.6815
        ],
        [
          0.9086,
          0.677
        ],
        [
          0.919,
          0.6697
        ],
        [
          0.929,
          0.6642
        ],
        [
          0.939,
          0.6574
        ],
        [
          0.949,
          0.652
        ],
        [
          0.9591,
          0.646
        ],
        [
          0.9695,
          0.6379
        ],
        [
          0.9795,
          0.6301
        ],
        [
          0.9895,
          0.6228
        ],
        [
          0.9995,
          0.6156
        ],
        [
          1.0,
          0.6151
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.13256734478216076,
        "difference": -0.0009882327489711272,
        "worse": 0.55,
        "kept_right": 0.13146698789835543,
        "kept_wrong": 0.0011003568838054247,
        "handed_off": 0.867432655217839
      },
      "usd_per_1000": 23.19997853675522
    },
    {
      "decision": 254,
      "frontier": 188,
      "frontier_accuracy": 0.9713375796178344,
      "in_sample": {
        "threshold": 0.9712557594616492,
        "answered": 0.10145586897179254,
        "accuracy": 0.9713375796178344
      },
      "curve": [
        [
          0.0,
          0.9713
        ],
        [
          0.01,
          0.9713
        ],
        [
          0.02,
          0.9713
        ],
        [
          0.03,
          0.9713
        ],
        [
          0.04,
          0.9713
        ],
        [
          0.05,
          0.9713
        ],
        [
          0.0601,
          0.9713
        ],
        [
          0.0705,
          0.9713
        ],
        [
          0.0805,
          0.9713
        ],
        [
          0.0905,
          0.9713
        ],
        [
          0.1005,
          0.9713
        ],
        [
          0.1106,
          0.9709
        ],
        [
          0.121,
          0.9709
        ],
        [
          0.131,
          0.9709
        ],
        [
          0.141,
          0.9709
        ],
        [
          0.151,
          0.9704
        ],
        [
          0.1611,
          0.9695
        ],
        [
          0.1715,
          0.9695
        ],
        [
          0.1815,
          0.9695
        ],
        [
          0.1915,
          0.9691
        ],
        [
          0.2015,
          0.9682
        ],
        [
          0.2116,
          0.9682
        ],
        [
          0.222,
          0.9672
        ],
        [
          0.232,
          0.9668
        ],
        [
          0.242,
          0.9668
        ],
        [
          0.252,
          0.9668
        ],
        [
          0.2621,
          0.9659
        ],
        [
          0.2725,
          0.9654
        ],
        [
          0.2825,
          0.9636
        ],
        [
          0.2925,
          0.9631
        ],
        [
          0.3025,
          0.9613
        ],
        [
          0.3126,
          0.9604
        ],
        [
          0.323,
          0.9577
        ],
        [
          0.333,
          0.9563
        ],
        [
          0.343,
          0.9545
        ],
        [
          0.353,
          0.9513
        ],
        [
          0.3631,
          0.949
        ],
        [
          0.3735,
          0.9454
        ],
        [
          0.3835,
          0.9404
        ],
        [
          0.3935,
          0.939
        ],
        [
          0.4035,
          0.9359
        ],
        [
          0.4136,
          0.9327
        ],
        [
          0.424,
          0.9295
        ],
        [
          0.434,
          0.9272
        ],
        [
          0.444,
          0.9231
        ],
        [
          0.454,
          0.9204
        ],
        [
          0.4641,
          0.9181
        ],
        [
          0.4745,
          0.9136
        ],
        [
          0.4845,
          0.9099
        ],
        [
          0.4945,
          0.9063
        ],
        [
          0.5045,
          0.9004
        ],
        [
          0.5146,
          0.8972
        ],
        [
          0.525,
          0.8944
        ],
        [
          0.535,
          0.8899
        ],
        [
          0.545,
          0.8844
        ],
        [
          0.5551,
          0.8799
        ],
        [
          0.5655,
          0.8763
        ],
        [
          0.5755,
          0.8717
        ],
        [
          0.5855,
          0.8667
        ],
        [
          0.5955,
          0.8626
        ],
        [
          0.6056,
          0.8576
        ],
        [
          0.616,
          0.8517
        ],
        [
          0.626,
          0.848
        ],
        [
          0.636,
          0.8426
        ],
        [
          0.646,
          0.8371
        ],
        [
          0.6561,
          0.833
        ],
        [
          0.6665,
          0.8253
        ],
        [
          0.6765,
          0.8198
        ],
        [
          0.6865,
          0.8139
        ],
        [
          0.6965,
          0.8076
        ],
        [
          0.7066,
          0.8016
        ],
        [
          0.717,
          0.7944
        ],
        [
          0.727,
          0.7866
        ],
        [
          0.737,
          0.7798
        ],
        [
          0.747,
          0.7734
        ],
        [
          0.7571,
          0.7662
        ],
        [
          0.7675,
          0.7593
        ],
        [
          0.7775,
          0.753
        ],
        [
          0.7875,
          0.748
        ],
        [
          0.7975,
          0.7439
        ],
        [
          0.8076,
          0.737
        ],
        [
          0.818,
          0.7302
        ],
        [
          0.828,
          0.7229
        ],
        [
          0.838,
          0.7152
        ],
        [
          0.848,
          0.7088
        ],
        [
          0.8581,
          0.7043
        ],
        [
          0.8685,
          0.6984
        ],
        [
          0.8785,
          0.692
        ],
        [
          0.8885,
          0.6852
        ],
        [
          0.8985,
          0.6806
        ],
        [
          0.9086,
          0.6761
        ],
        [
          0.919,
          0.6692
        ],
        [
          0.929,
          0.6638
        ],
        [
          0.939,
          0.6565
        ],
        [
          0.949,
          0.6506
        ],
        [
          0.9591,
          0.6447
        ],
        [
          0.9695,
          0.6369
        ],
        [
          0.9795,
          0.6288
        ],
        [
          0.9895,
          0.6224
        ],
        [
          0.9995,
          0.6156
        ],
        [
          1.0,
          0.6151
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.12909673563407098,
        "difference": -0.0009548553886090639,
        "worse": 0.55,
        "kept_right": 0.1281297562710043,
        "kept_wrong": 0.0009669793630666437,
        "handed_off": 0.8709032643659289
      },
      "usd_per_1000": 0.8458907197130637
    },
    {
      "decision": 254,
      "frontier": 190,
      "frontier_accuracy": 0.9672429481346679,
      "in_sample": {
        "threshold": 0.9712557594616492,
        "answered": 0.10145586897179254,
        "accuracy": 0.9672429481346679
      },
      "curve": [
        [
          0.0,
          0.9672
        ],
        [
          0.01,
          0.9672
        ],
        [
          0.02,
          0.9672
        ],
        [
          0.03,
          0.9672
        ],
        [
          0.04,
          0.9672
        ],
        [
          0.05,
          0.9672
        ],
        [
          0.0601,
          0.9672
        ],
        [
          0.0705,
          0.9672
        ],
        [
          0.0805,
          0.9672
        ],
        [
          0.0905,
          0.9672
        ],
        [
          0.1005,
          0.9672
        ],
        [
          0.1106,
          0.9668
        ],
        [
          0.121,
          0.9668
        ],
        [
          0.131,
          0.9668
        ],
        [
          0.141,
          0.9668
        ],
        [
          0.151,
          0.9663
        ],
        [
          0.1611,
          0.9659
        ],
        [
          0.1715,
          0.9663
        ],
        [
          0.1815,
          0.9663
        ],
        [
          0.1915,
          0.9659
        ],
        [
          0.2015,
          0.965
        ],
        [
          0.2116,
          0.965
        ],
        [
          0.222,
          0.9641
        ],
        [
          0.232,
          0.9631
        ],
        [
          0.242,
          0.9636
        ],
        [
          0.252,
          0.9641
        ],
        [
          0.2621,
          0.9627
        ],
        [
          0.2725,
          0.9622
        ],
        [
          0.2825,
          0.96
        ],
        [
          0.2925,
          0.9591
        ],
        [
          0.3025,
          0.9572
        ],
        [
          0.3126,
          0.9559
        ],
        [
          0.323,
          0.9536
        ],
        [
          0.333,
          0.9527
        ],
        [
          0.343,
          0.9509
        ],
        [
          0.353,
          0.9477
        ],
        [
          0.3631,
          0.9449
        ],
        [
          0.3735,
          0.9418
        ],
        [
          0.3835,
          0.9368
        ],
        [
          0.3935,
          0.9359
        ],
        [
          0.4035,
          0.9327
        ],
        [
          0.4136,
          0.9295
        ],
        [
          0.424,
          0.9258
        ],
        [
          0.434,
          0.9236
        ],
        [
          0.444,
          0.9195
        ],
        [
          0.454,
          0.9167
        ],
        [
          0.4641,
          0.9145
        ],
        [
          0.4745,
          0.9099
        ],
        [
          0.4845,
          0.9054
        ],
        [
          0.4945,
          0.9022
        ],
        [
          0.5045,
          0.8963
        ],
        [
          0.5146,
          0.8926
        ],
        [
          0.525,
          0.889
        ],
        [
          0.535,
          0.8849
        ],
        [
          0.545,
          0.8794
        ],
        [
          0.5551,
          0.8749
        ],
        [
          0.5655,
          0.8712
        ],
        [
          0.5755,
          0.8667
        ],
        [
          0.5855,
          0.8612
        ],
        [
          0.5955,
          0.8571
        ],
        [
          0.6056,
          0.8521
        ],
        [
          0.616,
          0.8458
        ],
        [
          0.626,
          0.8417
        ],
        [
          0.636,
          0.8362
        ],
        [
          0.646,
          0.8308
        ],
        [
          0.6561,
          0.8267
        ],
        [
          0.6665,
          0.8189
        ],
        [
          0.6765,
          0.8135
        ],
        [
          0.6865,
          0.8085
        ],
        [
          0.6965,
          0.8016
        ],
        [
          0.7066,
          0.7971
        ],
        [
          0.717,
          0.7898
        ],
        [
          0.727,
          0.783
        ],
        [
          0.737,
          0.7757
        ],
        [
          0.747,
          0.7693
        ],
        [
          0.7571,
          0.7621
        ],
        [
          0.7675,
          0.7552
        ],
        [
          0.7775,
          0.7498
        ],
        [
          0.7875,
          0.7452
        ],
        [
          0.7975,
          0.7402
        ],
        [
          0.8076,
          0.7334
        ],
        [
          0.818,
          0.7266
        ],
        [
          0.828,
          0.7193
        ],
        [
          0.838,
          0.712
        ],
        [
          0.848,
          0.7066
        ],
        [
          0.8581,
          0.7029
        ],
        [
          0.8685,
          0.697
        ],
        [
          0.8785,
          0.6902
        ],
        [
          0.8885,
          0.6833
        ],
        [
          0.8985,
          0.6797
        ],
        [
          0.9086,
          0.6752
        ],
        [
          0.919,
          0.6679
        ],
        [
          0.929,
          0.6624
        ],
        [
          0.939,
          0.6561
        ],
        [
          0.949,
          0.6506
        ],
        [
          0.9591,
          0.6447
        ],
        [
          0.9695,
          0.6365
        ],
        [
          0.9795,
          0.6292
        ],
        [
          0.9895,
          0.6228
        ],
        [
          0.9995,
          0.6156
        ],
        [
          1.0,
          0.6151
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.15207328912198712,
        "difference": -0.0017309831838691204,
        "worse": 0.6166666666666667,
        "kept_right": 0.14993914555935503,
        "kept_wrong": 0.002134143562632062,
        "handed_off": 0.847926710878013
      },
      "usd_per_1000": 4.728523840086779
    },
    {
      "decision": 254,
      "frontier": 124,
      "frontier_accuracy": 0.9667879890809827,
      "in_sample": {
        "threshold": 0.9712557594616492,
        "answered": 0.10145586897179254,
        "accuracy": 0.9667879890809827
      },
      "curve": [
        [
          0.0,
          0.9668
        ],
        [
          0.01,
          0.9668
        ],
        [
          0.02,
          0.9668
        ],
        [
          0.03,
          0.9668
        ],
        [
          0.04,
          0.9668
        ],
        [
          0.05,
          0.9668
        ],
        [
          0.0601,
          0.9668
        ],
        [
          0.0705,
          0.9668
        ],
        [
          0.0805,
          0.9668
        ],
        [
          0.0905,
          0.9668
        ],
        [
          0.1005,
          0.9668
        ],
        [
          0.1106,
          0.9663
        ],
        [
          0.121,
          0.9663
        ],
        [
          0.131,
          0.9663
        ],
        [
          0.141,
          0.9663
        ],
        [
          0.151,
          0.9659
        ],
        [
          0.1611,
          0.965
        ],
        [
          0.1715,
          0.965
        ],
        [
          0.1815,
          0.965
        ],
        [
          0.1915,
          0.9645
        ],
        [
          0.2015,
          0.9636
        ],
        [
          0.2116,
          0.9636
        ],
        [
          0.222,
          0.9627
        ],
        [
          0.232,
          0.9618
        ],
        [
          0.242,
          0.9618
        ],
        [
          0.252,
          0.9618
        ],
        [
          0.2621,
          0.9609
        ],
        [
          0.2725,
          0.9604
        ],
        [
          0.2825,
          0.9586
        ],
        [
          0.2925,
          0.9581
        ],
        [
          0.3025,
          0.9559
        ],
        [
          0.3126,
          0.9545
        ],
        [
          0.323,
          0.9522
        ],
        [
          0.333,
          0.9513
        ],
        [
          0.343,
          0.9495
        ],
        [
          0.353,
          0.9468
        ],
        [
          0.3631,
          0.944
        ],
        [
          0.3735,
          0.9409
        ],
        [
          0.3835,
          0.9359
        ],
        [
          0.3935,
          0.9349
        ],
        [
          0.4035,
          0.9318
        ],
        [
          0.4136,
          0.9286
        ],
        [
          0.424,
          0.9249
        ],
        [
          0.434,
          0.9227
        ],
        [
          0.444,
          0.9186
        ],
        [
          0.454,
          0.9158
        ],
        [
          0.4641,
          0.9136
        ],
        [
          0.4745,
          0.909
        ],
        [
          0.4845,
          0.9045
        ],
        [
          0.4945,
          0.9013
        ],
        [
          0.5045,
          0.8954
        ],
        [
          0.5146,
          0.8917
        ],
        [
          0.525,
          0.8885
        ],
        [
          0.535,
          0.8844
        ],
        [
          0.545,
          0.8794
        ],
        [
          0.5551,
          0.8749
        ],
        [
          0.5655,
          0.8717
        ],
        [
          0.5755,
          0.8672
        ],
        [
          0.5855,
          0.8621
        ],
        [
          0.5955,
          0.8581
        ],
        [
          0.6056,
          0.853
        ],
        [
          0.616,
          0.8467
        ],
        [
          0.626,
          0.8426
        ],
        [
          0.636,
          0.8371
        ],
        [
          0.646,
          0.8317
        ],
        [
          0.6561,
          0.828
        ],
        [
          0.6665,
          0.8203
        ],
        [
          0.6765,
          0.8157
        ],
        [
          0.6865,
          0.8103
        ],
        [
          0.6965,
          0.8035
        ],
        [
          0.7066,
          0.798
        ],
        [
          0.717,
          0.7907
        ],
        [
          0.727,
          0.7839
        ],
        [
          0.737,
          0.7766
        ],
        [
          0.747,
          0.7698
        ],
        [
          0.7571,
          0.7634
        ],
        [
          0.7675,
          0.7561
        ],
        [
          0.7775,
          0.7502
        ],
        [
          0.7875,
          0.7457
        ],
        [
          0.7975,
          0.7407
        ],
        [
          0.8076,
          0.7338
        ],
        [
          0.818,
          0.7275
        ],
        [
          0.828,
          0.7202
        ],
        [
          0.838,
          0.7129
        ],
        [
          0.848,
          0.707
        ],
        [
          0.8581,
          0.7025
        ],
        [
          0.8685,
          0.6965
        ],
        [
          0.8785,
          0.6897
        ],
        [
          0.8885,
          0.6829
        ],
        [
          0.8985,
          0.6793
        ],
        [
          0.9086,
          0.6761
        ],
        [
          0.919,
          0.6688
        ],
        [
          0.929,
          0.6633
        ],
        [
          0.939,
          0.6561
        ],
        [
          0.949,
          0.6501
        ],
        [
          0.9591,
          0.6442
        ],
        [
          0.9695,
          0.6365
        ],
        [
          0.9795,
          0.6283
        ],
        [
          0.9895,
          0.6224
        ],
        [
          0.9995,
          0.6156
        ],
        [
          1.0,
          0.6151
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.12909673563407098,
        "difference": -0.0009518223282511629,
        "worse": 0.55,
        "kept_right": 0.128138855452078,
        "kept_wrong": 0.0009578801819929402,
        "handed_off": 0.8709032643659289
      },
      "usd_per_1000": 0.9912734187957292
    },
    {
      "decision": 254,
      "frontier": 278,
      "frontier_accuracy": 0.9499545040946314,
      "in_sample": {
        "threshold": 0.9500754291205227,
        "answered": 0.15286624203821655,
        "accuracy": 0.9499545040946314
      },
      "curve": [
        [
          0.0,
          0.95
        ],
        [
          0.01,
          0.95
        ],
        [
          0.02,
          0.95
        ],
        [
          0.03,
          0.95
        ],
        [
          0.04,
          0.95
        ],
        [
          0.05,
          0.95
        ],
        [
          0.0601,
          0.95
        ],
        [
          0.0705,
          0.95
        ],
        [
          0.0805,
          0.95
        ],
        [
          0.0905,
          0.95
        ],
        [
          0.1005,
          0.95
        ],
        [
          0.1106,
          0.95
        ],
        [
          0.121,
          0.9504
        ],
        [
          0.131,
          0.9504
        ],
        [
          0.141,
          0.9504
        ],
        [
          0.151,
          0.95
        ],
        [
          0.1611,
          0.949
        ],
        [
          0.1715,
          0.949
        ],
        [
          0.1815,
          0.949
        ],
        [
          0.1915,
          0.949
        ],
        [
          0.2015,
          0.9481
        ],
        [
          0.2116,
          0.9481
        ],
        [
          0.222,
          0.9477
        ],
        [
          0.232,
          0.9468
        ],
        [
          0.242,
          0.9468
        ],
        [
          0.252,
          0.9463
        ],
        [
          0.2621,
          0.9454
        ],
        [
          0.2725,
          0.9454
        ],
        [
          0.2825,
          0.944
        ],
        [
          0.2925,
          0.9431
        ],
        [
          0.3025,
          0.9413
        ],
        [
          0.3126,
          0.9399
        ],
        [
          0.323,
          0.9372
        ],
        [
          0.333,
          0.9368
        ],
        [
          0.343,
          0.9349
        ],
        [
          0.353,
          0.9318
        ],
        [
          0.3631,
          0.9299
        ],
        [
          0.3735,
          0.9272
        ],
        [
          0.3835,
          0.9222
        ],
        [
          0.3935,
          0.9217
        ],
        [
          0.4035,
          0.9186
        ],
        [
          0.4136,
          0.9158
        ],
        [
          0.424,
          0.9126
        ],
        [
          0.434,
          0.9104
        ],
        [
          0.444,
          0.9063
        ],
        [
          0.454,
          0.904
        ],
        [
          0.4641,
          0.9017
        ],
        [
          0.4745,
          0.8972
        ],
        [
          0.4845,
          0.8935
        ],
        [
          0.4945,
          0.8899
        ],
        [
          0.5045,
          0.8849
        ],
        [
          0.5146,
          0.8822
        ],
        [
          0.525,
          0.879
        ],
        [
          0.535,
          0.8749
        ],
        [
          0.545,
          0.8703
        ],
        [
          0.5551,
          0.8662
        ],
        [
          0.5655,
          0.8631
        ],
        [
          0.5755,
          0.8599
        ],
        [
          0.5855,
          0.8549
        ],
        [
          0.5955,
          0.8508
        ],
        [
          0.6056,
          0.8462
        ],
        [
          0.616,
          0.8403
        ],
        [
          0.626,
          0.8367
        ],
        [
          0.636,
          0.8312
        ],
        [
          0.646,
          0.8258
        ],
        [
          0.6561,
          0.8221
        ],
        [
          0.6665,
          0.8157
        ],
        [
          0.6765,
          0.8107
        ],
        [
          0.6865,
          0.8062
        ],
        [
          0.6965,
          0.7998
        ],
        [
          0.7066,
          0.7944
        ],
        [
          0.717,
          0.7875
        ],
        [
          0.727,
          0.7807
        ],
        [
          0.737,
          0.7734
        ],
        [
          0.747,
          0.7666
        ],
        [
          0.7571,
          0.7598
        ],
        [
          0.7675,
          0.753
        ],
        [
          0.7775,
          0.747
        ],
        [
          0.7875,
          0.742
        ],
        [
          0.7975,
          0.7375
        ],
        [
          0.8076,
          0.7307
        ],
        [
          0.818,
          0.7243
        ],
        [
          0.828,
          0.717
        ],
        [
          0.838,
          0.7106
        ],
        [
          0.848,
          0.7056
        ],
        [
          0.8581,
          0.7011
        ],
        [
          0.8685,
          0.6952
        ],
        [
          0.8785,
          0.6888
        ],
        [
          0.8885,
          0.6829
        ],
        [
          0.8985,
          0.6793
        ],
        [
          0.9086,
          0.6747
        ],
        [
          0.919,
          0.6674
        ],
        [
          0.929,
          0.662
        ],
        [
          0.939,
          0.6551
        ],
        [
          0.949,
          0.6492
        ],
        [
          0.9591,
          0.6442
        ],
        [
          0.9695,
          0.6365
        ],
        [
          0.9795,
          0.6288
        ],
        [
          0.9895,
          0.6219
        ],
        [
          0.9995,
          0.6156
        ],
        [
          1.0,
          0.6151
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.16839548730317383,
        "difference": -0.001015638079106026,
        "worse": 0.52,
        "kept_right": 0.16618540660184766,
        "kept_wrong": 0.002210080701326273,
        "handed_off": 0.8316045126968256
      },
      "usd_per_1000": 5.419597145935748
    },
    {
      "decision": 254,
      "frontier": 275,
      "frontier_accuracy": 0.9426751592356688,
      "in_sample": {
        "threshold": 0.9565572389883911,
        "answered": 0.1410373066424022,
        "accuracy": 0.9426751592356688
      },
      "curve": [
        [
          0.0,
          0.9427
        ],
        [
          0.01,
          0.9427
        ],
        [
          0.02,
          0.9427
        ],
        [
          0.03,
          0.9427
        ],
        [
          0.04,
          0.9427
        ],
        [
          0.05,
          0.9427
        ],
        [
          0.0601,
          0.9427
        ],
        [
          0.0705,
          0.9427
        ],
        [
          0.0805,
          0.9427
        ],
        [
          0.0905,
          0.9427
        ],
        [
          0.1005,
          0.9427
        ],
        [
          0.1106,
          0.9427
        ],
        [
          0.121,
          0.9427
        ],
        [
          0.131,
          0.9427
        ],
        [
          0.141,
          0.9427
        ],
        [
          0.151,
          0.9422
        ],
        [
          0.1611,
          0.9413
        ],
        [
          0.1715,
          0.9413
        ],
        [
          0.1815,
          0.9413
        ],
        [
          0.1915,
          0.9413
        ],
        [
          0.2015,
          0.9404
        ],
        [
          0.2116,
          0.9404
        ],
        [
          0.222,
          0.9395
        ],
        [
          0.232,
          0.9386
        ],
        [
          0.242,
          0.9386
        ],
        [
          0.252,
          0.939
        ],
        [
          0.2621,
          0.9381
        ],
        [
          0.2725,
          0.9386
        ],
        [
          0.2825,
          0.9372
        ],
        [
          0.2925,
          0.9372
        ],
        [
          0.3025,
          0.9349
        ],
        [
          0.3126,
          0.9336
        ],
        [
          0.323,
          0.9308
        ],
        [
          0.333,
          0.9299
        ],
        [
          0.343,
          0.9281
        ],
        [
          0.353,
          0.9249
        ],
        [
          0.3631,
          0.9227
        ],
        [
          0.3735,
          0.9195
        ],
        [
          0.3835,
          0.9145
        ],
        [
          0.3935,
          0.9131
        ],
        [
          0.4035,
          0.9099
        ],
        [
          0.4136,
          0.9067
        ],
        [
          0.424,
          0.9035
        ],
        [
          0.434,
          0.9013
        ],
        [
          0.444,
          0.8972
        ],
        [
          0.454,
          0.8949
        ],
        [
          0.4641,
          0.8935
        ],
        [
          0.4745,
          0.8894
        ],
        [
          0.4845,
          0.8854
        ],
        [
          0.4945,
          0.8826
        ],
        [
          0.5045,
          0.8772
        ],
        [
          0.5146,
          0.874
        ],
        [
          0.525,
          0.8708
        ],
        [
          0.535,
          0.8662
        ],
        [
          0.545,
          0.8617
        ],
        [
          0.5551,
          0.8581
        ],
        [
          0.5655,
          0.8549
        ],
        [
          0.5755,
          0.8503
        ],
        [
          0.5855,
          0.8458
        ],
        [
          0.5955,
          0.8421
        ],
        [
          0.6056,
          0.838
        ],
        [
          0.616,
          0.8326
        ],
        [
          0.626,
          0.8285
        ],
        [
          0.636,
          0.823
        ],
        [
          0.646,
          0.818
        ],
        [
          0.6561,
          0.8144
        ],
        [
          0.6665,
          0.8066
        ],
        [
          0.6765,
          0.8016
        ],
        [
          0.6865,
          0.7971
        ],
        [
          0.6965,
          0.7907
        ],
        [
          0.7066,
          0.7857
        ],
        [
          0.717,
          0.7789
        ],
        [
          0.727,
          0.7721
        ],
        [
          0.737,
          0.7648
        ],
        [
          0.747,
          0.7593
        ],
        [
          0.7571,
          0.752
        ],
        [
          0.7675,
          0.7452
        ],
        [
          0.7775,
          0.7393
        ],
        [
          0.7875,
          0.7348
        ],
        [
          0.7975,
          0.7302
        ],
        [
          0.8076,
          0.7238
        ],
        [
          0.818,
          0.7184
        ],
        [
          0.828,
          0.7111
        ],
        [
          0.838,
          0.7038
        ],
        [
          0.848,
          0.6984
        ],
        [
          0.8581,
          0.6952
        ],
        [
          0.8685,
          0.6906
        ],
        [
          0.8785,
          0.6843
        ],
        [
          0.8885,
          0.6783
        ],
        [
          0.8985,
          0.6747
        ],
        [
          0.9086,
          0.6711
        ],
        [
          0.919,
          0.6647
        ],
        [
          0.929,
          0.6592
        ],
        [
          0.939,
          0.6524
        ],
        [
          0.949,
          0.6474
        ],
        [
          0.9591,
          0.6424
        ],
        [
          0.9695,
          0.6347
        ],
        [
          0.9795,
          0.6288
        ],
        [
          0.9895,
          0.6224
        ],
        [
          0.9995,
          0.6156
        ],
        [
          1.0,
          0.6151
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.15683399197827577,
        "difference": -0.0011490269303114844,
        "worse": 0.5133333333333333,
        "kept_right": 0.15500578193275813,
        "kept_wrong": 0.0018282100455177348,
        "handed_off": 0.8431660080217239
      },
      "usd_per_1000": 0.4406478609971647
    },
    {
      "decision": 254,
      "frontier": 192,
      "frontier_accuracy": 0.9331210191082803,
      "in_sample": {
        "threshold": 0.9565572389883911,
        "answered": 0.1410373066424022,
        "accuracy": 0.9331210191082803
      },
      "curve": [
        [
          0.0,
          0.9331
        ],
        [
          0.01,
          0.9331
        ],
        [
          0.02,
          0.9331
        ],
        [
          0.03,
          0.9331
        ],
        [
          0.04,
          0.9331
        ],
        [
          0.05,
          0.9331
        ],
        [
          0.0601,
          0.9331
        ],
        [
          0.0705,
          0.9331
        ],
        [
          0.0805,
          0.9331
        ],
        [
          0.0905,
          0.9331
        ],
        [
          0.1005,
          0.9331
        ],
        [
          0.1106,
          0.9331
        ],
        [
          0.121,
          0.9331
        ],
        [
          0.131,
          0.9331
        ],
        [
          0.141,
          0.9331
        ],
        [
          0.151,
          0.9327
        ],
        [
          0.1611,
          0.9318
        ],
        [
          0.1715,
          0.9318
        ],
        [
          0.1815,
          0.9318
        ],
        [
          0.1915,
          0.9318
        ],
        [
          0.2015,
          0.9308
        ],
        [
          0.2116,
          0.9308
        ],
        [
          0.222,
          0.9308
        ],
        [
          0.232,
          0.9299
        ],
        [
          0.242,
          0.9304
        ],
        [
          0.252,
          0.9304
        ],
        [
          0.2621,
          0.9295
        ],
        [
          0.2725,
          0.929
        ],
        [
          0.2825,
          0.9272
        ],
        [
          0.2925,
          0.9268
        ],
        [
          0.3025,
          0.9249
        ],
        [
          0.3126,
          0.924
        ],
        [
          0.323,
          0.9213
        ],
        [
          0.333,
          0.9204
        ],
        [
          0.343,
          0.9186
        ],
        [
          0.353,
          0.9154
        ],
        [
          0.3631,
          0.9136
        ],
        [
          0.3735,
          0.9104
        ],
        [
          0.3835,
          0.9058
        ],
        [
          0.3935,
          0.9058
        ],
        [
          0.4035,
          0.9031
        ],
        [
          0.4136,
          0.9004
        ],
        [
          0.424,
          0.8976
        ],
        [
          0.434,
          0.8954
        ],
        [
          0.444,
          0.8917
        ],
        [
          0.454,
          0.8899
        ],
        [
          0.4641,
          0.8876
        ],
        [
          0.4745,
          0.8835
        ],
        [
          0.4845,
          0.8817
        ],
        [
          0.4945,
          0.8785
        ],
        [
          0.5045,
          0.8731
        ],
        [
          0.5146,
          0.8699
        ],
        [
          0.525,
          0.8667
        ],
        [
          0.535,
          0.8631
        ],
        [
          0.545,
          0.8585
        ],
        [
          0.5551,
          0.8549
        ],
        [
          0.5655,
          0.8517
        ],
        [
          0.5755,
          0.8476
        ],
        [
          0.5855,
          0.843
        ],
        [
          0.5955,
          0.8389
        ],
        [
          0.6056,
          0.8344
        ],
        [
          0.616,
          0.8285
        ],
        [
          0.626,
          0.8253
        ],
        [
          0.636,
          0.8198
        ],
        [
          0.646,
          0.8148
        ],
        [
          0.6561,
          0.8112
        ],
        [
          0.6665,
          0.8053
        ],
        [
          0.6765,
          0.8012
        ],
        [
          0.6865,
          0.7971
        ],
        [
          0.6965,
          0.7903
        ],
        [
          0.7066,
          0.7857
        ],
        [
          0.717,
          0.7793
        ],
        [
          0.727,
          0.7725
        ],
        [
          0.737,
          0.7657
        ],
        [
          0.747,
          0.7593
        ],
        [
          0.7571,
          0.752
        ],
        [
          0.7675,
          0.7452
        ],
        [
          0.7775,
          0.7398
        ],
        [
          0.7875,
          0.7357
        ],
        [
          0.7975,
          0.7311
        ],
        [
          0.8076,
          0.7247
        ],
        [
          0.818,
          0.7184
        ],
        [
          0.828,
          0.712
        ],
        [
          0.838,
          0.7047
        ],
        [
          0.848,
          0.7006
        ],
        [
          0.8581,
          0.697
        ],
        [
          0.8685,
          0.6911
        ],
        [
          0.8785,
          0.6847
        ],
        [
          0.8885,
          0.6793
        ],
        [
          0.8985,
          0.6752
        ],
        [
          0.9086,
          0.6715
        ],
        [
          0.919,
          0.6652
        ],
        [
          0.929,
          0.6597
        ],
        [
          0.939,
          0.6529
        ],
        [
          0.949,
          0.6483
        ],
        [
          0.9591,
          0.6429
        ],
        [
          0.9695,
          0.6356
        ],
        [
          0.9795,
          0.6278
        ],
        [
          0.9895,
          0.6215
        ],
        [
          0.9995,
          0.6156
        ],
        [
          1.0,
          0.6151
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.16616130113022098,
        "difference": -0.0015400575847960745,
        "worse": 0.5433333333333333,
        "kept_right": 0.16369347890971794,
        "kept_wrong": 0.0024678222205031107,
        "handed_off": 0.8338386988697787
      },
      "usd_per_1000": 9.687332676274448
    },
    {
      "decision": 254,
      "frontier": 123,
      "frontier_accuracy": 0.9326660600545951,
      "in_sample": {
        "threshold": 0.9712557594616492,
        "answered": 0.10145586897179254,
        "accuracy": 0.9326660600545951
      },
      "curve": [
        [
          0.0,
          0.9327
        ],
        [
          0.01,
          0.9327
        ],
        [
          0.02,
          0.9327
        ],
        [
          0.03,
          0.9327
        ],
        [
          0.04,
          0.9327
        ],
        [
          0.05,
          0.9327
        ],
        [
          0.0601,
          0.9327
        ],
        [
          0.0705,
          0.9327
        ],
        [
          0.0805,
          0.9327
        ],
        [
          0.0905,
          0.9327
        ],
        [
          0.1005,
          0.9327
        ],
        [
          0.1106,
          0.9322
        ],
        [
          0.121,
          0.9322
        ],
        [
          0.131,
          0.9322
        ],
        [
          0.141,
          0.9322
        ],
        [
          0.151,
          0.9318
        ],
        [
          0.1611,
          0.9308
        ],
        [
          0.1715,
          0.9313
        ],
        [
          0.1815,
          0.9313
        ],
        [
          0.1915,
          0.9308
        ],
        [
          0.2015,
          0.9299
        ],
        [
          0.2116,
          0.9299
        ],
        [
          0.222,
          0.929
        ],
        [
          0.232,
          0.9286
        ],
        [
          0.242,
          0.9286
        ],
        [
          0.252,
          0.9281
        ],
        [
          0.2621,
          0.9272
        ],
        [
          0.2725,
          0.9272
        ],
        [
          0.2825,
          0.9254
        ],
        [
          0.2925,
          0.9249
        ],
        [
          0.3025,
          0.9227
        ],
        [
          0.3126,
          0.9227
        ],
        [
          0.323,
          0.9199
        ],
        [
          0.333,
          0.9195
        ],
        [
          0.343,
          0.9181
        ],
        [
          0.353,
          0.9158
        ],
        [
          0.3631,
          0.9136
        ],
        [
          0.3735,
          0.9108
        ],
        [
          0.3835,
          0.9063
        ],
        [
          0.3935,
          0.9054
        ],
        [
          0.4035,
          0.9026
        ],
        [
          0.4136,
          0.8995
        ],
        [
          0.424,
          0.8976
        ],
        [
          0.434,
          0.8949
        ],
        [
          0.444,
          0.8908
        ],
        [
          0.454,
          0.8881
        ],
        [
          0.4641,
          0.8858
        ],
        [
          0.4745,
          0.8826
        ],
        [
          0.4845,
          0.8794
        ],
        [
          0.4945,
          0.8758
        ],
        [
          0.5045,
          0.8708
        ],
        [
          0.5146,
          0.8681
        ],
        [
          0.525,
          0.8653
        ],
        [
          0.535,
          0.8608
        ],
        [
          0.545,
          0.8553
        ],
        [
          0.5551,
          0.8512
        ],
        [
          0.5655,
          0.8476
        ],
        [
          0.5755,
          0.843
        ],
        [
          0.5855,
          0.838
        ],
        [
          0.5955,
          0.8348
        ],
        [
          0.6056,
          0.8298
        ],
        [
          0.616,
          0.8248
        ],
        [
          0.626,
          0.8212
        ],
        [
          0.636,
          0.8157
        ],
        [
          0.646,
          0.8107
        ],
        [
          0.6561,
          0.8076
        ],
        [
          0.6665,
          0.8012
        ],
        [
          0.6765,
          0.7957
        ],
        [
          0.6865,
          0.7903
        ],
        [
          0.6965,
          0.7848
        ],
        [
          0.7066,
          0.7793
        ],
        [
          0.717,
          0.7716
        ],
        [
          0.727,
          0.7648
        ],
        [
          0.737,
          0.758
        ],
        [
          0.747,
          0.753
        ],
        [
          0.7571,
          0.7461
        ],
        [
          0.7675,
          0.7402
        ],
        [
          0.7775,
          0.7352
        ],
        [
          0.7875,
          0.7311
        ],
        [
          0.7975,
          0.7266
        ],
        [
          0.8076,
          0.7197
        ],
        [
          0.818,
          0.7147
        ],
        [
          0.828,
          0.7079
        ],
        [
          0.838,
          0.7011
        ],
        [
          0.848,
          0.6961
        ],
        [
          0.8581,
          0.6915
        ],
        [
          0.8685,
          0.6856
        ],
        [
          0.8785,
          0.6802
        ],
        [
          0.8885,
          0.6738
        ],
        [
          0.8985,
          0.6697
        ],
        [
          0.9086,
          0.667
        ],
        [
          0.919,
          0.6611
        ],
        [
          0.929,
          0.6561
        ],
        [
          0.939,
          0.6497
        ],
        [
          0.949,
          0.6447
        ],
        [
          0.9591,
          0.641
        ],
        [
          0.9695,
          0.6342
        ],
        [
          0.9795,
          0.6269
        ],
        [
          0.9895,
          0.6215
        ],
        [
          0.9995,
          0.6156
        ],
        [
          1.0,
          0.6151
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.13891511701125794,
        "difference": -0.0013673328280979181,
        "worse": 0.5933333333333334,
        "kept_right": 0.13747803475397885,
        "kept_wrong": 0.0014370822572790246,
        "handed_off": 0.8610848829887418
      },
      "usd_per_1000": 0.23810139579626774
    },
    {
      "decision": 254,
      "frontier": 274,
      "frontier_accuracy": 0.9085532302092811,
      "in_sample": {
        "threshold": 0.9565572389883911,
        "answered": 0.1410373066424022,
        "accuracy": 0.9085532302092811
      },
      "curve": [
        [
          0.0,
          0.9086
        ],
        [
          0.01,
          0.9086
        ],
        [
          0.02,
          0.9086
        ],
        [
          0.03,
          0.9086
        ],
        [
          0.04,
          0.9086
        ],
        [
          0.05,
          0.9086
        ],
        [
          0.0601,
          0.9086
        ],
        [
          0.0705,
          0.9086
        ],
        [
          0.0805,
          0.9086
        ],
        [
          0.0905,
          0.9086
        ],
        [
          0.1005,
          0.9086
        ],
        [
          0.1106,
          0.9086
        ],
        [
          0.121,
          0.9086
        ],
        [
          0.131,
          0.9086
        ],
        [
          0.141,
          0.9086
        ],
        [
          0.151,
          0.9081
        ],
        [
          0.1611,
          0.9072
        ],
        [
          0.1715,
          0.9072
        ],
        [
          0.1815,
          0.9072
        ],
        [
          0.1915,
          0.9076
        ],
        [
          0.2015,
          0.9072
        ],
        [
          0.2116,
          0.9072
        ],
        [
          0.222,
          0.9063
        ],
        [
          0.232,
          0.9054
        ],
        [
          0.242,
          0.9054
        ],
        [
          0.252,
          0.9054
        ],
        [
          0.2621,
          0.904
        ],
        [
          0.2725,
          0.9045
        ],
        [
          0.2825,
          0.9031
        ],
        [
          0.2925,
          0.9035
        ],
        [
          0.3025,
          0.9017
        ],
        [
          0.3126,
          0.9004
        ],
        [
          0.323,
          0.8985
        ],
        [
          0.333,
          0.8976
        ],
        [
          0.343,
          0.8958
        ],
        [
          0.353,
          0.8931
        ],
        [
          0.3631,
          0.8917
        ],
        [
          0.3735,
          0.8885
        ],
        [
          0.3835,
          0.884
        ],
        [
          0.3935,
          0.8826
        ],
        [
          0.4035,
          0.8799
        ],
        [
          0.4136,
          0.8767
        ],
        [
          0.424,
          0.874
        ],
        [
          0.434,
          0.8717
        ],
        [
          0.444,
          0.8681
        ],
        [
          0.454,
          0.8662
        ],
        [
          0.4641,
          0.8644
        ],
        [
          0.4745,
          0.8612
        ],
        [
          0.4845,
          0.8581
        ],
        [
          0.4945,
          0.8553
        ],
        [
          0.5045,
          0.8499
        ],
        [
          0.5146,
          0.8476
        ],
        [
          0.525,
          0.8449
        ],
        [
          0.535,
          0.8403
        ],
        [
          0.545,
          0.8358
        ],
        [
          0.5551,
          0.8317
        ],
        [
          0.5655,
          0.8285
        ],
        [
          0.5755,
          0.8239
        ],
        [
          0.5855,
          0.8194
        ],
        [
          0.5955,
          0.8162
        ],
        [
          0.6056,
          0.8121
        ],
        [
          0.616,
          0.8057
        ],
        [
          0.626,
          0.803
        ],
        [
          0.636,
          0.7975
        ],
        [
          0.646,
          0.7925
        ],
        [
          0.6561,
          0.7898
        ],
        [
          0.6665,
          0.7825
        ],
        [
          0.6765,
          0.7775
        ],
        [
          0.6865,
          0.773
        ],
        [
          0.6965,
          0.7671
        ],
        [
          0.7066,
          0.7621
        ],
        [
          0.717,
          0.7557
        ],
        [
          0.727,
          0.7489
        ],
        [
          0.737,
          0.742
        ],
        [
          0.747,
          0.737
        ],
        [
          0.7571,
          0.7302
        ],
        [
          0.7675,
          0.7238
        ],
        [
          0.7775,
          0.7179
        ],
        [
          0.7875,
          0.7138
        ],
        [
          0.7975,
          0.7097
        ],
        [
          0.8076,
          0.7038
        ],
        [
          0.818,
          0.6993
        ],
        [
          0.828,
          0.6934
        ],
        [
          0.838,
          0.687
        ],
        [
          0.848,
          0.682
        ],
        [
          0.8581,
          0.6788
        ],
        [
          0.8685,
          0.6742
        ],
        [
          0.8785,
          0.6697
        ],
        [
          0.8885,
          0.6647
        ],
        [
          0.8985,
          0.6629
        ],
        [
          0.9086,
          0.6601
        ],
        [
          0.919,
          0.6542
        ],
        [
          0.929,
          0.6501
        ],
        [
          0.939,
          0.6438
        ],
        [
          0.949,
          0.6401
        ],
        [
          0.9591,
          0.6356
        ],
        [
          0.9695,
          0.6292
        ],
        [
          0.9795,
          0.6233
        ],
        [
          0.9895,
          0.6187
        ],
        [
          0.9995,
          0.6151
        ],
        [
          1.0,
          0.6151
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.17003764016338158,
        "difference": -0.0014613802955888275,
        "worse": 0.5733333333333334,
        "kept_right": 0.16751505129923666,
        "kept_wrong": 0.0025225888641449068,
        "handed_off": 0.8299623598366184
      },
      "usd_per_1000": 0.3605956939866286
    },
    {
      "decision": 254,
      "frontier": 47,
      "frontier_accuracy": 0.9044585987261147,
      "in_sample": {
        "threshold": 0.9565572389883911,
        "answered": 0.1410373066424022,
        "accuracy": 0.9044585987261147
      },
      "curve": [
        [
          0.0,
          0.9045
        ],
        [
          0.01,
          0.9045
        ],
        [
          0.02,
          0.9045
        ],
        [
          0.03,
          0.9045
        ],
        [
          0.04,
          0.9045
        ],
        [
          0.05,
          0.9045
        ],
        [
          0.0601,
          0.9045
        ],
        [
          0.0705,
          0.9045
        ],
        [
          0.0805,
          0.9045
        ],
        [
          0.0905,
          0.9045
        ],
        [
          0.1005,
          0.9045
        ],
        [
          0.1106,
          0.9045
        ],
        [
          0.121,
          0.9045
        ],
        [
          0.131,
          0.9045
        ],
        [
          0.141,
          0.9045
        ],
        [
          0.151,
          0.904
        ],
        [
          0.1611,
          0.9035
        ],
        [
          0.1715,
          0.904
        ],
        [
          0.1815,
          0.904
        ],
        [
          0.1915,
          0.904
        ],
        [
          0.2015,
          0.9031
        ],
        [
          0.2116,
          0.9031
        ],
        [
          0.222,
          0.9026
        ],
        [
          0.232,
          0.9022
        ],
        [
          0.242,
          0.9022
        ],
        [
          0.252,
          0.9022
        ],
        [
          0.2621,
          0.9017
        ],
        [
          0.2725,
          0.9008
        ],
        [
          0.2825,
          0.899
        ],
        [
          0.2925,
          0.8995
        ],
        [
          0.3025,
          0.8972
        ],
        [
          0.3126,
          0.8958
        ],
        [
          0.323,
          0.8931
        ],
        [
          0.333,
          0.8926
        ],
        [
          0.343,
          0.8908
        ],
        [
          0.353,
          0.8881
        ],
        [
          0.3631,
          0.8863
        ],
        [
          0.3735,
          0.884
        ],
        [
          0.3835,
          0.879
        ],
        [
          0.3935,
          0.8785
        ],
        [
          0.4035,
          0.8753
        ],
        [
          0.4136,
          0.8726
        ],
        [
          0.424,
          0.8703
        ],
        [
          0.434,
          0.8676
        ],
        [
          0.444,
          0.8649
        ],
        [
          0.454,
          0.8621
        ],
        [
          0.4641,
          0.8603
        ],
        [
          0.4745,
          0.8562
        ],
        [
          0.4845,
          0.8526
        ],
        [
          0.4945,
          0.8494
        ],
        [
          0.5045,
          0.8449
        ],
        [
          0.5146,
          0.8421
        ],
        [
          0.525,
          0.8394
        ],
        [
          0.535,
          0.8362
        ],
        [
          0.545,
          0.8321
        ],
        [
          0.5551,
          0.8294
        ],
        [
          0.5655,
          0.8262
        ],
        [
          0.5755,
          0.8226
        ],
        [
          0.5855,
          0.8176
        ],
        [
          0.5955,
          0.8148
        ],
        [
          0.6056,
          0.8107
        ],
        [
          0.616,
          0.8062
        ],
        [
          0.626,
          0.8021
        ],
        [
          0.636,
          0.7975
        ],
        [
          0.646,
          0.7921
        ],
        [
          0.6561,
          0.7898
        ],
        [
          0.6665,
          0.7839
        ],
        [
          0.6765,
          0.7789
        ],
        [
          0.6865,
          0.7743
        ],
        [
          0.6965,
          0.7698
        ],
        [
          0.7066,
          0.7662
        ],
        [
          0.717,
          0.7607
        ],
        [
          0.727,
          0.7557
        ],
        [
          0.737,
          0.7489
        ],
        [
          0.747,
          0.7429
        ],
        [
          0.7571,
          0.737
        ],
        [
          0.7675,
          0.7307
        ],
        [
          0.7775,
          0.7252
        ],
        [
          0.7875,
          0.7216
        ],
        [
          0.7975,
          0.7175
        ],
        [
          0.8076,
          0.7111
        ],
        [
          0.818,
          0.7061
        ],
        [
          0.828,
          0.7002
        ],
        [
          0.838,
          0.6943
        ],
        [
          0.848,
          0.6897
        ],
        [
          0.8581,
          0.6856
        ],
        [
          0.8685,
          0.6797
        ],
        [
          0.8785,
          0.6747
        ],
        [
          0.8885,
          0.6697
        ],
        [
          0.8985,
          0.667
        ],
        [
          0.9086,
          0.6629
        ],
        [
          0.919,
          0.6579
        ],
        [
          0.929,
          0.6533
        ],
        [
          0.939,
          0.6479
        ],
        [
          0.949,
          0.6429
        ],
        [
          0.9591,
          0.6392
        ],
        [
          0.9695,
          0.6347
        ],
        [
          0.9795,
          0.6274
        ],
        [
          0.9895,
          0.6219
        ],
        [
          0.9995,
          0.6156
        ],
        [
          1.0,
          0.6151
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.18731760328748223,
        "difference": -0.0017645457024150371,
        "worse": 0.6533333333333333,
        "kept_right": 0.18377937130649347,
        "kept_wrong": 0.003538231980988858,
        "handed_off": 0.8126823967125175
      },
      "usd_per_1000": 4.013828250909134
    },
    {
      "decision": 254,
      "frontier": 193,
      "frontier_accuracy": 0.89171974522293,
      "in_sample": {
        "threshold": 0.8275601621380464,
        "answered": 0.2770700636942675,
        "accuracy": 0.89171974522293
      },
      "curve": [
        [
          0.0,
          0.8917
        ],
        [
          0.01,
          0.8917
        ],
        [
          0.02,
          0.8917
        ],
        [
          0.03,
          0.8917
        ],
        [
          0.04,
          0.8917
        ],
        [
          0.05,
          0.8917
        ],
        [
          0.0601,
          0.8917
        ],
        [
          0.0705,
          0.8917
        ],
        [
          0.0805,
          0.8917
        ],
        [
          0.0905,
          0.8917
        ],
        [
          0.1005,
          0.8917
        ],
        [
          0.1106,
          0.8917
        ],
        [
          0.121,
          0.8922
        ],
        [
          0.131,
          0.8926
        ],
        [
          0.141,
          0.8926
        ],
        [
          0.151,
          0.8926
        ],
        [
          0.1611,
          0.8926
        ],
        [
          0.1715,
          0.8926
        ],
        [
          0.1815,
          0.8926
        ],
        [
          0.1915,
          0.8926
        ],
        [
          0.2015,
          0.8922
        ],
        [
          0.2116,
          0.8922
        ],
        [
          0.222,
          0.8922
        ],
        [
          0.232,
          0.8913
        ],
        [
          0.242,
          0.8913
        ],
        [
          0.252,
          0.8917
        ],
        [
          0.2621,
          0.8922
        ],
        [
          0.2725,
          0.8917
        ],
        [
          0.2825,
          0.8904
        ],
        [
          0.2925,
          0.8904
        ],
        [
          0.3025,
          0.8885
        ],
        [
          0.3126,
          0.8872
        ],
        [
          0.323,
          0.8854
        ],
        [
          0.333,
          0.8854
        ],
        [
          0.343,
          0.884
        ],
        [
          0.353,
          0.8813
        ],
        [
          0.3631,
          0.8794
        ],
        [
          0.3735,
          0.8767
        ],
        [
          0.3835,
          0.8717
        ],
        [
          0.3935,
          0.8717
        ],
        [
          0.4035,
          0.8685
        ],
        [
          0.4136,
          0.8653
        ],
        [
          0.424,
          0.8626
        ],
        [
          0.434,
          0.8599
        ],
        [
          0.444,
          0.8558
        ],
        [
          0.454,
          0.8544
        ],
        [
          0.4641,
          0.853
        ],
        [
          0.4745,
          0.8494
        ],
        [
          0.4845,
          0.8476
        ],
        [
          0.4945,
          0.8444
        ],
        [
          0.5045,
          0.8394
        ],
        [
          0.5146,
          0.8376
        ],
        [
          0.525,
          0.8348
        ],
        [
          0.535,
          0.8326
        ],
        [
          0.545,
          0.8276
        ],
        [
          0.5551,
          0.8239
        ],
        [
          0.5655,
          0.8207
        ],
        [
          0.5755,
          0.8171
        ],
        [
          0.5855,
          0.813
        ],
        [
          0.5955,
          0.8098
        ],
        [
          0.6056,
          0.8062
        ],
        [
          0.616,
          0.8016
        ],
        [
          0.626,
          0.7989
        ],
        [
          0.636,
          0.7948
        ],
        [
          0.646,
          0.7898
        ],
        [
          0.6561,
          0.7875
        ],
        [
          0.6665,
          0.7812
        ],
        [
          0.6765,
          0.7775
        ],
        [
          0.6865,
          0.7739
        ],
        [
          0.6965,
          0.768
        ],
        [
          0.7066,
          0.7639
        ],
        [
          0.717,
          0.758
        ],
        [
          0.727,
          0.753
        ],
        [
          0.737,
          0.7457
        ],
        [
          0.747,
          0.7393
        ],
        [
          0.7571,
          0.7334
        ],
        [
          0.7675,
          0.7266
        ],
        [
          0.7775,
          0.7216
        ],
        [
          0.7875,
          0.717
        ],
        [
          0.7975,
          0.7125
        ],
        [
          0.8076,
          0.7066
        ],
        [
          0.818,
          0.7015
        ],
        [
          0.828,
          0.6947
        ],
        [
          0.838,
          0.6888
        ],
        [
          0.848,
          0.6838
        ],
        [
          0.8581,
          0.6806
        ],
        [
          0.8685,
          0.6761
        ],
        [
          0.8785,
          0.672
        ],
        [
          0.8885,
          0.6674
        ],
        [
          0.8985,
          0.6647
        ],
        [
          0.9086,
          0.6624
        ],
        [
          0.919,
          0.6561
        ],
        [
          0.929,
          0.6506
        ],
        [
          0.939,
          0.6451
        ],
        [
          0.949,
          0.6419
        ],
        [
          0.9591,
          0.6374
        ],
        [
          0.9695,
          0.6319
        ],
        [
          0.9795,
          0.6265
        ],
        [
          0.9895,
          0.621
        ],
        [
          0.9995,
          0.6151
        ],
        [
          1.0,
          0.6151
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.2776988936144221,
        "difference": -0.0022436532101598994,
        "worse": 0.49666666666666665,
        "kept_right": 0.2660963606588657,
        "kept_wrong": 0.011602532955556537,
        "handed_off": 0.7223011063855778
      },
      "usd_per_1000": 5.602920647545214
    },
    {
      "decision": 254,
      "frontier": 55,
      "frontier_accuracy": 0.8503184713375797,
      "in_sample": {
        "threshold": 0.7170641991290364,
        "answered": 0.36032757051865333,
        "accuracy": 0.8503184713375797
      },
      "curve": [
        [
          0.0,
          0.8503
        ],
        [
          0.01,
          0.8503
        ],
        [
          0.02,
          0.8503
        ],
        [
          0.03,
          0.8503
        ],
        [
          0.04,
          0.8503
        ],
        [
          0.05,
          0.8503
        ],
        [
          0.0601,
          0.8508
        ],
        [
          0.0705,
          0.8508
        ],
        [
          0.0805,
          0.8508
        ],
        [
          0.0905,
          0.8508
        ],
        [
          0.1005,
          0.8508
        ],
        [
          0.1106,
          0.8508
        ],
        [
          0.121,
          0.8512
        ],
        [
          0.131,
          0.8517
        ],
        [
          0.141,
          0.8526
        ],
        [
          0.151,
          0.8535
        ],
        [
          0.1611,
          0.854
        ],
        [
          0.1715,
          0.854
        ],
        [
          0.1815,
          0.8549
        ],
        [
          0.1915,
          0.8549
        ],
        [
          0.2015,
          0.8544
        ],
        [
          0.2116,
          0.8553
        ],
        [
          0.222,
          0.8558
        ],
        [
          0.232,
          0.8558
        ],
        [
          0.242,
          0.8562
        ],
        [
          0.252,
          0.8562
        ],
        [
          0.2621,
          0.8562
        ],
        [
          0.2725,
          0.8562
        ],
        [
          0.2825,
          0.8553
        ],
        [
          0.2925,
          0.8558
        ],
        [
          0.3025,
          0.8544
        ],
        [
          0.3126,
          0.8535
        ],
        [
          0.323,
          0.8521
        ],
        [
          0.333,
          0.8521
        ],
        [
          0.343,
          0.8517
        ],
        [
          0.353,
          0.8499
        ],
        [
          0.3631,
          0.8499
        ],
        [
          0.3735,
          0.8485
        ],
        [
          0.3835,
          0.8439
        ],
        [
          0.3935,
          0.8439
        ],
        [
          0.4035,
          0.8417
        ],
        [
          0.4136,
          0.8389
        ],
        [
          0.424,
          0.8371
        ],
        [
          0.434,
          0.8358
        ],
        [
          0.444,
          0.8321
        ],
        [
          0.454,
          0.8303
        ],
        [
          0.4641,
          0.8289
        ],
        [
          0.4745,
          0.8253
        ],
        [
          0.4845,
          0.8235
        ],
        [
          0.4945,
          0.8207
        ],
        [
          0.5045,
          0.8171
        ],
        [
          0.5146,
          0.8157
        ],
        [
          0.525,
          0.8144
        ],
        [
          0.535,
          0.813
        ],
        [
          0.545,
          0.8085
        ],
        [
          0.5551,
          0.8048
        ],
        [
          0.5655,
          0.8021
        ],
        [
          0.5755,
          0.7989
        ],
        [
          0.5855,
          0.7944
        ],
        [
          0.5955,
          0.7916
        ],
        [
          0.6056,
          0.788
        ],
        [
          0.616,
          0.7848
        ],
        [
          0.626,
          0.7821
        ],
        [
          0.636,
          0.7793
        ],
        [
          0.646,
          0.7757
        ],
        [
          0.6561,
          0.7743
        ],
        [
          0.6665,
          0.768
        ],
        [
          0.6765,
          0.7643
        ],
        [
          0.6865,
          0.7602
        ],
        [
          0.6965,
          0.7543
        ],
        [
          0.7066,
          0.7511
        ],
        [
          0.717,
          0.7466
        ],
        [
          0.727,
          0.742
        ],
        [
          0.737,
          0.7361
        ],
        [
          0.747,
          0.7311
        ],
        [
          0.7571,
          0.7257
        ],
        [
          0.7675,
          0.7184
        ],
        [
          0.7775,
          0.7147
        ],
        [
          0.7875,
          0.7106
        ],
        [
          0.7975,
          0.7066
        ],
        [
          0.8076,
          0.7006
        ],
        [
          0.818,
          0.6961
        ],
        [
          0.828,
          0.6897
        ],
        [
          0.838,
          0.6847
        ],
        [
          0.848,
          0.6793
        ],
        [
          0.8581,
          0.6752
        ],
        [
          0.8685,
          0.6711
        ],
        [
          0.8785,
          0.6679
        ],
        [
          0.8885,
          0.6638
        ],
        [
          0.8985,
          0.6615
        ],
        [
          0.9086,
          0.6588
        ],
        [
          0.919,
          0.6533
        ],
        [
          0.929,
          0.6488
        ],
        [
          0.939,
          0.6438
        ],
        [
          0.949,
          0.6406
        ],
        [
          0.9591,
          0.6374
        ],
        [
          0.9695,
          0.6319
        ],
        [
          0.9795,
          0.626
        ],
        [
          0.9895,
          0.621
        ],
        [
          0.9995,
          0.6151
        ],
        [
          1.0,
          0.6151
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.35499698160883814,
        "difference": -0.001440233439716853,
        "worse": 0.45666666666666667,
        "kept_right": 0.32641147113746993,
        "kept_wrong": 0.028585510471368304,
        "handed_off": 0.6450030183911623
      },
      "usd_per_1000": 2.363884040345653
    },
    {
      "decision": 150,
      "frontier": 140,
      "frontier_accuracy": 0.9808917197452229,
      "in_sample": {
        "threshold": 0.91,
        "answered": 0.17925386715195632,
        "accuracy": 0.9808917197452229
      },
      "curve": [
        [
          0.0,
          0.9809
        ],
        [
          0.0191,
          0.9809
        ],
        [
          0.056,
          0.9809
        ],
        [
          0.096,
          0.9804
        ],
        [
          0.1319,
          0.9809
        ],
        [
          0.1565,
          0.9809
        ],
        [
          0.1793,
          0.9809
        ],
        [
          0.1965,
          0.9804
        ],
        [
          0.2111,
          0.9804
        ],
        [
          0.2261,
          0.9795
        ],
        [
          0.2466,
          0.9773
        ],
        [
          0.2575,
          0.9759
        ],
        [
          0.2784,
          0.9727
        ],
        [
          0.2934,
          0.9682
        ],
        [
          0.3057,
          0.9677
        ],
        [
          0.3198,
          0.9645
        ],
        [
          0.3298,
          0.9618
        ],
        [
          0.3435,
          0.9581
        ],
        [
          0.3549,
          0.954
        ],
        [
          0.3703,
          0.9495
        ],
        [
          0.3844,
          0.9454
        ],
        [
          0.4008,
          0.9395
        ],
        [
          0.4158,
          0.9322
        ],
        [
          0.4318,
          0.9249
        ],
        [
          0.444,
          0.9208
        ],
        [
          0.4604,
          0.9131
        ],
        [
          0.4777,
          0.9063
        ],
        [
          0.4891,
          0.9022
        ],
        [
          0.5127,
          0.8926
        ],
        [
          0.5232,
          0.8881
        ],
        [
          0.5359,
          0.8831
        ],
        [
          0.5605,
          0.8744
        ],
        [
          0.5769,
          0.8676
        ],
        [
          0.5919,
          0.8617
        ],
        [
          0.6056,
          0.8544
        ],
        [
          0.6197,
          0.8471
        ],
        [
          0.6365,
          0.8399
        ],
        [
          0.6501,
          0.8312
        ],
        [
          0.667,
          0.8203
        ],
        [
          0.6838,
          0.813
        ],
        [
          0.6993,
          0.8039
        ],
        [
          0.7161,
          0.7921
        ],
        [
          0.7352,
          0.7798
        ],
        [
          0.752,
          0.773
        ],
        [
          0.7702,
          0.7607
        ],
        [
          0.7862,
          0.7534
        ],
        [
          0.8021,
          0.7429
        ],
        [
          0.8235,
          0.7275
        ],
        [
          0.8399,
          0.7157
        ],
        [
          0.8576,
          0.7061
        ],
        [
          0.8726,
          0.6961
        ],
        [
          0.889,
          0.6865
        ],
        [
          0.9004,
          0.6783
        ],
        [
          0.9145,
          0.6697
        ],
        [
          0.9295,
          0.657
        ],
        [
          0.9472,
          0.6429
        ],
        [
          0.9604,
          0.6338
        ],
        [
          0.9722,
          0.6251
        ],
        [
          0.9873,
          0.6137
        ],
        [
          0.9986,
          0.606
        ],
        [
          1.0,
          0.6046
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.17449170930623278,
        "difference": -0.0011549849428810293,
        "worse": 0.53,
        "kept_right": 0.1718028212801597,
        "kept_wrong": 0.0026888880260732893,
        "handed_off": 0.8255082906937666
      },
      "usd_per_1000": 2.012266240358793
    },
    {
      "decision": 150,
      "frontier": 277,
      "frontier_accuracy": 0.9795268425841674,
      "in_sample": {
        "threshold": 0.92,
        "answered": 0.1565059144676979,
        "accuracy": 0.9795268425841674
      },
      "curve": [
        [
          0.0,
          0.9795
        ],
        [
          0.0191,
          0.9795
        ],
        [
          0.056,
          0.9795
        ],
        [
          0.096,
          0.9791
        ],
        [
          0.1319,
          0.9795
        ],
        [
          0.1565,
          0.9795
        ],
        [
          0.1793,
          0.9791
        ],
        [
          0.1965,
          0.9782
        ],
        [
          0.2111,
          0.9782
        ],
        [
          0.2261,
          0.9773
        ],
        [
          0.2466,
          0.975
        ],
        [
          0.2575,
          0.9741
        ],
        [
          0.2784,
          0.9709
        ],
        [
          0.2934,
          0.9668
        ],
        [
          0.3057,
          0.9663
        ],
        [
          0.3198,
          0.9631
        ],
        [
          0.3298,
          0.9604
        ],
        [
          0.3435,
          0.9572
        ],
        [
          0.3549,
          0.9531
        ],
        [
          0.3703,
          0.9486
        ],
        [
          0.3844,
          0.9445
        ],
        [
          0.4008,
          0.9386
        ],
        [
          0.4158,
          0.9304
        ],
        [
          0.4318,
          0.9231
        ],
        [
          0.444,
          0.9195
        ],
        [
          0.4604,
          0.9113
        ],
        [
          0.4777,
          0.904
        ],
        [
          0.4891,
          0.8995
        ],
        [
          0.5127,
          0.8904
        ],
        [
          0.5232,
          0.8858
        ],
        [
          0.5359,
          0.8813
        ],
        [
          0.5605,
          0.8726
        ],
        [
          0.5769,
          0.8658
        ],
        [
          0.5919,
          0.8599
        ],
        [
          0.6056,
          0.8526
        ],
        [
          0.6197,
          0.8453
        ],
        [
          0.6365,
          0.838
        ],
        [
          0.6501,
          0.8289
        ],
        [
          0.667,
          0.818
        ],
        [
          0.6838,
          0.8107
        ],
        [
          0.6993,
          0.8021
        ],
        [
          0.7161,
          0.7907
        ],
        [
          0.7352,
          0.7789
        ],
        [
          0.752,
          0.7721
        ],
        [
          0.7702,
          0.7598
        ],
        [
          0.7862,
          0.7525
        ],
        [
          0.8021,
          0.7429
        ],
        [
          0.8235,
          0.7275
        ],
        [
          0.8399,
          0.7161
        ],
        [
          0.8576,
          0.7061
        ],
        [
          0.8726,
          0.6961
        ],
        [
          0.889,
          0.6865
        ],
        [
          0.9004,
          0.6788
        ],
        [
          0.9145,
          0.6702
        ],
        [
          0.9295,
          0.6574
        ],
        [
          0.9472,
          0.6433
        ],
        [
          0.9604,
          0.6342
        ],
        [
          0.9722,
          0.6256
        ],
        [
          0.9873,
          0.6142
        ],
        [
          0.9986,
          0.606
        ],
        [
          1.0,
          0.6046
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.15910766758317016,
        "difference": -0.0013155994056847035,
        "worse": 0.5166666666666667,
        "kept_right": 0.15686442758202498,
        "kept_wrong": 0.002243240001145108,
        "handed_off": 0.8408923324168301
      },
      "usd_per_1000": 5.115755257548267
    },
    {
      "decision": 150,
      "frontier": 276,
      "frontier_accuracy": 0.9777070063694268,
      "in_sample": {
        "threshold": 0.92,
        "answered": 0.1565059144676979,
        "accuracy": 0.9777070063694268
      },
      "curve": [
        [
          0.0,
          0.9777
        ],
        [
          0.0191,
          0.9777
        ],
        [
          0.056,
          0.9777
        ],
        [
          0.096,
          0.9773
        ],
        [
          0.1319,
          0.9777
        ],
        [
          0.1565,
          0.9777
        ],
        [
          0.1793,
          0.9773
        ],
        [
          0.1965,
          0.9763
        ],
        [
          0.2111,
          0.9763
        ],
        [
          0.2261,
          0.9754
        ],
        [
          0.2466,
          0.9732
        ],
        [
          0.2575,
          0.9722
        ],
        [
          0.2784,
          0.9691
        ],
        [
          0.2934,
          0.9654
        ],
        [
          0.3057,
          0.965
        ],
        [
          0.3198,
          0.9618
        ],
        [
          0.3298,
          0.9595
        ],
        [
          0.3435,
          0.9563
        ],
        [
          0.3549,
          0.9522
        ],
        [
          0.3703,
          0.9477
        ],
        [
          0.3844,
          0.9436
        ],
        [
          0.4008,
          0.9372
        ],
        [
          0.4158,
          0.929
        ],
        [
          0.4318,
          0.9217
        ],
        [
          0.444,
          0.9186
        ],
        [
          0.4604,
          0.9104
        ],
        [
          0.4777,
          0.9031
        ],
        [
          0.4891,
          0.8985
        ],
        [
          0.5127,
          0.8894
        ],
        [
          0.5232,
          0.8849
        ],
        [
          0.5359,
          0.8808
        ],
        [
          0.5605,
          0.8726
        ],
        [
          0.5769,
          0.8658
        ],
        [
          0.5919,
          0.8599
        ],
        [
          0.6056,
          0.8526
        ],
        [
          0.6197,
          0.8453
        ],
        [
          0.6365,
          0.838
        ],
        [
          0.6501,
          0.8289
        ],
        [
          0.667,
          0.818
        ],
        [
          0.6838,
          0.8112
        ],
        [
          0.6993,
          0.8021
        ],
        [
          0.7161,
          0.7907
        ],
        [
          0.7352,
          0.7789
        ],
        [
          0.752,
          0.7725
        ],
        [
          0.7702,
          0.7602
        ],
        [
          0.7862,
          0.753
        ],
        [
          0.8021,
          0.7439
        ],
        [
          0.8235,
          0.7284
        ],
        [
          0.8399,
          0.7166
        ],
        [
          0.8576,
          0.7066
        ],
        [
          0.8726,
          0.6965
        ],
        [
          0.889,
          0.687
        ],
        [
          0.9004,
          0.6793
        ],
        [
          0.9145,
          0.6706
        ],
        [
          0.9295,
          0.6579
        ],
        [
          0.9472,
          0.6438
        ],
        [
          0.9604,
          0.6347
        ],
        [
          0.9722,
          0.6256
        ],
        [
          0.9873,
          0.6142
        ],
        [
          0.9986,
          0.606
        ],
        [
          1.0,
          0.6046
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.15917730124953405,
        "difference": -0.0013307371592420757,
        "worse": 0.5166666666666667,
        "kept_right": 0.15691892349483152,
        "kept_wrong": 0.00225837775470248,
        "handed_off": 0.8408226987504662
      },
      "usd_per_1000": 1.3792508431640096
    },
    {
      "decision": 150,
      "frontier": 48,
      "frontier_accuracy": 0.9758871701546861,
      "in_sample": {
        "threshold": 0.92,
        "answered": 0.1565059144676979,
        "accuracy": 0.9758871701546861
      },
      "curve": [
        [
          0.0,
          0.9759
        ],
        [
          0.0191,
          0.9759
        ],
        [
          0.056,
          0.9759
        ],
        [
          0.096,
          0.9754
        ],
        [
          0.1319,
          0.9759
        ],
        [
          0.1565,
          0.9759
        ],
        [
          0.1793,
          0.9754
        ],
        [
          0.1965,
          0.9745
        ],
        [
          0.2111,
          0.9745
        ],
        [
          0.2261,
          0.9736
        ],
        [
          0.2466,
          0.9713
        ],
        [
          0.2575,
          0.9704
        ],
        [
          0.2784,
          0.9672
        ],
        [
          0.2934,
          0.9631
        ],
        [
          0.3057,
          0.9627
        ],
        [
          0.3198,
          0.9595
        ],
        [
          0.3298,
          0.9568
        ],
        [
          0.3435,
          0.9536
        ],
        [
          0.3549,
          0.9495
        ],
        [
          0.3703,
          0.9449
        ],
        [
          0.3844,
          0.9409
        ],
        [
          0.4008,
          0.9345
        ],
        [
          0.4158,
          0.9263
        ],
        [
          0.4318,
          0.9195
        ],
        [
          0.444,
          0.9163
        ],
        [
          0.4604,
          0.9081
        ],
        [
          0.4777,
          0.9013
        ],
        [
          0.4891,
          0.8972
        ],
        [
          0.5127,
          0.8881
        ],
        [
          0.5232,
          0.8835
        ],
        [
          0.5359,
          0.8794
        ],
        [
          0.5605,
          0.8708
        ],
        [
          0.5769,
          0.864
        ],
        [
          0.5919,
          0.8581
        ],
        [
          0.6056,
          0.8508
        ],
        [
          0.6197,
          0.8435
        ],
        [
          0.6365,
          0.8367
        ],
        [
          0.6501,
          0.8276
        ],
        [
          0.667,
          0.8167
        ],
        [
          0.6838,
          0.8094
        ],
        [
          0.6993,
          0.8016
        ],
        [
          0.7161,
          0.7903
        ],
        [
          0.7352,
          0.7784
        ],
        [
          0.752,
          0.7725
        ],
        [
          0.7702,
          0.7602
        ],
        [
          0.7862,
          0.7534
        ],
        [
          0.8021,
          0.7439
        ],
        [
          0.8235,
          0.7284
        ],
        [
          0.8399,
          0.7166
        ],
        [
          0.8576,
          0.7066
        ],
        [
          0.8726,
          0.6965
        ],
        [
          0.889,
          0.687
        ],
        [
          0.9004,
          0.6793
        ],
        [
          0.9145,
          0.6702
        ],
        [
          0.9295,
          0.6574
        ],
        [
          0.9472,
          0.6433
        ],
        [
          0.9604,
          0.6342
        ],
        [
          0.9722,
          0.6256
        ],
        [
          0.9873,
          0.6142
        ],
        [
          0.9986,
          0.606
        ],
        [
          1.0,
          0.6046
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.15910766758317016,
        "difference": -0.0013155994056847035,
        "worse": 0.5166666666666667,
        "kept_right": 0.15686442758202498,
        "kept_wrong": 0.002243240001145108,
        "handed_off": 0.8408923324168301
      },
      "usd_per_1000": 4.537785734984571
    },
    {
      "decision": 150,
      "frontier": 191,
      "frontier_accuracy": 0.9740673339399454,
      "in_sample": {
        "threshold": 0.91,
        "answered": 0.17925386715195632,
        "accuracy": 0.9740673339399454
      },
      "curve": [
        [
          0.0,
          0.9741
        ],
        [
          0.0191,
          0.9741
        ],
        [
          0.056,
          0.9741
        ],
        [
          0.096,
          0.9736
        ],
        [
          0.1319,
          0.9741
        ],
        [
          0.1565,
          0.9741
        ],
        [
          0.1793,
          0.9741
        ],
        [
          0.1965,
          0.9732
        ],
        [
          0.2111,
          0.9732
        ],
        [
          0.2261,
          0.9722
        ],
        [
          0.2466,
          0.97
        ],
        [
          0.2575,
          0.9691
        ],
        [
          0.2784,
          0.9659
        ],
        [
          0.2934,
          0.9618
        ],
        [
          0.3057,
          0.9613
        ],
        [
          0.3198,
          0.9581
        ],
        [
          0.3298,
          0.9563
        ],
        [
          0.3435,
          0.9527
        ],
        [
          0.3549,
          0.9486
        ],
        [
          0.3703,
          0.944
        ],
        [
          0.3844,
          0.9399
        ],
        [
          0.4008,
          0.9336
        ],
        [
          0.4158,
          0.9254
        ],
        [
          0.4318,
          0.9186
        ],
        [
          0.444,
          0.9149
        ],
        [
          0.4604,
          0.9067
        ],
        [
          0.4777,
          0.8999
        ],
        [
          0.4891,
          0.8954
        ],
        [
          0.5127,
          0.8858
        ],
        [
          0.5232,
          0.8808
        ],
        [
          0.5359,
          0.8767
        ],
        [
          0.5605,
          0.8681
        ],
        [
          0.5769,
          0.8617
        ],
        [
          0.5919,
          0.8558
        ],
        [
          0.6056,
          0.8485
        ],
        [
          0.6197,
          0.8412
        ],
        [
          0.6365,
          0.8339
        ],
        [
          0.6501,
          0.8248
        ],
        [
          0.667,
          0.8139
        ],
        [
          0.6838,
          0.8071
        ],
        [
          0.6993,
          0.7985
        ],
        [
          0.7161,
          0.7871
        ],
        [
          0.7352,
          0.7762
        ],
        [
          0.752,
          0.7698
        ],
        [
          0.7702,
          0.7584
        ],
        [
          0.7862,
          0.7511
        ],
        [
          0.8021,
          0.7416
        ],
        [
          0.8235,
          0.7261
        ],
        [
          0.8399,
          0.7143
        ],
        [
          0.8576,
          0.7043
        ],
        [
          0.8726,
          0.6947
        ],
        [
          0.889,
          0.6852
        ],
        [
          0.9004,
          0.6774
        ],
        [
          0.9145,
          0.6692
        ],
        [
          0.9295,
          0.6565
        ],
        [
          0.9472,
          0.6429
        ],
        [
          0.9604,
          0.6338
        ],
        [
          0.9722,
          0.6256
        ],
        [
          0.9873,
          0.6142
        ],
        [
          0.9986,
          0.606
        ],
        [
          1.0,
          0.6046
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1718794246275747,
        "difference": -0.001391387317027107,
        "worse": 0.4533333333333333,
        "kept_right": 0.1692178287303661,
        "kept_wrong": 0.0026615958972086213,
        "handed_off": 0.8281205753724253
      },
      "usd_per_1000": 22.15731279210393
    },
    {
      "decision": 150,
      "frontier": 188,
      "frontier_accuracy": 0.9713375796178344,
      "in_sample": {
        "threshold": 0.95,
        "answered": 0.05595996360327571,
        "accuracy": 0.9713375796178344
      },
      "curve": [
        [
          0.0,
          0.9713
        ],
        [
          0.0191,
          0.9713
        ],
        [
          0.056,
          0.9713
        ],
        [
          0.096,
          0.9709
        ],
        [
          0.1319,
          0.9709
        ],
        [
          0.1565,
          0.9709
        ],
        [
          0.1793,
          0.9709
        ],
        [
          0.1965,
          0.9704
        ],
        [
          0.2111,
          0.9704
        ],
        [
          0.2261,
          0.9695
        ],
        [
          0.2466,
          0.9672
        ],
        [
          0.2575,
          0.9659
        ],
        [
          0.2784,
          0.9627
        ],
        [
          0.2934,
          0.9581
        ],
        [
          0.3057,
          0.9581
        ],
        [
          0.3198,
          0.955
        ],
        [
          0.3298,
          0.9522
        ],
        [
          0.3435,
          0.9486
        ],
        [
          0.3549,
          0.9445
        ],
        [
          0.3703,
          0.9409
        ],
        [
          0.3844,
          0.9372
        ],
        [
          0.4008,
          0.9313
        ],
        [
          0.4158,
          0.9236
        ],
        [
          0.4318,
          0.9163
        ],
        [
          0.444,
          0.9126
        ],
        [
          0.4604,
          0.9049
        ],
        [
          0.4777,
          0.8981
        ],
        [
          0.4891,
          0.8935
        ],
        [
          0.5127,
          0.8844
        ],
        [
          0.5232,
          0.8799
        ],
        [
          0.5359,
          0.8753
        ],
        [
          0.5605,
          0.8672
        ],
        [
          0.5769,
          0.8603
        ],
        [
          0.5919,
          0.8544
        ],
        [
          0.6056,
          0.8476
        ],
        [
          0.6197,
          0.8403
        ],
        [
          0.6365,
          0.833
        ],
        [
          0.6501,
          0.8248
        ],
        [
          0.667,
          0.8139
        ],
        [
          0.6838,
          0.8076
        ],
        [
          0.6993,
          0.7989
        ],
        [
          0.7161,
          0.7875
        ],
        [
          0.7352,
          0.7757
        ],
        [
          0.752,
          0.7693
        ],
        [
          0.7702,
          0.7575
        ],
        [
          0.7862,
          0.7507
        ],
        [
          0.8021,
          0.7411
        ],
        [
          0.8235,
          0.7257
        ],
        [
          0.8399,
          0.7138
        ],
        [
          0.8576,
          0.7043
        ],
        [
          0.8726,
          0.6947
        ],
        [
          0.889,
          0.6856
        ],
        [
          0.9004,
          0.6774
        ],
        [
          0.9145,
          0.6688
        ],
        [
          0.9295,
          0.6561
        ],
        [
          0.9472,
          0.6419
        ],
        [
          0.9604,
          0.6324
        ],
        [
          0.9722,
          0.6242
        ],
        [
          0.9873,
          0.6137
        ],
        [
          0.9986,
          0.606
        ],
        [
          1.0,
          0.6046
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.14241299952691944,
        "difference": -0.0011307590626042437,
        "worse": 0.63,
        "kept_right": 0.1404153080611312,
        "kept_wrong": 0.0019976914657882654,
        "handed_off": 0.85758700047308
      },
      "usd_per_1000": 0.8415099994868356
    },
    {
      "decision": 150,
      "frontier": 190,
      "frontier_accuracy": 0.9672429481346679,
      "in_sample": {
        "threshold": 0.92,
        "answered": 0.1565059144676979,
        "accuracy": 0.9672429481346679
      },
      "curve": [
        [
          0.0,
          0.9672
        ],
        [
          0.0191,
          0.9672
        ],
        [
          0.056,
          0.9672
        ],
        [
          0.096,
          0.9668
        ],
        [
          0.1319,
          0.9672
        ],
        [
          0.1565,
          0.9672
        ],
        [
          0.1793,
          0.9668
        ],
        [
          0.1965,
          0.9659
        ],
        [
          0.2111,
          0.9659
        ],
        [
          0.2261,
          0.965
        ],
        [
          0.2466,
          0.9627
        ],
        [
          0.2575,
          0.9618
        ],
        [
          0.2784,
          0.9586
        ],
        [
          0.2934,
          0.9545
        ],
        [
          0.3057,
          0.954
        ],
        [
          0.3198,
          0.9513
        ],
        [
          0.3298,
          0.95
        ],
        [
          0.3435,
          0.9468
        ],
        [
          0.3549,
          0.9427
        ],
        [
          0.3703,
          0.9381
        ],
        [
          0.3844,
          0.934
        ],
        [
          0.4008,
          0.9277
        ],
        [
          0.4158,
          0.9195
        ],
        [
          0.4318,
          0.9122
        ],
        [
          0.444,
          0.909
        ],
        [
          0.4604,
          0.9008
        ],
        [
          0.4777,
          0.894
        ],
        [
          0.4891,
          0.8894
        ],
        [
          0.5127,
          0.8799
        ],
        [
          0.5232,
          0.8749
        ],
        [
          0.5359,
          0.8708
        ],
        [
          0.5605,
          0.8631
        ],
        [
          0.5769,
          0.8567
        ],
        [
          0.5919,
          0.8512
        ],
        [
          0.6056,
          0.8439
        ],
        [
          0.6197,
          0.8367
        ],
        [
          0.6365,
          0.8303
        ],
        [
          0.6501,
          0.8212
        ],
        [
          0.667,
          0.8103
        ],
        [
          0.6838,
          0.8048
        ],
        [
          0.6993,
          0.7971
        ],
        [
          0.7161,
          0.7857
        ],
        [
          0.7352,
          0.7753
        ],
        [
          0.752,
          0.7698
        ],
        [
          0.7702,
          0.7584
        ],
        [
          0.7862,
          0.7516
        ],
        [
          0.8021,
          0.742
        ],
        [
          0.8235,
          0.7266
        ],
        [
          0.8399,
          0.7152
        ],
        [
          0.8576,
          0.7052
        ],
        [
          0.8726,
          0.6952
        ],
        [
          0.889,
          0.6856
        ],
        [
          0.9004,
          0.6774
        ],
        [
          0.9145,
          0.6692
        ],
        [
          0.9295,
          0.6565
        ],
        [
          0.9472,
          0.6424
        ],
        [
          0.9604,
          0.6333
        ],
        [
          0.9722,
          0.6251
        ],
        [
          0.9873,
          0.6137
        ],
        [
          0.9986,
          0.606
        ],
        [
          1.0,
          0.6046
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.15910766758317016,
        "difference": -0.0013155994056847035,
        "worse": 0.5166666666666667,
        "kept_right": 0.15686442758202498,
        "kept_wrong": 0.002243240001145108,
        "handed_off": 0.8408923324168301
      },
      "usd_per_1000": 4.697801468667983
    },
    {
      "decision": 150,
      "frontier": 124,
      "frontier_accuracy": 0.9667879890809827,
      "in_sample": {
        "threshold": 0.92,
        "answered": 0.1565059144676979,
        "accuracy": 0.9667879890809827
      },
      "curve": [
        [
          0.0,
          0.9668
        ],
        [
          0.0191,
          0.9668
        ],
        [
          0.056,
          0.9668
        ],
        [
          0.096,
          0.9663
        ],
        [
          0.1319,
          0.9668
        ],
        [
          0.1565,
          0.9668
        ],
        [
          0.1793,
          0.9663
        ],
        [
          0.1965,
          0.9654
        ],
        [
          0.2111,
          0.9654
        ],
        [
          0.2261,
          0.9645
        ],
        [
          0.2466,
          0.9622
        ],
        [
          0.2575,
          0.9618
        ],
        [
          0.2784,
          0.9586
        ],
        [
          0.2934,
          0.9545
        ],
        [
          0.3057,
          0.954
        ],
        [
          0.3198,
          0.9509
        ],
        [
          0.3298,
          0.9486
        ],
        [
          0.3435,
          0.9454
        ],
        [
          0.3549,
          0.9413
        ],
        [
          0.3703,
          0.9368
        ],
        [
          0.3844,
          0.9327
        ],
        [
          0.4008,
          0.9268
        ],
        [
          0.4158,
          0.919
        ],
        [
          0.4318,
          0.9126
        ],
        [
          0.444,
          0.9095
        ],
        [
          0.4604,
          0.9013
        ],
        [
          0.4777,
          0.8944
        ],
        [
          0.4891,
          0.8899
        ],
        [
          0.5127,
          0.8808
        ],
        [
          0.5232,
          0.8763
        ],
        [
          0.5359,
          0.8726
        ],
        [
          0.5605,
          0.8644
        ],
        [
          0.5769,
          0.8581
        ],
        [
          0.5919,
          0.8521
        ],
        [
          0.6056,
          0.8449
        ],
        [
          0.6197,
          0.8376
        ],
        [
          0.6365,
          0.8308
        ],
        [
          0.6501,
          0.8217
        ],
        [
          0.667,
          0.8107
        ],
        [
          0.6838,
          0.8039
        ],
        [
          0.6993,
          0.7957
        ],
        [
          0.7161,
          0.7848
        ],
        [
          0.7352,
          0.7743
        ],
        [
          0.752,
          0.768
        ],
        [
          0.7702,
          0.7566
        ],
        [
          0.7862,
          0.7493
        ],
        [
          0.8021,
          0.7398
        ],
        [
          0.8235,
          0.7243
        ],
        [
          0.8399,
          0.7134
        ],
        [
          0.8576,
          0.7034
        ],
        [
          0.8726,
          0.6943
        ],
        [
          0.889,
          0.6847
        ],
        [
          0.9004,
          0.6765
        ],
        [
          0.9145,
          0.6674
        ],
        [
          0.9295,
          0.6547
        ],
        [
          0.9472,
          0.6415
        ],
        [
          0.9604,
          0.6324
        ],
        [
          0.9722,
          0.6242
        ],
        [
          0.9873,
          0.6137
        ],
        [
          0.9986,
          0.606
        ],
        [
          1.0,
          0.6046
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.15917730124953405,
        "difference": -0.0013277096085306014,
        "worse": 0.5166666666666667,
        "kept_right": 0.15691892349483152,
        "kept_wrong": 0.00225837775470248,
        "handed_off": 0.8408226987504662
      },
      "usd_per_1000": 0.9657198564183156
    },
    {
      "decision": 150,
      "frontier": 278,
      "frontier_accuracy": 0.9499545040946314,
      "in_sample": {
        "threshold": 0.91,
        "answered": 0.17925386715195632,
        "accuracy": 0.9499545040946314
      },
      "curve": [
        [
          0.0,
          0.95
        ],
        [
          0.0191,
          0.95
        ],
        [
          0.056,
          0.95
        ],
        [
          0.096,
          0.9495
        ],
        [
          0.1319,
          0.95
        ],
        [
          0.1565,
          0.95
        ],
        [
          0.1793,
          0.95
        ],
        [
          0.1965,
          0.9495
        ],
        [
          0.2111,
          0.9495
        ],
        [
          0.2261,
          0.9486
        ],
        [
          0.2466,
          0.9463
        ],
        [
          0.2575,
          0.9459
        ],
        [
          0.2784,
          0.9427
        ],
        [
          0.2934,
          0.939
        ],
        [
          0.3057,
          0.9386
        ],
        [
          0.3198,
          0.9359
        ],
        [
          0.3298,
          0.934
        ],
        [
          0.3435,
          0.9308
        ],
        [
          0.3549,
          0.9268
        ],
        [
          0.3703,
          0.9227
        ],
        [
          0.3844,
          0.9195
        ],
        [
          0.4008,
          0.9136
        ],
        [
          0.4158,
          0.9058
        ],
        [
          0.4318,
          0.8999
        ],
        [
          0.444,
          0.8967
        ],
        [
          0.4604,
          0.889
        ],
        [
          0.4777,
          0.8826
        ],
        [
          0.4891,
          0.8785
        ],
        [
          0.5127,
          0.869
        ],
        [
          0.5232,
          0.8653
        ],
        [
          0.5359,
          0.8608
        ],
        [
          0.5605,
          0.853
        ],
        [
          0.5769,
          0.8471
        ],
        [
          0.5919,
          0.8421
        ],
        [
          0.6056,
          0.8353
        ],
        [
          0.6197,
          0.8285
        ],
        [
          0.6365,
          0.8217
        ],
        [
          0.6501,
          0.8126
        ],
        [
          0.667,
          0.8025
        ],
        [
          0.6838,
          0.7966
        ],
        [
          0.6993,
          0.7884
        ],
        [
          0.7161,
          0.778
        ],
        [
          0.7352,
          0.7675
        ],
        [
          0.752,
          0.7607
        ],
        [
          0.7702,
          0.7498
        ],
        [
          0.7862,
          0.7434
        ],
        [
          0.8021,
          0.7361
        ],
        [
          0.8235,
          0.7211
        ],
        [
          0.8399,
          0.7106
        ],
        [
          0.8576,
          0.7015
        ],
        [
          0.8726,
          0.692
        ],
        [
          0.889,
          0.6833
        ],
        [
          0.9004,
          0.6756
        ],
        [
          0.9145,
          0.6674
        ],
        [
          0.9295,
          0.6547
        ],
        [
          0.9472,
          0.641
        ],
        [
          0.9604,
          0.6328
        ],
        [
          0.9722,
          0.6251
        ],
        [
          0.9873,
          0.6142
        ],
        [
          0.9986,
          0.606
        ],
        [
          1.0,
          0.6046
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.17730753406137378,
        "difference": -0.0014035804155834043,
        "worse": 0.51,
        "kept_right": 0.17437605597152597,
        "kept_wrong": 0.002931478089847832,
        "handed_off": 0.8226924659386264
      },
      "usd_per_1000": 5.370038880456257
    },
    {
      "decision": 150,
      "frontier": 275,
      "frontier_accuracy": 0.9426751592356688,
      "in_sample": {
        "threshold": 0.89,
        "answered": 0.2111010009099181,
        "accuracy": 0.9426751592356688
      },
      "curve": [
        [
          0.0,
          0.9427
        ],
        [
          0.0191,
          0.9427
        ],
        [
          0.056,
          0.9427
        ],
        [
          0.096,
          0.9422
        ],
        [
          0.1319,
          0.9427
        ],
        [
          0.1565,
          0.9427
        ],
        [
          0.1793,
          0.9431
        ],
        [
          0.1965,
          0.9427
        ],
        [
          0.2111,
          0.9427
        ],
        [
          0.2261,
          0.9418
        ],
        [
          0.2466,
          0.9399
        ],
        [
          0.2575,
          0.939
        ],
        [
          0.2784,
          0.9363
        ],
        [
          0.2934,
          0.9327
        ],
        [
          0.3057,
          0.9322
        ],
        [
          0.3198,
          0.9295
        ],
        [
          0.3298,
          0.9286
        ],
        [
          0.3435,
          0.9254
        ],
        [
          0.3549,
          0.9217
        ],
        [
          0.3703,
          0.9172
        ],
        [
          0.3844,
          0.9131
        ],
        [
          0.4008,
          0.9072
        ],
        [
          0.4158,
          0.9004
        ],
        [
          0.4318,
          0.8944
        ],
        [
          0.444,
          0.8913
        ],
        [
          0.4604,
          0.8835
        ],
        [
          0.4777,
          0.8767
        ],
        [
          0.4891,
          0.8726
        ],
        [
          0.5127,
          0.864
        ],
        [
          0.5232,
          0.8599
        ],
        [
          0.5359,
          0.8553
        ],
        [
          0.5605,
          0.8462
        ],
        [
          0.5769,
          0.8399
        ],
        [
          0.5919,
          0.8344
        ],
        [
          0.6056,
          0.828
        ],
        [
          0.6197,
          0.8217
        ],
        [
          0.6365,
          0.8157
        ],
        [
          0.6501,
          0.8071
        ],
        [
          0.667,
          0.7975
        ],
        [
          0.6838,
          0.7912
        ],
        [
          0.6993,
          0.7843
        ],
        [
          0.7161,
          0.773
        ],
        [
          0.7352,
          0.7621
        ],
        [
          0.752,
          0.7571
        ],
        [
          0.7702,
          0.7457
        ],
        [
          0.7862,
          0.7393
        ],
        [
          0.8021,
          0.7316
        ],
        [
          0.8235,
          0.7175
        ],
        [
          0.8399,
          0.7075
        ],
        [
          0.8576,
          0.6988
        ],
        [
          0.8726,
          0.6906
        ],
        [
          0.889,
          0.682
        ],
        [
          0.9004,
          0.6738
        ],
        [
          0.9145,
          0.6652
        ],
        [
          0.9295,
          0.6524
        ],
        [
          0.9472,
          0.6397
        ],
        [
          0.9604,
          0.6315
        ],
        [
          0.9722,
          0.6242
        ],
        [
          0.9873,
          0.6133
        ],
        [
          0.9986,
          0.606
        ],
        [
          1.0,
          0.6046
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1954349847959352,
        "difference": -0.0014612058354380854,
        "worse": 0.45,
        "kept_right": 0.19176685268722699,
        "kept_wrong": 0.003668132108708287,
        "handed_off": 0.8045650152040644
      },
      "usd_per_1000": 0.42923583971532164
    },
    {
      "decision": 150,
      "frontier": 192,
      "frontier_accuracy": 0.9331210191082803,
      "in_sample": {
        "threshold": 0.95,
        "answered": 0.05595996360327571,
        "accuracy": 0.9331210191082803
      },
      "curve": [
        [
          0.0,
          0.9331
        ],
        [
          0.0191,
          0.9331
        ],
        [
          0.056,
          0.9331
        ],
        [
          0.096,
          0.9327
        ],
        [
          0.1319,
          0.9327
        ],
        [
          0.1565,
          0.9327
        ],
        [
          0.1793,
          0.9327
        ],
        [
          0.1965,
          0.9322
        ],
        [
          0.2111,
          0.9322
        ],
        [
          0.2261,
          0.9313
        ],
        [
          0.2466,
          0.929
        ],
        [
          0.2575,
          0.9286
        ],
        [
          0.2784,
          0.9263
        ],
        [
          0.2934,
          0.9222
        ],
        [
          0.3057,
          0.9222
        ],
        [
          0.3198,
          0.9199
        ],
        [
          0.3298,
          0.9186
        ],
        [
          0.3435,
          0.9154
        ],
        [
          0.3549,
          0.9113
        ],
        [
          0.3703,
          0.9076
        ],
        [
          0.3844,
          0.9049
        ],
        [
          0.4008,
          0.8995
        ],
        [
          0.4158,
          0.8917
        ],
        [
          0.4318,
          0.8867
        ],
        [
          0.444,
          0.8835
        ],
        [
          0.4604,
          0.8753
        ],
        [
          0.4777,
          0.869
        ],
        [
          0.4891,
          0.8653
        ],
        [
          0.5127,
          0.8562
        ],
        [
          0.5232,
          0.8521
        ],
        [
          0.5359,
          0.8476
        ],
        [
          0.5605,
          0.8399
        ],
        [
          0.5769,
          0.8348
        ],
        [
          0.5919,
          0.8303
        ],
        [
          0.6056,
          0.8235
        ],
        [
          0.6197,
          0.8167
        ],
        [
          0.6365,
          0.8098
        ],
        [
          0.6501,
          0.8012
        ],
        [
          0.667,
          0.7912
        ],
        [
          0.6838,
          0.7862
        ],
        [
          0.6993,
          0.7775
        ],
        [
          0.7161,
          0.7666
        ],
        [
          0.7352,
          0.758
        ],
        [
          0.752,
          0.753
        ],
        [
          0.7702,
          0.7425
        ],
        [
          0.7862,
          0.7366
        ],
        [
          0.8021,
          0.7284
        ],
        [
          0.8235,
          0.7138
        ],
        [
          0.8399,
          0.7047
        ],
        [
          0.8576,
          0.6961
        ],
        [
          0.8726,
          0.6874
        ],
        [
          0.889,
          0.6802
        ],
        [
          0.9004,
          0.6724
        ],
        [
          0.9145,
          0.6661
        ],
        [
          0.9295,
          0.6538
        ],
        [
          0.9472,
          0.6406
        ],
        [
          0.9604,
          0.6324
        ],
        [
          0.9722,
          0.6247
        ],
        [
          0.9873,
          0.6137
        ],
        [
          0.9986,
          0.606
        ],
        [
          1.0,
          0.6046
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1565225925278809,
        "difference": -0.001743164871612903,
        "worse": 0.7166666666666667,
        "kept_right": 0.15390943725632045,
        "kept_wrong": 0.002613155271560405,
        "handed_off": 0.8434774074721189
      },
      "usd_per_1000": 9.80768259232769
    },
    {
      "decision": 150,
      "frontier": 123,
      "frontier_accuracy": 0.9326660600545951,
      "in_sample": {
        "threshold": 0.89,
        "answered": 0.2111010009099181,
        "accuracy": 0.9331210191082803
      },
      "curve": [
        [
          0.0,
          0.9327
        ],
        [
          0.0191,
          0.9327
        ],
        [
          0.056,
          0.9327
        ],
        [
          0.096,
          0.9322
        ],
        [
          0.1319,
          0.9327
        ],
        [
          0.1565,
          0.9327
        ],
        [
          0.1793,
          0.9331
        ],
        [
          0.1965,
          0.9331
        ],
        [
          0.2111,
          0.9331
        ],
        [
          0.2261,
          0.9322
        ],
        [
          0.2466,
          0.9299
        ],
        [
          0.2575,
          0.9286
        ],
        [
          0.2784,
          0.9254
        ],
        [
          0.2934,
          0.9217
        ],
        [
          0.3057,
          0.9213
        ],
        [
          0.3198,
          0.9186
        ],
        [
          0.3298,
          0.9158
        ],
        [
          0.3435,
          0.9126
        ],
        [
          0.3549,
          0.9086
        ],
        [
          0.3703,
          0.9045
        ],
        [
          0.3844,
          0.9004
        ],
        [
          0.4008,
          0.8944
        ],
        [
          0.4158,
          0.8872
        ],
        [
          0.4318,
          0.8822
        ],
        [
          0.444,
          0.8785
        ],
        [
          0.4604,
          0.8708
        ],
        [
          0.4777,
          0.8653
        ],
        [
          0.4891,
          0.8617
        ],
        [
          0.5127,
          0.8521
        ],
        [
          0.5232,
          0.8476
        ],
        [
          0.5359,
          0.8426
        ],
        [
          0.5605,
          0.8339
        ],
        [
          0.5769,
          0.8276
        ],
        [
          0.5919,
          0.8217
        ],
        [
          0.6056,
          0.8157
        ],
        [
          0.6197,
          0.8094
        ],
        [
          0.6365,
          0.8035
        ],
        [
          0.6501,
          0.7957
        ],
        [
          0.667,
          0.7866
        ],
        [
          0.6838,
          0.7807
        ],
        [
          0.6993,
          0.7739
        ],
        [
          0.7161,
          0.763
        ],
        [
          0.7352,
          0.753
        ],
        [
          0.752,
          0.747
        ],
        [
          0.7702,
          0.7361
        ],
        [
          0.7862,
          0.7316
        ],
        [
          0.8021,
          0.7229
        ],
        [
          0.8235,
          0.7106
        ],
        [
          0.8399,
          0.6993
        ],
        [
          0.8576,
          0.6915
        ],
        [
          0.8726,
          0.6829
        ],
        [
          0.889,
          0.6747
        ],
        [
          0.9004,
          0.6665
        ],
        [
          0.9145,
          0.6592
        ],
        [
          0.9295,
          0.6479
        ],
        [
          0.9472,
          0.6365
        ],
        [
          0.9604,
          0.6292
        ],
        [
          0.9722,
          0.6224
        ],
        [
          0.9873,
          0.6128
        ],
        [
          0.9986,
          0.606
        ],
        [
          1.0,
          0.6046
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.2017548643877892,
        "difference": -0.001179304657541324,
        "worse": 0.5466666666666666,
        "kept_right": 0.19806855323406233,
        "kept_wrong": 0.0036863111537269273,
        "handed_off": 0.7982451356122103
      },
      "usd_per_1000": 0.2296723476778115
    },
    {
      "decision": 150,
      "frontier": 274,
      "frontier_accuracy": 0.9085532302092811,
      "in_sample": {
        "threshold": 0.88,
        "answered": 0.22611464968152867,
        "accuracy": 0.9085532302092811
      },
      "curve": [
        [
          0.0,
          0.9086
        ],
        [
          0.0191,
          0.9086
        ],
        [
          0.056,
          0.9086
        ],
        [
          0.096,
          0.9081
        ],
        [
          0.1319,
          0.9086
        ],
        [
          0.1565,
          0.9086
        ],
        [
          0.1793,
          0.909
        ],
        [
          0.1965,
          0.909
        ],
        [
          0.2111,
          0.9095
        ],
        [
          0.2261,
          0.9086
        ],
        [
          0.2466,
          0.9063
        ],
        [
          0.2575,
          0.9054
        ],
        [
          0.2784,
          0.9026
        ],
        [
          0.2934,
          0.8995
        ],
        [
          0.3057,
          0.899
        ],
        [
          0.3198,
          0.8963
        ],
        [
          0.3298,
          0.8954
        ],
        [
          0.3435,
          0.8922
        ],
        [
          0.3549,
          0.8881
        ],
        [
          0.3703,
          0.8844
        ],
        [
          0.3844,
          0.8822
        ],
        [
          0.4008,
          0.8776
        ],
        [
          0.4158,
          0.8708
        ],
        [
          0.4318,
          0.8644
        ],
        [
          0.444,
          0.8617
        ],
        [
          0.4604,
          0.8553
        ],
        [
          0.4777,
          0.849
        ],
        [
          0.4891,
          0.8453
        ],
        [
          0.5127,
          0.838
        ],
        [
          0.5232,
          0.833
        ],
        [
          0.5359,
          0.8285
        ],
        [
          0.5605,
          0.8207
        ],
        [
          0.5769,
          0.8148
        ],
        [
          0.5919,
          0.8098
        ],
        [
          0.6056,
          0.8035
        ],
        [
          0.6197,
          0.7971
        ],
        [
          0.6365,
          0.7916
        ],
        [
          0.6501,
          0.7834
        ],
        [
          0.667,
          0.7757
        ],
        [
          0.6838,
          0.7712
        ],
        [
          0.6993,
          0.7648
        ],
        [
          0.7161,
          0.7548
        ],
        [
          0.7352,
          0.7466
        ],
        [
          0.752,
          0.7416
        ],
        [
          0.7702,
          0.732
        ],
        [
          0.7862,
          0.727
        ],
        [
          0.8021,
          0.7197
        ],
        [
          0.8235,
          0.7061
        ],
        [
          0.8399,
          0.697
        ],
        [
          0.8576,
          0.6893
        ],
        [
          0.8726,
          0.6815
        ],
        [
          0.889,
          0.6738
        ],
        [
          0.9004,
          0.6656
        ],
        [
          0.9145,
          0.6601
        ],
        [
          0.9295,
          0.6479
        ],
        [
          0.9472,
          0.6365
        ],
        [
          0.9604,
          0.6288
        ],
        [
          0.9722,
          0.6219
        ],
        [
          0.9873,
          0.6128
        ],
        [
          0.9986,
          0.606
        ],
        [
          1.0,
          0.6046
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.21198654624611407,
        "difference": -0.0011973349930462097,
        "worse": 0.43666666666666665,
        "kept_right": 0.2077121404249117,
        "kept_wrong": 0.004274405821202528,
        "handed_off": 0.7880134537538851
      },
      "usd_per_1000": 0.3511638528937992
    },
    {
      "decision": 150,
      "frontier": 47,
      "frontier_accuracy": 0.9044585987261147,
      "in_sample": {
        "threshold": 0.91,
        "answered": 0.17925386715195632,
        "accuracy": 0.9044585987261147
      },
      "curve": [
        [
          0.0,
          0.9045
        ],
        [
          0.0191,
          0.9045
        ],
        [
          0.056,
          0.9045
        ],
        [
          0.096,
          0.904
        ],
        [
          0.1319,
          0.9045
        ],
        [
          0.1565,
          0.904
        ],
        [
          0.1793,
          0.9045
        ],
        [
          0.1965,
          0.9035
        ],
        [
          0.2111,
          0.904
        ],
        [
          0.2261,
          0.9031
        ],
        [
          0.2466,
          0.9008
        ],
        [
          0.2575,
          0.8999
        ],
        [
          0.2784,
          0.899
        ],
        [
          0.2934,
          0.8954
        ],
        [
          0.3057,
          0.8949
        ],
        [
          0.3198,
          0.8922
        ],
        [
          0.3298,
          0.8908
        ],
        [
          0.3435,
          0.8881
        ],
        [
          0.3549,
          0.8849
        ],
        [
          0.3703,
          0.8813
        ],
        [
          0.3844,
          0.8781
        ],
        [
          0.4008,
          0.8735
        ],
        [
          0.4158,
          0.8672
        ],
        [
          0.4318,
          0.8631
        ],
        [
          0.444,
          0.8603
        ],
        [
          0.4604,
          0.853
        ],
        [
          0.4777,
          0.8471
        ],
        [
          0.4891,
          0.8439
        ],
        [
          0.5127,
          0.8376
        ],
        [
          0.5232,
          0.8339
        ],
        [
          0.5359,
          0.8317
        ],
        [
          0.5605,
          0.8253
        ],
        [
          0.5769,
          0.8194
        ],
        [
          0.5919,
          0.8148
        ],
        [
          0.6056,
          0.8076
        ],
        [
          0.6197,
          0.8021
        ],
        [
          0.6365,
          0.7957
        ],
        [
          0.6501,
          0.7875
        ],
        [
          0.667,
          0.7784
        ],
        [
          0.6838,
          0.7748
        ],
        [
          0.6993,
          0.7684
        ],
        [
          0.7161,
          0.7584
        ],
        [
          0.7352,
          0.7484
        ],
        [
          0.752,
          0.7448
        ],
        [
          0.7702,
          0.7338
        ],
        [
          0.7862,
          0.7288
        ],
        [
          0.8021,
          0.7216
        ],
        [
          0.8235,
          0.7079
        ],
        [
          0.8399,
          0.6984
        ],
        [
          0.8576,
          0.6906
        ],
        [
          0.8726,
          0.682
        ],
        [
          0.889,
          0.6756
        ],
        [
          0.9004,
          0.6683
        ],
        [
          0.9145,
          0.6606
        ],
        [
          0.9295,
          0.6497
        ],
        [
          0.9472,
          0.6374
        ],
        [
          0.9604,
          0.6301
        ],
        [
          0.9722,
          0.6219
        ],
        [
          0.9873,
          0.6119
        ],
        [
          0.9986,
          0.6056
        ],
        [
          1.0,
          0.6046
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.17770056634059753,
        "difference": -0.0014790706496107407,
        "worse": 0.56,
        "kept_right": 0.17464816271749184,
        "kept_wrong": 0.0030524036231058195,
        "handed_off": 0.8222994336594024
      },
      "usd_per_1000": 4.069694577196937
    },
    {
      "decision": 150,
      "frontier": 193,
      "frontier_accuracy": 0.89171974522293,
      "in_sample": {
        "threshold": 0.87,
        "answered": 0.2356687898089172,
        "accuracy": 0.89171974522293
      },
      "curve": [
        [
          0.0,
          0.8917
        ],
        [
          0.0191,
          0.8917
        ],
        [
          0.056,
          0.8917
        ],
        [
          0.096,
          0.8917
        ],
        [
          0.1319,
          0.8926
        ],
        [
          0.1565,
          0.8922
        ],
        [
          0.1793,
          0.8922
        ],
        [
          0.1965,
          0.8922
        ],
        [
          0.2111,
          0.8926
        ],
        [
          0.2261,
          0.8926
        ],
        [
          0.2466,
          0.8904
        ],
        [
          0.2575,
          0.8899
        ],
        [
          0.2784,
          0.8885
        ],
        [
          0.2934,
          0.8844
        ],
        [
          0.3057,
          0.8849
        ],
        [
          0.3198,
          0.8831
        ],
        [
          0.3298,
          0.8826
        ],
        [
          0.3435,
          0.8799
        ],
        [
          0.3549,
          0.8763
        ],
        [
          0.3703,
          0.8726
        ],
        [
          0.3844,
          0.8694
        ],
        [
          0.4008,
          0.8635
        ],
        [
          0.4158,
          0.8581
        ],
        [
          0.4318,
          0.8544
        ],
        [
          0.444,
          0.8517
        ],
        [
          0.4604,
          0.8458
        ],
        [
          0.4777,
          0.8394
        ],
        [
          0.4891,
          0.8367
        ],
        [
          0.5127,
          0.8298
        ],
        [
          0.5232,
          0.8267
        ],
        [
          0.5359,
          0.8235
        ],
        [
          0.5605,
          0.8167
        ],
        [
          0.5769,
          0.8116
        ],
        [
          0.5919,
          0.8066
        ],
        [
          0.6056,
          0.8007
        ],
        [
          0.6197,
          0.7944
        ],
        [
          0.6365,
          0.7875
        ],
        [
          0.6501,
          0.7798
        ],
        [
          0.667,
          0.7707
        ],
        [
          0.6838,
          0.7657
        ],
        [
          0.6993,
          0.7575
        ],
        [
          0.7161,
          0.7493
        ],
        [
          0.7352,
          0.7416
        ],
        [
          0.752,
          0.737
        ],
        [
          0.7702,
          0.7284
        ],
        [
          0.7862,
          0.7229
        ],
        [
          0.8021,
          0.7161
        ],
        [
          0.8235,
          0.7029
        ],
        [
          0.8399,
          0.6929
        ],
        [
          0.8576,
          0.6847
        ],
        [
          0.8726,
          0.6779
        ],
        [
          0.889,
          0.6706
        ],
        [
          0.9004,
          0.6642
        ],
        [
          0.9145,
          0.6579
        ],
        [
          0.9295,
          0.6474
        ],
        [
          0.9472,
          0.6356
        ],
        [
          0.9604,
          0.6292
        ],
        [
          0.9722,
          0.6228
        ],
        [
          0.9873,
          0.6128
        ],
        [
          0.9986,
          0.606
        ],
        [
          1.0,
          0.6046
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.2332708964748188,
        "difference": -0.0011698415434113524,
        "worse": 0.43,
        "kept_right": 0.22732635068607673,
        "kept_wrong": 0.005944545788742501,
        "handed_off": 0.76672910352518
      },
      "usd_per_1000": 5.955579252316881
    },
    {
      "decision": 150,
      "frontier": 55,
      "frontier_accuracy": 0.8503184713375797,
      "in_sample": {
        "threshold": 0.76,
        "answered": 0.3434940855323021,
        "accuracy": 0.8507734303912647
      },
      "curve": [
        [
          0.0,
          0.8503
        ],
        [
          0.0191,
          0.8503
        ],
        [
          0.056,
          0.8512
        ],
        [
          0.096,
          0.8517
        ],
        [
          0.1319,
          0.8521
        ],
        [
          0.1565,
          0.8526
        ],
        [
          0.1793,
          0.8544
        ],
        [
          0.1965,
          0.8553
        ],
        [
          0.2111,
          0.8571
        ],
        [
          0.2261,
          0.8567
        ],
        [
          0.2466,
          0.8567
        ],
        [
          0.2575,
          0.8558
        ],
        [
          0.2784,
          0.8544
        ],
        [
          0.2934,
          0.853
        ],
        [
          0.3057,
          0.854
        ],
        [
          0.3198,
          0.8526
        ],
        [
          0.3298,
          0.853
        ],
        [
          0.3435,
          0.8508
        ],
        [
          0.3549,
          0.8471
        ],
        [
          0.3703,
          0.843
        ],
        [
          0.3844,
          0.8412
        ],
        [
          0.4008,
          0.8376
        ],
        [
          0.4158,
          0.833
        ],
        [
          0.4318,
          0.828
        ],
        [
          0.444,
          0.8267
        ],
        [
          0.4604,
          0.8198
        ],
        [
          0.4777,
          0.8139
        ],
        [
          0.4891,
          0.8121
        ],
        [
          0.5127,
          0.8071
        ],
        [
          0.5232,
          0.8044
        ],
        [
          0.5359,
          0.8025
        ],
        [
          0.5605,
          0.7962
        ],
        [
          0.5769,
          0.7903
        ],
        [
          0.5919,
          0.7853
        ],
        [
          0.6056,
          0.7803
        ],
        [
          0.6197,
          0.7748
        ],
        [
          0.6365,
          0.7689
        ],
        [
          0.6501,
          0.7621
        ],
        [
          0.667,
          0.7543
        ],
        [
          0.6838,
          0.7507
        ],
        [
          0.6993,
          0.7448
        ],
        [
          0.7161,
          0.737
        ],
        [
          0.7352,
          0.7302
        ],
        [
          0.752,
          0.7266
        ],
        [
          0.7702,
          0.7184
        ],
        [
          0.7862,
          0.7147
        ],
        [
          0.8021,
          0.7106
        ],
        [
          0.8235,
          0.6975
        ],
        [
          0.8399,
          0.6879
        ],
        [
          0.8576,
          0.6797
        ],
        [
          0.8726,
          0.6742
        ],
        [
          0.889,
          0.6674
        ],
        [
          0.9004,
          0.6615
        ],
        [
          0.9145,
          0.6551
        ],
        [
          0.9295,
          0.6442
        ],
        [
          0.9472,
          0.6338
        ],
        [
          0.9604,
          0.6269
        ],
        [
          0.9722,
          0.6201
        ],
        [
          0.9873,
          0.6119
        ],
        [
          0.9986,
          0.606
        ],
        [
          1.0,
          0.6046
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.33148620608755414,
        "difference": 6.641646244469182e-05,
        "worse": 0.3933333333333333,
        "kept_right": 0.3075695158831251,
        "kept_wrong": 0.02391669020442912,
        "handed_off": 0.6685137939124455
      },
      "usd_per_1000": 2.458248956580333
    },
    {
      "decision": 44,
      "frontier": 140,
      "frontier_accuracy": 0.9808917197452229,
      "in_sample": {
        "threshold": 0.9156852342507505,
        "answered": 0.17015468607825296,
        "accuracy": 0.9808917197452229
      },
      "curve": [
        [
          0.0,
          0.9809
        ],
        [
          0.01,
          0.9809
        ],
        [
          0.02,
          0.9809
        ],
        [
          0.03,
          0.9809
        ],
        [
          0.04,
          0.9809
        ],
        [
          0.05,
          0.9809
        ],
        [
          0.0601,
          0.9809
        ],
        [
          0.0705,
          0.9809
        ],
        [
          0.0805,
          0.9809
        ],
        [
          0.0905,
          0.9809
        ],
        [
          0.1005,
          0.9809
        ],
        [
          0.1106,
          0.9809
        ],
        [
          0.121,
          0.9809
        ],
        [
          0.131,
          0.9809
        ],
        [
          0.141,
          0.9809
        ],
        [
          0.151,
          0.9809
        ],
        [
          0.1611,
          0.9809
        ],
        [
          0.1715,
          0.9804
        ],
        [
          0.1815,
          0.9804
        ],
        [
          0.1915,
          0.98
        ],
        [
          0.2015,
          0.98
        ],
        [
          0.2116,
          0.9795
        ],
        [
          0.222,
          0.9791
        ],
        [
          0.232,
          0.9786
        ],
        [
          0.242,
          0.9773
        ],
        [
          0.252,
          0.9768
        ],
        [
          0.2621,
          0.9759
        ],
        [
          0.2725,
          0.9745
        ],
        [
          0.2825,
          0.9732
        ],
        [
          0.2925,
          0.9709
        ],
        [
          0.3025,
          0.9695
        ],
        [
          0.3126,
          0.9682
        ],
        [
          0.323,
          0.965
        ],
        [
          0.333,
          0.9631
        ],
        [
          0.343,
          0.9622
        ],
        [
          0.353,
          0.96
        ],
        [
          0.3631,
          0.9572
        ],
        [
          0.3735,
          0.955
        ],
        [
          0.3835,
          0.9527
        ],
        [
          0.3935,
          0.9495
        ],
        [
          0.4035,
          0.9468
        ],
        [
          0.4136,
          0.9449
        ],
        [
          0.424,
          0.9395
        ],
        [
          0.434,
          0.9354
        ],
        [
          0.444,
          0.9331
        ],
        [
          0.454,
          0.9277
        ],
        [
          0.4641,
          0.9231
        ],
        [
          0.4745,
          0.9199
        ],
        [
          0.4845,
          0.9167
        ],
        [
          0.4945,
          0.9113
        ],
        [
          0.5045,
          0.9067
        ],
        [
          0.5146,
          0.9022
        ],
        [
          0.525,
          0.8981
        ],
        [
          0.535,
          0.8944
        ],
        [
          0.545,
          0.8913
        ],
        [
          0.5551,
          0.8881
        ],
        [
          0.5655,
          0.8822
        ],
        [
          0.5755,
          0.8781
        ],
        [
          0.5855,
          0.8722
        ],
        [
          0.5955,
          0.8676
        ],
        [
          0.6056,
          0.8635
        ],
        [
          0.616,
          0.8562
        ],
        [
          0.626,
          0.853
        ],
        [
          0.636,
          0.8476
        ],
        [
          0.646,
          0.8403
        ],
        [
          0.6561,
          0.8344
        ],
        [
          0.6665,
          0.828
        ],
        [
          0.6765,
          0.8203
        ],
        [
          0.6865,
          0.8153
        ],
        [
          0.6965,
          0.8089
        ],
        [
          0.7066,
          0.8044
        ],
        [
          0.717,
          0.7989
        ],
        [
          0.727,
          0.7934
        ],
        [
          0.737,
          0.7871
        ],
        [
          0.747,
          0.7807
        ],
        [
          0.7571,
          0.7762
        ],
        [
          0.7675,
          0.7675
        ],
        [
          0.7775,
          0.763
        ],
        [
          0.7875,
          0.7566
        ],
        [
          0.7975,
          0.752
        ],
        [
          0.8076,
          0.747
        ],
        [
          0.818,
          0.7402
        ],
        [
          0.828,
          0.7329
        ],
        [
          0.838,
          0.7252
        ],
        [
          0.848,
          0.7188
        ],
        [
          0.8581,
          0.712
        ],
        [
          0.8685,
          0.7043
        ],
        [
          0.8785,
          0.697
        ],
        [
          0.8885,
          0.6888
        ],
        [
          0.8985,
          0.6824
        ],
        [
          0.9086,
          0.6752
        ],
        [
          0.919,
          0.667
        ],
        [
          0.929,
          0.6597
        ],
        [
          0.939,
          0.6515
        ],
        [
          0.949,
          0.6442
        ],
        [
          0.9591,
          0.6365
        ],
        [
          0.9695,
          0.6278
        ],
        [
          0.9795,
          0.6187
        ],
        [
          0.9895,
          0.6119
        ],
        [
          0.9995,
          0.6037
        ],
        [
          1.0,
          0.6037
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.18782827290377466,
        "difference": -0.0010974178736330603,
        "worse": 0.54,
        "kept_right": 0.18651863421811848,
        "kept_wrong": 0.0013096386856561925,
        "handed_off": 0.8121717270962256
      },
      "usd_per_1000": 1.9802645143443653
    },
    {
      "decision": 44,
      "frontier": 277,
      "frontier_accuracy": 0.9795268425841674,
      "in_sample": {
        "threshold": 0.9156852342507505,
        "answered": 0.17015468607825296,
        "accuracy": 0.9795268425841674
      },
      "curve": [
        [
          0.0,
          0.9795
        ],
        [
          0.01,
          0.9795
        ],
        [
          0.02,
          0.9795
        ],
        [
          0.03,
          0.9795
        ],
        [
          0.04,
          0.9795
        ],
        [
          0.05,
          0.9795
        ],
        [
          0.0601,
          0.9795
        ],
        [
          0.0705,
          0.9795
        ],
        [
          0.0805,
          0.9795
        ],
        [
          0.0905,
          0.9795
        ],
        [
          0.1005,
          0.9795
        ],
        [
          0.1106,
          0.9795
        ],
        [
          0.121,
          0.9795
        ],
        [
          0.131,
          0.9795
        ],
        [
          0.141,
          0.9795
        ],
        [
          0.151,
          0.9795
        ],
        [
          0.1611,
          0.9795
        ],
        [
          0.1715,
          0.9791
        ],
        [
          0.1815,
          0.9791
        ],
        [
          0.1915,
          0.9786
        ],
        [
          0.2015,
          0.9786
        ],
        [
          0.2116,
          0.9777
        ],
        [
          0.222,
          0.9773
        ],
        [
          0.232,
          0.9768
        ],
        [
          0.242,
          0.9754
        ],
        [
          0.252,
          0.975
        ],
        [
          0.2621,
          0.9741
        ],
        [
          0.2725,
          0.9727
        ],
        [
          0.2825,
          0.9713
        ],
        [
          0.2925,
          0.9691
        ],
        [
          0.3025,
          0.9677
        ],
        [
          0.3126,
          0.9663
        ],
        [
          0.323,
          0.9641
        ],
        [
          0.333,
          0.9622
        ],
        [
          0.343,
          0.9604
        ],
        [
          0.353,
          0.9581
        ],
        [
          0.3631,
          0.9554
        ],
        [
          0.3735,
          0.9536
        ],
        [
          0.3835,
          0.9513
        ],
        [
          0.3935,
          0.9481
        ],
        [
          0.4035,
          0.9449
        ],
        [
          0.4136,
          0.9431
        ],
        [
          0.424,
          0.9381
        ],
        [
          0.434,
          0.9345
        ],
        [
          0.444,
          0.9322
        ],
        [
          0.454,
          0.9268
        ],
        [
          0.4641,
          0.9222
        ],
        [
          0.4745,
          0.9186
        ],
        [
          0.4845,
          0.9154
        ],
        [
          0.4945,
          0.9099
        ],
        [
          0.5045,
          0.9054
        ],
        [
          0.5146,
          0.9008
        ],
        [
          0.525,
          0.8967
        ],
        [
          0.535,
          0.8931
        ],
        [
          0.545,
          0.8894
        ],
        [
          0.5551,
          0.8863
        ],
        [
          0.5655,
          0.8808
        ],
        [
          0.5755,
          0.8767
        ],
        [
          0.5855,
          0.8708
        ],
        [
          0.5955,
          0.8662
        ],
        [
          0.6056,
          0.8626
        ],
        [
          0.616,
          0.8553
        ],
        [
          0.626,
          0.8517
        ],
        [
          0.636,
          0.8458
        ],
        [
          0.646,
          0.8385
        ],
        [
          0.6561,
          0.833
        ],
        [
          0.6665,
          0.8262
        ],
        [
          0.6765,
          0.8189
        ],
        [
          0.6865,
          0.8139
        ],
        [
          0.6965,
          0.808
        ],
        [
          0.7066,
          0.8035
        ],
        [
          0.717,
          0.798
        ],
        [
          0.727,
          0.7925
        ],
        [
          0.737,
          0.7862
        ],
        [
          0.747,
          0.7803
        ],
        [
          0.7571,
          0.7753
        ],
        [
          0.7675,
          0.7671
        ],
        [
          0.7775,
          0.7625
        ],
        [
          0.7875,
          0.7557
        ],
        [
          0.7975,
          0.7511
        ],
        [
          0.8076,
          0.7461
        ],
        [
          0.818,
          0.7393
        ],
        [
          0.828,
          0.7316
        ],
        [
          0.838,
          0.7238
        ],
        [
          0.848,
          0.7175
        ],
        [
          0.8581,
          0.7106
        ],
        [
          0.8685,
          0.7029
        ],
        [
          0.8785,
          0.6956
        ],
        [
          0.8885,
          0.6874
        ],
        [
          0.8985,
          0.6815
        ],
        [
          0.9086,
          0.6742
        ],
        [
          0.919,
          0.6665
        ],
        [
          0.929,
          0.6592
        ],
        [
          0.939,
          0.6515
        ],
        [
          0.949,
          0.6442
        ],
        [
          0.9591,
          0.6365
        ],
        [
          0.9695,
          0.6278
        ],
        [
          0.9795,
          0.6187
        ],
        [
          0.9895,
          0.6119
        ],
        [
          0.9995,
          0.6037
        ],
        [
          1.0,
          0.6037
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.18573384182685432,
        "difference": -0.001015585734672651,
        "worse": 0.54,
        "kept_right": 0.18456364143857015,
        "kept_wrong": 0.0011702003882840923,
        "handed_off": 0.814266158173146
      },
      "usd_per_1000": 4.9545135290552365
    },
    {
      "decision": 44,
      "frontier": 276,
      "frontier_accuracy": 0.9777070063694268,
      "in_sample": {
        "threshold": 0.8952663292322481,
        "answered": 0.18789808917197454,
        "accuracy": 0.9777070063694268
      },
      "curve": [
        [
          0.0,
          0.9777
        ],
        [
          0.01,
          0.9777
        ],
        [
          0.02,
          0.9777
        ],
        [
          0.03,
          0.9777
        ],
        [
          0.04,
          0.9777
        ],
        [
          0.05,
          0.9777
        ],
        [
          0.0601,
          0.9777
        ],
        [
          0.0705,
          0.9777
        ],
        [
          0.0805,
          0.9777
        ],
        [
          0.0905,
          0.9777
        ],
        [
          0.1005,
          0.9777
        ],
        [
          0.1106,
          0.9777
        ],
        [
          0.121,
          0.9777
        ],
        [
          0.131,
          0.9777
        ],
        [
          0.141,
          0.9777
        ],
        [
          0.151,
          0.9777
        ],
        [
          0.1611,
          0.9777
        ],
        [
          0.1715,
          0.9777
        ],
        [
          0.1815,
          0.9777
        ],
        [
          0.1915,
          0.9773
        ],
        [
          0.2015,
          0.9773
        ],
        [
          0.2116,
          0.9763
        ],
        [
          0.222,
          0.9759
        ],
        [
          0.232,
          0.9754
        ],
        [
          0.242,
          0.9745
        ],
        [
          0.252,
          0.9745
        ],
        [
          0.2621,
          0.9736
        ],
        [
          0.2725,
          0.9718
        ],
        [
          0.2825,
          0.9704
        ],
        [
          0.2925,
          0.9682
        ],
        [
          0.3025,
          0.9668
        ],
        [
          0.3126,
          0.9659
        ],
        [
          0.323,
          0.9641
        ],
        [
          0.333,
          0.9622
        ],
        [
          0.343,
          0.9604
        ],
        [
          0.353,
          0.9581
        ],
        [
          0.3631,
          0.9554
        ],
        [
          0.3735,
          0.9536
        ],
        [
          0.3835,
          0.9513
        ],
        [
          0.3935,
          0.9481
        ],
        [
          0.4035,
          0.9449
        ],
        [
          0.4136,
          0.9431
        ],
        [
          0.424,
          0.9381
        ],
        [
          0.434,
          0.9345
        ],
        [
          0.444,
          0.9322
        ],
        [
          0.454,
          0.9268
        ],
        [
          0.4641,
          0.9222
        ],
        [
          0.4745,
          0.9186
        ],
        [
          0.4845,
          0.9154
        ],
        [
          0.4945,
          0.9099
        ],
        [
          0.5045,
          0.9054
        ],
        [
          0.5146,
          0.9008
        ],
        [
          0.525,
          0.8967
        ],
        [
          0.535,
          0.8931
        ],
        [
          0.545,
          0.8894
        ],
        [
          0.5551,
          0.8863
        ],
        [
          0.5655,
          0.8817
        ],
        [
          0.5755,
          0.8776
        ],
        [
          0.5855,
          0.8722
        ],
        [
          0.5955,
          0.8676
        ],
        [
          0.6056,
          0.864
        ],
        [
          0.616,
          0.8567
        ],
        [
          0.626,
          0.853
        ],
        [
          0.636,
          0.8476
        ],
        [
          0.646,
          0.8403
        ],
        [
          0.6561,
          0.8348
        ],
        [
          0.6665,
          0.828
        ],
        [
          0.6765,
          0.8203
        ],
        [
          0.6865,
          0.8153
        ],
        [
          0.6965,
          0.8094
        ],
        [
          0.7066,
          0.8048
        ],
        [
          0.717,
          0.7994
        ],
        [
          0.727,
          0.7939
        ],
        [
          0.737,
          0.7875
        ],
        [
          0.747,
          0.7812
        ],
        [
          0.7571,
          0.7762
        ],
        [
          0.7675,
          0.768
        ],
        [
          0.7775,
          0.7634
        ],
        [
          0.7875,
          0.7566
        ],
        [
          0.7975,
          0.752
        ],
        [
          0.8076,
          0.747
        ],
        [
          0.818,
          0.7402
        ],
        [
          0.828,
          0.7325
        ],
        [
          0.838,
          0.7247
        ],
        [
          0.848,
          0.7184
        ],
        [
          0.8581,
          0.7116
        ],
        [
          0.8685,
          0.7038
        ],
        [
          0.8785,
          0.6965
        ],
        [
          0.8885,
          0.6884
        ],
        [
          0.8985,
          0.682
        ],
        [
          0.9086,
          0.6747
        ],
        [
          0.919,
          0.6665
        ],
        [
          0.929,
          0.6592
        ],
        [
          0.939,
          0.6515
        ],
        [
          0.949,
          0.6442
        ],
        [
          0.9591,
          0.6365
        ],
        [
          0.9695,
          0.6278
        ],
        [
          0.9795,
          0.6187
        ],
        [
          0.9895,
          0.6119
        ],
        [
          0.9995,
          0.6037
        ],
        [
          1.0,
          0.6037
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.19812744622855716,
        "difference": -0.001067142291756824,
        "worse": 0.5466666666666666,
        "kept_right": 0.1963145422794909,
        "kept_wrong": 0.0018129039490661766,
        "handed_off": 0.801872553771443
      },
      "usd_per_1000": 1.3163270979007526
    },
    {
      "decision": 44,
      "frontier": 48,
      "frontier_accuracy": 0.9758871701546861,
      "in_sample": {
        "threshold": 0.9156852342507505,
        "answered": 0.17015468607825296,
        "accuracy": 0.9758871701546861
      },
      "curve": [
        [
          0.0,
          0.9759
        ],
        [
          0.01,
          0.9759
        ],
        [
          0.02,
          0.9759
        ],
        [
          0.03,
          0.9759
        ],
        [
          0.04,
          0.9759
        ],
        [
          0.05,
          0.9759
        ],
        [
          0.0601,
          0.9759
        ],
        [
          0.0705,
          0.9759
        ],
        [
          0.0805,
          0.9759
        ],
        [
          0.0905,
          0.9759
        ],
        [
          0.1005,
          0.9759
        ],
        [
          0.1106,
          0.9759
        ],
        [
          0.121,
          0.9759
        ],
        [
          0.131,
          0.9759
        ],
        [
          0.141,
          0.9759
        ],
        [
          0.151,
          0.9759
        ],
        [
          0.1611,
          0.9759
        ],
        [
          0.1715,
          0.9754
        ],
        [
          0.1815,
          0.9754
        ],
        [
          0.1915,
          0.975
        ],
        [
          0.2015,
          0.975
        ],
        [
          0.2116,
          0.9741
        ],
        [
          0.222,
          0.9736
        ],
        [
          0.232,
          0.9732
        ],
        [
          0.242,
          0.9718
        ],
        [
          0.252,
          0.9713
        ],
        [
          0.2621,
          0.9704
        ],
        [
          0.2725,
          0.9686
        ],
        [
          0.2825,
          0.9672
        ],
        [
          0.2925,
          0.965
        ],
        [
          0.3025,
          0.9636
        ],
        [
          0.3126,
          0.9622
        ],
        [
          0.323,
          0.96
        ],
        [
          0.333,
          0.9581
        ],
        [
          0.343,
          0.9563
        ],
        [
          0.353,
          0.9545
        ],
        [
          0.3631,
          0.9518
        ],
        [
          0.3735,
          0.95
        ],
        [
          0.3835,
          0.9477
        ],
        [
          0.3935,
          0.9445
        ],
        [
          0.4035,
          0.9413
        ],
        [
          0.4136,
          0.9395
        ],
        [
          0.424,
          0.9345
        ],
        [
          0.434,
          0.9308
        ],
        [
          0.444,
          0.9286
        ],
        [
          0.454,
          0.9236
        ],
        [
          0.4641,
          0.919
        ],
        [
          0.4745,
          0.9154
        ],
        [
          0.4845,
          0.9122
        ],
        [
          0.4945,
          0.9067
        ],
        [
          0.5045,
          0.9022
        ],
        [
          0.5146,
          0.8976
        ],
        [
          0.525,
          0.8935
        ],
        [
          0.535,
          0.8899
        ],
        [
          0.545,
          0.8867
        ],
        [
          0.5551,
          0.8831
        ],
        [
          0.5655,
          0.8781
        ],
        [
          0.5755,
          0.874
        ],
        [
          0.5855,
          0.8685
        ],
        [
          0.5955,
          0.864
        ],
        [
          0.6056,
          0.8603
        ],
        [
          0.616,
          0.853
        ],
        [
          0.626,
          0.8494
        ],
        [
          0.636,
          0.8435
        ],
        [
          0.646,
          0.8362
        ],
        [
          0.6561,
          0.8308
        ],
        [
          0.6665,
          0.8239
        ],
        [
          0.6765,
          0.8167
        ],
        [
          0.6865,
          0.8116
        ],
        [
          0.6965,
          0.8057
        ],
        [
          0.7066,
          0.8012
        ],
        [
          0.717,
          0.7957
        ],
        [
          0.727,
          0.7907
        ],
        [
          0.737,
          0.7843
        ],
        [
          0.747,
          0.7784
        ],
        [
          0.7571,
          0.7739
        ],
        [
          0.7675,
          0.7657
        ],
        [
          0.7775,
          0.7611
        ],
        [
          0.7875,
          0.7548
        ],
        [
          0.7975,
          0.7507
        ],
        [
          0.8076,
          0.7457
        ],
        [
          0.818,
          0.7393
        ],
        [
          0.828,
          0.7316
        ],
        [
          0.838,
          0.7238
        ],
        [
          0.848,
          0.7175
        ],
        [
          0.8581,
          0.7111
        ],
        [
          0.8685,
          0.7034
        ],
        [
          0.8785,
          0.6961
        ],
        [
          0.8885,
          0.6879
        ],
        [
          0.8985,
          0.682
        ],
        [
          0.9086,
          0.6752
        ],
        [
          0.919,
          0.667
        ],
        [
          0.929,
          0.6592
        ],
        [
          0.939,
          0.6515
        ],
        [
          0.949,
          0.6442
        ],
        [
          0.9591,
          0.6365
        ],
        [
          0.9695,
          0.6278
        ],
        [
          0.9795,
          0.6187
        ],
        [
          0.9895,
          0.6119
        ],
        [
          0.9995,
          0.6037
        ],
        [
          1.0,
          0.6037
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.18573384182685432,
        "difference": -0.001015585734672651,
        "worse": 0.54,
        "kept_right": 0.18456364143857015,
        "kept_wrong": 0.0011702003882840923,
        "handed_off": 0.814266158173146
      },
      "usd_per_1000": 4.394844943034855
    },
    {
      "decision": 44,
      "frontier": 191,
      "frontier_accuracy": 0.9740673339399454,
      "in_sample": {
        "threshold": 0.9156852342507505,
        "answered": 0.17015468607825296,
        "accuracy": 0.9740673339399454
      },
      "curve": [
        [
          0.0,
          0.9741
        ],
        [
          0.01,
          0.9741
        ],
        [
          0.02,
          0.9741
        ],
        [
          0.03,
          0.9741
        ],
        [
          0.04,
          0.9741
        ],
        [
          0.05,
          0.9741
        ],
        [
          0.0601,
          0.9741
        ],
        [
          0.0705,
          0.9741
        ],
        [
          0.0805,
          0.9741
        ],
        [
          0.0905,
          0.9741
        ],
        [
          0.1005,
          0.9741
        ],
        [
          0.1106,
          0.9741
        ],
        [
          0.121,
          0.9741
        ],
        [
          0.131,
          0.9741
        ],
        [
          0.141,
          0.9741
        ],
        [
          0.151,
          0.9741
        ],
        [
          0.1611,
          0.9741
        ],
        [
          0.1715,
          0.9736
        ],
        [
          0.1815,
          0.9736
        ],
        [
          0.1915,
          0.9732
        ],
        [
          0.2015,
          0.9732
        ],
        [
          0.2116,
          0.9722
        ],
        [
          0.222,
          0.9718
        ],
        [
          0.232,
          0.9713
        ],
        [
          0.242,
          0.97
        ],
        [
          0.252,
          0.9695
        ],
        [
          0.2621,
          0.9686
        ],
        [
          0.2725,
          0.9668
        ],
        [
          0.2825,
          0.9654
        ],
        [
          0.2925,
          0.9631
        ],
        [
          0.3025,
          0.9618
        ],
        [
          0.3126,
          0.9609
        ],
        [
          0.323,
          0.9581
        ],
        [
          0.333,
          0.9568
        ],
        [
          0.343,
          0.9554
        ],
        [
          0.353,
          0.9531
        ],
        [
          0.3631,
          0.9504
        ],
        [
          0.3735,
          0.9481
        ],
        [
          0.3835,
          0.9459
        ],
        [
          0.3935,
          0.9427
        ],
        [
          0.4035,
          0.9395
        ],
        [
          0.4136,
          0.9377
        ],
        [
          0.424,
          0.9327
        ],
        [
          0.434,
          0.929
        ],
        [
          0.444,
          0.9263
        ],
        [
          0.454,
          0.9208
        ],
        [
          0.4641,
          0.9163
        ],
        [
          0.4745,
          0.9126
        ],
        [
          0.4845,
          0.9095
        ],
        [
          0.4945,
          0.9045
        ],
        [
          0.5045,
          0.8999
        ],
        [
          0.5146,
          0.8954
        ],
        [
          0.525,
          0.8913
        ],
        [
          0.535,
          0.8876
        ],
        [
          0.545,
          0.8844
        ],
        [
          0.5551,
          0.8808
        ],
        [
          0.5655,
          0.8763
        ],
        [
          0.5755,
          0.8722
        ],
        [
          0.5855,
          0.8667
        ],
        [
          0.5955,
          0.8626
        ],
        [
          0.6056,
          0.8585
        ],
        [
          0.616,
          0.8517
        ],
        [
          0.626,
          0.848
        ],
        [
          0.636,
          0.8426
        ],
        [
          0.646,
          0.8353
        ],
        [
          0.6561,
          0.8298
        ],
        [
          0.6665,
          0.823
        ],
        [
          0.6765,
          0.8157
        ],
        [
          0.6865,
          0.8116
        ],
        [
          0.6965,
          0.8057
        ],
        [
          0.7066,
          0.8012
        ],
        [
          0.717,
          0.7957
        ],
        [
          0.727,
          0.7903
        ],
        [
          0.737,
          0.7839
        ],
        [
          0.747,
          0.7775
        ],
        [
          0.7571,
          0.773
        ],
        [
          0.7675,
          0.7648
        ],
        [
          0.7775,
          0.7602
        ],
        [
          0.7875,
          0.7543
        ],
        [
          0.7975,
          0.7498
        ],
        [
          0.8076,
          0.7448
        ],
        [
          0.818,
          0.7384
        ],
        [
          0.828,
          0.7307
        ],
        [
          0.838,
          0.7229
        ],
        [
          0.848,
          0.7166
        ],
        [
          0.8581,
          0.7102
        ],
        [
          0.8685,
          0.7025
        ],
        [
          0.8785,
          0.6947
        ],
        [
          0.8885,
          0.687
        ],
        [
          0.8985,
          0.6811
        ],
        [
          0.9086,
          0.6742
        ],
        [
          0.919,
          0.6661
        ],
        [
          0.929,
          0.6588
        ],
        [
          0.939,
          0.651
        ],
        [
          0.949,
          0.6442
        ],
        [
          0.9591,
          0.6365
        ],
        [
          0.9695,
          0.6278
        ],
        [
          0.9795,
          0.6187
        ],
        [
          0.9895,
          0.6119
        ],
        [
          0.9995,
          0.6037
        ],
        [
          1.0,
          0.6037
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.18573384182685432,
        "difference": -0.001015585734672651,
        "worse": 0.54,
        "kept_right": 0.18456364143857015,
        "kept_wrong": 0.0011702003882840923,
        "handed_off": 0.814266158173146
      },
      "usd_per_1000": 21.78713839271634
    },
    {
      "decision": 44,
      "frontier": 188,
      "frontier_accuracy": 0.9713375796178344,
      "in_sample": {
        "threshold": 0.9156852342507505,
        "answered": 0.17015468607825296,
        "accuracy": 0.9713375796178344
      },
      "curve": [
        [
          0.0,
          0.9713
        ],
        [
          0.01,
          0.9713
        ],
        [
          0.02,
          0.9713
        ],
        [
          0.03,
          0.9713
        ],
        [
          0.04,
          0.9713
        ],
        [
          0.05,
          0.9713
        ],
        [
          0.0601,
          0.9713
        ],
        [
          0.0705,
          0.9713
        ],
        [
          0.0805,
          0.9713
        ],
        [
          0.0905,
          0.9713
        ],
        [
          0.1005,
          0.9713
        ],
        [
          0.1106,
          0.9713
        ],
        [
          0.121,
          0.9713
        ],
        [
          0.131,
          0.9713
        ],
        [
          0.141,
          0.9713
        ],
        [
          0.151,
          0.9713
        ],
        [
          0.1611,
          0.9713
        ],
        [
          0.1715,
          0.9709
        ],
        [
          0.1815,
          0.9709
        ],
        [
          0.1915,
          0.9704
        ],
        [
          0.2015,
          0.9704
        ],
        [
          0.2116,
          0.97
        ],
        [
          0.222,
          0.9704
        ],
        [
          0.232,
          0.97
        ],
        [
          0.242,
          0.9686
        ],
        [
          0.252,
          0.9682
        ],
        [
          0.2621,
          0.9672
        ],
        [
          0.2725,
          0.9659
        ],
        [
          0.2825,
          0.9645
        ],
        [
          0.2925,
          0.9622
        ],
        [
          0.3025,
          0.9609
        ],
        [
          0.3126,
          0.96
        ],
        [
          0.323,
          0.9577
        ],
        [
          0.333,
          0.9559
        ],
        [
          0.343,
          0.955
        ],
        [
          0.353,
          0.9527
        ],
        [
          0.3631,
          0.95
        ],
        [
          0.3735,
          0.9477
        ],
        [
          0.3835,
          0.9454
        ],
        [
          0.3935,
          0.9422
        ],
        [
          0.4035,
          0.9395
        ],
        [
          0.4136,
          0.9381
        ],
        [
          0.424,
          0.9331
        ],
        [
          0.434,
          0.929
        ],
        [
          0.444,
          0.9263
        ],
        [
          0.454,
          0.9208
        ],
        [
          0.4641,
          0.9163
        ],
        [
          0.4745,
          0.9131
        ],
        [
          0.4845,
          0.9099
        ],
        [
          0.4945,
          0.9049
        ],
        [
          0.5045,
          0.9004
        ],
        [
          0.5146,
          0.8958
        ],
        [
          0.525,
          0.8917
        ],
        [
          0.535,
          0.8881
        ],
        [
          0.545,
          0.8854
        ],
        [
          0.5551,
          0.8822
        ],
        [
          0.5655,
          0.8767
        ],
        [
          0.5755,
          0.8726
        ],
        [
          0.5855,
          0.8667
        ],
        [
          0.5955,
          0.8621
        ],
        [
          0.6056,
          0.8585
        ],
        [
          0.616,
          0.8517
        ],
        [
          0.626,
          0.8485
        ],
        [
          0.636,
          0.843
        ],
        [
          0.646,
          0.8358
        ],
        [
          0.6561,
          0.8298
        ],
        [
          0.6665,
          0.823
        ],
        [
          0.6765,
          0.8157
        ],
        [
          0.6865,
          0.8107
        ],
        [
          0.6965,
          0.8048
        ],
        [
          0.7066,
          0.8003
        ],
        [
          0.717,
          0.7948
        ],
        [
          0.727,
          0.7894
        ],
        [
          0.737,
          0.783
        ],
        [
          0.747,
          0.7766
        ],
        [
          0.7571,
          0.7716
        ],
        [
          0.7675,
          0.763
        ],
        [
          0.7775,
          0.7584
        ],
        [
          0.7875,
          0.752
        ],
        [
          0.7975,
          0.748
        ],
        [
          0.8076,
          0.7429
        ],
        [
          0.818,
          0.737
        ],
        [
          0.828,
          0.7293
        ],
        [
          0.838,
          0.722
        ],
        [
          0.848,
          0.7157
        ],
        [
          0.8581,
          0.7088
        ],
        [
          0.8685,
          0.7011
        ],
        [
          0.8785,
          0.6938
        ],
        [
          0.8885,
          0.6856
        ],
        [
          0.8985,
          0.6797
        ],
        [
          0.9086,
          0.6724
        ],
        [
          0.919,
          0.6652
        ],
        [
          0.929,
          0.6579
        ],
        [
          0.939,
          0.6506
        ],
        [
          0.949,
          0.6442
        ],
        [
          0.9591,
          0.6365
        ],
        [
          0.9695,
          0.6278
        ],
        [
          0.9795,
          0.6187
        ],
        [
          0.9895,
          0.6119
        ],
        [
          0.9995,
          0.6037
        ],
        [
          1.0,
          0.6037
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.2047039025429684,
        "difference": -0.0015945120465445603,
        "worse": 0.69,
        "kept_right": 0.20225453505271784,
        "kept_wrong": 0.002449367490250333,
        "handed_off": 0.7952960974570318
      },
      "usd_per_1000": 0.7817572595249216
    },
    {
      "decision": 44,
      "frontier": 190,
      "frontier_accuracy": 0.9672429481346679,
      "in_sample": {
        "threshold": 0.8952663292322481,
        "answered": 0.18789808917197454,
        "accuracy": 0.9672429481346679
      },
      "curve": [
        [
          0.0,
          0.9672
        ],
        [
          0.01,
          0.9672
        ],
        [
          0.02,
          0.9672
        ],
        [
          0.03,
          0.9672
        ],
        [
          0.04,
          0.9672
        ],
        [
          0.05,
          0.9672
        ],
        [
          0.0601,
          0.9672
        ],
        [
          0.0705,
          0.9672
        ],
        [
          0.0805,
          0.9672
        ],
        [
          0.0905,
          0.9672
        ],
        [
          0.1005,
          0.9672
        ],
        [
          0.1106,
          0.9672
        ],
        [
          0.121,
          0.9672
        ],
        [
          0.131,
          0.9672
        ],
        [
          0.141,
          0.9672
        ],
        [
          0.151,
          0.9672
        ],
        [
          0.1611,
          0.9672
        ],
        [
          0.1715,
          0.9668
        ],
        [
          0.1815,
          0.9672
        ],
        [
          0.1915,
          0.9668
        ],
        [
          0.2015,
          0.9668
        ],
        [
          0.2116,
          0.9659
        ],
        [
          0.222,
          0.9654
        ],
        [
          0.232,
          0.965
        ],
        [
          0.242,
          0.9641
        ],
        [
          0.252,
          0.9636
        ],
        [
          0.2621,
          0.9627
        ],
        [
          0.2725,
          0.9613
        ],
        [
          0.2825,
          0.96
        ],
        [
          0.2925,
          0.9577
        ],
        [
          0.3025,
          0.9563
        ],
        [
          0.3126,
          0.9554
        ],
        [
          0.323,
          0.9527
        ],
        [
          0.333,
          0.9513
        ],
        [
          0.343,
          0.9495
        ],
        [
          0.353,
          0.9472
        ],
        [
          0.3631,
          0.9445
        ],
        [
          0.3735,
          0.9427
        ],
        [
          0.3835,
          0.9404
        ],
        [
          0.3935,
          0.9372
        ],
        [
          0.4035,
          0.934
        ],
        [
          0.4136,
          0.9322
        ],
        [
          0.424,
          0.9272
        ],
        [
          0.434,
          0.9236
        ],
        [
          0.444,
          0.9217
        ],
        [
          0.454,
          0.9163
        ],
        [
          0.4641,
          0.9117
        ],
        [
          0.4745,
          0.9081
        ],
        [
          0.4845,
          0.9054
        ],
        [
          0.4945,
          0.9004
        ],
        [
          0.5045,
          0.8958
        ],
        [
          0.5146,
          0.8913
        ],
        [
          0.525,
          0.8872
        ],
        [
          0.535,
          0.8835
        ],
        [
          0.545,
          0.8803
        ],
        [
          0.5551,
          0.8772
        ],
        [
          0.5655,
          0.8726
        ],
        [
          0.5755,
          0.8685
        ],
        [
          0.5855,
          0.8635
        ],
        [
          0.5955,
          0.8599
        ],
        [
          0.6056,
          0.8562
        ],
        [
          0.616,
          0.849
        ],
        [
          0.626,
          0.8453
        ],
        [
          0.636,
          0.8394
        ],
        [
          0.646,
          0.8326
        ],
        [
          0.6561,
          0.8271
        ],
        [
          0.6665,
          0.8203
        ],
        [
          0.6765,
          0.813
        ],
        [
          0.6865,
          0.8089
        ],
        [
          0.6965,
          0.803
        ],
        [
          0.7066,
          0.7985
        ],
        [
          0.717,
          0.793
        ],
        [
          0.727,
          0.7875
        ],
        [
          0.737,
          0.7812
        ],
        [
          0.747,
          0.7753
        ],
        [
          0.7571,
          0.7707
        ],
        [
          0.7675,
          0.763
        ],
        [
          0.7775,
          0.7584
        ],
        [
          0.7875,
          0.7525
        ],
        [
          0.7975,
          0.7484
        ],
        [
          0.8076,
          0.7434
        ],
        [
          0.818,
          0.7366
        ],
        [
          0.828,
          0.7288
        ],
        [
          0.838,
          0.7211
        ],
        [
          0.848,
          0.7147
        ],
        [
          0.8581,
          0.7084
        ],
        [
          0.8685,
          0.7006
        ],
        [
          0.8785,
          0.6929
        ],
        [
          0.8885,
          0.6852
        ],
        [
          0.8985,
          0.6793
        ],
        [
          0.9086,
          0.6724
        ],
        [
          0.919,
          0.6647
        ],
        [
          0.929,
          0.6579
        ],
        [
          0.939,
          0.6497
        ],
        [
          0.949,
          0.6429
        ],
        [
          0.9591,
          0.6356
        ],
        [
          0.9695,
          0.6278
        ],
        [
          0.9795,
          0.6187
        ],
        [
          0.9895,
          0.6119
        ],
        [
          0.9995,
          0.6037
        ],
        [
          1.0,
          0.6037
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.19751957025730268,
        "difference": -0.0012398672733696264,
        "worse": 0.54,
        "kept_right": 0.19572793617815226,
        "kept_wrong": 0.0017916340791503508,
        "handed_off": 0.8024804297426975
      },
      "usd_per_1000": 4.484164855290982
    },
    {
      "decision": 44,
      "frontier": 124,
      "frontier_accuracy": 0.9667879890809827,
      "in_sample": {
        "threshold": 0.9156852342507505,
        "answered": 0.17015468607825296,
        "accuracy": 0.9667879890809827
      },
      "curve": [
        [
          0.0,
          0.9668
        ],
        [
          0.01,
          0.9668
        ],
        [
          0.02,
          0.9668
        ],
        [
          0.03,
          0.9668
        ],
        [
          0.04,
          0.9668
        ],
        [
          0.05,
          0.9668
        ],
        [
          0.0601,
          0.9668
        ],
        [
          0.0705,
          0.9668
        ],
        [
          0.0805,
          0.9668
        ],
        [
          0.0905,
          0.9668
        ],
        [
          0.1005,
          0.9668
        ],
        [
          0.1106,
          0.9668
        ],
        [
          0.121,
          0.9668
        ],
        [
          0.131,
          0.9668
        ],
        [
          0.141,
          0.9668
        ],
        [
          0.151,
          0.9668
        ],
        [
          0.1611,
          0.9668
        ],
        [
          0.1715,
          0.9663
        ],
        [
          0.1815,
          0.9663
        ],
        [
          0.1915,
          0.9659
        ],
        [
          0.2015,
          0.9659
        ],
        [
          0.2116,
          0.965
        ],
        [
          0.222,
          0.9645
        ],
        [
          0.232,
          0.9641
        ],
        [
          0.242,
          0.9631
        ],
        [
          0.252,
          0.9631
        ],
        [
          0.2621,
          0.9622
        ],
        [
          0.2725,
          0.9609
        ],
        [
          0.2825,
          0.9595
        ],
        [
          0.2925,
          0.9572
        ],
        [
          0.3025,
          0.9559
        ],
        [
          0.3126,
          0.9545
        ],
        [
          0.323,
          0.9527
        ],
        [
          0.333,
          0.9509
        ],
        [
          0.343,
          0.949
        ],
        [
          0.353,
          0.9472
        ],
        [
          0.3631,
          0.9445
        ],
        [
          0.3735,
          0.9427
        ],
        [
          0.3835,
          0.9404
        ],
        [
          0.3935,
          0.9372
        ],
        [
          0.4035,
          0.934
        ],
        [
          0.4136,
          0.9322
        ],
        [
          0.424,
          0.9272
        ],
        [
          0.434,
          0.9236
        ],
        [
          0.444,
          0.9213
        ],
        [
          0.454,
          0.9163
        ],
        [
          0.4641,
          0.9117
        ],
        [
          0.4745,
          0.9081
        ],
        [
          0.4845,
          0.9049
        ],
        [
          0.4945,
          0.8995
        ],
        [
          0.5045,
          0.8949
        ],
        [
          0.5146,
          0.8904
        ],
        [
          0.525,
          0.8863
        ],
        [
          0.535,
          0.8826
        ],
        [
          0.545,
          0.8799
        ],
        [
          0.5551,
          0.8767
        ],
        [
          0.5655,
          0.8717
        ],
        [
          0.5755,
          0.8676
        ],
        [
          0.5855,
          0.8621
        ],
        [
          0.5955,
          0.8581
        ],
        [
          0.6056,
          0.854
        ],
        [
          0.616,
          0.8471
        ],
        [
          0.626,
          0.8439
        ],
        [
          0.636,
          0.8385
        ],
        [
          0.646,
          0.8312
        ],
        [
          0.6561,
          0.8258
        ],
        [
          0.6665,
          0.8194
        ],
        [
          0.6765,
          0.8121
        ],
        [
          0.6865,
          0.808
        ],
        [
          0.6965,
          0.8025
        ],
        [
          0.7066,
          0.798
        ],
        [
          0.717,
          0.7925
        ],
        [
          0.727,
          0.7875
        ],
        [
          0.737,
          0.7812
        ],
        [
          0.747,
          0.7753
        ],
        [
          0.7571,
          0.7707
        ],
        [
          0.7675,
          0.7625
        ],
        [
          0.7775,
          0.758
        ],
        [
          0.7875,
          0.752
        ],
        [
          0.7975,
          0.7475
        ],
        [
          0.8076,
          0.7425
        ],
        [
          0.818,
          0.7357
        ],
        [
          0.828,
          0.7279
        ],
        [
          0.838,
          0.7202
        ],
        [
          0.848,
          0.7138
        ],
        [
          0.8581,
          0.7079
        ],
        [
          0.8685,
          0.7006
        ],
        [
          0.8785,
          0.6934
        ],
        [
          0.8885,
          0.6856
        ],
        [
          0.8985,
          0.6793
        ],
        [
          0.9086,
          0.6724
        ],
        [
          0.919,
          0.6652
        ],
        [
          0.929,
          0.6579
        ],
        [
          0.939,
          0.6506
        ],
        [
          0.949,
          0.6433
        ],
        [
          0.9591,
          0.636
        ],
        [
          0.9695,
          0.6278
        ],
        [
          0.9795,
          0.6187
        ],
        [
          0.9895,
          0.6119
        ],
        [
          0.9995,
          0.6037
        ],
        [
          1.0,
          0.6037
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.18584313144434064,
        "difference": -0.0010155940066691302,
        "worse": 0.54,
        "kept_right": 0.1846607877652247,
        "kept_wrong": 0.0011823436791159076,
        "handed_off": 0.8141568685556596
      },
      "usd_per_1000": 0.9358383173264084
    },
    {
      "decision": 44,
      "frontier": 278,
      "frontier_accuracy": 0.9499545040946314,
      "in_sample": {
        "threshold": 0.9156852342507505,
        "answered": 0.17015468607825296,
        "accuracy": 0.9499545040946314
      },
      "curve": [
        [
          0.0,
          0.95
        ],
        [
          0.01,
          0.95
        ],
        [
          0.02,
          0.95
        ],
        [
          0.03,
          0.95
        ],
        [
          0.04,
          0.95
        ],
        [
          0.05,
          0.95
        ],
        [
          0.0601,
          0.95
        ],
        [
          0.0705,
          0.95
        ],
        [
          0.0805,
          0.95
        ],
        [
          0.0905,
          0.95
        ],
        [
          0.1005,
          0.95
        ],
        [
          0.1106,
          0.95
        ],
        [
          0.121,
          0.95
        ],
        [
          0.131,
          0.95
        ],
        [
          0.141,
          0.95
        ],
        [
          0.151,
          0.95
        ],
        [
          0.1611,
          0.95
        ],
        [
          0.1715,
          0.9495
        ],
        [
          0.1815,
          0.9495
        ],
        [
          0.1915,
          0.949
        ],
        [
          0.2015,
          0.949
        ],
        [
          0.2116,
          0.9486
        ],
        [
          0.222,
          0.9481
        ],
        [
          0.232,
          0.9477
        ],
        [
          0.242,
          0.9463
        ],
        [
          0.252,
          0.9459
        ],
        [
          0.2621,
          0.9454
        ],
        [
          0.2725,
          0.944
        ],
        [
          0.2825,
          0.9431
        ],
        [
          0.2925,
          0.9418
        ],
        [
          0.3025,
          0.9404
        ],
        [
          0.3126,
          0.9395
        ],
        [
          0.323,
          0.9377
        ],
        [
          0.333,
          0.9359
        ],
        [
          0.343,
          0.9345
        ],
        [
          0.353,
          0.9327
        ],
        [
          0.3631,
          0.9304
        ],
        [
          0.3735,
          0.929
        ],
        [
          0.3835,
          0.9268
        ],
        [
          0.3935,
          0.9236
        ],
        [
          0.4035,
          0.9208
        ],
        [
          0.4136,
          0.9195
        ],
        [
          0.424,
          0.9149
        ],
        [
          0.434,
          0.9108
        ],
        [
          0.444,
          0.9086
        ],
        [
          0.454,
          0.904
        ],
        [
          0.4641,
          0.8995
        ],
        [
          0.4745,
          0.8963
        ],
        [
          0.4845,
          0.8935
        ],
        [
          0.4945,
          0.8881
        ],
        [
          0.5045,
          0.8849
        ],
        [
          0.5146,
          0.8803
        ],
        [
          0.525,
          0.8763
        ],
        [
          0.535,
          0.8726
        ],
        [
          0.545,
          0.8703
        ],
        [
          0.5551,
          0.8672
        ],
        [
          0.5655,
          0.8631
        ],
        [
          0.5755,
          0.859
        ],
        [
          0.5855,
          0.853
        ],
        [
          0.5955,
          0.849
        ],
        [
          0.6056,
          0.8453
        ],
        [
          0.616,
          0.8389
        ],
        [
          0.626,
          0.8362
        ],
        [
          0.636,
          0.8312
        ],
        [
          0.646,
          0.8244
        ],
        [
          0.6561,
          0.8189
        ],
        [
          0.6665,
          0.8126
        ],
        [
          0.6765,
          0.8066
        ],
        [
          0.6865,
          0.8016
        ],
        [
          0.6965,
          0.7962
        ],
        [
          0.7066,
          0.7916
        ],
        [
          0.717,
          0.7871
        ],
        [
          0.727,
          0.7821
        ],
        [
          0.737,
          0.7757
        ],
        [
          0.747,
          0.7698
        ],
        [
          0.7571,
          0.7648
        ],
        [
          0.7675,
          0.7566
        ],
        [
          0.7775,
          0.752
        ],
        [
          0.7875,
          0.7461
        ],
        [
          0.7975,
          0.742
        ],
        [
          0.8076,
          0.737
        ],
        [
          0.818,
          0.7316
        ],
        [
          0.828,
          0.7243
        ],
        [
          0.838,
          0.7166
        ],
        [
          0.848,
          0.7102
        ],
        [
          0.8581,
          0.7038
        ],
        [
          0.8685,
          0.6984
        ],
        [
          0.8785,
          0.692
        ],
        [
          0.8885,
          0.6838
        ],
        [
          0.8985,
          0.6779
        ],
        [
          0.9086,
          0.6706
        ],
        [
          0.919,
          0.6629
        ],
        [
          0.929,
          0.657
        ],
        [
          0.939,
          0.6497
        ],
        [
          0.949,
          0.6424
        ],
        [
          0.9591,
          0.6347
        ],
        [
          0.9695,
          0.626
        ],
        [
          0.9795,
          0.6174
        ],
        [
          0.9895,
          0.6115
        ],
        [
          0.9995,
          0.6037
        ],
        [
          1.0,
          0.6037
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1876594592817764,
        "difference": -0.0010489851937530734,
        "worse": 0.54,
        "kept_right": 0.18633156995000147,
        "kept_wrong": 0.0013278893317747297,
        "handed_off": 0.8123405407182239
      },
      "usd_per_1000": 5.302920794515058
    },
    {
      "decision": 44,
      "frontier": 275,
      "frontier_accuracy": 0.9426751592356688,
      "in_sample": {
        "threshold": 0.9156852342507505,
        "answered": 0.17015468607825296,
        "accuracy": 0.9426751592356688
      },
      "curve": [
        [
          0.0,
          0.9427
        ],
        [
          0.01,
          0.9427
        ],
        [
          0.02,
          0.9427
        ],
        [
          0.03,
          0.9427
        ],
        [
          0.04,
          0.9427
        ],
        [
          0.05,
          0.9427
        ],
        [
          0.0601,
          0.9427
        ],
        [
          0.0705,
          0.9427
        ],
        [
          0.0805,
          0.9427
        ],
        [
          0.0905,
          0.9427
        ],
        [
          0.1005,
          0.9427
        ],
        [
          0.1106,
          0.9427
        ],
        [
          0.121,
          0.9427
        ],
        [
          0.131,
          0.9427
        ],
        [
          0.141,
          0.9427
        ],
        [
          0.151,
          0.9427
        ],
        [
          0.1611,
          0.9427
        ],
        [
          0.1715,
          0.9422
        ],
        [
          0.1815,
          0.9422
        ],
        [
          0.1915,
          0.9418
        ],
        [
          0.2015,
          0.9418
        ],
        [
          0.2116,
          0.9409
        ],
        [
          0.222,
          0.9404
        ],
        [
          0.232,
          0.9399
        ],
        [
          0.242,
          0.9386
        ],
        [
          0.252,
          0.9381
        ],
        [
          0.2621,
          0.9377
        ],
        [
          0.2725,
          0.9363
        ],
        [
          0.2825,
          0.9349
        ],
        [
          0.2925,
          0.9331
        ],
        [
          0.3025,
          0.9318
        ],
        [
          0.3126,
          0.9308
        ],
        [
          0.323,
          0.929
        ],
        [
          0.333,
          0.9272
        ],
        [
          0.343,
          0.9263
        ],
        [
          0.353,
          0.924
        ],
        [
          0.3631,
          0.9217
        ],
        [
          0.3735,
          0.9199
        ],
        [
          0.3835,
          0.9181
        ],
        [
          0.3935,
          0.9163
        ],
        [
          0.4035,
          0.9136
        ],
        [
          0.4136,
          0.9117
        ],
        [
          0.424,
          0.9076
        ],
        [
          0.434,
          0.904
        ],
        [
          0.444,
          0.9013
        ],
        [
          0.454,
          0.8963
        ],
        [
          0.4641,
          0.8917
        ],
        [
          0.4745,
          0.8885
        ],
        [
          0.4845,
          0.8854
        ],
        [
          0.4945,
          0.8799
        ],
        [
          0.5045,
          0.8767
        ],
        [
          0.5146,
          0.8726
        ],
        [
          0.525,
          0.8685
        ],
        [
          0.535,
          0.8649
        ],
        [
          0.545,
          0.8626
        ],
        [
          0.5551,
          0.8599
        ],
        [
          0.5655,
          0.8558
        ],
        [
          0.5755,
          0.8517
        ],
        [
          0.5855,
          0.8467
        ],
        [
          0.5955,
          0.8435
        ],
        [
          0.6056,
          0.8399
        ],
        [
          0.616,
          0.8335
        ],
        [
          0.626,
          0.8303
        ],
        [
          0.636,
          0.8248
        ],
        [
          0.646,
          0.8176
        ],
        [
          0.6561,
          0.813
        ],
        [
          0.6665,
          0.8062
        ],
        [
          0.6765,
          0.7994
        ],
        [
          0.6865,
          0.7953
        ],
        [
          0.6965,
          0.7907
        ],
        [
          0.7066,
          0.7871
        ],
        [
          0.717,
          0.7821
        ],
        [
          0.727,
          0.7775
        ],
        [
          0.737,
          0.7716
        ],
        [
          0.747,
          0.7657
        ],
        [
          0.7571,
          0.7607
        ],
        [
          0.7675,
          0.7525
        ],
        [
          0.7775,
          0.7489
        ],
        [
          0.7875,
          0.7429
        ],
        [
          0.7975,
          0.7384
        ],
        [
          0.8076,
          0.7343
        ],
        [
          0.818,
          0.7284
        ],
        [
          0.828,
          0.7207
        ],
        [
          0.838,
          0.7129
        ],
        [
          0.848,
          0.7075
        ],
        [
          0.8581,
          0.702
        ],
        [
          0.8685,
          0.6961
        ],
        [
          0.8785,
          0.6902
        ],
        [
          0.8885,
          0.6824
        ],
        [
          0.8985,
          0.677
        ],
        [
          0.9086,
          0.6711
        ],
        [
          0.919,
          0.6633
        ],
        [
          0.929,
          0.6565
        ],
        [
          0.939,
          0.6483
        ],
        [
          0.949,
          0.641
        ],
        [
          0.9591,
          0.6338
        ],
        [
          0.9695,
          0.6251
        ],
        [
          0.9795,
          0.6169
        ],
        [
          0.9895,
          0.611
        ],
        [
          0.9995,
          0.6037
        ],
        [
          1.0,
          0.6037
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.18573384182685432,
        "difference": -0.001015585734672651,
        "worse": 0.54,
        "kept_right": 0.18456364143857015,
        "kept_wrong": 0.0011702003882840923,
        "handed_off": 0.814266158173146
      },
      "usd_per_1000": 0.4344880029221141
    },
    {
      "decision": 44,
      "frontier": 192,
      "frontier_accuracy": 0.9331210191082803,
      "in_sample": {
        "threshold": 0.8952663292322481,
        "answered": 0.18789808917197454,
        "accuracy": 0.9331210191082803
      },
      "curve": [
        [
          0.0,
          0.9331
        ],
        [
          0.01,
          0.9331
        ],
        [
          0.02,
          0.9331
        ],
        [
          0.03,
          0.9331
        ],
        [
          0.04,
          0.9331
        ],
        [
          0.05,
          0.9331
        ],
        [
          0.0601,
          0.9331
        ],
        [
          0.0705,
          0.9331
        ],
        [
          0.0805,
          0.9331
        ],
        [
          0.0905,
          0.9331
        ],
        [
          0.1005,
          0.9331
        ],
        [
          0.1106,
          0.9331
        ],
        [
          0.121,
          0.9331
        ],
        [
          0.131,
          0.9331
        ],
        [
          0.141,
          0.9331
        ],
        [
          0.151,
          0.9331
        ],
        [
          0.1611,
          0.9331
        ],
        [
          0.1715,
          0.9327
        ],
        [
          0.1815,
          0.9331
        ],
        [
          0.1915,
          0.9327
        ],
        [
          0.2015,
          0.9327
        ],
        [
          0.2116,
          0.9322
        ],
        [
          0.222,
          0.9318
        ],
        [
          0.232,
          0.9313
        ],
        [
          0.242,
          0.9299
        ],
        [
          0.252,
          0.9299
        ],
        [
          0.2621,
          0.929
        ],
        [
          0.2725,
          0.9272
        ],
        [
          0.2825,
          0.9254
        ],
        [
          0.2925,
          0.924
        ],
        [
          0.3025,
          0.9227
        ],
        [
          0.3126,
          0.9222
        ],
        [
          0.323,
          0.9204
        ],
        [
          0.333,
          0.919
        ],
        [
          0.343,
          0.9172
        ],
        [
          0.353,
          0.9158
        ],
        [
          0.3631,
          0.9131
        ],
        [
          0.3735,
          0.9122
        ],
        [
          0.3835,
          0.9099
        ],
        [
          0.3935,
          0.9072
        ],
        [
          0.4035,
          0.904
        ],
        [
          0.4136,
          0.9022
        ],
        [
          0.424,
          0.8985
        ],
        [
          0.434,
          0.8958
        ],
        [
          0.444,
          0.8931
        ],
        [
          0.454,
          0.8885
        ],
        [
          0.4641,
          0.8844
        ],
        [
          0.4745,
          0.8813
        ],
        [
          0.4845,
          0.879
        ],
        [
          0.4945,
          0.8744
        ],
        [
          0.5045,
          0.8708
        ],
        [
          0.5146,
          0.8667
        ],
        [
          0.525,
          0.8626
        ],
        [
          0.535,
          0.859
        ],
        [
          0.545,
          0.8562
        ],
        [
          0.5551,
          0.853
        ],
        [
          0.5655,
          0.8494
        ],
        [
          0.5755,
          0.8453
        ],
        [
          0.5855,
          0.8408
        ],
        [
          0.5955,
          0.8376
        ],
        [
          0.6056,
          0.8344
        ],
        [
          0.616,
          0.828
        ],
        [
          0.626,
          0.8253
        ],
        [
          0.636,
          0.8198
        ],
        [
          0.646,
          0.8135
        ],
        [
          0.6561,
          0.8076
        ],
        [
          0.6665,
          0.8025
        ],
        [
          0.6765,
          0.7966
        ],
        [
          0.6865,
          0.793
        ],
        [
          0.6965,
          0.7875
        ],
        [
          0.7066,
          0.783
        ],
        [
          0.717,
          0.7784
        ],
        [
          0.727,
          0.773
        ],
        [
          0.737,
          0.7671
        ],
        [
          0.747,
          0.7621
        ],
        [
          0.7571,
          0.7571
        ],
        [
          0.7675,
          0.7493
        ],
        [
          0.7775,
          0.7452
        ],
        [
          0.7875,
          0.7393
        ],
        [
          0.7975,
          0.7352
        ],
        [
          0.8076,
          0.7316
        ],
        [
          0.818,
          0.7257
        ],
        [
          0.828,
          0.7193
        ],
        [
          0.838,
          0.712
        ],
        [
          0.848,
          0.7061
        ],
        [
          0.8581,
          0.7002
        ],
        [
          0.8685,
          0.6938
        ],
        [
          0.8785,
          0.6874
        ],
        [
          0.8885,
          0.6797
        ],
        [
          0.8985,
          0.6738
        ],
        [
          0.9086,
          0.667
        ],
        [
          0.919,
          0.6601
        ],
        [
          0.929,
          0.6556
        ],
        [
          0.939,
          0.6483
        ],
        [
          0.949,
          0.641
        ],
        [
          0.9591,
          0.6342
        ],
        [
          0.9695,
          0.626
        ],
        [
          0.9795,
          0.6174
        ],
        [
          0.9895,
          0.6115
        ],
        [
          0.9995,
          0.6037
        ],
        [
          1.0,
          0.6037
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.20148350783099828,
        "difference": -0.0013640352289786394,
        "worse": 0.5433333333333333,
        "kept_right": 0.19936159528248365,
        "kept_wrong": 0.002121912548514412,
        "handed_off": 0.798516492169002
      },
      "usd_per_1000": 9.285966705878371
    },
    {
      "decision": 44,
      "frontier": 123,
      "frontier_accuracy": 0.9326660600545951,
      "in_sample": {
        "threshold": 0.8952663292322481,
        "answered": 0.18789808917197454,
        "accuracy": 0.9326660600545951
      },
      "curve": [
        [
          0.0,
          0.9327
        ],
        [
          0.01,
          0.9327
        ],
        [
          0.02,
          0.9327
        ],
        [
          0.03,
          0.9327
        ],
        [
          0.04,
          0.9327
        ],
        [
          0.05,
          0.9327
        ],
        [
          0.0601,
          0.9327
        ],
        [
          0.0705,
          0.9327
        ],
        [
          0.0805,
          0.9327
        ],
        [
          0.0905,
          0.9327
        ],
        [
          0.1005,
          0.9327
        ],
        [
          0.1106,
          0.9327
        ],
        [
          0.121,
          0.9327
        ],
        [
          0.131,
          0.9327
        ],
        [
          0.141,
          0.9327
        ],
        [
          0.151,
          0.9327
        ],
        [
          0.1611,
          0.9331
        ],
        [
          0.1715,
          0.9327
        ],
        [
          0.1815,
          0.9327
        ],
        [
          0.1915,
          0.9322
        ],
        [
          0.2015,
          0.9322
        ],
        [
          0.2116,
          0.9318
        ],
        [
          0.222,
          0.9318
        ],
        [
          0.232,
          0.9313
        ],
        [
          0.242,
          0.9299
        ],
        [
          0.252,
          0.9295
        ],
        [
          0.2621,
          0.9286
        ],
        [
          0.2725,
          0.9272
        ],
        [
          0.2825,
          0.9258
        ],
        [
          0.2925,
          0.9236
        ],
        [
          0.3025,
          0.9222
        ],
        [
          0.3126,
          0.9213
        ],
        [
          0.323,
          0.919
        ],
        [
          0.333,
          0.9172
        ],
        [
          0.343,
          0.9163
        ],
        [
          0.353,
          0.9145
        ],
        [
          0.3631,
          0.9117
        ],
        [
          0.3735,
          0.9095
        ],
        [
          0.3835,
          0.9072
        ],
        [
          0.3935,
          0.9045
        ],
        [
          0.4035,
          0.9017
        ],
        [
          0.4136,
          0.8999
        ],
        [
          0.424,
          0.8954
        ],
        [
          0.434,
          0.8913
        ],
        [
          0.444,
          0.8894
        ],
        [
          0.454,
          0.884
        ],
        [
          0.4641,
          0.8803
        ],
        [
          0.4745,
          0.8772
        ],
        [
          0.4845,
          0.874
        ],
        [
          0.4945,
          0.8685
        ],
        [
          0.5045,
          0.8649
        ],
        [
          0.5146,
          0.8603
        ],
        [
          0.525,
          0.8567
        ],
        [
          0.535,
          0.853
        ],
        [
          0.545,
          0.8512
        ],
        [
          0.5551,
          0.8476
        ],
        [
          0.5655,
          0.843
        ],
        [
          0.5755,
          0.8389
        ],
        [
          0.5855,
          0.8344
        ],
        [
          0.5955,
          0.8298
        ],
        [
          0.6056,
          0.8262
        ],
        [
          0.616,
          0.8194
        ],
        [
          0.626,
          0.8167
        ],
        [
          0.636,
          0.8121
        ],
        [
          0.646,
          0.8053
        ],
        [
          0.6561,
          0.7994
        ],
        [
          0.6665,
          0.793
        ],
        [
          0.6765,
          0.7862
        ],
        [
          0.6865,
          0.7821
        ],
        [
          0.6965,
          0.7771
        ],
        [
          0.7066,
          0.7725
        ],
        [
          0.717,
          0.768
        ],
        [
          0.727,
          0.763
        ],
        [
          0.737,
          0.7575
        ],
        [
          0.747,
          0.7511
        ],
        [
          0.7571,
          0.747
        ],
        [
          0.7675,
          0.7389
        ],
        [
          0.7775,
          0.7357
        ],
        [
          0.7875,
          0.7293
        ],
        [
          0.7975,
          0.7261
        ],
        [
          0.8076,
          0.722
        ],
        [
          0.818,
          0.7161
        ],
        [
          0.828,
          0.7106
        ],
        [
          0.838,
          0.7038
        ],
        [
          0.848,
          0.6975
        ],
        [
          0.8581,
          0.6915
        ],
        [
          0.8685,
          0.6856
        ],
        [
          0.8785,
          0.6802
        ],
        [
          0.8885,
          0.6729
        ],
        [
          0.8985,
          0.6679
        ],
        [
          0.9086,
          0.6624
        ],
        [
          0.919,
          0.6556
        ],
        [
          0.929,
          0.6492
        ],
        [
          0.939,
          0.6433
        ],
        [
          0.949,
          0.6374
        ],
        [
          0.9591,
          0.631
        ],
        [
          0.9695,
          0.6237
        ],
        [
          0.9795,
          0.6165
        ],
        [
          0.9895,
          0.611
        ],
        [
          0.9995,
          0.6037
        ],
        [
          1.0,
          0.6037
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.20810972655026141,
        "difference": -0.0013883602881186888,
        "worse": 0.62,
        "kept_right": 0.20551784093772688,
        "kept_wrong": 0.0025918856125343825,
        "handed_off": 0.791890273449739
      },
      "usd_per_1000": 0.22822633091454325
    },
    {
      "decision": 44,
      "frontier": 274,
      "frontier_accuracy": 0.9085532302092811,
      "in_sample": {
        "threshold": 0.8952663292322481,
        "answered": 0.18789808917197454,
        "accuracy": 0.9085532302092811
      },
      "curve": [
        [
          0.0,
          0.9086
        ],
        [
          0.01,
          0.9086
        ],
        [
          0.02,
          0.9086
        ],
        [
          0.03,
          0.9086
        ],
        [
          0.04,
          0.9086
        ],
        [
          0.05,
          0.9086
        ],
        [
          0.0601,
          0.9086
        ],
        [
          0.0705,
          0.9086
        ],
        [
          0.0805,
          0.9086
        ],
        [
          0.0905,
          0.9086
        ],
        [
          0.1005,
          0.9086
        ],
        [
          0.1106,
          0.9086
        ],
        [
          0.121,
          0.9086
        ],
        [
          0.131,
          0.9086
        ],
        [
          0.141,
          0.9086
        ],
        [
          0.151,
          0.9086
        ],
        [
          0.1611,
          0.9086
        ],
        [
          0.1715,
          0.9086
        ],
        [
          0.1815,
          0.9086
        ],
        [
          0.1915,
          0.9081
        ],
        [
          0.2015,
          0.9081
        ],
        [
          0.2116,
          0.9076
        ],
        [
          0.222,
          0.9076
        ],
        [
          0.232,
          0.9072
        ],
        [
          0.242,
          0.9067
        ],
        [
          0.252,
          0.9063
        ],
        [
          0.2621,
          0.9058
        ],
        [
          0.2725,
          0.9045
        ],
        [
          0.2825,
          0.9035
        ],
        [
          0.2925,
          0.9017
        ],
        [
          0.3025,
          0.9004
        ],
        [
          0.3126,
          0.8995
        ],
        [
          0.323,
          0.8972
        ],
        [
          0.333,
          0.8954
        ],
        [
          0.343,
          0.8949
        ],
        [
          0.353,
          0.8926
        ],
        [
          0.3631,
          0.8899
        ],
        [
          0.3735,
          0.8885
        ],
        [
          0.3835,
          0.8872
        ],
        [
          0.3935,
          0.8844
        ],
        [
          0.4035,
          0.8817
        ],
        [
          0.4136,
          0.8799
        ],
        [
          0.424,
          0.8758
        ],
        [
          0.434,
          0.8726
        ],
        [
          0.444,
          0.8703
        ],
        [
          0.454,
          0.8653
        ],
        [
          0.4641,
          0.8617
        ],
        [
          0.4745,
          0.8585
        ],
        [
          0.4845,
          0.8553
        ],
        [
          0.4945,
          0.8503
        ],
        [
          0.5045,
          0.8467
        ],
        [
          0.5146,
          0.8426
        ],
        [
          0.525,
          0.8385
        ],
        [
          0.535,
          0.8344
        ],
        [
          0.545,
          0.8317
        ],
        [
          0.5551,
          0.8289
        ],
        [
          0.5655,
          0.8258
        ],
        [
          0.5755,
          0.8217
        ],
        [
          0.5855,
          0.8176
        ],
        [
          0.5955,
          0.8139
        ],
        [
          0.6056,
          0.8103
        ],
        [
          0.616,
          0.8035
        ],
        [
          0.626,
          0.8016
        ],
        [
          0.636,
          0.7971
        ],
        [
          0.646,
          0.7903
        ],
        [
          0.6561,
          0.7853
        ],
        [
          0.6665,
          0.7784
        ],
        [
          0.6765,
          0.7721
        ],
        [
          0.6865,
          0.7689
        ],
        [
          0.6965,
          0.7643
        ],
        [
          0.7066,
          0.7616
        ],
        [
          0.717,
          0.7566
        ],
        [
          0.727,
          0.752
        ],
        [
          0.737,
          0.7484
        ],
        [
          0.747,
          0.7439
        ],
        [
          0.7571,
          0.7398
        ],
        [
          0.7675,
          0.732
        ],
        [
          0.7775,
          0.7284
        ],
        [
          0.7875,
          0.7234
        ],
        [
          0.7975,
          0.7207
        ],
        [
          0.8076,
          0.717
        ],
        [
          0.818,
          0.7111
        ],
        [
          0.828,
          0.7043
        ],
        [
          0.838,
          0.697
        ],
        [
          0.848,
          0.6924
        ],
        [
          0.8581,
          0.6874
        ],
        [
          0.8685,
          0.6815
        ],
        [
          0.8785,
          0.6756
        ],
        [
          0.8885,
          0.6706
        ],
        [
          0.8985,
          0.6656
        ],
        [
          0.9086,
          0.6601
        ],
        [
          0.919,
          0.6538
        ],
        [
          0.929,
          0.6488
        ],
        [
          0.939,
          0.6424
        ],
        [
          0.949,
          0.636
        ],
        [
          0.9591,
          0.6288
        ],
        [
          0.9695,
          0.6206
        ],
        [
          0.9795,
          0.6142
        ],
        [
          0.9895,
          0.6096
        ],
        [
          0.9995,
          0.6037
        ],
        [
          1.0,
          0.6037
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.2118854797967008,
        "difference": -0.0014612943412437567,
        "worse": 0.6433333333333333,
        "kept_right": 0.20900542548452036,
        "kept_wrong": 0.002880054312180311,
        "handed_off": 0.7881145202032991
      },
      "usd_per_1000": 0.3514677291851797
    },
    {
      "decision": 44,
      "frontier": 47,
      "frontier_accuracy": 0.9044585987261147,
      "in_sample": {
        "threshold": 0.9156852342507505,
        "answered": 0.17015468607825296,
        "accuracy": 0.9044585987261147
      },
      "curve": [
        [
          0.0,
          0.9045
        ],
        [
          0.01,
          0.9045
        ],
        [
          0.02,
          0.9045
        ],
        [
          0.03,
          0.9045
        ],
        [
          0.04,
          0.9045
        ],
        [
          0.05,
          0.9045
        ],
        [
          0.0601,
          0.9045
        ],
        [
          0.0705,
          0.9045
        ],
        [
          0.0805,
          0.9045
        ],
        [
          0.0905,
          0.9045
        ],
        [
          0.1005,
          0.9045
        ],
        [
          0.1106,
          0.9045
        ],
        [
          0.121,
          0.9045
        ],
        [
          0.131,
          0.9045
        ],
        [
          0.141,
          0.9045
        ],
        [
          0.151,
          0.9045
        ],
        [
          0.1611,
          0.9045
        ],
        [
          0.1715,
          0.904
        ],
        [
          0.1815,
          0.904
        ],
        [
          0.1915,
          0.9035
        ],
        [
          0.2015,
          0.904
        ],
        [
          0.2116,
          0.9035
        ],
        [
          0.222,
          0.9031
        ],
        [
          0.232,
          0.9026
        ],
        [
          0.242,
          0.9013
        ],
        [
          0.252,
          0.9008
        ],
        [
          0.2621,
          0.9004
        ],
        [
          0.2725,
          0.8995
        ],
        [
          0.2825,
          0.899
        ],
        [
          0.2925,
          0.8976
        ],
        [
          0.3025,
          0.8967
        ],
        [
          0.3126,
          0.8963
        ],
        [
          0.323,
          0.8949
        ],
        [
          0.333,
          0.8931
        ],
        [
          0.343,
          0.8913
        ],
        [
          0.353,
          0.8894
        ],
        [
          0.3631,
          0.8876
        ],
        [
          0.3735,
          0.8867
        ],
        [
          0.3835,
          0.8844
        ],
        [
          0.3935,
          0.8813
        ],
        [
          0.4035,
          0.8781
        ],
        [
          0.4136,
          0.8763
        ],
        [
          0.424,
          0.8717
        ],
        [
          0.434,
          0.869
        ],
        [
          0.444,
          0.8667
        ],
        [
          0.454,
          0.8621
        ],
        [
          0.4641,
          0.8581
        ],
        [
          0.4745,
          0.8553
        ],
        [
          0.4845,
          0.8526
        ],
        [
          0.4945,
          0.848
        ],
        [
          0.5045,
          0.8449
        ],
        [
          0.5146,
          0.8412
        ],
        [
          0.525,
          0.838
        ],
        [
          0.535,
          0.8339
        ],
        [
          0.545,
          0.8321
        ],
        [
          0.5551,
          0.8294
        ],
        [
          0.5655,
          0.8258
        ],
        [
          0.5755,
          0.8221
        ],
        [
          0.5855,
          0.818
        ],
        [
          0.5955,
          0.8144
        ],
        [
          0.6056,
          0.8116
        ],
        [
          0.616,
          0.8053
        ],
        [
          0.626,
          0.8025
        ],
        [
          0.636,
          0.7975
        ],
        [
          0.646,
          0.7925
        ],
        [
          0.6561,
          0.7875
        ],
        [
          0.6665,
          0.7812
        ],
        [
          0.6765,
          0.7743
        ],
        [
          0.6865,
          0.7712
        ],
        [
          0.6965,
          0.7675
        ],
        [
          0.7066,
          0.7634
        ],
        [
          0.717,
          0.7584
        ],
        [
          0.727,
          0.7539
        ],
        [
          0.737,
          0.748
        ],
        [
          0.747,
          0.7425
        ],
        [
          0.7571,
          0.7379
        ],
        [
          0.7675,
          0.7311
        ],
        [
          0.7775,
          0.7275
        ],
        [
          0.7875,
          0.7229
        ],
        [
          0.7975,
          0.7197
        ],
        [
          0.8076,
          0.7152
        ],
        [
          0.818,
          0.7116
        ],
        [
          0.828,
          0.7056
        ],
        [
          0.838,
          0.6988
        ],
        [
          0.848,
          0.6943
        ],
        [
          0.8581,
          0.6893
        ],
        [
          0.8685,
          0.6852
        ],
        [
          0.8785,
          0.6793
        ],
        [
          0.8885,
          0.6724
        ],
        [
          0.8985,
          0.6665
        ],
        [
          0.9086,
          0.6611
        ],
        [
          0.919,
          0.6542
        ],
        [
          0.929,
          0.6492
        ],
        [
          0.939,
          0.6429
        ],
        [
          0.949,
          0.6369
        ],
        [
          0.9591,
          0.6306
        ],
        [
          0.9695,
          0.6228
        ],
        [
          0.9795,
          0.6151
        ],
        [
          0.9895,
          0.611
        ],
        [
          0.9995,
          0.6037
        ],
        [
          1.0,
          0.6037
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.20000221226426917,
        "difference": -0.0014672418308540206,
        "worse": 0.67,
        "kept_right": 0.19782564967310817,
        "kept_wrong": 0.0021765625911608732,
        "handed_off": 0.7999977877357308
      },
      "usd_per_1000": 3.959995191788337
    },
    {
      "decision": 44,
      "frontier": 193,
      "frontier_accuracy": 0.89171974522293,
      "in_sample": {
        "threshold": 0.8476270590247672,
        "answered": 0.2256596906278435,
        "accuracy": 0.89171974522293
      },
      "curve": [
        [
          0.0,
          0.8917
        ],
        [
          0.01,
          0.8917
        ],
        [
          0.02,
          0.8917
        ],
        [
          0.03,
          0.8917
        ],
        [
          0.04,
          0.8917
        ],
        [
          0.05,
          0.8917
        ],
        [
          0.0601,
          0.8917
        ],
        [
          0.0705,
          0.8917
        ],
        [
          0.0805,
          0.8917
        ],
        [
          0.0905,
          0.8917
        ],
        [
          0.1005,
          0.8917
        ],
        [
          0.1106,
          0.8917
        ],
        [
          0.121,
          0.8917
        ],
        [
          0.131,
          0.8917
        ],
        [
          0.141,
          0.8917
        ],
        [
          0.151,
          0.8917
        ],
        [
          0.1611,
          0.8917
        ],
        [
          0.1715,
          0.8913
        ],
        [
          0.1815,
          0.8913
        ],
        [
          0.1915,
          0.8913
        ],
        [
          0.2015,
          0.8922
        ],
        [
          0.2116,
          0.8917
        ],
        [
          0.222,
          0.8917
        ],
        [
          0.232,
          0.8913
        ],
        [
          0.242,
          0.8908
        ],
        [
          0.252,
          0.8908
        ],
        [
          0.2621,
          0.8904
        ],
        [
          0.2725,
          0.8885
        ],
        [
          0.2825,
          0.8881
        ],
        [
          0.2925,
          0.8863
        ],
        [
          0.3025,
          0.8849
        ],
        [
          0.3126,
          0.884
        ],
        [
          0.323,
          0.8831
        ],
        [
          0.333,
          0.8822
        ],
        [
          0.343,
          0.8803
        ],
        [
          0.353,
          0.8794
        ],
        [
          0.3631,
          0.8767
        ],
        [
          0.3735,
          0.8763
        ],
        [
          0.3835,
          0.8744
        ],
        [
          0.3935,
          0.8731
        ],
        [
          0.4035,
          0.8703
        ],
        [
          0.4136,
          0.8685
        ],
        [
          0.424,
          0.8644
        ],
        [
          0.434,
          0.8608
        ],
        [
          0.444,
          0.8594
        ],
        [
          0.454,
          0.8544
        ],
        [
          0.4641,
          0.8512
        ],
        [
          0.4745,
          0.848
        ],
        [
          0.4845,
          0.8458
        ],
        [
          0.4945,
          0.8417
        ],
        [
          0.5045,
          0.8394
        ],
        [
          0.5146,
          0.8353
        ],
        [
          0.525,
          0.8321
        ],
        [
          0.535,
          0.8285
        ],
        [
          0.545,
          0.8271
        ],
        [
          0.5551,
          0.8239
        ],
        [
          0.5655,
          0.8198
        ],
        [
          0.5755,
          0.8167
        ],
        [
          0.5855,
          0.8121
        ],
        [
          0.5955,
          0.8094
        ],
        [
          0.6056,
          0.8066
        ],
        [
          0.616,
          0.7998
        ],
        [
          0.626,
          0.798
        ],
        [
          0.636,
          0.7934
        ],
        [
          0.646,
          0.7871
        ],
        [
          0.6561,
          0.7821
        ],
        [
          0.6665,
          0.7771
        ],
        [
          0.6765,
          0.7716
        ],
        [
          0.6865,
          0.7671
        ],
        [
          0.6965,
          0.7625
        ],
        [
          0.7066,
          0.758
        ],
        [
          0.717,
          0.7534
        ],
        [
          0.727,
          0.748
        ],
        [
          0.737,
          0.7429
        ],
        [
          0.747,
          0.7389
        ],
        [
          0.7571,
          0.7338
        ],
        [
          0.7675,
          0.7275
        ],
        [
          0.7775,
          0.7229
        ],
        [
          0.7875,
          0.7179
        ],
        [
          0.7975,
          0.7147
        ],
        [
          0.8076,
          0.712
        ],
        [
          0.818,
          0.707
        ],
        [
          0.828,
          0.7011
        ],
        [
          0.838,
          0.6938
        ],
        [
          0.848,
          0.6893
        ],
        [
          0.8581,
          0.6852
        ],
        [
          0.8685,
          0.6811
        ],
        [
          0.8785,
          0.6756
        ],
        [
          0.8885,
          0.6697
        ],
        [
          0.8985,
          0.6656
        ],
        [
          0.9086,
          0.6592
        ],
        [
          0.919,
          0.6524
        ],
        [
          0.929,
          0.6474
        ],
        [
          0.939,
          0.641
        ],
        [
          0.949,
          0.6347
        ],
        [
          0.9591,
          0.6274
        ],
        [
          0.9695,
          0.6197
        ],
        [
          0.9795,
          0.6115
        ],
        [
          0.9895,
          0.6087
        ],
        [
          0.9995,
          0.6033
        ],
        [
          1.0,
          0.6037
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.23866122306654233,
        "difference": -0.001524406654748163,
        "worse": 0.5766666666666667,
        "kept_right": 0.23375369270816024,
        "kept_wrong": 0.004907530358381949,
        "handed_off": 0.7613387769334582
      },
      "usd_per_1000": 5.9140780091212966
    },
    {
      "decision": 44,
      "frontier": 55,
      "frontier_accuracy": 0.8503184713375797,
      "in_sample": {
        "threshold": 0.6621500580464365,
        "answered": 0.3548680618744313,
        "accuracy": 0.8503184713375797
      },
      "curve": [
        [
          0.0,
          0.8503
        ],
        [
          0.01,
          0.8503
        ],
        [
          0.02,
          0.8512
        ],
        [
          0.03,
          0.8512
        ],
        [
          0.04,
          0.8512
        ],
        [
          0.05,
          0.8512
        ],
        [
          0.0601,
          0.8512
        ],
        [
          0.0705,
          0.8512
        ],
        [
          0.0805,
          0.8512
        ],
        [
          0.0905,
          0.8512
        ],
        [
          0.1005,
          0.8517
        ],
        [
          0.1106,
          0.8521
        ],
        [
          0.121,
          0.8521
        ],
        [
          0.131,
          0.8521
        ],
        [
          0.141,
          0.8521
        ],
        [
          0.151,
          0.8521
        ],
        [
          0.1611,
          0.8521
        ],
        [
          0.1715,
          0.8521
        ],
        [
          0.1815,
          0.8526
        ],
        [
          0.1915,
          0.8526
        ],
        [
          0.2015,
          0.854
        ],
        [
          0.2116,
          0.8535
        ],
        [
          0.222,
          0.8535
        ],
        [
          0.232,
          0.854
        ],
        [
          0.242,
          0.8535
        ],
        [
          0.252,
          0.8553
        ],
        [
          0.2621,
          0.8558
        ],
        [
          0.2725,
          0.8553
        ],
        [
          0.2825,
          0.8553
        ],
        [
          0.2925,
          0.8544
        ],
        [
          0.3025,
          0.854
        ],
        [
          0.3126,
          0.853
        ],
        [
          0.323,
          0.8521
        ],
        [
          0.333,
          0.8517
        ],
        [
          0.343,
          0.8512
        ],
        [
          0.353,
          0.8512
        ],
        [
          0.3631,
          0.849
        ],
        [
          0.3735,
          0.8476
        ],
        [
          0.3835,
          0.8467
        ],
        [
          0.3935,
          0.8462
        ],
        [
          0.4035,
          0.8467
        ],
        [
          0.4136,
          0.8449
        ],
        [
          0.424,
          0.8412
        ],
        [
          0.434,
          0.838
        ],
        [
          0.444,
          0.838
        ],
        [
          0.454,
          0.8335
        ],
        [
          0.4641,
          0.8308
        ],
        [
          0.4745,
          0.828
        ],
        [
          0.4845,
          0.8262
        ],
        [
          0.4945,
          0.8226
        ],
        [
          0.5045,
          0.8207
        ],
        [
          0.5146,
          0.8171
        ],
        [
          0.525,
          0.8162
        ],
        [
          0.535,
          0.8144
        ],
        [
          0.545,
          0.8126
        ],
        [
          0.5551,
          0.8116
        ],
        [
          0.5655,
          0.808
        ],
        [
          0.5755,
          0.8048
        ],
        [
          0.5855,
          0.8012
        ],
        [
          0.5955,
          0.7975
        ],
        [
          0.6056,
          0.7939
        ],
        [
          0.616,
          0.788
        ],
        [
          0.626,
          0.7853
        ],
        [
          0.636,
          0.7825
        ],
        [
          0.646,
          0.7771
        ],
        [
          0.6561,
          0.7721
        ],
        [
          0.6665,
          0.7666
        ],
        [
          0.6765,
          0.7611
        ],
        [
          0.6865,
          0.7571
        ],
        [
          0.6965,
          0.753
        ],
        [
          0.7066,
          0.7489
        ],
        [
          0.717,
          0.7448
        ],
        [
          0.727,
          0.7402
        ],
        [
          0.737,
          0.7352
        ],
        [
          0.747,
          0.7316
        ],
        [
          0.7571,
          0.727
        ],
        [
          0.7675,
          0.7207
        ],
        [
          0.7775,
          0.717
        ],
        [
          0.7875,
          0.7125
        ],
        [
          0.7975,
          0.7097
        ],
        [
          0.8076,
          0.7061
        ],
        [
          0.818,
          0.7025
        ],
        [
          0.828,
          0.6979
        ],
        [
          0.838,
          0.6924
        ],
        [
          0.848,
          0.6879
        ],
        [
          0.8581,
          0.6843
        ],
        [
          0.8685,
          0.6797
        ],
        [
          0.8785,
          0.6747
        ],
        [
          0.8885,
          0.6688
        ],
        [
          0.8985,
          0.6642
        ],
        [
          0.9086,
          0.6583
        ],
        [
          0.919,
          0.6506
        ],
        [
          0.929,
          0.6465
        ],
        [
          0.939,
          0.6406
        ],
        [
          0.949,
          0.6338
        ],
        [
          0.9591,
          0.6269
        ],
        [
          0.9695,
          0.6201
        ],
        [
          0.9795,
          0.6133
        ],
        [
          0.9895,
          0.6101
        ],
        [
          0.9995,
          0.6037
        ],
        [
          1.0,
          0.6037
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.3624596960942165,
        "difference": -0.0021259214927042687,
        "worse": 0.5033333333333333,
        "kept_right": 0.33313089916726724,
        "kept_wrong": 0.029328796926948907,
        "handed_off": 0.6375403039057833
      },
      "usd_per_1000": 2.345322287877194
    },
    {
      "decision": 264,
      "frontier": 140,
      "frontier_accuracy": 0.9808917197452229,
      "in_sample": {
        "threshold": 0.892259,
        "answered": 0.15332120109190173,
        "accuracy": 0.9808917197452229
      },
      "curve": [
        [
          0.0,
          0.9809
        ],
        [
          0.01,
          0.9809
        ],
        [
          0.02,
          0.9809
        ],
        [
          0.03,
          0.9809
        ],
        [
          0.04,
          0.9809
        ],
        [
          0.05,
          0.9809
        ],
        [
          0.0601,
          0.9809
        ],
        [
          0.0705,
          0.9809
        ],
        [
          0.0805,
          0.9809
        ],
        [
          0.0905,
          0.9809
        ],
        [
          0.1005,
          0.9809
        ],
        [
          0.1106,
          0.9809
        ],
        [
          0.121,
          0.9809
        ],
        [
          0.131,
          0.9809
        ],
        [
          0.141,
          0.9809
        ],
        [
          0.151,
          0.9809
        ],
        [
          0.1611,
          0.9804
        ],
        [
          0.1715,
          0.9804
        ],
        [
          0.1815,
          0.98
        ],
        [
          0.1915,
          0.98
        ],
        [
          0.2015,
          0.9795
        ],
        [
          0.2116,
          0.9786
        ],
        [
          0.222,
          0.9773
        ],
        [
          0.232,
          0.9768
        ],
        [
          0.242,
          0.9768
        ],
        [
          0.252,
          0.9768
        ],
        [
          0.2621,
          0.9754
        ],
        [
          0.2725,
          0.9727
        ],
        [
          0.2825,
          0.9722
        ],
        [
          0.2925,
          0.9695
        ],
        [
          0.3025,
          0.9672
        ],
        [
          0.3126,
          0.9659
        ],
        [
          0.323,
          0.9641
        ],
        [
          0.333,
          0.9631
        ],
        [
          0.343,
          0.9613
        ],
        [
          0.353,
          0.96
        ],
        [
          0.3631,
          0.9581
        ],
        [
          0.3735,
          0.9559
        ],
        [
          0.3835,
          0.9527
        ],
        [
          0.3935,
          0.9486
        ],
        [
          0.4035,
          0.9454
        ],
        [
          0.4136,
          0.9413
        ],
        [
          0.424,
          0.9372
        ],
        [
          0.434,
          0.9349
        ],
        [
          0.444,
          0.9304
        ],
        [
          0.454,
          0.9272
        ],
        [
          0.4641,
          0.9217
        ],
        [
          0.4745,
          0.9172
        ],
        [
          0.4845,
          0.9117
        ],
        [
          0.4945,
          0.9072
        ],
        [
          0.5045,
          0.9035
        ],
        [
          0.5146,
          0.8972
        ],
        [
          0.525,
          0.8913
        ],
        [
          0.535,
          0.8876
        ],
        [
          0.545,
          0.8817
        ],
        [
          0.5551,
          0.8776
        ],
        [
          0.5655,
          0.874
        ],
        [
          0.5755,
          0.8681
        ],
        [
          0.5855,
          0.8599
        ],
        [
          0.5955,
          0.8553
        ],
        [
          0.6056,
          0.8485
        ],
        [
          0.616,
          0.8426
        ],
        [
          0.626,
          0.8376
        ],
        [
          0.636,
          0.8335
        ],
        [
          0.646,
          0.828
        ],
        [
          0.6561,
          0.823
        ],
        [
          0.6665,
          0.8162
        ],
        [
          0.6765,
          0.8112
        ],
        [
          0.6865,
          0.8048
        ],
        [
          0.6965,
          0.7989
        ],
        [
          0.7066,
          0.793
        ],
        [
          0.717,
          0.7871
        ],
        [
          0.727,
          0.7816
        ],
        [
          0.737,
          0.7739
        ],
        [
          0.747,
          0.7693
        ],
        [
          0.7571,
          0.7639
        ],
        [
          0.7675,
          0.7589
        ],
        [
          0.7775,
          0.7534
        ],
        [
          0.7875,
          0.747
        ],
        [
          0.7975,
          0.7402
        ],
        [
          0.8076,
          0.7338
        ],
        [
          0.818,
          0.7266
        ],
        [
          0.828,
          0.7179
        ],
        [
          0.838,
          0.7111
        ],
        [
          0.848,
          0.7034
        ],
        [
          0.8581,
          0.6975
        ],
        [
          0.8685,
          0.6897
        ],
        [
          0.8785,
          0.682
        ],
        [
          0.8885,
          0.6747
        ],
        [
          0.8985,
          0.6674
        ],
        [
          0.9086,
          0.6606
        ],
        [
          0.919,
          0.6533
        ],
        [
          0.929,
          0.6465
        ],
        [
          0.939,
          0.6388
        ],
        [
          0.949,
          0.631
        ],
        [
          0.9591,
          0.6247
        ],
        [
          0.9695,
          0.6174
        ],
        [
          0.9795,
          0.6115
        ],
        [
          0.9895,
          0.6037
        ],
        [
          0.9995,
          0.5965
        ],
        [
          1.0,
          0.596
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1712602336642977,
        "difference": -0.0010276656521176093,
        "worse": 0.54,
        "kept_right": 0.17000819840950446,
        "kept_wrong": 0.0012520352547932376,
        "handed_off": 0.8287397663357021
      },
      "usd_per_1000": 2.004806989189183
    },
    {
      "decision": 264,
      "frontier": 277,
      "frontier_accuracy": 0.9795268425841674,
      "in_sample": {
        "threshold": 0.892259,
        "answered": 0.15332120109190173,
        "accuracy": 0.9795268425841674
      },
      "curve": [
        [
          0.0,
          0.9795
        ],
        [
          0.01,
          0.9795
        ],
        [
          0.02,
          0.9795
        ],
        [
          0.03,
          0.9795
        ],
        [
          0.04,
          0.9795
        ],
        [
          0.05,
          0.9795
        ],
        [
          0.0601,
          0.9795
        ],
        [
          0.0705,
          0.9795
        ],
        [
          0.0805,
          0.9795
        ],
        [
          0.0905,
          0.9795
        ],
        [
          0.1005,
          0.9795
        ],
        [
          0.1106,
          0.9795
        ],
        [
          0.121,
          0.9795
        ],
        [
          0.131,
          0.9795
        ],
        [
          0.141,
          0.9795
        ],
        [
          0.151,
          0.9795
        ],
        [
          0.1611,
          0.9791
        ],
        [
          0.1715,
          0.9791
        ],
        [
          0.1815,
          0.9786
        ],
        [
          0.1915,
          0.9786
        ],
        [
          0.2015,
          0.9782
        ],
        [
          0.2116,
          0.9777
        ],
        [
          0.222,
          0.9763
        ],
        [
          0.232,
          0.9759
        ],
        [
          0.242,
          0.9759
        ],
        [
          0.252,
          0.9759
        ],
        [
          0.2621,
          0.9745
        ],
        [
          0.2725,
          0.9718
        ],
        [
          0.2825,
          0.9713
        ],
        [
          0.2925,
          0.9686
        ],
        [
          0.3025,
          0.9668
        ],
        [
          0.3126,
          0.9654
        ],
        [
          0.323,
          0.9636
        ],
        [
          0.333,
          0.9622
        ],
        [
          0.343,
          0.9604
        ],
        [
          0.353,
          0.9595
        ],
        [
          0.3631,
          0.9577
        ],
        [
          0.3735,
          0.9554
        ],
        [
          0.3835,
          0.9522
        ],
        [
          0.3935,
          0.9481
        ],
        [
          0.4035,
          0.9449
        ],
        [
          0.4136,
          0.9409
        ],
        [
          0.424,
          0.9363
        ],
        [
          0.434,
          0.9349
        ],
        [
          0.444,
          0.9304
        ],
        [
          0.454,
          0.9272
        ],
        [
          0.4641,
          0.9222
        ],
        [
          0.4745,
          0.9177
        ],
        [
          0.4845,
          0.9126
        ],
        [
          0.4945,
          0.9076
        ],
        [
          0.5045,
          0.904
        ],
        [
          0.5146,
          0.8976
        ],
        [
          0.525,
          0.8908
        ],
        [
          0.535,
          0.8872
        ],
        [
          0.545,
          0.8813
        ],
        [
          0.5551,
          0.8772
        ],
        [
          0.5655,
          0.8735
        ],
        [
          0.5755,
          0.8676
        ],
        [
          0.5855,
          0.8594
        ],
        [
          0.5955,
          0.8549
        ],
        [
          0.6056,
          0.848
        ],
        [
          0.616,
          0.8412
        ],
        [
          0.626,
          0.8367
        ],
        [
          0.636,
          0.8326
        ],
        [
          0.646,
          0.8276
        ],
        [
          0.6561,
          0.8226
        ],
        [
          0.6665,
          0.8157
        ],
        [
          0.6765,
          0.8107
        ],
        [
          0.6865,
          0.8044
        ],
        [
          0.6965,
          0.7985
        ],
        [
          0.7066,
          0.7925
        ],
        [
          0.717,
          0.7866
        ],
        [
          0.727,
          0.7816
        ],
        [
          0.737,
          0.7739
        ],
        [
          0.747,
          0.7693
        ],
        [
          0.7571,
          0.7639
        ],
        [
          0.7675,
          0.7584
        ],
        [
          0.7775,
          0.753
        ],
        [
          0.7875,
          0.7466
        ],
        [
          0.7975,
          0.7398
        ],
        [
          0.8076,
          0.7334
        ],
        [
          0.818,
          0.7261
        ],
        [
          0.828,
          0.7175
        ],
        [
          0.838,
          0.7102
        ],
        [
          0.848,
          0.7029
        ],
        [
          0.8581,
          0.6965
        ],
        [
          0.8685,
          0.6893
        ],
        [
          0.8785,
          0.6815
        ],
        [
          0.8885,
          0.6747
        ],
        [
          0.8985,
          0.6674
        ],
        [
          0.9086,
          0.6606
        ],
        [
          0.919,
          0.6533
        ],
        [
          0.929,
          0.6465
        ],
        [
          0.939,
          0.6388
        ],
        [
          0.949,
          0.631
        ],
        [
          0.9591,
          0.6251
        ],
        [
          0.9695,
          0.6174
        ],
        [
          0.9795,
          0.6115
        ],
        [
          0.9895,
          0.6037
        ],
        [
          0.9995,
          0.5965
        ],
        [
          1.0,
          0.596
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.17290046177295224,
        "difference": -0.0011550157252883586,
        "worse": 0.5433333333333333,
        "kept_right": 0.17147865767717638,
        "kept_wrong": 0.0014218040957759176,
        "handed_off": 0.8270995382270475
      },
      "usd_per_1000": 5.0168177809833665
    },
    {
      "decision": 264,
      "frontier": 276,
      "frontier_accuracy": 0.9777070063694268,
      "in_sample": {
        "threshold": 0.873492,
        "answered": 0.17151956323930848,
        "accuracy": 0.9777070063694268
      },
      "curve": [
        [
          0.0,
          0.9777
        ],
        [
          0.01,
          0.9777
        ],
        [
          0.02,
          0.9777
        ],
        [
          0.03,
          0.9777
        ],
        [
          0.04,
          0.9777
        ],
        [
          0.05,
          0.9777
        ],
        [
          0.0601,
          0.9777
        ],
        [
          0.0705,
          0.9777
        ],
        [
          0.0805,
          0.9777
        ],
        [
          0.0905,
          0.9777
        ],
        [
          0.1005,
          0.9777
        ],
        [
          0.1106,
          0.9777
        ],
        [
          0.121,
          0.9777
        ],
        [
          0.131,
          0.9777
        ],
        [
          0.141,
          0.9777
        ],
        [
          0.151,
          0.9777
        ],
        [
          0.1611,
          0.9773
        ],
        [
          0.1715,
          0.9777
        ],
        [
          0.1815,
          0.9773
        ],
        [
          0.1915,
          0.9773
        ],
        [
          0.2015,
          0.9768
        ],
        [
          0.2116,
          0.9763
        ],
        [
          0.222,
          0.975
        ],
        [
          0.232,
          0.9745
        ],
        [
          0.242,
          0.9745
        ],
        [
          0.252,
          0.9745
        ],
        [
          0.2621,
          0.9741
        ],
        [
          0.2725,
          0.9713
        ],
        [
          0.2825,
          0.9709
        ],
        [
          0.2925,
          0.9682
        ],
        [
          0.3025,
          0.9663
        ],
        [
          0.3126,
          0.965
        ],
        [
          0.323,
          0.9627
        ],
        [
          0.333,
          0.9613
        ],
        [
          0.343,
          0.9595
        ],
        [
          0.353,
          0.9586
        ],
        [
          0.3631,
          0.9568
        ],
        [
          0.3735,
          0.9545
        ],
        [
          0.3835,
          0.9513
        ],
        [
          0.3935,
          0.9472
        ],
        [
          0.4035,
          0.944
        ],
        [
          0.4136,
          0.9399
        ],
        [
          0.424,
          0.9354
        ],
        [
          0.434,
          0.9345
        ],
        [
          0.444,
          0.9299
        ],
        [
          0.454,
          0.9272
        ],
        [
          0.4641,
          0.9217
        ],
        [
          0.4745,
          0.9172
        ],
        [
          0.4845,
          0.9126
        ],
        [
          0.4945,
          0.9076
        ],
        [
          0.5045,
          0.904
        ],
        [
          0.5146,
          0.8976
        ],
        [
          0.525,
          0.8908
        ],
        [
          0.535,
          0.8872
        ],
        [
          0.545,
          0.8813
        ],
        [
          0.5551,
          0.8772
        ],
        [
          0.5655,
          0.8735
        ],
        [
          0.5755,
          0.8676
        ],
        [
          0.5855,
          0.8594
        ],
        [
          0.5955,
          0.8549
        ],
        [
          0.6056,
          0.8485
        ],
        [
          0.616,
          0.8421
        ],
        [
          0.626,
          0.8376
        ],
        [
          0.636,
          0.8335
        ],
        [
          0.646,
          0.828
        ],
        [
          0.6561,
          0.8235
        ],
        [
          0.6665,
          0.8167
        ],
        [
          0.6765,
          0.8116
        ],
        [
          0.6865,
          0.8053
        ],
        [
          0.6965,
          0.7994
        ],
        [
          0.7066,
          0.7934
        ],
        [
          0.717,
          0.7875
        ],
        [
          0.727,
          0.7825
        ],
        [
          0.737,
          0.7748
        ],
        [
          0.747,
          0.7702
        ],
        [
          0.7571,
          0.7648
        ],
        [
          0.7675,
          0.7593
        ],
        [
          0.7775,
          0.7539
        ],
        [
          0.7875,
          0.7475
        ],
        [
          0.7975,
          0.7407
        ],
        [
          0.8076,
          0.7343
        ],
        [
          0.818,
          0.727
        ],
        [
          0.828,
          0.7184
        ],
        [
          0.838,
          0.7111
        ],
        [
          0.848,
          0.7034
        ],
        [
          0.8581,
          0.697
        ],
        [
          0.8685,
          0.6897
        ],
        [
          0.8785,
          0.682
        ],
        [
          0.8885,
          0.6752
        ],
        [
          0.8985,
          0.6679
        ],
        [
          0.9086,
          0.6611
        ],
        [
          0.919,
          0.6538
        ],
        [
          0.929,
          0.647
        ],
        [
          0.939,
          0.6392
        ],
        [
          0.949,
          0.6315
        ],
        [
          0.9591,
          0.6251
        ],
        [
          0.9695,
          0.6174
        ],
        [
          0.9795,
          0.6115
        ],
        [
          0.9895,
          0.6037
        ],
        [
          0.9995,
          0.5965
        ],
        [
          1.0,
          0.596
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.18830868807397477,
        "difference": -0.0014643162916436563,
        "worse": 0.5266666666666666,
        "kept_right": 0.18607434853001584,
        "kept_wrong": 0.002234339543958837,
        "handed_off": 0.8116913119260256
      },
      "usd_per_1000": 1.3167176696220768
    },
    {
      "decision": 264,
      "frontier": 48,
      "frontier_accuracy": 0.9758871701546861,
      "in_sample": {
        "threshold": 0.892259,
        "answered": 0.15332120109190173,
        "accuracy": 0.9758871701546861
      },
      "curve": [
        [
          0.0,
          0.9759
        ],
        [
          0.01,
          0.9759
        ],
        [
          0.02,
          0.9759
        ],
        [
          0.03,
          0.9759
        ],
        [
          0.04,
          0.9759
        ],
        [
          0.05,
          0.9759
        ],
        [
          0.0601,
          0.9759
        ],
        [
          0.0705,
          0.9759
        ],
        [
          0.0805,
          0.9759
        ],
        [
          0.0905,
          0.9759
        ],
        [
          0.1005,
          0.9759
        ],
        [
          0.1106,
          0.9759
        ],
        [
          0.121,
          0.9759
        ],
        [
          0.131,
          0.9759
        ],
        [
          0.141,
          0.9759
        ],
        [
          0.151,
          0.9759
        ],
        [
          0.1611,
          0.9754
        ],
        [
          0.1715,
          0.9754
        ],
        [
          0.1815,
          0.975
        ],
        [
          0.1915,
          0.975
        ],
        [
          0.2015,
          0.9745
        ],
        [
          0.2116,
          0.9741
        ],
        [
          0.222,
          0.9727
        ],
        [
          0.232,
          0.9722
        ],
        [
          0.242,
          0.9722
        ],
        [
          0.252,
          0.9722
        ],
        [
          0.2621,
          0.9713
        ],
        [
          0.2725,
          0.9686
        ],
        [
          0.2825,
          0.9682
        ],
        [
          0.2925,
          0.9654
        ],
        [
          0.3025,
          0.9641
        ],
        [
          0.3126,
          0.9627
        ],
        [
          0.323,
          0.9604
        ],
        [
          0.333,
          0.9591
        ],
        [
          0.343,
          0.9572
        ],
        [
          0.353,
          0.9563
        ],
        [
          0.3631,
          0.9545
        ],
        [
          0.3735,
          0.9522
        ],
        [
          0.3835,
          0.949
        ],
        [
          0.3935,
          0.9449
        ],
        [
          0.4035,
          0.9418
        ],
        [
          0.4136,
          0.9381
        ],
        [
          0.424,
          0.9336
        ],
        [
          0.434,
          0.9322
        ],
        [
          0.444,
          0.9277
        ],
        [
          0.454,
          0.9245
        ],
        [
          0.4641,
          0.9195
        ],
        [
          0.4745,
          0.9149
        ],
        [
          0.4845,
          0.9099
        ],
        [
          0.4945,
          0.9049
        ],
        [
          0.5045,
          0.9008
        ],
        [
          0.5146,
          0.8949
        ],
        [
          0.525,
          0.8881
        ],
        [
          0.535,
          0.8844
        ],
        [
          0.545,
          0.8785
        ],
        [
          0.5551,
          0.874
        ],
        [
          0.5655,
          0.8703
        ],
        [
          0.5755,
          0.8644
        ],
        [
          0.5855,
          0.8562
        ],
        [
          0.5955,
          0.8517
        ],
        [
          0.6056,
          0.8449
        ],
        [
          0.616,
          0.8385
        ],
        [
          0.626,
          0.8339
        ],
        [
          0.636,
          0.8298
        ],
        [
          0.646,
          0.8248
        ],
        [
          0.6561,
          0.8203
        ],
        [
          0.6665,
          0.8135
        ],
        [
          0.6765,
          0.8085
        ],
        [
          0.6865,
          0.8021
        ],
        [
          0.6965,
          0.7962
        ],
        [
          0.7066,
          0.7903
        ],
        [
          0.717,
          0.7843
        ],
        [
          0.727,
          0.7789
        ],
        [
          0.737,
          0.7712
        ],
        [
          0.747,
          0.7666
        ],
        [
          0.7571,
          0.7611
        ],
        [
          0.7675,
          0.7566
        ],
        [
          0.7775,
          0.7511
        ],
        [
          0.7875,
          0.7457
        ],
        [
          0.7975,
          0.7384
        ],
        [
          0.8076,
          0.7329
        ],
        [
          0.818,
          0.7257
        ],
        [
          0.828,
          0.717
        ],
        [
          0.838,
          0.7097
        ],
        [
          0.848,
          0.702
        ],
        [
          0.8581,
          0.6956
        ],
        [
          0.8685,
          0.6884
        ],
        [
          0.8785,
          0.6806
        ],
        [
          0.8885,
          0.6738
        ],
        [
          0.8985,
          0.6665
        ],
        [
          0.9086,
          0.6597
        ],
        [
          0.919,
          0.6529
        ],
        [
          0.929,
          0.646
        ],
        [
          0.939,
          0.6383
        ],
        [
          0.949,
          0.6306
        ],
        [
          0.9591,
          0.6247
        ],
        [
          0.9695,
          0.6174
        ],
        [
          0.9795,
          0.6115
        ],
        [
          0.9895,
          0.6037
        ],
        [
          0.9995,
          0.5965
        ],
        [
          1.0,
          0.596
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.17290046177295224,
        "difference": -0.0011550157252883586,
        "worse": 0.5433333333333333,
        "kept_right": 0.17147865767717638,
        "kept_wrong": 0.0014218040957759176,
        "handed_off": 0.8270995382270475
      },
      "usd_per_1000": 4.448328443177422
    },
    {
      "decision": 264,
      "frontier": 191,
      "frontier_accuracy": 0.9740673339399454,
      "in_sample": {
        "threshold": 0.892259,
        "answered": 0.15332120109190173,
        "accuracy": 0.9740673339399454
      },
      "curve": [
        [
          0.0,
          0.9741
        ],
        [
          0.01,
          0.9741
        ],
        [
          0.02,
          0.9741
        ],
        [
          0.03,
          0.9741
        ],
        [
          0.04,
          0.9741
        ],
        [
          0.05,
          0.9741
        ],
        [
          0.0601,
          0.9741
        ],
        [
          0.0705,
          0.9741
        ],
        [
          0.0805,
          0.9741
        ],
        [
          0.0905,
          0.9741
        ],
        [
          0.1005,
          0.9741
        ],
        [
          0.1106,
          0.9741
        ],
        [
          0.121,
          0.9741
        ],
        [
          0.131,
          0.9741
        ],
        [
          0.141,
          0.9741
        ],
        [
          0.151,
          0.9741
        ],
        [
          0.1611,
          0.9736
        ],
        [
          0.1715,
          0.9736
        ],
        [
          0.1815,
          0.9732
        ],
        [
          0.1915,
          0.9732
        ],
        [
          0.2015,
          0.9727
        ],
        [
          0.2116,
          0.9718
        ],
        [
          0.222,
          0.9704
        ],
        [
          0.232,
          0.97
        ],
        [
          0.242,
          0.97
        ],
        [
          0.252,
          0.97
        ],
        [
          0.2621,
          0.9691
        ],
        [
          0.2725,
          0.9663
        ],
        [
          0.2825,
          0.9663
        ],
        [
          0.2925,
          0.9636
        ],
        [
          0.3025,
          0.9618
        ],
        [
          0.3126,
          0.9604
        ],
        [
          0.323,
          0.9581
        ],
        [
          0.333,
          0.9568
        ],
        [
          0.343,
          0.9554
        ],
        [
          0.353,
          0.955
        ],
        [
          0.3631,
          0.9531
        ],
        [
          0.3735,
          0.9509
        ],
        [
          0.3835,
          0.9477
        ],
        [
          0.3935,
          0.9436
        ],
        [
          0.4035,
          0.9399
        ],
        [
          0.4136,
          0.9359
        ],
        [
          0.424,
          0.9313
        ],
        [
          0.434,
          0.9299
        ],
        [
          0.444,
          0.9254
        ],
        [
          0.454,
          0.9222
        ],
        [
          0.4641,
          0.9167
        ],
        [
          0.4745,
          0.9122
        ],
        [
          0.4845,
          0.9076
        ],
        [
          0.4945,
          0.9026
        ],
        [
          0.5045,
          0.8985
        ],
        [
          0.5146,
          0.8922
        ],
        [
          0.525,
          0.8854
        ],
        [
          0.535,
          0.8817
        ],
        [
          0.545,
          0.8758
        ],
        [
          0.5551,
          0.8712
        ],
        [
          0.5655,
          0.8676
        ],
        [
          0.5755,
          0.8621
        ],
        [
          0.5855,
          0.8544
        ],
        [
          0.5955,
          0.8499
        ],
        [
          0.6056,
          0.8435
        ],
        [
          0.616,
          0.8371
        ],
        [
          0.626,
          0.8326
        ],
        [
          0.636,
          0.8285
        ],
        [
          0.646,
          0.8235
        ],
        [
          0.6561,
          0.8185
        ],
        [
          0.6665,
          0.8121
        ],
        [
          0.6765,
          0.8071
        ],
        [
          0.6865,
          0.8007
        ],
        [
          0.6965,
          0.7948
        ],
        [
          0.7066,
          0.7889
        ],
        [
          0.717,
          0.783
        ],
        [
          0.727,
          0.778
        ],
        [
          0.737,
          0.7702
        ],
        [
          0.747,
          0.7657
        ],
        [
          0.7571,
          0.7602
        ],
        [
          0.7675,
          0.7552
        ],
        [
          0.7775,
          0.7502
        ],
        [
          0.7875,
          0.7443
        ],
        [
          0.7975,
          0.7375
        ],
        [
          0.8076,
          0.7316
        ],
        [
          0.818,
          0.7243
        ],
        [
          0.828,
          0.7157
        ],
        [
          0.838,
          0.7093
        ],
        [
          0.848,
          0.7015
        ],
        [
          0.8581,
          0.6952
        ],
        [
          0.8685,
          0.6879
        ],
        [
          0.8785,
          0.6802
        ],
        [
          0.8885,
          0.6733
        ],
        [
          0.8985,
          0.6661
        ],
        [
          0.9086,
          0.6592
        ],
        [
          0.919,
          0.6515
        ],
        [
          0.929,
          0.6447
        ],
        [
          0.939,
          0.6374
        ],
        [
          0.949,
          0.6301
        ],
        [
          0.9591,
          0.6242
        ],
        [
          0.9695,
          0.6169
        ],
        [
          0.9795,
          0.611
        ],
        [
          0.9895,
          0.6037
        ],
        [
          0.9995,
          0.5965
        ],
        [
          1.0,
          0.596
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1712602336642977,
        "difference": -0.0010276656521176093,
        "worse": 0.54,
        "kept_right": 0.17000819840950446,
        "kept_wrong": 0.0012520352547932376,
        "handed_off": 0.8287397663357021
      },
      "usd_per_1000": 22.158592007787234
    },
    {
      "decision": 264,
      "frontier": 188,
      "frontier_accuracy": 0.9713375796178344,
      "in_sample": {
        "threshold": 0.892259,
        "answered": 0.15332120109190173,
        "accuracy": 0.9713375796178344
      },
      "curve": [
        [
          0.0,
          0.9713
        ],
        [
          0.01,
          0.9713
        ],
        [
          0.02,
          0.9713
        ],
        [
          0.03,
          0.9713
        ],
        [
          0.04,
          0.9713
        ],
        [
          0.05,
          0.9713
        ],
        [
          0.0601,
          0.9713
        ],
        [
          0.0705,
          0.9713
        ],
        [
          0.0805,
          0.9713
        ],
        [
          0.0905,
          0.9713
        ],
        [
          0.1005,
          0.9713
        ],
        [
          0.1106,
          0.9713
        ],
        [
          0.121,
          0.9713
        ],
        [
          0.131,
          0.9713
        ],
        [
          0.141,
          0.9713
        ],
        [
          0.151,
          0.9713
        ],
        [
          0.1611,
          0.9709
        ],
        [
          0.1715,
          0.9709
        ],
        [
          0.1815,
          0.9709
        ],
        [
          0.1915,
          0.9704
        ],
        [
          0.2015,
          0.97
        ],
        [
          0.2116,
          0.9691
        ],
        [
          0.222,
          0.9677
        ],
        [
          0.232,
          0.9672
        ],
        [
          0.242,
          0.9672
        ],
        [
          0.252,
          0.9672
        ],
        [
          0.2621,
          0.9663
        ],
        [
          0.2725,
          0.9641
        ],
        [
          0.2825,
          0.9636
        ],
        [
          0.2925,
          0.9609
        ],
        [
          0.3025,
          0.9591
        ],
        [
          0.3126,
          0.9581
        ],
        [
          0.323,
          0.9563
        ],
        [
          0.333,
          0.9559
        ],
        [
          0.343,
          0.954
        ],
        [
          0.353,
          0.9527
        ],
        [
          0.3631,
          0.9509
        ],
        [
          0.3735,
          0.9486
        ],
        [
          0.3835,
          0.9454
        ],
        [
          0.3935,
          0.9413
        ],
        [
          0.4035,
          0.9377
        ],
        [
          0.4136,
          0.9336
        ],
        [
          0.424,
          0.9295
        ],
        [
          0.434,
          0.9277
        ],
        [
          0.444,
          0.9231
        ],
        [
          0.454,
          0.9199
        ],
        [
          0.4641,
          0.9145
        ],
        [
          0.4745,
          0.9099
        ],
        [
          0.4845,
          0.9058
        ],
        [
          0.4945,
          0.9013
        ],
        [
          0.5045,
          0.8976
        ],
        [
          0.5146,
          0.8917
        ],
        [
          0.525,
          0.8858
        ],
        [
          0.535,
          0.8822
        ],
        [
          0.545,
          0.8763
        ],
        [
          0.5551,
          0.8726
        ],
        [
          0.5655,
          0.869
        ],
        [
          0.5755,
          0.8631
        ],
        [
          0.5855,
          0.8549
        ],
        [
          0.5955,
          0.8503
        ],
        [
          0.6056,
          0.8435
        ],
        [
          0.616,
          0.8371
        ],
        [
          0.626,
          0.8321
        ],
        [
          0.636,
          0.828
        ],
        [
          0.646,
          0.8226
        ],
        [
          0.6561,
          0.8176
        ],
        [
          0.6665,
          0.8107
        ],
        [
          0.6765,
          0.8057
        ],
        [
          0.6865,
          0.7994
        ],
        [
          0.6965,
          0.7934
        ],
        [
          0.7066,
          0.788
        ],
        [
          0.717,
          0.7825
        ],
        [
          0.727,
          0.7766
        ],
        [
          0.737,
          0.7693
        ],
        [
          0.747,
          0.7648
        ],
        [
          0.7571,
          0.7593
        ],
        [
          0.7675,
          0.7543
        ],
        [
          0.7775,
          0.7493
        ],
        [
          0.7875,
          0.7434
        ],
        [
          0.7975,
          0.7361
        ],
        [
          0.8076,
          0.7298
        ],
        [
          0.818,
          0.7225
        ],
        [
          0.828,
          0.7138
        ],
        [
          0.838,
          0.707
        ],
        [
          0.848,
          0.7002
        ],
        [
          0.8581,
          0.6943
        ],
        [
          0.8685,
          0.6865
        ],
        [
          0.8785,
          0.6793
        ],
        [
          0.8885,
          0.6724
        ],
        [
          0.8985,
          0.6652
        ],
        [
          0.9086,
          0.6583
        ],
        [
          0.919,
          0.6515
        ],
        [
          0.929,
          0.6451
        ],
        [
          0.939,
          0.6374
        ],
        [
          0.949,
          0.6297
        ],
        [
          0.9591,
          0.6237
        ],
        [
          0.9695,
          0.6165
        ],
        [
          0.9795,
          0.6106
        ],
        [
          0.9895,
          0.6037
        ],
        [
          0.9995,
          0.5965
        ],
        [
          1.0,
          0.596
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1774271162348015,
        "difference": -0.0013065915127884143,
        "worse": 0.6466666666666666,
        "kept_right": 0.1758900697123412,
        "kept_wrong": 0.0015370465224603593,
        "handed_off": 0.8225728837651982
      },
      "usd_per_1000": 0.7924994806670599
    },
    {
      "decision": 264,
      "frontier": 190,
      "frontier_accuracy": 0.9672429481346679,
      "in_sample": {
        "threshold": 0.892259,
        "answered": 0.15332120109190173,
        "accuracy": 0.9672429481346679
      },
      "curve": [
        [
          0.0,
          0.9672
        ],
        [
          0.01,
          0.9672
        ],
        [
          0.02,
          0.9672
        ],
        [
          0.03,
          0.9672
        ],
        [
          0.04,
          0.9672
        ],
        [
          0.05,
          0.9672
        ],
        [
          0.0601,
          0.9672
        ],
        [
          0.0705,
          0.9672
        ],
        [
          0.0805,
          0.9672
        ],
        [
          0.0905,
          0.9672
        ],
        [
          0.1005,
          0.9672
        ],
        [
          0.1106,
          0.9672
        ],
        [
          0.121,
          0.9672
        ],
        [
          0.131,
          0.9672
        ],
        [
          0.141,
          0.9672
        ],
        [
          0.151,
          0.9672
        ],
        [
          0.1611,
          0.9668
        ],
        [
          0.1715,
          0.9668
        ],
        [
          0.1815,
          0.9663
        ],
        [
          0.1915,
          0.9663
        ],
        [
          0.2015,
          0.9659
        ],
        [
          0.2116,
          0.9654
        ],
        [
          0.222,
          0.9641
        ],
        [
          0.232,
          0.9636
        ],
        [
          0.242,
          0.9641
        ],
        [
          0.252,
          0.9641
        ],
        [
          0.2621,
          0.9631
        ],
        [
          0.2725,
          0.9604
        ],
        [
          0.2825,
          0.96
        ],
        [
          0.2925,
          0.9572
        ],
        [
          0.3025,
          0.9554
        ],
        [
          0.3126,
          0.9545
        ],
        [
          0.323,
          0.9522
        ],
        [
          0.333,
          0.9509
        ],
        [
          0.343,
          0.9495
        ],
        [
          0.353,
          0.949
        ],
        [
          0.3631,
          0.9472
        ],
        [
          0.3735,
          0.9449
        ],
        [
          0.3835,
          0.9418
        ],
        [
          0.3935,
          0.9377
        ],
        [
          0.4035,
          0.9345
        ],
        [
          0.4136,
          0.9304
        ],
        [
          0.424,
          0.9258
        ],
        [
          0.434,
          0.9245
        ],
        [
          0.444,
          0.9199
        ],
        [
          0.454,
          0.9172
        ],
        [
          0.4641,
          0.9122
        ],
        [
          0.4745,
          0.9076
        ],
        [
          0.4845,
          0.9031
        ],
        [
          0.4945,
          0.8985
        ],
        [
          0.5045,
          0.8949
        ],
        [
          0.5146,
          0.8885
        ],
        [
          0.525,
          0.8817
        ],
        [
          0.535,
          0.8785
        ],
        [
          0.545,
          0.8726
        ],
        [
          0.5551,
          0.8685
        ],
        [
          0.5655,
          0.8649
        ],
        [
          0.5755,
          0.8594
        ],
        [
          0.5855,
          0.8517
        ],
        [
          0.5955,
          0.8471
        ],
        [
          0.6056,
          0.8408
        ],
        [
          0.616,
          0.8344
        ],
        [
          0.626,
          0.8298
        ],
        [
          0.636,
          0.8258
        ],
        [
          0.646,
          0.8207
        ],
        [
          0.6561,
          0.8162
        ],
        [
          0.6665,
          0.8094
        ],
        [
          0.6765,
          0.8044
        ],
        [
          0.6865,
          0.798
        ],
        [
          0.6965,
          0.7921
        ],
        [
          0.7066,
          0.7862
        ],
        [
          0.717,
          0.7803
        ],
        [
          0.727,
          0.7753
        ],
        [
          0.737,
          0.7675
        ],
        [
          0.747,
          0.7634
        ],
        [
          0.7571,
          0.758
        ],
        [
          0.7675,
          0.7534
        ],
        [
          0.7775,
          0.7484
        ],
        [
          0.7875,
          0.7425
        ],
        [
          0.7975,
          0.7357
        ],
        [
          0.8076,
          0.7298
        ],
        [
          0.818,
          0.7225
        ],
        [
          0.828,
          0.7143
        ],
        [
          0.838,
          0.7079
        ],
        [
          0.848,
          0.7006
        ],
        [
          0.8581,
          0.6943
        ],
        [
          0.8685,
          0.6865
        ],
        [
          0.8785,
          0.6793
        ],
        [
          0.8885,
          0.6724
        ],
        [
          0.8985,
          0.6652
        ],
        [
          0.9086,
          0.6588
        ],
        [
          0.919,
          0.6515
        ],
        [
          0.929,
          0.6447
        ],
        [
          0.939,
          0.6379
        ],
        [
          0.949,
          0.6301
        ],
        [
          0.9591,
          0.6242
        ],
        [
          0.9695,
          0.6165
        ],
        [
          0.9795,
          0.6106
        ],
        [
          0.9895,
          0.6033
        ],
        [
          0.9995,
          0.5965
        ],
        [
          1.0,
          0.596
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.17516549109937318,
        "difference": -0.0013248039475396655,
        "worse": 0.55,
        "kept_right": 0.17355570039912246,
        "kept_wrong": 0.0016097907002507645,
        "handed_off": 0.8248345089006266
      },
      "usd_per_1000": 4.593106648928916
    },
    {
      "decision": 264,
      "frontier": 124,
      "frontier_accuracy": 0.9667879890809827,
      "in_sample": {
        "threshold": 0.892259,
        "answered": 0.15332120109190173,
        "accuracy": 0.9667879890809827
      },
      "curve": [
        [
          0.0,
          0.9668
        ],
        [
          0.01,
          0.9668
        ],
        [
          0.02,
          0.9668
        ],
        [
          0.03,
          0.9668
        ],
        [
          0.04,
          0.9668
        ],
        [
          0.05,
          0.9668
        ],
        [
          0.0601,
          0.9668
        ],
        [
          0.0705,
          0.9668
        ],
        [
          0.0805,
          0.9668
        ],
        [
          0.0905,
          0.9668
        ],
        [
          0.1005,
          0.9668
        ],
        [
          0.1106,
          0.9668
        ],
        [
          0.121,
          0.9668
        ],
        [
          0.131,
          0.9668
        ],
        [
          0.141,
          0.9668
        ],
        [
          0.151,
          0.9668
        ],
        [
          0.1611,
          0.9663
        ],
        [
          0.1715,
          0.9663
        ],
        [
          0.1815,
          0.9659
        ],
        [
          0.1915,
          0.9659
        ],
        [
          0.2015,
          0.9654
        ],
        [
          0.2116,
          0.965
        ],
        [
          0.222,
          0.9636
        ],
        [
          0.232,
          0.9631
        ],
        [
          0.242,
          0.9631
        ],
        [
          0.252,
          0.9631
        ],
        [
          0.2621,
          0.9627
        ],
        [
          0.2725,
          0.96
        ],
        [
          0.2825,
          0.9595
        ],
        [
          0.2925,
          0.9568
        ],
        [
          0.3025,
          0.9554
        ],
        [
          0.3126,
          0.954
        ],
        [
          0.323,
          0.9522
        ],
        [
          0.333,
          0.9509
        ],
        [
          0.343,
          0.9495
        ],
        [
          0.353,
          0.9486
        ],
        [
          0.3631,
          0.9468
        ],
        [
          0.3735,
          0.9445
        ],
        [
          0.3835,
          0.9413
        ],
        [
          0.3935,
          0.9372
        ],
        [
          0.4035,
          0.934
        ],
        [
          0.4136,
          0.9304
        ],
        [
          0.424,
          0.9258
        ],
        [
          0.434,
          0.9245
        ],
        [
          0.444,
          0.9199
        ],
        [
          0.454,
          0.9172
        ],
        [
          0.4641,
          0.9122
        ],
        [
          0.4745,
          0.9076
        ],
        [
          0.4845,
          0.9026
        ],
        [
          0.4945,
          0.8976
        ],
        [
          0.5045,
          0.894
        ],
        [
          0.5146,
          0.8881
        ],
        [
          0.525,
          0.8813
        ],
        [
          0.535,
          0.8776
        ],
        [
          0.545,
          0.8717
        ],
        [
          0.5551,
          0.8681
        ],
        [
          0.5655,
          0.8644
        ],
        [
          0.5755,
          0.8585
        ],
        [
          0.5855,
          0.8508
        ],
        [
          0.5955,
          0.8462
        ],
        [
          0.6056,
          0.8403
        ],
        [
          0.616,
          0.8344
        ],
        [
          0.626,
          0.8298
        ],
        [
          0.636,
          0.8258
        ],
        [
          0.646,
          0.8207
        ],
        [
          0.6561,
          0.8162
        ],
        [
          0.6665,
          0.8098
        ],
        [
          0.6765,
          0.8048
        ],
        [
          0.6865,
          0.7985
        ],
        [
          0.6965,
          0.7925
        ],
        [
          0.7066,
          0.7871
        ],
        [
          0.717,
          0.7812
        ],
        [
          0.727,
          0.7762
        ],
        [
          0.737,
          0.7684
        ],
        [
          0.747,
          0.7639
        ],
        [
          0.7571,
          0.7589
        ],
        [
          0.7675,
          0.7543
        ],
        [
          0.7775,
          0.7493
        ],
        [
          0.7875,
          0.7439
        ],
        [
          0.7975,
          0.7366
        ],
        [
          0.8076,
          0.7307
        ],
        [
          0.818,
          0.7234
        ],
        [
          0.828,
          0.7152
        ],
        [
          0.838,
          0.7084
        ],
        [
          0.848,
          0.7015
        ],
        [
          0.8581,
          0.6956
        ],
        [
          0.8685,
          0.6884
        ],
        [
          0.8785,
          0.6811
        ],
        [
          0.8885,
          0.6742
        ],
        [
          0.8985,
          0.667
        ],
        [
          0.9086,
          0.6601
        ],
        [
          0.919,
          0.6529
        ],
        [
          0.929,
          0.646
        ],
        [
          0.939,
          0.6388
        ],
        [
          0.949,
          0.631
        ],
        [
          0.9591,
          0.6247
        ],
        [
          0.9695,
          0.6169
        ],
        [
          0.9795,
          0.611
        ],
        [
          0.9895,
          0.6037
        ],
        [
          0.9995,
          0.5965
        ],
        [
          1.0,
          0.596
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.17359206082356768,
        "difference": -0.001203550250872065,
        "worse": 0.5466666666666666,
        "kept_right": 0.1721126202537619,
        "kept_wrong": 0.0014794405698058343,
        "handed_off": 0.8264079391764321
      },
      "usd_per_1000": 0.9341492763240203
    },
    {
      "decision": 264,
      "frontier": 278,
      "frontier_accuracy": 0.9499545040946314,
      "in_sample": {
        "threshold": 0.892259,
        "answered": 0.15332120109190173,
        "accuracy": 0.9499545040946314
      },
      "curve": [
        [
          0.0,
          0.95
        ],
        [
          0.01,
          0.95
        ],
        [
          0.02,
          0.95
        ],
        [
          0.03,
          0.95
        ],
        [
          0.04,
          0.95
        ],
        [
          0.05,
          0.95
        ],
        [
          0.0601,
          0.95
        ],
        [
          0.0705,
          0.95
        ],
        [
          0.0805,
          0.95
        ],
        [
          0.0905,
          0.95
        ],
        [
          0.1005,
          0.95
        ],
        [
          0.1106,
          0.95
        ],
        [
          0.121,
          0.95
        ],
        [
          0.131,
          0.95
        ],
        [
          0.141,
          0.95
        ],
        [
          0.151,
          0.95
        ],
        [
          0.1611,
          0.9495
        ],
        [
          0.1715,
          0.9495
        ],
        [
          0.1815,
          0.949
        ],
        [
          0.1915,
          0.949
        ],
        [
          0.2015,
          0.9486
        ],
        [
          0.2116,
          0.9481
        ],
        [
          0.222,
          0.9468
        ],
        [
          0.232,
          0.9463
        ],
        [
          0.242,
          0.9463
        ],
        [
          0.252,
          0.9463
        ],
        [
          0.2621,
          0.9459
        ],
        [
          0.2725,
          0.9436
        ],
        [
          0.2825,
          0.9431
        ],
        [
          0.2925,
          0.9404
        ],
        [
          0.3025,
          0.9399
        ],
        [
          0.3126,
          0.9386
        ],
        [
          0.323,
          0.9372
        ],
        [
          0.333,
          0.9359
        ],
        [
          0.343,
          0.9345
        ],
        [
          0.353,
          0.9336
        ],
        [
          0.3631,
          0.9322
        ],
        [
          0.3735,
          0.9304
        ],
        [
          0.3835,
          0.9272
        ],
        [
          0.3935,
          0.924
        ],
        [
          0.4035,
          0.9208
        ],
        [
          0.4136,
          0.9167
        ],
        [
          0.424,
          0.9126
        ],
        [
          0.434,
          0.9113
        ],
        [
          0.444,
          0.9067
        ],
        [
          0.454,
          0.9035
        ],
        [
          0.4641,
          0.8995
        ],
        [
          0.4745,
          0.8958
        ],
        [
          0.4845,
          0.8931
        ],
        [
          0.4945,
          0.889
        ],
        [
          0.5045,
          0.8854
        ],
        [
          0.5146,
          0.8794
        ],
        [
          0.525,
          0.8731
        ],
        [
          0.535,
          0.8694
        ],
        [
          0.545,
          0.8635
        ],
        [
          0.5551,
          0.8594
        ],
        [
          0.5655,
          0.8562
        ],
        [
          0.5755,
          0.8508
        ],
        [
          0.5855,
          0.843
        ],
        [
          0.5955,
          0.8385
        ],
        [
          0.6056,
          0.8321
        ],
        [
          0.616,
          0.8258
        ],
        [
          0.626,
          0.8212
        ],
        [
          0.636,
          0.8176
        ],
        [
          0.646,
          0.8139
        ],
        [
          0.6561,
          0.8094
        ],
        [
          0.6665,
          0.8025
        ],
        [
          0.6765,
          0.7975
        ],
        [
          0.6865,
          0.7912
        ],
        [
          0.6965,
          0.7857
        ],
        [
          0.7066,
          0.7807
        ],
        [
          0.717,
          0.7753
        ],
        [
          0.727,
          0.7702
        ],
        [
          0.737,
          0.7625
        ],
        [
          0.747,
          0.7584
        ],
        [
          0.7571,
          0.7539
        ],
        [
          0.7675,
          0.7493
        ],
        [
          0.7775,
          0.7452
        ],
        [
          0.7875,
          0.7393
        ],
        [
          0.7975,
          0.7329
        ],
        [
          0.8076,
          0.7266
        ],
        [
          0.818,
          0.7193
        ],
        [
          0.828,
          0.7116
        ],
        [
          0.838,
          0.7047
        ],
        [
          0.848,
          0.697
        ],
        [
          0.8581,
          0.6911
        ],
        [
          0.8685,
          0.6843
        ],
        [
          0.8785,
          0.677
        ],
        [
          0.8885,
          0.6706
        ],
        [
          0.8985,
          0.6638
        ],
        [
          0.9086,
          0.657
        ],
        [
          0.919,
          0.6506
        ],
        [
          0.929,
          0.6442
        ],
        [
          0.939,
          0.6369
        ],
        [
          0.949,
          0.6292
        ],
        [
          0.9591,
          0.6233
        ],
        [
          0.9695,
          0.6165
        ],
        [
          0.9795,
          0.6115
        ],
        [
          0.9895,
          0.6037
        ],
        [
          0.9995,
          0.5965
        ],
        [
          1.0,
          0.596
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.17223059033786037,
        "difference": -0.0010216242269475827,
        "worse": 0.54,
        "kept_right": 0.17089668721920964,
        "kept_wrong": 0.0013339031186506208,
        "handed_off": 0.8277694096621395
      },
      "usd_per_1000": 5.387807301157559
    },
    {
      "decision": 264,
      "frontier": 275,
      "frontier_accuracy": 0.9426751592356688,
      "in_sample": {
        "threshold": 0.892259,
        "answered": 0.15332120109190173,
        "accuracy": 0.9426751592356688
      },
      "curve": [
        [
          0.0,
          0.9427
        ],
        [
          0.01,
          0.9427
        ],
        [
          0.02,
          0.9427
        ],
        [
          0.03,
          0.9427
        ],
        [
          0.04,
          0.9427
        ],
        [
          0.05,
          0.9427
        ],
        [
          0.0601,
          0.9427
        ],
        [
          0.0705,
          0.9427
        ],
        [
          0.0805,
          0.9427
        ],
        [
          0.0905,
          0.9427
        ],
        [
          0.1005,
          0.9427
        ],
        [
          0.1106,
          0.9427
        ],
        [
          0.121,
          0.9427
        ],
        [
          0.131,
          0.9427
        ],
        [
          0.141,
          0.9427
        ],
        [
          0.151,
          0.9427
        ],
        [
          0.1611,
          0.9422
        ],
        [
          0.1715,
          0.9422
        ],
        [
          0.1815,
          0.9418
        ],
        [
          0.1915,
          0.9418
        ],
        [
          0.2015,
          0.9413
        ],
        [
          0.2116,
          0.9409
        ],
        [
          0.222,
          0.9395
        ],
        [
          0.232,
          0.939
        ],
        [
          0.242,
          0.939
        ],
        [
          0.252,
          0.939
        ],
        [
          0.2621,
          0.9386
        ],
        [
          0.2725,
          0.9363
        ],
        [
          0.2825,
          0.9363
        ],
        [
          0.2925,
          0.934
        ],
        [
          0.3025,
          0.9322
        ],
        [
          0.3126,
          0.9308
        ],
        [
          0.323,
          0.9299
        ],
        [
          0.333,
          0.9286
        ],
        [
          0.343,
          0.9277
        ],
        [
          0.353,
          0.9272
        ],
        [
          0.3631,
          0.9263
        ],
        [
          0.3735,
          0.924
        ],
        [
          0.3835,
          0.9208
        ],
        [
          0.3935,
          0.9172
        ],
        [
          0.4035,
          0.9136
        ],
        [
          0.4136,
          0.9099
        ],
        [
          0.424,
          0.9058
        ],
        [
          0.434,
          0.9045
        ],
        [
          0.444,
          0.9004
        ],
        [
          0.454,
          0.8972
        ],
        [
          0.4641,
          0.8917
        ],
        [
          0.4745,
          0.8876
        ],
        [
          0.4845,
          0.884
        ],
        [
          0.4945,
          0.8799
        ],
        [
          0.5045,
          0.8767
        ],
        [
          0.5146,
          0.8703
        ],
        [
          0.525,
          0.864
        ],
        [
          0.535,
          0.8608
        ],
        [
          0.545,
          0.8553
        ],
        [
          0.5551,
          0.8512
        ],
        [
          0.5655,
          0.8476
        ],
        [
          0.5755,
          0.843
        ],
        [
          0.5855,
          0.8358
        ],
        [
          0.5955,
          0.8321
        ],
        [
          0.6056,
          0.8262
        ],
        [
          0.616,
          0.8207
        ],
        [
          0.626,
          0.8162
        ],
        [
          0.636,
          0.8126
        ],
        [
          0.646,
          0.8085
        ],
        [
          0.6561,
          0.8044
        ],
        [
          0.6665,
          0.7989
        ],
        [
          0.6765,
          0.7939
        ],
        [
          0.6865,
          0.7884
        ],
        [
          0.6965,
          0.7825
        ],
        [
          0.7066,
          0.7784
        ],
        [
          0.717,
          0.773
        ],
        [
          0.727,
          0.7684
        ],
        [
          0.737,
          0.7607
        ],
        [
          0.747,
          0.7566
        ],
        [
          0.7571,
          0.7516
        ],
        [
          0.7675,
          0.7466
        ],
        [
          0.7775,
          0.742
        ],
        [
          0.7875,
          0.7366
        ],
        [
          0.7975,
          0.7298
        ],
        [
          0.8076,
          0.7243
        ],
        [
          0.818,
          0.7166
        ],
        [
          0.828,
          0.7088
        ],
        [
          0.838,
          0.7025
        ],
        [
          0.848,
          0.6956
        ],
        [
          0.8581,
          0.6893
        ],
        [
          0.8685,
          0.682
        ],
        [
          0.8785,
          0.6742
        ],
        [
          0.8885,
          0.6679
        ],
        [
          0.8985,
          0.6611
        ],
        [
          0.9086,
          0.6547
        ],
        [
          0.919,
          0.6483
        ],
        [
          0.929,
          0.6415
        ],
        [
          0.939,
          0.6351
        ],
        [
          0.949,
          0.6283
        ],
        [
          0.9591,
          0.6233
        ],
        [
          0.9695,
          0.616
        ],
        [
          0.9795,
          0.6101
        ],
        [
          0.9895,
          0.6024
        ],
        [
          0.9995,
          0.5965
        ],
        [
          1.0,
          0.596
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.17290046177295224,
        "difference": -0.0011550157252883586,
        "worse": 0.5433333333333333,
        "kept_right": 0.17147865767717638,
        "kept_wrong": 0.0014218040957759176,
        "handed_off": 0.8270995382270475
      },
      "usd_per_1000": 0.42555362501408933
    },
    {
      "decision": 264,
      "frontier": 192,
      "frontier_accuracy": 0.9331210191082803,
      "in_sample": {
        "threshold": 0.892259,
        "answered": 0.15332120109190173,
        "accuracy": 0.9331210191082803
      },
      "curve": [
        [
          0.0,
          0.9331
        ],
        [
          0.01,
          0.9331
        ],
        [
          0.02,
          0.9331
        ],
        [
          0.03,
          0.9331
        ],
        [
          0.04,
          0.9331
        ],
        [
          0.05,
          0.9331
        ],
        [
          0.0601,
          0.9331
        ],
        [
          0.0705,
          0.9331
        ],
        [
          0.0805,
          0.9331
        ],
        [
          0.0905,
          0.9331
        ],
        [
          0.1005,
          0.9331
        ],
        [
          0.1106,
          0.9331
        ],
        [
          0.121,
          0.9331
        ],
        [
          0.131,
          0.9331
        ],
        [
          0.141,
          0.9331
        ],
        [
          0.151,
          0.9331
        ],
        [
          0.1611,
          0.9327
        ],
        [
          0.1715,
          0.9327
        ],
        [
          0.1815,
          0.9322
        ],
        [
          0.1915,
          0.9318
        ],
        [
          0.2015,
          0.9313
        ],
        [
          0.2116,
          0.9322
        ],
        [
          0.222,
          0.9313
        ],
        [
          0.232,
          0.9308
        ],
        [
          0.242,
          0.9308
        ],
        [
          0.252,
          0.9308
        ],
        [
          0.2621,
          0.9304
        ],
        [
          0.2725,
          0.9277
        ],
        [
          0.2825,
          0.9272
        ],
        [
          0.2925,
          0.9245
        ],
        [
          0.3025,
          0.9231
        ],
        [
          0.3126,
          0.9217
        ],
        [
          0.323,
          0.9199
        ],
        [
          0.333,
          0.9186
        ],
        [
          0.343,
          0.9177
        ],
        [
          0.353,
          0.9167
        ],
        [
          0.3631,
          0.9154
        ],
        [
          0.3735,
          0.9136
        ],
        [
          0.3835,
          0.9104
        ],
        [
          0.3935,
          0.9072
        ],
        [
          0.4035,
          0.904
        ],
        [
          0.4136,
          0.9004
        ],
        [
          0.424,
          0.8958
        ],
        [
          0.434,
          0.8944
        ],
        [
          0.444,
          0.8904
        ],
        [
          0.454,
          0.8872
        ],
        [
          0.4641,
          0.8835
        ],
        [
          0.4745,
          0.8808
        ],
        [
          0.4845,
          0.8781
        ],
        [
          0.4945,
          0.8744
        ],
        [
          0.5045,
          0.8717
        ],
        [
          0.5146,
          0.8658
        ],
        [
          0.525,
          0.8594
        ],
        [
          0.535,
          0.8558
        ],
        [
          0.545,
          0.8499
        ],
        [
          0.5551,
          0.8458
        ],
        [
          0.5655,
          0.8426
        ],
        [
          0.5755,
          0.8376
        ],
        [
          0.5855,
          0.8312
        ],
        [
          0.5955,
          0.8267
        ],
        [
          0.6056,
          0.8207
        ],
        [
          0.616,
          0.8157
        ],
        [
          0.626,
          0.8112
        ],
        [
          0.636,
          0.8076
        ],
        [
          0.646,
          0.8025
        ],
        [
          0.6561,
          0.7985
        ],
        [
          0.6665,
          0.7921
        ],
        [
          0.6765,
          0.7875
        ],
        [
          0.6865,
          0.7816
        ],
        [
          0.6965,
          0.7757
        ],
        [
          0.7066,
          0.7712
        ],
        [
          0.717,
          0.7662
        ],
        [
          0.727,
          0.7616
        ],
        [
          0.737,
          0.7543
        ],
        [
          0.747,
          0.7507
        ],
        [
          0.7571,
          0.7457
        ],
        [
          0.7675,
          0.7416
        ],
        [
          0.7775,
          0.7379
        ],
        [
          0.7875,
          0.7316
        ],
        [
          0.7975,
          0.7252
        ],
        [
          0.8076,
          0.7193
        ],
        [
          0.818,
          0.7125
        ],
        [
          0.828,
          0.7052
        ],
        [
          0.838,
          0.6988
        ],
        [
          0.848,
          0.6915
        ],
        [
          0.8581,
          0.6856
        ],
        [
          0.8685,
          0.6793
        ],
        [
          0.8785,
          0.6724
        ],
        [
          0.8885,
          0.6661
        ],
        [
          0.8985,
          0.6597
        ],
        [
          0.9086,
          0.6529
        ],
        [
          0.919,
          0.6456
        ],
        [
          0.929,
          0.6392
        ],
        [
          0.939,
          0.6328
        ],
        [
          0.949,
          0.6256
        ],
        [
          0.9591,
          0.6201
        ],
        [
          0.9695,
          0.6142
        ],
        [
          0.9795,
          0.6087
        ],
        [
          0.9895,
          0.6028
        ],
        [
          0.9995,
          0.5965
        ],
        [
          1.0,
          0.596
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.19229404225078497,
        "difference": -0.002052427813394649,
        "worse": 0.6633333333333333,
        "kept_right": 0.18931998736711592,
        "kept_wrong": 0.0029740548836687873,
        "handed_off": 0.8077059577492149
      },
      "usd_per_1000": 9.377115085632157
    },
    {
      "decision": 264,
      "frontier": 123,
      "frontier_accuracy": 0.9326660600545951,
      "in_sample": {
        "threshold": 0.892259,
        "answered": 0.15332120109190173,
        "accuracy": 0.9326660600545951
      },
      "curve": [
        [
          0.0,
          0.9327
        ],
        [
          0.01,
          0.9327
        ],
        [
          0.02,
          0.9327
        ],
        [
          0.03,
          0.9327
        ],
        [
          0.04,
          0.9327
        ],
        [
          0.05,
          0.9327
        ],
        [
          0.0601,
          0.9327
        ],
        [
          0.0705,
          0.9327
        ],
        [
          0.0805,
          0.9327
        ],
        [
          0.0905,
          0.9327
        ],
        [
          0.1005,
          0.9327
        ],
        [
          0.1106,
          0.9327
        ],
        [
          0.121,
          0.9327
        ],
        [
          0.131,
          0.9327
        ],
        [
          0.141,
          0.9327
        ],
        [
          0.151,
          0.9327
        ],
        [
          0.1611,
          0.9322
        ],
        [
          0.1715,
          0.9322
        ],
        [
          0.1815,
          0.9322
        ],
        [
          0.1915,
          0.9322
        ],
        [
          0.2015,
          0.9318
        ],
        [
          0.2116,
          0.9308
        ],
        [
          0.222,
          0.9295
        ],
        [
          0.232,
          0.929
        ],
        [
          0.242,
          0.929
        ],
        [
          0.252,
          0.9295
        ],
        [
          0.2621,
          0.9286
        ],
        [
          0.2725,
          0.9258
        ],
        [
          0.2825,
          0.9258
        ],
        [
          0.2925,
          0.9231
        ],
        [
          0.3025,
          0.9217
        ],
        [
          0.3126,
          0.9204
        ],
        [
          0.323,
          0.9186
        ],
        [
          0.333,
          0.9177
        ],
        [
          0.343,
          0.9167
        ],
        [
          0.353,
          0.9154
        ],
        [
          0.3631,
          0.9145
        ],
        [
          0.3735,
          0.9122
        ],
        [
          0.3835,
          0.909
        ],
        [
          0.3935,
          0.9049
        ],
        [
          0.4035,
          0.9013
        ],
        [
          0.4136,
          0.8976
        ],
        [
          0.424,
          0.8935
        ],
        [
          0.434,
          0.8917
        ],
        [
          0.444,
          0.8876
        ],
        [
          0.454,
          0.8844
        ],
        [
          0.4641,
          0.879
        ],
        [
          0.4745,
          0.8749
        ],
        [
          0.4845,
          0.8703
        ],
        [
          0.4945,
          0.8658
        ],
        [
          0.5045,
          0.8617
        ],
        [
          0.5146,
          0.8553
        ],
        [
          0.525,
          0.8494
        ],
        [
          0.535,
          0.8462
        ],
        [
          0.545,
          0.8403
        ],
        [
          0.5551,
          0.8367
        ],
        [
          0.5655,
          0.8339
        ],
        [
          0.5755,
          0.8289
        ],
        [
          0.5855,
          0.8212
        ],
        [
          0.5955,
          0.8167
        ],
        [
          0.6056,
          0.8112
        ],
        [
          0.616,
          0.8062
        ],
        [
          0.626,
          0.8012
        ],
        [
          0.636,
          0.7975
        ],
        [
          0.646,
          0.7925
        ],
        [
          0.6561,
          0.788
        ],
        [
          0.6665,
          0.7825
        ],
        [
          0.6765,
          0.7784
        ],
        [
          0.6865,
          0.7725
        ],
        [
          0.6965,
          0.7671
        ],
        [
          0.7066,
          0.7625
        ],
        [
          0.717,
          0.7571
        ],
        [
          0.727,
          0.7525
        ],
        [
          0.737,
          0.7461
        ],
        [
          0.747,
          0.7425
        ],
        [
          0.7571,
          0.737
        ],
        [
          0.7675,
          0.7325
        ],
        [
          0.7775,
          0.7279
        ],
        [
          0.7875,
          0.7225
        ],
        [
          0.7975,
          0.7166
        ],
        [
          0.8076,
          0.7116
        ],
        [
          0.818,
          0.7043
        ],
        [
          0.828,
          0.697
        ],
        [
          0.838,
          0.6915
        ],
        [
          0.848,
          0.6852
        ],
        [
          0.8581,
          0.6806
        ],
        [
          0.8685,
          0.6747
        ],
        [
          0.8785,
          0.6679
        ],
        [
          0.8885,
          0.6615
        ],
        [
          0.8985,
          0.6556
        ],
        [
          0.9086,
          0.6506
        ],
        [
          0.919,
          0.6447
        ],
        [
          0.929,
          0.6388
        ],
        [
          0.939,
          0.6319
        ],
        [
          0.949,
          0.6256
        ],
        [
          0.9591,
          0.6206
        ],
        [
          0.9695,
          0.6146
        ],
        [
          0.9795,
          0.6096
        ],
        [
          0.9895,
          0.6028
        ],
        [
          0.9995,
          0.5965
        ],
        [
          1.0,
          0.596
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1830248686635396,
        "difference": -0.0014459057799036971,
        "worse": 0.6466666666666666,
        "kept_right": 0.18104532833397557,
        "kept_wrong": 0.0019795403295639847,
        "handed_off": 0.8169751313364606
      },
      "usd_per_1000": 0.21942639961176122
    },
    {
      "decision": 264,
      "frontier": 274,
      "frontier_accuracy": 0.9085532302092811,
      "in_sample": {
        "threshold": 0.84836,
        "answered": 0.19563239308462238,
        "accuracy": 0.9085532302092811
      },
      "curve": [
        [
          0.0,
          0.9086
        ],
        [
          0.01,
          0.9086
        ],
        [
          0.02,
          0.9086
        ],
        [
          0.03,
          0.9086
        ],
        [
          0.04,
          0.9086
        ],
        [
          0.05,
          0.9086
        ],
        [
          0.0601,
          0.9086
        ],
        [
          0.0705,
          0.9086
        ],
        [
          0.0805,
          0.9086
        ],
        [
          0.0905,
          0.9086
        ],
        [
          0.1005,
          0.909
        ],
        [
          0.1106,
          0.909
        ],
        [
          0.121,
          0.909
        ],
        [
          0.131,
          0.909
        ],
        [
          0.141,
          0.909
        ],
        [
          0.151,
          0.909
        ],
        [
          0.1611,
          0.9086
        ],
        [
          0.1715,
          0.909
        ],
        [
          0.1815,
          0.9086
        ],
        [
          0.1915,
          0.9086
        ],
        [
          0.2015,
          0.9081
        ],
        [
          0.2116,
          0.9081
        ],
        [
          0.222,
          0.9067
        ],
        [
          0.232,
          0.9063
        ],
        [
          0.242,
          0.9063
        ],
        [
          0.252,
          0.9063
        ],
        [
          0.2621,
          0.9054
        ],
        [
          0.2725,
          0.9035
        ],
        [
          0.2825,
          0.9035
        ],
        [
          0.2925,
          0.9008
        ],
        [
          0.3025,
          0.899
        ],
        [
          0.3126,
          0.8981
        ],
        [
          0.323,
          0.8972
        ],
        [
          0.333,
          0.8958
        ],
        [
          0.343,
          0.8944
        ],
        [
          0.353,
          0.894
        ],
        [
          0.3631,
          0.8931
        ],
        [
          0.3735,
          0.8913
        ],
        [
          0.3835,
          0.8885
        ],
        [
          0.3935,
          0.8854
        ],
        [
          0.4035,
          0.8817
        ],
        [
          0.4136,
          0.8781
        ],
        [
          0.424,
          0.8744
        ],
        [
          0.434,
          0.8731
        ],
        [
          0.444,
          0.8685
        ],
        [
          0.454,
          0.8658
        ],
        [
          0.4641,
          0.8608
        ],
        [
          0.4745,
          0.8567
        ],
        [
          0.4845,
          0.8535
        ],
        [
          0.4945,
          0.849
        ],
        [
          0.5045,
          0.8458
        ],
        [
          0.5146,
          0.8394
        ],
        [
          0.525,
          0.8335
        ],
        [
          0.535,
          0.8308
        ],
        [
          0.545,
          0.8248
        ],
        [
          0.5551,
          0.8207
        ],
        [
          0.5655,
          0.8176
        ],
        [
          0.5755,
          0.8148
        ],
        [
          0.5855,
          0.8089
        ],
        [
          0.5955,
          0.8053
        ],
        [
          0.6056,
          0.8007
        ],
        [
          0.616,
          0.7953
        ],
        [
          0.626,
          0.7903
        ],
        [
          0.636,
          0.7871
        ],
        [
          0.646,
          0.7825
        ],
        [
          0.6561,
          0.7793
        ],
        [
          0.6665,
          0.7739
        ],
        [
          0.6765,
          0.7693
        ],
        [
          0.6865,
          0.7634
        ],
        [
          0.6965,
          0.758
        ],
        [
          0.7066,
          0.7543
        ],
        [
          0.717,
          0.7498
        ],
        [
          0.727,
          0.7461
        ],
        [
          0.737,
          0.7398
        ],
        [
          0.747,
          0.7366
        ],
        [
          0.7571,
          0.7316
        ],
        [
          0.7675,
          0.7275
        ],
        [
          0.7775,
          0.7243
        ],
        [
          0.7875,
          0.7188
        ],
        [
          0.7975,
          0.7125
        ],
        [
          0.8076,
          0.7075
        ],
        [
          0.818,
          0.7006
        ],
        [
          0.828,
          0.6934
        ],
        [
          0.838,
          0.687
        ],
        [
          0.848,
          0.6806
        ],
        [
          0.8581,
          0.6747
        ],
        [
          0.8685,
          0.6688
        ],
        [
          0.8785,
          0.6624
        ],
        [
          0.8885,
          0.6565
        ],
        [
          0.8985,
          0.6497
        ],
        [
          0.9086,
          0.6438
        ],
        [
          0.919,
          0.6374
        ],
        [
          0.929,
          0.6319
        ],
        [
          0.939,
          0.6256
        ],
        [
          0.949,
          0.6192
        ],
        [
          0.9591,
          0.6151
        ],
        [
          0.9695,
          0.6087
        ],
        [
          0.9795,
          0.6056
        ],
        [
          0.9895,
          0.6015
        ],
        [
          0.9995,
          0.5965
        ],
        [
          1.0,
          0.596
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.20542154535312102,
        "difference": -0.0014402200763933514,
        "worse": 0.6433333333333333,
        "kept_right": 0.20200166924427398,
        "kept_wrong": 0.0034198761088468027,
        "handed_off": 0.7945784546468787
      },
      "usd_per_1000": 0.338685632036945
    },
    {
      "decision": 264,
      "frontier": 47,
      "frontier_accuracy": 0.9044585987261147,
      "in_sample": {
        "threshold": 0.892259,
        "answered": 0.15332120109190173,
        "accuracy": 0.9044585987261147
      },
      "curve": [
        [
          0.0,
          0.9045
        ],
        [
          0.01,
          0.9045
        ],
        [
          0.02,
          0.9045
        ],
        [
          0.03,
          0.9045
        ],
        [
          0.04,
          0.9045
        ],
        [
          0.05,
          0.9045
        ],
        [
          0.0601,
          0.9045
        ],
        [
          0.0705,
          0.9045
        ],
        [
          0.0805,
          0.9045
        ],
        [
          0.0905,
          0.9045
        ],
        [
          0.1005,
          0.9045
        ],
        [
          0.1106,
          0.9045
        ],
        [
          0.121,
          0.9045
        ],
        [
          0.131,
          0.9045
        ],
        [
          0.141,
          0.9045
        ],
        [
          0.151,
          0.9045
        ],
        [
          0.1611,
          0.904
        ],
        [
          0.1715,
          0.904
        ],
        [
          0.1815,
          0.9035
        ],
        [
          0.1915,
          0.9035
        ],
        [
          0.2015,
          0.9031
        ],
        [
          0.2116,
          0.9035
        ],
        [
          0.222,
          0.9022
        ],
        [
          0.232,
          0.9017
        ],
        [
          0.242,
          0.9017
        ],
        [
          0.252,
          0.9017
        ],
        [
          0.2621,
          0.9017
        ],
        [
          0.2725,
          0.8995
        ],
        [
          0.2825,
          0.8999
        ],
        [
          0.2925,
          0.8972
        ],
        [
          0.3025,
          0.8963
        ],
        [
          0.3126,
          0.8949
        ],
        [
          0.323,
          0.8935
        ],
        [
          0.333,
          0.8931
        ],
        [
          0.343,
          0.8926
        ],
        [
          0.353,
          0.8922
        ],
        [
          0.3631,
          0.8908
        ],
        [
          0.3735,
          0.889
        ],
        [
          0.3835,
          0.8872
        ],
        [
          0.3935,
          0.884
        ],
        [
          0.4035,
          0.8813
        ],
        [
          0.4136,
          0.8785
        ],
        [
          0.424,
          0.8744
        ],
        [
          0.434,
          0.8731
        ],
        [
          0.444,
          0.8685
        ],
        [
          0.454,
          0.8653
        ],
        [
          0.4641,
          0.8612
        ],
        [
          0.4745,
          0.8585
        ],
        [
          0.4845,
          0.8549
        ],
        [
          0.4945,
          0.8499
        ],
        [
          0.5045,
          0.8467
        ],
        [
          0.5146,
          0.8403
        ],
        [
          0.525,
          0.8339
        ],
        [
          0.535,
          0.8303
        ],
        [
          0.545,
          0.8248
        ],
        [
          0.5551,
          0.8217
        ],
        [
          0.5655,
          0.8185
        ],
        [
          0.5755,
          0.813
        ],
        [
          0.5855,
          0.8066
        ],
        [
          0.5955,
          0.8021
        ],
        [
          0.6056,
          0.7966
        ],
        [
          0.616,
          0.7912
        ],
        [
          0.626,
          0.7866
        ],
        [
          0.636,
          0.7834
        ],
        [
          0.646,
          0.7793
        ],
        [
          0.6561,
          0.7753
        ],
        [
          0.6665,
          0.7702
        ],
        [
          0.6765,
          0.7662
        ],
        [
          0.6865,
          0.7607
        ],
        [
          0.6965,
          0.7548
        ],
        [
          0.7066,
          0.7507
        ],
        [
          0.717,
          0.7457
        ],
        [
          0.727,
          0.742
        ],
        [
          0.737,
          0.7343
        ],
        [
          0.747,
          0.7311
        ],
        [
          0.7571,
          0.7266
        ],
        [
          0.7675,
          0.722
        ],
        [
          0.7775,
          0.7179
        ],
        [
          0.7875,
          0.7129
        ],
        [
          0.7975,
          0.7079
        ],
        [
          0.8076,
          0.7034
        ],
        [
          0.818,
          0.6975
        ],
        [
          0.828,
          0.6911
        ],
        [
          0.838,
          0.6861
        ],
        [
          0.848,
          0.6793
        ],
        [
          0.8581,
          0.6738
        ],
        [
          0.8685,
          0.6679
        ],
        [
          0.8785,
          0.6611
        ],
        [
          0.8885,
          0.6551
        ],
        [
          0.8985,
          0.6488
        ],
        [
          0.9086,
          0.6433
        ],
        [
          0.919,
          0.6369
        ],
        [
          0.929,
          0.6315
        ],
        [
          0.939,
          0.6251
        ],
        [
          0.949,
          0.6183
        ],
        [
          0.9591,
          0.6133
        ],
        [
          0.9695,
          0.6096
        ],
        [
          0.9795,
          0.6056
        ],
        [
          0.9895,
          0.601
        ],
        [
          0.9995,
          0.5965
        ],
        [
          1.0,
          0.596
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.19034972199993516,
        "difference": -0.0018584547296489902,
        "worse": 0.62,
        "kept_right": 0.18746343080380184,
        "kept_wrong": 0.002886291196133159,
        "handed_off": 0.8096502780000644
      },
      "usd_per_1000": 3.9920503036217716
    },
    {
      "decision": 264,
      "frontier": 193,
      "frontier_accuracy": 0.89171974522293,
      "in_sample": {
        "threshold": 0.892259,
        "answered": 0.15332120109190173,
        "accuracy": 0.89171974522293
      },
      "curve": [
        [
          0.0,
          0.8917
        ],
        [
          0.01,
          0.8917
        ],
        [
          0.02,
          0.8917
        ],
        [
          0.03,
          0.8917
        ],
        [
          0.04,
          0.8917
        ],
        [
          0.05,
          0.8917
        ],
        [
          0.0601,
          0.8917
        ],
        [
          0.0705,
          0.8917
        ],
        [
          0.0805,
          0.8917
        ],
        [
          0.0905,
          0.8917
        ],
        [
          0.1005,
          0.8917
        ],
        [
          0.1106,
          0.8917
        ],
        [
          0.121,
          0.8917
        ],
        [
          0.131,
          0.8917
        ],
        [
          0.141,
          0.8917
        ],
        [
          0.151,
          0.8917
        ],
        [
          0.1611,
          0.8913
        ],
        [
          0.1715,
          0.8913
        ],
        [
          0.1815,
          0.8908
        ],
        [
          0.1915,
          0.8908
        ],
        [
          0.2015,
          0.8908
        ],
        [
          0.2116,
          0.8913
        ],
        [
          0.222,
          0.8904
        ],
        [
          0.232,
          0.8899
        ],
        [
          0.242,
          0.8904
        ],
        [
          0.252,
          0.8904
        ],
        [
          0.2621,
          0.8904
        ],
        [
          0.2725,
          0.8881
        ],
        [
          0.2825,
          0.8881
        ],
        [
          0.2925,
          0.8854
        ],
        [
          0.3025,
          0.8844
        ],
        [
          0.3126,
          0.884
        ],
        [
          0.323,
          0.8822
        ],
        [
          0.333,
          0.8813
        ],
        [
          0.343,
          0.8803
        ],
        [
          0.353,
          0.879
        ],
        [
          0.3631,
          0.8776
        ],
        [
          0.3735,
          0.8758
        ],
        [
          0.3835,
          0.8735
        ],
        [
          0.3935,
          0.8703
        ],
        [
          0.4035,
          0.8681
        ],
        [
          0.4136,
          0.8644
        ],
        [
          0.424,
          0.8612
        ],
        [
          0.434,
          0.8603
        ],
        [
          0.444,
          0.8571
        ],
        [
          0.454,
          0.8549
        ],
        [
          0.4641,
          0.8503
        ],
        [
          0.4745,
          0.8471
        ],
        [
          0.4845,
          0.8439
        ],
        [
          0.4945,
          0.8403
        ],
        [
          0.5045,
          0.8371
        ],
        [
          0.5146,
          0.8312
        ],
        [
          0.525,
          0.8253
        ],
        [
          0.535,
          0.8226
        ],
        [
          0.545,
          0.8171
        ],
        [
          0.5551,
          0.813
        ],
        [
          0.5655,
          0.8112
        ],
        [
          0.5755,
          0.8066
        ],
        [
          0.5855,
          0.8003
        ],
        [
          0.5955,
          0.7971
        ],
        [
          0.6056,
          0.7907
        ],
        [
          0.616,
          0.7848
        ],
        [
          0.626,
          0.7812
        ],
        [
          0.636,
          0.778
        ],
        [
          0.646,
          0.7739
        ],
        [
          0.6561,
          0.7712
        ],
        [
          0.6665,
          0.7666
        ],
        [
          0.6765,
          0.7625
        ],
        [
          0.6865,
          0.7575
        ],
        [
          0.6965,
          0.7525
        ],
        [
          0.7066,
          0.7484
        ],
        [
          0.717,
          0.7443
        ],
        [
          0.727,
          0.7411
        ],
        [
          0.737,
          0.7352
        ],
        [
          0.747,
          0.7325
        ],
        [
          0.7571,
          0.7288
        ],
        [
          0.7675,
          0.7247
        ],
        [
          0.7775,
          0.7225
        ],
        [
          0.7875,
          0.717
        ],
        [
          0.7975,
          0.7111
        ],
        [
          0.8076,
          0.7061
        ],
        [
          0.818,
          0.7002
        ],
        [
          0.828,
          0.6938
        ],
        [
          0.838,
          0.6874
        ],
        [
          0.848,
          0.6811
        ],
        [
          0.8581,
          0.6765
        ],
        [
          0.8685,
          0.6697
        ],
        [
          0.8785,
          0.6638
        ],
        [
          0.8885,
          0.6579
        ],
        [
          0.8985,
          0.6515
        ],
        [
          0.9086,
          0.6456
        ],
        [
          0.919,
          0.6397
        ],
        [
          0.929,
          0.6356
        ],
        [
          0.939,
          0.6297
        ],
        [
          0.949,
          0.6233
        ],
        [
          0.9591,
          0.6178
        ],
        [
          0.9695,
          0.6115
        ],
        [
          0.9795,
          0.6074
        ],
        [
          0.9895,
          0.6019
        ],
        [
          0.9995,
          0.5965
        ],
        [
          1.0,
          0.596
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.21980844063478397,
        "difference": -0.002361659200403628,
        "worse": 0.7833333333333333,
        "kept_right": 0.21489426514073068,
        "kept_wrong": 0.004914175494053202,
        "handed_off": 0.7801915593652151
      },
      "usd_per_1000": 6.04460431632325
    },
    {
      "decision": 264,
      "frontier": 55,
      "frontier_accuracy": 0.8503184713375797,
      "in_sample": {
        "threshold": 0.681298,
        "answered": 0.32393084622383983,
        "accuracy": 0.8503184713375797
      },
      "curve": [
        [
          0.0,
          0.8503
        ],
        [
          0.01,
          0.8508
        ],
        [
          0.02,
          0.8512
        ],
        [
          0.03,
          0.8517
        ],
        [
          0.04,
          0.8517
        ],
        [
          0.05,
          0.8517
        ],
        [
          0.0601,
          0.8521
        ],
        [
          0.0705,
          0.8521
        ],
        [
          0.0805,
          0.8521
        ],
        [
          0.0905,
          0.8521
        ],
        [
          0.1005,
          0.8521
        ],
        [
          0.1106,
          0.8521
        ],
        [
          0.121,
          0.8521
        ],
        [
          0.131,
          0.8521
        ],
        [
          0.141,
          0.8521
        ],
        [
          0.151,
          0.8521
        ],
        [
          0.1611,
          0.8517
        ],
        [
          0.1715,
          0.8517
        ],
        [
          0.1815,
          0.8512
        ],
        [
          0.1915,
          0.8517
        ],
        [
          0.2015,
          0.8521
        ],
        [
          0.2116,
          0.853
        ],
        [
          0.222,
          0.853
        ],
        [
          0.232,
          0.8526
        ],
        [
          0.242,
          0.8535
        ],
        [
          0.252,
          0.8544
        ],
        [
          0.2621,
          0.8544
        ],
        [
          0.2725,
          0.853
        ],
        [
          0.2825,
          0.8535
        ],
        [
          0.2925,
          0.8517
        ],
        [
          0.3025,
          0.8512
        ],
        [
          0.3126,
          0.8508
        ],
        [
          0.323,
          0.8503
        ],
        [
          0.333,
          0.8494
        ],
        [
          0.343,
          0.849
        ],
        [
          0.353,
          0.8485
        ],
        [
          0.3631,
          0.848
        ],
        [
          0.3735,
          0.8476
        ],
        [
          0.3835,
          0.8458
        ],
        [
          0.3935,
          0.8439
        ],
        [
          0.4035,
          0.8417
        ],
        [
          0.4136,
          0.8394
        ],
        [
          0.424,
          0.8358
        ],
        [
          0.434,
          0.8358
        ],
        [
          0.444,
          0.833
        ],
        [
          0.454,
          0.8303
        ],
        [
          0.4641,
          0.8262
        ],
        [
          0.4745,
          0.8244
        ],
        [
          0.4845,
          0.8203
        ],
        [
          0.4945,
          0.8167
        ],
        [
          0.5045,
          0.8153
        ],
        [
          0.5146,
          0.8098
        ],
        [
          0.525,
          0.8071
        ],
        [
          0.535,
          0.8048
        ],
        [
          0.545,
          0.8003
        ],
        [
          0.5551,
          0.7975
        ],
        [
          0.5655,
          0.7953
        ],
        [
          0.5755,
          0.7916
        ],
        [
          0.5855,
          0.7853
        ],
        [
          0.5955,
          0.7812
        ],
        [
          0.6056,
          0.7762
        ],
        [
          0.616,
          0.7712
        ],
        [
          0.626,
          0.7684
        ],
        [
          0.636,
          0.7657
        ],
        [
          0.646,
          0.7621
        ],
        [
          0.6561,
          0.7598
        ],
        [
          0.6665,
          0.7543
        ],
        [
          0.6765,
          0.7516
        ],
        [
          0.6865,
          0.7475
        ],
        [
          0.6965,
          0.7434
        ],
        [
          0.7066,
          0.7411
        ],
        [
          0.717,
          0.7366
        ],
        [
          0.727,
          0.7329
        ],
        [
          0.737,
          0.7261
        ],
        [
          0.747,
          0.7238
        ],
        [
          0.7571,
          0.7202
        ],
        [
          0.7675,
          0.717
        ],
        [
          0.7775,
          0.7143
        ],
        [
          0.7875,
          0.7084
        ],
        [
          0.7975,
          0.7034
        ],
        [
          0.8076,
          0.6984
        ],
        [
          0.818,
          0.6929
        ],
        [
          0.828,
          0.6861
        ],
        [
          0.838,
          0.6806
        ],
        [
          0.848,
          0.6738
        ],
        [
          0.8581,
          0.6702
        ],
        [
          0.8685,
          0.6647
        ],
        [
          0.8785,
          0.6597
        ],
        [
          0.8885,
          0.6547
        ],
        [
          0.8985,
          0.6497
        ],
        [
          0.9086,
          0.6438
        ],
        [
          0.919,
          0.6392
        ],
        [
          0.929,
          0.6342
        ],
        [
          0.939,
          0.6278
        ],
        [
          0.949,
          0.6215
        ],
        [
          0.9591,
          0.616
        ],
        [
          0.9695,
          0.611
        ],
        [
          0.9795,
          0.6069
        ],
        [
          0.9895,
          0.601
        ],
        [
          0.9995,
          0.5965
        ],
        [
          1.0,
          0.596
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.3365048500769049,
        "difference": -0.0018491923862877177,
        "worse": 0.5566666666666666,
        "kept_right": 0.31396524590247515,
        "kept_wrong": 0.022539604174429875,
        "handed_off": 0.6634951499230953
      },
      "usd_per_1000": 2.424632633010862
    },
    {
      "decision": 194,
      "frontier": 140,
      "frontier_accuracy": 0.9808917197452229,
      "in_sample": {
        "threshold": 1.0,
        "answered": 0.09190172884440401,
        "accuracy": 0.9808917197452229
      },
      "curve": [
        [
          0.0,
          0.9809
        ],
        [
          0.0919,
          0.9809
        ],
        [
          0.1183,
          0.9804
        ],
        [
          0.1392,
          0.98
        ],
        [
          0.1538,
          0.98
        ],
        [
          0.1661,
          0.9795
        ],
        [
          0.1774,
          0.9791
        ],
        [
          0.1975,
          0.9782
        ],
        [
          0.2084,
          0.9773
        ],
        [
          0.2229,
          0.9754
        ],
        [
          0.2389,
          0.9736
        ],
        [
          0.2543,
          0.9722
        ],
        [
          0.2698,
          0.9695
        ],
        [
          0.2803,
          0.9686
        ],
        [
          0.2944,
          0.9659
        ],
        [
          0.3076,
          0.9631
        ],
        [
          0.3239,
          0.9595
        ],
        [
          0.3353,
          0.9572
        ],
        [
          0.3499,
          0.9504
        ],
        [
          0.3626,
          0.9463
        ],
        [
          0.3758,
          0.944
        ],
        [
          0.3899,
          0.9377
        ],
        [
          0.3999,
          0.9345
        ],
        [
          0.4149,
          0.929
        ],
        [
          0.434,
          0.9227
        ],
        [
          0.4513,
          0.9163
        ],
        [
          0.4704,
          0.9081
        ],
        [
          0.4909,
          0.8999
        ],
        [
          0.5041,
          0.8954
        ],
        [
          0.515,
          0.8885
        ],
        [
          0.525,
          0.8831
        ],
        [
          0.5369,
          0.8781
        ],
        [
          0.5487,
          0.8703
        ],
        [
          0.5705,
          0.8585
        ],
        [
          0.5842,
          0.8512
        ],
        [
          0.5983,
          0.8444
        ],
        [
          0.6106,
          0.8376
        ],
        [
          0.6288,
          0.8271
        ],
        [
          0.6529,
          0.813
        ],
        [
          0.6702,
          0.8021
        ],
        [
          0.6874,
          0.7912
        ],
        [
          0.707,
          0.7803
        ],
        [
          0.722,
          0.7725
        ],
        [
          0.7429,
          0.758
        ],
        [
          0.7593,
          0.7498
        ],
        [
          0.7857,
          0.7329
        ],
        [
          0.8085,
          0.7179
        ],
        [
          0.8294,
          0.7061
        ],
        [
          0.8458,
          0.6965
        ],
        [
          0.8731,
          0.6761
        ],
        [
          0.8908,
          0.6656
        ],
        [
          0.9049,
          0.6556
        ],
        [
          0.9208,
          0.6438
        ],
        [
          0.9409,
          0.6319
        ],
        [
          0.9518,
          0.6256
        ],
        [
          0.9704,
          0.6119
        ],
        [
          0.9891,
          0.5978
        ],
        [
          1.0,
          0.5919
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.11822002932606367,
        "difference": -0.0009246624860789604,
        "worse": 0.5166666666666667,
        "kept_right": 0.11729536683998466,
        "kept_wrong": 0.0009246624860789604,
        "handed_off": 0.8817799706739365
      },
      "usd_per_1000": 2.161954282094078
    },
    {
      "decision": 194,
      "frontier": 277,
      "frontier_accuracy": 0.9795268425841674,
      "in_sample": {
        "threshold": 1.0,
        "answered": 0.09190172884440401,
        "accuracy": 0.9795268425841674
      },
      "curve": [
        [
          0.0,
          0.9795
        ],
        [
          0.0919,
          0.9795
        ],
        [
          0.1183,
          0.9791
        ],
        [
          0.1392,
          0.9786
        ],
        [
          0.1538,
          0.9786
        ],
        [
          0.1661,
          0.9782
        ],
        [
          0.1774,
          0.9777
        ],
        [
          0.1975,
          0.9768
        ],
        [
          0.2084,
          0.9759
        ],
        [
          0.2229,
          0.9745
        ],
        [
          0.2389,
          0.9732
        ],
        [
          0.2543,
          0.9718
        ],
        [
          0.2698,
          0.9691
        ],
        [
          0.2803,
          0.9682
        ],
        [
          0.2944,
          0.9645
        ],
        [
          0.3076,
          0.9613
        ],
        [
          0.3239,
          0.9577
        ],
        [
          0.3353,
          0.9559
        ],
        [
          0.3499,
          0.949
        ],
        [
          0.3626,
          0.9454
        ],
        [
          0.3758,
          0.9431
        ],
        [
          0.3899,
          0.9372
        ],
        [
          0.3999,
          0.9336
        ],
        [
          0.4149,
          0.9277
        ],
        [
          0.434,
          0.9213
        ],
        [
          0.4513,
          0.9149
        ],
        [
          0.4704,
          0.9072
        ],
        [
          0.4909,
          0.8995
        ],
        [
          0.5041,
          0.8949
        ],
        [
          0.515,
          0.8881
        ],
        [
          0.525,
          0.8826
        ],
        [
          0.5369,
          0.8776
        ],
        [
          0.5487,
          0.8699
        ],
        [
          0.5705,
          0.8576
        ],
        [
          0.5842,
          0.8508
        ],
        [
          0.5983,
          0.8439
        ],
        [
          0.6106,
          0.8371
        ],
        [
          0.6288,
          0.8262
        ],
        [
          0.6529,
          0.8121
        ],
        [
          0.6702,
          0.8021
        ],
        [
          0.6874,
          0.7912
        ],
        [
          0.707,
          0.7793
        ],
        [
          0.722,
          0.7716
        ],
        [
          0.7429,
          0.7571
        ],
        [
          0.7593,
          0.7489
        ],
        [
          0.7857,
          0.7325
        ],
        [
          0.8085,
          0.7179
        ],
        [
          0.8294,
          0.7056
        ],
        [
          0.8458,
          0.6965
        ],
        [
          0.8731,
          0.6761
        ],
        [
          0.8908,
          0.6647
        ],
        [
          0.9049,
          0.6547
        ],
        [
          0.9208,
          0.6424
        ],
        [
          0.9409,
          0.6306
        ],
        [
          0.9518,
          0.6247
        ],
        [
          0.9704,
          0.611
        ],
        [
          0.9891,
          0.5974
        ],
        [
          1.0,
          0.5919
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.11822002932606367,
        "difference": -0.0009246624860789604,
        "worse": 0.5166666666666667,
        "kept_right": 0.11729536683998466,
        "kept_wrong": 0.0009246624860789604,
        "handed_off": 0.8817799706739365
      },
      "usd_per_1000": 5.377322521029245
    },
    {
      "decision": 194,
      "frontier": 276,
      "frontier_accuracy": 0.9777070063694268,
      "in_sample": {
        "threshold": 1.0,
        "answered": 0.09190172884440401,
        "accuracy": 0.9777070063694268
      },
      "curve": [
        [
          0.0,
          0.9777
        ],
        [
          0.0919,
          0.9777
        ],
        [
          0.1183,
          0.9773
        ],
        [
          0.1392,
          0.9768
        ],
        [
          0.1538,
          0.9768
        ],
        [
          0.1661,
          0.9763
        ],
        [
          0.1774,
          0.9759
        ],
        [
          0.1975,
          0.975
        ],
        [
          0.2084,
          0.9741
        ],
        [
          0.2229,
          0.9727
        ],
        [
          0.2389,
          0.9718
        ],
        [
          0.2543,
          0.9704
        ],
        [
          0.2698,
          0.9682
        ],
        [
          0.2803,
          0.9672
        ],
        [
          0.2944,
          0.9631
        ],
        [
          0.3076,
          0.96
        ],
        [
          0.3239,
          0.9563
        ],
        [
          0.3353,
          0.954
        ],
        [
          0.3499,
          0.9472
        ],
        [
          0.3626,
          0.944
        ],
        [
          0.3758,
          0.9422
        ],
        [
          0.3899,
          0.9363
        ],
        [
          0.3999,
          0.9327
        ],
        [
          0.4149,
          0.9268
        ],
        [
          0.434,
          0.9208
        ],
        [
          0.4513,
          0.9145
        ],
        [
          0.4704,
          0.9063
        ],
        [
          0.4909,
          0.8985
        ],
        [
          0.5041,
          0.8944
        ],
        [
          0.515,
          0.8876
        ],
        [
          0.525,
          0.8822
        ],
        [
          0.5369,
          0.8772
        ],
        [
          0.5487,
          0.8694
        ],
        [
          0.5705,
          0.8571
        ],
        [
          0.5842,
          0.8503
        ],
        [
          0.5983,
          0.8435
        ],
        [
          0.6106,
          0.8367
        ],
        [
          0.6288,
          0.8258
        ],
        [
          0.6529,
          0.8116
        ],
        [
          0.6702,
          0.8021
        ],
        [
          0.6874,
          0.7912
        ],
        [
          0.707,
          0.7793
        ],
        [
          0.722,
          0.7716
        ],
        [
          0.7429,
          0.7575
        ],
        [
          0.7593,
          0.7493
        ],
        [
          0.7857,
          0.7325
        ],
        [
          0.8085,
          0.7179
        ],
        [
          0.8294,
          0.7056
        ],
        [
          0.8458,
          0.6965
        ],
        [
          0.8731,
          0.6761
        ],
        [
          0.8908,
          0.6652
        ],
        [
          0.9049,
          0.6551
        ],
        [
          0.9208,
          0.6429
        ],
        [
          0.9409,
          0.631
        ],
        [
          0.9518,
          0.6251
        ],
        [
          0.9704,
          0.6115
        ],
        [
          0.9891,
          0.5974
        ],
        [
          1.0,
          0.5919
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.11822002932606367,
        "difference": -0.0009246624860789604,
        "worse": 0.5166666666666667,
        "kept_right": 0.11729536683998466,
        "kept_wrong": 0.0009246624860789604,
        "handed_off": 0.8817799706739365
      },
      "usd_per_1000": 1.4592523699525701
    },
    {
      "decision": 194,
      "frontier": 48,
      "frontier_accuracy": 0.9758871701546861,
      "in_sample": {
        "threshold": 1.0,
        "answered": 0.09190172884440401,
        "accuracy": 0.9758871701546861
      },
      "curve": [
        [
          0.0,
          0.9759
        ],
        [
          0.0919,
          0.9759
        ],
        [
          0.1183,
          0.9754
        ],
        [
          0.1392,
          0.975
        ],
        [
          0.1538,
          0.975
        ],
        [
          0.1661,
          0.9745
        ],
        [
          0.1774,
          0.9741
        ],
        [
          0.1975,
          0.9732
        ],
        [
          0.2084,
          0.9722
        ],
        [
          0.2229,
          0.9709
        ],
        [
          0.2389,
          0.9695
        ],
        [
          0.2543,
          0.9682
        ],
        [
          0.2698,
          0.9654
        ],
        [
          0.2803,
          0.9645
        ],
        [
          0.2944,
          0.9609
        ],
        [
          0.3076,
          0.9577
        ],
        [
          0.3239,
          0.954
        ],
        [
          0.3353,
          0.9518
        ],
        [
          0.3499,
          0.9449
        ],
        [
          0.3626,
          0.9409
        ],
        [
          0.3758,
          0.9386
        ],
        [
          0.3899,
          0.9327
        ],
        [
          0.3999,
          0.9286
        ],
        [
          0.4149,
          0.9227
        ],
        [
          0.434,
          0.9163
        ],
        [
          0.4513,
          0.9099
        ],
        [
          0.4704,
          0.9026
        ],
        [
          0.4909,
          0.8949
        ],
        [
          0.5041,
          0.8904
        ],
        [
          0.515,
          0.8835
        ],
        [
          0.525,
          0.8781
        ],
        [
          0.5369,
          0.8731
        ],
        [
          0.5487,
          0.8653
        ],
        [
          0.5705,
          0.853
        ],
        [
          0.5842,
          0.8462
        ],
        [
          0.5983,
          0.8394
        ],
        [
          0.6106,
          0.8335
        ],
        [
          0.6288,
          0.823
        ],
        [
          0.6529,
          0.8094
        ],
        [
          0.6702,
          0.7998
        ],
        [
          0.6874,
          0.7889
        ],
        [
          0.707,
          0.7775
        ],
        [
          0.722,
          0.7698
        ],
        [
          0.7429,
          0.7557
        ],
        [
          0.7593,
          0.7475
        ],
        [
          0.7857,
          0.7311
        ],
        [
          0.8085,
          0.717
        ],
        [
          0.8294,
          0.7047
        ],
        [
          0.8458,
          0.6956
        ],
        [
          0.8731,
          0.6756
        ],
        [
          0.8908,
          0.6647
        ],
        [
          0.9049,
          0.6547
        ],
        [
          0.9208,
          0.6424
        ],
        [
          0.9409,
          0.6306
        ],
        [
          0.9518,
          0.6247
        ],
        [
          0.9704,
          0.611
        ],
        [
          0.9891,
          0.5974
        ],
        [
          1.0,
          0.5919
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.11822002932606367,
        "difference": -0.0009246624860789604,
        "worse": 0.5166666666666667,
        "kept_right": 0.11729536683998466,
        "kept_wrong": 0.0009246624860789604,
        "handed_off": 0.8817799706739365
      },
      "usd_per_1000": 4.771249746894332
    },
    {
      "decision": 194,
      "frontier": 191,
      "frontier_accuracy": 0.9740673339399454,
      "in_sample": {
        "threshold": 1.0,
        "answered": 0.09190172884440401,
        "accuracy": 0.9740673339399454
      },
      "curve": [
        [
          0.0,
          0.9741
        ],
        [
          0.0919,
          0.9741
        ],
        [
          0.1183,
          0.9736
        ],
        [
          0.1392,
          0.9732
        ],
        [
          0.1538,
          0.9732
        ],
        [
          0.1661,
          0.9727
        ],
        [
          0.1774,
          0.9722
        ],
        [
          0.1975,
          0.9713
        ],
        [
          0.2084,
          0.9709
        ],
        [
          0.2229,
          0.9695
        ],
        [
          0.2389,
          0.9682
        ],
        [
          0.2543,
          0.9672
        ],
        [
          0.2698,
          0.965
        ],
        [
          0.2803,
          0.9636
        ],
        [
          0.2944,
          0.9595
        ],
        [
          0.3076,
          0.9563
        ],
        [
          0.3239,
          0.9527
        ],
        [
          0.3353,
          0.9504
        ],
        [
          0.3499,
          0.9436
        ],
        [
          0.3626,
          0.9395
        ],
        [
          0.3758,
          0.9377
        ],
        [
          0.3899,
          0.9318
        ],
        [
          0.3999,
          0.9277
        ],
        [
          0.4149,
          0.9217
        ],
        [
          0.434,
          0.9158
        ],
        [
          0.4513,
          0.9095
        ],
        [
          0.4704,
          0.9017
        ],
        [
          0.4909,
          0.8949
        ],
        [
          0.5041,
          0.8908
        ],
        [
          0.515,
          0.884
        ],
        [
          0.525,
          0.8785
        ],
        [
          0.5369,
          0.8735
        ],
        [
          0.5487,
          0.8662
        ],
        [
          0.5705,
          0.854
        ],
        [
          0.5842,
          0.8467
        ],
        [
          0.5983,
          0.8399
        ],
        [
          0.6106,
          0.8335
        ],
        [
          0.6288,
          0.823
        ],
        [
          0.6529,
          0.8089
        ],
        [
          0.6702,
          0.7994
        ],
        [
          0.6874,
          0.7884
        ],
        [
          0.707,
          0.7771
        ],
        [
          0.722,
          0.7693
        ],
        [
          0.7429,
          0.7552
        ],
        [
          0.7593,
          0.747
        ],
        [
          0.7857,
          0.7311
        ],
        [
          0.8085,
          0.7166
        ],
        [
          0.8294,
          0.7043
        ],
        [
          0.8458,
          0.6956
        ],
        [
          0.8731,
          0.6752
        ],
        [
          0.8908,
          0.6642
        ],
        [
          0.9049,
          0.6542
        ],
        [
          0.9208,
          0.6419
        ],
        [
          0.9409,
          0.6301
        ],
        [
          0.9518,
          0.6242
        ],
        [
          0.9704,
          0.611
        ],
        [
          0.9891,
          0.5974
        ],
        [
          1.0,
          0.5919
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.11827462441250589,
        "difference": -0.0009276983037644597,
        "worse": 0.5166666666666667,
        "kept_right": 0.11734389580571108,
        "kept_wrong": 0.0009307286067947627,
        "handed_off": 0.8817253755874943
      },
      "usd_per_1000": 23.604142691843816
    },
    {
      "decision": 194,
      "frontier": 188,
      "frontier_accuracy": 0.9713375796178344,
      "in_sample": {
        "threshold": 1.0,
        "answered": 0.09190172884440401,
        "accuracy": 0.9713375796178344
      },
      "curve": [
        [
          0.0,
          0.9713
        ],
        [
          0.0919,
          0.9713
        ],
        [
          0.1183,
          0.9709
        ],
        [
          0.1392,
          0.9704
        ],
        [
          0.1538,
          0.9709
        ],
        [
          0.1661,
          0.9704
        ],
        [
          0.1774,
          0.97
        ],
        [
          0.1975,
          0.9691
        ],
        [
          0.2084,
          0.9682
        ],
        [
          0.2229,
          0.9663
        ],
        [
          0.2389,
          0.965
        ],
        [
          0.2543,
          0.9636
        ],
        [
          0.2698,
          0.9618
        ],
        [
          0.2803,
          0.9604
        ],
        [
          0.2944,
          0.9577
        ],
        [
          0.3076,
          0.955
        ],
        [
          0.3239,
          0.9513
        ],
        [
          0.3353,
          0.9486
        ],
        [
          0.3499,
          0.9418
        ],
        [
          0.3626,
          0.9381
        ],
        [
          0.3758,
          0.9359
        ],
        [
          0.3899,
          0.9295
        ],
        [
          0.3999,
          0.9263
        ],
        [
          0.4149,
          0.9208
        ],
        [
          0.434,
          0.9149
        ],
        [
          0.4513,
          0.909
        ],
        [
          0.4704,
          0.9013
        ],
        [
          0.4909,
          0.894
        ],
        [
          0.5041,
          0.8899
        ],
        [
          0.515,
          0.8831
        ],
        [
          0.525,
          0.8776
        ],
        [
          0.5369,
          0.8726
        ],
        [
          0.5487,
          0.8649
        ],
        [
          0.5705,
          0.853
        ],
        [
          0.5842,
          0.8462
        ],
        [
          0.5983,
          0.8394
        ],
        [
          0.6106,
          0.833
        ],
        [
          0.6288,
          0.8226
        ],
        [
          0.6529,
          0.8089
        ],
        [
          0.6702,
          0.798
        ],
        [
          0.6874,
          0.7875
        ],
        [
          0.707,
          0.7771
        ],
        [
          0.722,
          0.7693
        ],
        [
          0.7429,
          0.7552
        ],
        [
          0.7593,
          0.7466
        ],
        [
          0.7857,
          0.7302
        ],
        [
          0.8085,
          0.717
        ],
        [
          0.8294,
          0.7052
        ],
        [
          0.8458,
          0.6956
        ],
        [
          0.8731,
          0.6752
        ],
        [
          0.8908,
          0.6647
        ],
        [
          0.9049,
          0.6551
        ],
        [
          0.9208,
          0.6429
        ],
        [
          0.9409,
          0.631
        ],
        [
          0.9518,
          0.6247
        ],
        [
          0.9704,
          0.611
        ],
        [
          0.9891,
          0.5974
        ],
        [
          1.0,
          0.5919
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.13640319039577622,
        "difference": -0.0012914123136884591,
        "worse": 0.6633333333333333,
        "kept_right": 0.13499353551746737,
        "kept_wrong": 0.0014096548783087762,
        "handed_off": 0.8635968096042236
      },
      "usd_per_1000": 0.860861131750635
    },
    {
      "decision": 194,
      "frontier": 190,
      "frontier_accuracy": 0.9672429481346679,
      "in_sample": {
        "threshold": 1.0,
        "answered": 0.09190172884440401,
        "accuracy": 0.9672429481346679
      },
      "curve": [
        [
          0.0,
          0.9672
        ],
        [
          0.0919,
          0.9672
        ],
        [
          0.1183,
          0.9668
        ],
        [
          0.1392,
          0.9668
        ],
        [
          0.1538,
          0.9668
        ],
        [
          0.1661,
          0.9663
        ],
        [
          0.1774,
          0.9659
        ],
        [
          0.1975,
          0.965
        ],
        [
          0.2084,
          0.9641
        ],
        [
          0.2229,
          0.9627
        ],
        [
          0.2389,
          0.9613
        ],
        [
          0.2543,
          0.9604
        ],
        [
          0.2698,
          0.9581
        ],
        [
          0.2803,
          0.9572
        ],
        [
          0.2944,
          0.9536
        ],
        [
          0.3076,
          0.9504
        ],
        [
          0.3239,
          0.9468
        ],
        [
          0.3353,
          0.9445
        ],
        [
          0.3499,
          0.9377
        ],
        [
          0.3626,
          0.9345
        ],
        [
          0.3758,
          0.9322
        ],
        [
          0.3899,
          0.9268
        ],
        [
          0.3999,
          0.9236
        ],
        [
          0.4149,
          0.9177
        ],
        [
          0.434,
          0.9122
        ],
        [
          0.4513,
          0.9058
        ],
        [
          0.4704,
          0.8981
        ],
        [
          0.4909,
          0.8904
        ],
        [
          0.5041,
          0.8867
        ],
        [
          0.515,
          0.8799
        ],
        [
          0.525,
          0.8744
        ],
        [
          0.5369,
          0.8694
        ],
        [
          0.5487,
          0.8621
        ],
        [
          0.5705,
          0.8499
        ],
        [
          0.5842,
          0.843
        ],
        [
          0.5983,
          0.8367
        ],
        [
          0.6106,
          0.8298
        ],
        [
          0.6288,
          0.8194
        ],
        [
          0.6529,
          0.8057
        ],
        [
          0.6702,
          0.7962
        ],
        [
          0.6874,
          0.7853
        ],
        [
          0.707,
          0.7739
        ],
        [
          0.722,
          0.7662
        ],
        [
          0.7429,
          0.7516
        ],
        [
          0.7593,
          0.7434
        ],
        [
          0.7857,
          0.7288
        ],
        [
          0.8085,
          0.7143
        ],
        [
          0.8294,
          0.702
        ],
        [
          0.8458,
          0.6934
        ],
        [
          0.8731,
          0.6729
        ],
        [
          0.8908,
          0.6633
        ],
        [
          0.9049,
          0.6538
        ],
        [
          0.9208,
          0.6415
        ],
        [
          0.9409,
          0.6301
        ],
        [
          0.9518,
          0.6237
        ],
        [
          0.9704,
          0.6106
        ],
        [
          0.9891,
          0.5974
        ],
        [
          1.0,
          0.5919
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.13709807956109246,
        "difference": -0.0013338669470675988,
        "worse": 0.65,
        "kept_right": 0.13563991494293162,
        "kept_wrong": 0.0014581646181607406,
        "handed_off": 0.8629019204389078
      },
      "usd_per_1000": 4.833923320897759
    },
    {
      "decision": 194,
      "frontier": 124,
      "frontier_accuracy": 0.9667879890809827,
      "in_sample": {
        "threshold": 1.0,
        "answered": 0.09190172884440401,
        "accuracy": 0.9667879890809827
      },
      "curve": [
        [
          0.0,
          0.9668
        ],
        [
          0.0919,
          0.9668
        ],
        [
          0.1183,
          0.9663
        ],
        [
          0.1392,
          0.9659
        ],
        [
          0.1538,
          0.9659
        ],
        [
          0.1661,
          0.9654
        ],
        [
          0.1774,
          0.965
        ],
        [
          0.1975,
          0.9641
        ],
        [
          0.2084,
          0.9631
        ],
        [
          0.2229,
          0.9618
        ],
        [
          0.2389,
          0.9609
        ],
        [
          0.2543,
          0.9595
        ],
        [
          0.2698,
          0.9568
        ],
        [
          0.2803,
          0.9559
        ],
        [
          0.2944,
          0.9522
        ],
        [
          0.3076,
          0.949
        ],
        [
          0.3239,
          0.9454
        ],
        [
          0.3353,
          0.9436
        ],
        [
          0.3499,
          0.9368
        ],
        [
          0.3626,
          0.9336
        ],
        [
          0.3758,
          0.9318
        ],
        [
          0.3899,
          0.9263
        ],
        [
          0.3999,
          0.9227
        ],
        [
          0.4149,
          0.9167
        ],
        [
          0.434,
          0.9108
        ],
        [
          0.4513,
          0.9045
        ],
        [
          0.4704,
          0.8972
        ],
        [
          0.4909,
          0.8899
        ],
        [
          0.5041,
          0.8854
        ],
        [
          0.515,
          0.8785
        ],
        [
          0.525,
          0.8731
        ],
        [
          0.5369,
          0.8681
        ],
        [
          0.5487,
          0.8603
        ],
        [
          0.5705,
          0.848
        ],
        [
          0.5842,
          0.8408
        ],
        [
          0.5983,
          0.8339
        ],
        [
          0.6106,
          0.8276
        ],
        [
          0.6288,
          0.8171
        ],
        [
          0.6529,
          0.8039
        ],
        [
          0.6702,
          0.7944
        ],
        [
          0.6874,
          0.7834
        ],
        [
          0.707,
          0.7716
        ],
        [
          0.722,
          0.7639
        ],
        [
          0.7429,
          0.7498
        ],
        [
          0.7593,
          0.742
        ],
        [
          0.7857,
          0.7275
        ],
        [
          0.8085,
          0.7134
        ],
        [
          0.8294,
          0.7011
        ],
        [
          0.8458,
          0.6924
        ],
        [
          0.8731,
          0.6724
        ],
        [
          0.8908,
          0.6624
        ],
        [
          0.9049,
          0.6529
        ],
        [
          0.9208,
          0.641
        ],
        [
          0.9409,
          0.6301
        ],
        [
          0.9518,
          0.6237
        ],
        [
          0.9704,
          0.611
        ],
        [
          0.9891,
          0.5974
        ],
        [
          1.0,
          0.5919
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.11822002932606367,
        "difference": -0.0009246624860789604,
        "worse": 0.5166666666666667,
        "kept_right": 0.11729536683998466,
        "kept_wrong": 0.0009246624860789604,
        "handed_off": 0.8817799706739365
      },
      "usd_per_1000": 1.0255778962409625
    },
    {
      "decision": 194,
      "frontier": 278,
      "frontier_accuracy": 0.9499545040946314,
      "in_sample": {
        "threshold": 1.0,
        "answered": 0.09190172884440401,
        "accuracy": 0.9499545040946314
      },
      "curve": [
        [
          0.0,
          0.95
        ],
        [
          0.0919,
          0.95
        ],
        [
          0.1183,
          0.9495
        ],
        [
          0.1392,
          0.9495
        ],
        [
          0.1538,
          0.9495
        ],
        [
          0.1661,
          0.949
        ],
        [
          0.1774,
          0.9486
        ],
        [
          0.1975,
          0.9486
        ],
        [
          0.2084,
          0.9477
        ],
        [
          0.2229,
          0.9463
        ],
        [
          0.2389,
          0.9445
        ],
        [
          0.2543,
          0.9431
        ],
        [
          0.2698,
          0.9413
        ],
        [
          0.2803,
          0.9404
        ],
        [
          0.2944,
          0.9372
        ],
        [
          0.3076,
          0.9349
        ],
        [
          0.3239,
          0.9313
        ],
        [
          0.3353,
          0.9295
        ],
        [
          0.3499,
          0.9227
        ],
        [
          0.3626,
          0.919
        ],
        [
          0.3758,
          0.9172
        ],
        [
          0.3899,
          0.9122
        ],
        [
          0.3999,
          0.9086
        ],
        [
          0.4149,
          0.9026
        ],
        [
          0.434,
          0.8967
        ],
        [
          0.4513,
          0.8908
        ],
        [
          0.4704,
          0.8844
        ],
        [
          0.4909,
          0.8776
        ],
        [
          0.5041,
          0.8749
        ],
        [
          0.515,
          0.8685
        ],
        [
          0.525,
          0.8631
        ],
        [
          0.5369,
          0.8581
        ],
        [
          0.5487,
          0.8512
        ],
        [
          0.5705,
          0.8394
        ],
        [
          0.5842,
          0.833
        ],
        [
          0.5983,
          0.8262
        ],
        [
          0.6106,
          0.8203
        ],
        [
          0.6288,
          0.8103
        ],
        [
          0.6529,
          0.7971
        ],
        [
          0.6702,
          0.7871
        ],
        [
          0.6874,
          0.7775
        ],
        [
          0.707,
          0.7671
        ],
        [
          0.722,
          0.7598
        ],
        [
          0.7429,
          0.7466
        ],
        [
          0.7593,
          0.7389
        ],
        [
          0.7857,
          0.7238
        ],
        [
          0.8085,
          0.7097
        ],
        [
          0.8294,
          0.6975
        ],
        [
          0.8458,
          0.6893
        ],
        [
          0.8731,
          0.6688
        ],
        [
          0.8908,
          0.6583
        ],
        [
          0.9049,
          0.6492
        ],
        [
          0.9208,
          0.6379
        ],
        [
          0.9409,
          0.6265
        ],
        [
          0.9518,
          0.6215
        ],
        [
          0.9704,
          0.6096
        ],
        [
          0.9891,
          0.5974
        ],
        [
          1.0,
          0.5919
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.144647692628193,
        "difference": -0.0013733000006834408,
        "worse": 0.5766666666666667,
        "kept_right": 0.14276209163748527,
        "kept_wrong": 0.0018856009907076174,
        "handed_off": 0.8553523073718077
      },
      "usd_per_1000": 5.59617715798657
    },
    {
      "decision": 194,
      "frontier": 275,
      "frontier_accuracy": 0.9426751592356688,
      "in_sample": {
        "threshold": 1.0,
        "answered": 0.09190172884440401,
        "accuracy": 0.9426751592356688
      },
      "curve": [
        [
          0.0,
          0.9427
        ],
        [
          0.0919,
          0.9427
        ],
        [
          0.1183,
          0.9422
        ],
        [
          0.1392,
          0.9418
        ],
        [
          0.1538,
          0.9418
        ],
        [
          0.1661,
          0.9413
        ],
        [
          0.1774,
          0.9409
        ],
        [
          0.1975,
          0.9404
        ],
        [
          0.2084,
          0.9395
        ],
        [
          0.2229,
          0.9386
        ],
        [
          0.2389,
          0.9377
        ],
        [
          0.2543,
          0.9368
        ],
        [
          0.2698,
          0.934
        ],
        [
          0.2803,
          0.9327
        ],
        [
          0.2944,
          0.929
        ],
        [
          0.3076,
          0.9263
        ],
        [
          0.3239,
          0.9231
        ],
        [
          0.3353,
          0.9213
        ],
        [
          0.3499,
          0.9145
        ],
        [
          0.3626,
          0.9108
        ],
        [
          0.3758,
          0.9095
        ],
        [
          0.3899,
          0.904
        ],
        [
          0.3999,
          0.9004
        ],
        [
          0.4149,
          0.8949
        ],
        [
          0.434,
          0.889
        ],
        [
          0.4513,
          0.8826
        ],
        [
          0.4704,
          0.8753
        ],
        [
          0.4909,
          0.869
        ],
        [
          0.5041,
          0.8649
        ],
        [
          0.515,
          0.8594
        ],
        [
          0.525,
          0.854
        ],
        [
          0.5369,
          0.849
        ],
        [
          0.5487,
          0.8421
        ],
        [
          0.5705,
          0.8312
        ],
        [
          0.5842,
          0.8248
        ],
        [
          0.5983,
          0.8194
        ],
        [
          0.6106,
          0.8135
        ],
        [
          0.6288,
          0.8035
        ],
        [
          0.6529,
          0.7916
        ],
        [
          0.6702,
          0.7816
        ],
        [
          0.6874,
          0.7716
        ],
        [
          0.707,
          0.7602
        ],
        [
          0.722,
          0.7539
        ],
        [
          0.7429,
          0.7411
        ],
        [
          0.7593,
          0.7338
        ],
        [
          0.7857,
          0.7202
        ],
        [
          0.8085,
          0.707
        ],
        [
          0.8294,
          0.6947
        ],
        [
          0.8458,
          0.6884
        ],
        [
          0.8731,
          0.6706
        ],
        [
          0.8908,
          0.6606
        ],
        [
          0.9049,
          0.6515
        ],
        [
          0.9208,
          0.6397
        ],
        [
          0.9409,
          0.6278
        ],
        [
          0.9518,
          0.6224
        ],
        [
          0.9704,
          0.6092
        ],
        [
          0.9891,
          0.5974
        ],
        [
          1.0,
          0.5919
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.11984641023754876,
        "difference": -0.001033671005770921,
        "worse": 0.52,
        "kept_right": 0.11876422675483835,
        "kept_wrong": 0.0010821834827104255,
        "handed_off": 0.8801535897624521
      },
      "usd_per_1000": 0.48168820139324664
    },
    {
      "decision": 194,
      "frontier": 192,
      "frontier_accuracy": 0.9331210191082803,
      "in_sample": {
        "threshold": 1.0,
        "answered": 0.09190172884440401,
        "accuracy": 0.9331210191082803
      },
      "curve": [
        [
          0.0,
          0.9331
        ],
        [
          0.0919,
          0.9331
        ],
        [
          0.1183,
          0.9327
        ],
        [
          0.1392,
          0.9327
        ],
        [
          0.1538,
          0.9327
        ],
        [
          0.1661,
          0.9322
        ],
        [
          0.1774,
          0.9318
        ],
        [
          0.1975,
          0.9308
        ],
        [
          0.2084,
          0.9308
        ],
        [
          0.2229,
          0.9299
        ],
        [
          0.2389,
          0.9295
        ],
        [
          0.2543,
          0.929
        ],
        [
          0.2698,
          0.9277
        ],
        [
          0.2803,
          0.9263
        ],
        [
          0.2944,
          0.9231
        ],
        [
          0.3076,
          0.9199
        ],
        [
          0.3239,
          0.9163
        ],
        [
          0.3353,
          0.9136
        ],
        [
          0.3499,
          0.9067
        ],
        [
          0.3626,
          0.904
        ],
        [
          0.3758,
          0.9022
        ],
        [
          0.3899,
          0.8976
        ],
        [
          0.3999,
          0.8944
        ],
        [
          0.4149,
          0.8885
        ],
        [
          0.434,
          0.8831
        ],
        [
          0.4513,
          0.8772
        ],
        [
          0.4704,
          0.8703
        ],
        [
          0.4909,
          0.8649
        ],
        [
          0.5041,
          0.8621
        ],
        [
          0.515,
          0.8562
        ],
        [
          0.525,
          0.8508
        ],
        [
          0.5369,
          0.8458
        ],
        [
          0.5487,
          0.8394
        ],
        [
          0.5705,
          0.828
        ],
        [
          0.5842,
          0.8217
        ],
        [
          0.5983,
          0.8148
        ],
        [
          0.6106,
          0.8107
        ],
        [
          0.6288,
          0.8003
        ],
        [
          0.6529,
          0.7875
        ],
        [
          0.6702,
          0.7789
        ],
        [
          0.6874,
          0.7698
        ],
        [
          0.707,
          0.7593
        ],
        [
          0.722,
          0.752
        ],
        [
          0.7429,
          0.7393
        ],
        [
          0.7593,
          0.7316
        ],
        [
          0.7857,
          0.7166
        ],
        [
          0.8085,
          0.7038
        ],
        [
          0.8294,
          0.6934
        ],
        [
          0.8458,
          0.6847
        ],
        [
          0.8731,
          0.6652
        ],
        [
          0.8908,
          0.6561
        ],
        [
          0.9049,
          0.647
        ],
        [
          0.9208,
          0.636
        ],
        [
          0.9409,
          0.6251
        ],
        [
          0.9518,
          0.6197
        ],
        [
          0.9704,
          0.6092
        ],
        [
          0.9891,
          0.5969
        ],
        [
          1.0,
          0.5919
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1337786420369597,
        "difference": -0.0010974370146174556,
        "worse": 0.52,
        "kept_right": 0.13228109427317936,
        "kept_wrong": 0.0014975477637803165,
        "handed_off": 0.8662213579630405
      },
      "usd_per_1000": 10.085291014228696
    },
    {
      "decision": 194,
      "frontier": 123,
      "frontier_accuracy": 0.9326660600545951,
      "in_sample": {
        "threshold": 0.97,
        "answered": 0.1537761601455869,
        "accuracy": 0.9326660600545951
      },
      "curve": [
        [
          0.0,
          0.9327
        ],
        [
          0.0919,
          0.9327
        ],
        [
          0.1183,
          0.9327
        ],
        [
          0.1392,
          0.9322
        ],
        [
          0.1538,
          0.9327
        ],
        [
          0.1661,
          0.9322
        ],
        [
          0.1774,
          0.9318
        ],
        [
          0.1975,
          0.9308
        ],
        [
          0.2084,
          0.9299
        ],
        [
          0.2229,
          0.9286
        ],
        [
          0.2389,
          0.9272
        ],
        [
          0.2543,
          0.9258
        ],
        [
          0.2698,
          0.9236
        ],
        [
          0.2803,
          0.9222
        ],
        [
          0.2944,
          0.9199
        ],
        [
          0.3076,
          0.9167
        ],
        [
          0.3239,
          0.9131
        ],
        [
          0.3353,
          0.9108
        ],
        [
          0.3499,
          0.9045
        ],
        [
          0.3626,
          0.9017
        ],
        [
          0.3758,
          0.8999
        ],
        [
          0.3899,
          0.8949
        ],
        [
          0.3999,
          0.8913
        ],
        [
          0.4149,
          0.8863
        ],
        [
          0.434,
          0.8803
        ],
        [
          0.4513,
          0.874
        ],
        [
          0.4704,
          0.8672
        ],
        [
          0.4909,
          0.8608
        ],
        [
          0.5041,
          0.8571
        ],
        [
          0.515,
          0.8508
        ],
        [
          0.525,
          0.8453
        ],
        [
          0.5369,
          0.8412
        ],
        [
          0.5487,
          0.8348
        ],
        [
          0.5705,
          0.8235
        ],
        [
          0.5842,
          0.8167
        ],
        [
          0.5983,
          0.8103
        ],
        [
          0.6106,
          0.8048
        ],
        [
          0.6288,
          0.7948
        ],
        [
          0.6529,
          0.7843
        ],
        [
          0.6702,
          0.773
        ],
        [
          0.6874,
          0.763
        ],
        [
          0.707,
          0.7539
        ],
        [
          0.722,
          0.7461
        ],
        [
          0.7429,
          0.7329
        ],
        [
          0.7593,
          0.7252
        ],
        [
          0.7857,
          0.7106
        ],
        [
          0.8085,
          0.6979
        ],
        [
          0.8294,
          0.6879
        ],
        [
          0.8458,
          0.6811
        ],
        [
          0.8731,
          0.6638
        ],
        [
          0.8908,
          0.6556
        ],
        [
          0.9049,
          0.6474
        ],
        [
          0.9208,
          0.6369
        ],
        [
          0.9409,
          0.6269
        ],
        [
          0.9518,
          0.6206
        ],
        [
          0.9704,
          0.6083
        ],
        [
          0.9891,
          0.596
        ],
        [
          1.0,
          0.5919
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.155009369322237,
        "difference": -0.00137326096645166,
        "worse": 0.52,
        "kept_right": 0.15306614327307486,
        "kept_wrong": 0.0019432260491619845,
        "handed_off": 0.8449906306777633
      },
      "usd_per_1000": 0.25578847968336704
    },
    {
      "decision": 194,
      "frontier": 274,
      "frontier_accuracy": 0.9085532302092811,
      "in_sample": {
        "threshold": 1.0,
        "answered": 0.09190172884440401,
        "accuracy": 0.9085532302092811
      },
      "curve": [
        [
          0.0,
          0.9086
        ],
        [
          0.0919,
          0.9086
        ],
        [
          0.1183,
          0.9081
        ],
        [
          0.1392,
          0.9076
        ],
        [
          0.1538,
          0.9076
        ],
        [
          0.1661,
          0.9076
        ],
        [
          0.1774,
          0.9072
        ],
        [
          0.1975,
          0.9067
        ],
        [
          0.2084,
          0.9067
        ],
        [
          0.2229,
          0.9058
        ],
        [
          0.2389,
          0.9058
        ],
        [
          0.2543,
          0.9045
        ],
        [
          0.2698,
          0.9022
        ],
        [
          0.2803,
          0.9008
        ],
        [
          0.2944,
          0.8985
        ],
        [
          0.3076,
          0.8958
        ],
        [
          0.3239,
          0.8922
        ],
        [
          0.3353,
          0.8904
        ],
        [
          0.3499,
          0.8849
        ],
        [
          0.3626,
          0.8817
        ],
        [
          0.3758,
          0.8803
        ],
        [
          0.3899,
          0.8749
        ],
        [
          0.3999,
          0.8717
        ],
        [
          0.4149,
          0.8676
        ],
        [
          0.434,
          0.8635
        ],
        [
          0.4513,
          0.8576
        ],
        [
          0.4704,
          0.8508
        ],
        [
          0.4909,
          0.8458
        ],
        [
          0.5041,
          0.8426
        ],
        [
          0.515,
          0.8371
        ],
        [
          0.525,
          0.8317
        ],
        [
          0.5369,
          0.8267
        ],
        [
          0.5487,
          0.8198
        ],
        [
          0.5705,
          0.8098
        ],
        [
          0.5842,
          0.8035
        ],
        [
          0.5983,
          0.7998
        ],
        [
          0.6106,
          0.7944
        ],
        [
          0.6288,
          0.7853
        ],
        [
          0.6529,
          0.7748
        ],
        [
          0.6702,
          0.7652
        ],
        [
          0.6874,
          0.7566
        ],
        [
          0.707,
          0.747
        ],
        [
          0.722,
          0.7407
        ],
        [
          0.7429,
          0.7293
        ],
        [
          0.7593,
          0.722
        ],
        [
          0.7857,
          0.7084
        ],
        [
          0.8085,
          0.697
        ],
        [
          0.8294,
          0.6856
        ],
        [
          0.8458,
          0.6793
        ],
        [
          0.8731,
          0.6606
        ],
        [
          0.8908,
          0.6533
        ],
        [
          0.9049,
          0.6447
        ],
        [
          0.9208,
          0.6333
        ],
        [
          0.9409,
          0.6219
        ],
        [
          0.9518,
          0.616
        ],
        [
          0.9704,
          0.6051
        ],
        [
          0.9891,
          0.596
        ],
        [
          1.0,
          0.5919
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.14264372958807467,
        "difference": -0.0017068154831746212,
        "worse": 0.6133333333333333,
        "kept_right": 0.14022449968303696,
        "kept_wrong": 0.0024192299050375438,
        "handed_off": 0.8573562704119253
      },
      "usd_per_1000": 0.3942819715530882
    },
    {
      "decision": 194,
      "frontier": 47,
      "frontier_accuracy": 0.9044585987261147,
      "in_sample": {
        "threshold": 1.0,
        "answered": 0.09190172884440401,
        "accuracy": 0.9044585987261147
      },
      "curve": [
        [
          0.0,
          0.9045
        ],
        [
          0.0919,
          0.9045
        ],
        [
          0.1183,
          0.904
        ],
        [
          0.1392,
          0.904
        ],
        [
          0.1538,
          0.904
        ],
        [
          0.1661,
          0.904
        ],
        [
          0.1774,
          0.904
        ],
        [
          0.1975,
          0.9031
        ],
        [
          0.2084,
          0.9026
        ],
        [
          0.2229,
          0.9017
        ],
        [
          0.2389,
          0.9008
        ],
        [
          0.2543,
          0.8995
        ],
        [
          0.2698,
          0.8972
        ],
        [
          0.2803,
          0.8963
        ],
        [
          0.2944,
          0.8931
        ],
        [
          0.3076,
          0.8908
        ],
        [
          0.3239,
          0.8881
        ],
        [
          0.3353,
          0.8863
        ],
        [
          0.3499,
          0.8799
        ],
        [
          0.3626,
          0.8772
        ],
        [
          0.3758,
          0.8758
        ],
        [
          0.3899,
          0.8717
        ],
        [
          0.3999,
          0.8681
        ],
        [
          0.4149,
          0.8626
        ],
        [
          0.434,
          0.8585
        ],
        [
          0.4513,
          0.853
        ],
        [
          0.4704,
          0.8471
        ],
        [
          0.4909,
          0.8412
        ],
        [
          0.5041,
          0.8389
        ],
        [
          0.515,
          0.8335
        ],
        [
          0.525,
          0.8285
        ],
        [
          0.5369,
          0.8235
        ],
        [
          0.5487,
          0.8176
        ],
        [
          0.5705,
          0.8057
        ],
        [
          0.5842,
          0.7994
        ],
        [
          0.5983,
          0.7934
        ],
        [
          0.6106,
          0.7889
        ],
        [
          0.6288,
          0.7803
        ],
        [
          0.6529,
          0.7693
        ],
        [
          0.6702,
          0.7607
        ],
        [
          0.6874,
          0.752
        ],
        [
          0.707,
          0.7429
        ],
        [
          0.722,
          0.7361
        ],
        [
          0.7429,
          0.7243
        ],
        [
          0.7593,
          0.7175
        ],
        [
          0.7857,
          0.7038
        ],
        [
          0.8085,
          0.6934
        ],
        [
          0.8294,
          0.6829
        ],
        [
          0.8458,
          0.6752
        ],
        [
          0.8731,
          0.6579
        ],
        [
          0.8908,
          0.6497
        ],
        [
          0.9049,
          0.6424
        ],
        [
          0.9208,
          0.6338
        ],
        [
          0.9409,
          0.6233
        ],
        [
          0.9518,
          0.6174
        ],
        [
          0.9704,
          0.6065
        ],
        [
          0.9891,
          0.596
        ],
        [
          1.0,
          0.5919
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.15949696473732958,
        "difference": -0.0015218732293165477,
        "worse": 0.7133333333333334,
        "kept_right": 0.1570656988717545,
        "kept_wrong": 0.0024312658655750484,
        "handed_off": 0.8405030352626708
      },
      "usd_per_1000": 4.173010042683523
    },
    {
      "decision": 194,
      "frontier": 193,
      "frontier_accuracy": 0.89171974522293,
      "in_sample": {
        "threshold": 1.0,
        "answered": 0.09190172884440401,
        "accuracy": 0.89171974522293
      },
      "curve": [
        [
          0.0,
          0.8917
        ],
        [
          0.0919,
          0.8917
        ],
        [
          0.1183,
          0.8913
        ],
        [
          0.1392,
          0.8908
        ],
        [
          0.1538,
          0.8908
        ],
        [
          0.1661,
          0.8904
        ],
        [
          0.1774,
          0.8899
        ],
        [
          0.1975,
          0.889
        ],
        [
          0.2084,
          0.889
        ],
        [
          0.2229,
          0.8881
        ],
        [
          0.2389,
          0.8872
        ],
        [
          0.2543,
          0.8876
        ],
        [
          0.2698,
          0.8872
        ],
        [
          0.2803,
          0.8867
        ],
        [
          0.2944,
          0.8849
        ],
        [
          0.3076,
          0.8831
        ],
        [
          0.3239,
          0.8799
        ],
        [
          0.3353,
          0.8776
        ],
        [
          0.3499,
          0.8717
        ],
        [
          0.3626,
          0.869
        ],
        [
          0.3758,
          0.8676
        ],
        [
          0.3899,
          0.8635
        ],
        [
          0.3999,
          0.8608
        ],
        [
          0.4149,
          0.8558
        ],
        [
          0.434,
          0.8512
        ],
        [
          0.4513,
          0.8458
        ],
        [
          0.4704,
          0.8394
        ],
        [
          0.4909,
          0.8335
        ],
        [
          0.5041,
          0.8303
        ],
        [
          0.515,
          0.8258
        ],
        [
          0.525,
          0.8207
        ],
        [
          0.5369,
          0.8157
        ],
        [
          0.5487,
          0.8098
        ],
        [
          0.5705,
          0.7989
        ],
        [
          0.5842,
          0.7921
        ],
        [
          0.5983,
          0.788
        ],
        [
          0.6106,
          0.7834
        ],
        [
          0.6288,
          0.7743
        ],
        [
          0.6529,
          0.763
        ],
        [
          0.6702,
          0.7552
        ],
        [
          0.6874,
          0.748
        ],
        [
          0.707,
          0.7384
        ],
        [
          0.722,
          0.7334
        ],
        [
          0.7429,
          0.7216
        ],
        [
          0.7593,
          0.7157
        ],
        [
          0.7857,
          0.7015
        ],
        [
          0.8085,
          0.6902
        ],
        [
          0.8294,
          0.6797
        ],
        [
          0.8458,
          0.672
        ],
        [
          0.8731,
          0.6561
        ],
        [
          0.8908,
          0.6483
        ],
        [
          0.9049,
          0.641
        ],
        [
          0.9208,
          0.6315
        ],
        [
          0.9409,
          0.6228
        ],
        [
          0.9518,
          0.6178
        ],
        [
          0.9704,
          0.606
        ],
        [
          0.9891,
          0.596
        ],
        [
          1.0,
          0.5919
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1235223560711811,
        "difference": -0.0012308859400446367,
        "worse": 0.53,
        "kept_right": 0.12203981143298198,
        "kept_wrong": 0.0014825446381990905,
        "handed_off": 0.8764776439288192
      },
      "usd_per_1000": 6.81942698823023
    },
    {
      "decision": 194,
      "frontier": 55,
      "frontier_accuracy": 0.8503184713375797,
      "in_sample": {
        "threshold": 0.79,
        "answered": 0.3002729754322111,
        "accuracy": 0.8507734303912647
      },
      "curve": [
        [
          0.0,
          0.8503
        ],
        [
          0.0919,
          0.8512
        ],
        [
          0.1183,
          0.8512
        ],
        [
          0.1392,
          0.8521
        ],
        [
          0.1538,
          0.8521
        ],
        [
          0.1661,
          0.853
        ],
        [
          0.1774,
          0.853
        ],
        [
          0.1975,
          0.854
        ],
        [
          0.2084,
          0.8544
        ],
        [
          0.2229,
          0.8535
        ],
        [
          0.2389,
          0.8521
        ],
        [
          0.2543,
          0.8517
        ],
        [
          0.2698,
          0.8512
        ],
        [
          0.2803,
          0.8508
        ],
        [
          0.2944,
          0.8512
        ],
        [
          0.3076,
          0.849
        ],
        [
          0.3239,
          0.8476
        ],
        [
          0.3353,
          0.8462
        ],
        [
          0.3499,
          0.8412
        ],
        [
          0.3626,
          0.8394
        ],
        [
          0.3758,
          0.8389
        ],
        [
          0.3899,
          0.8353
        ],
        [
          0.3999,
          0.8321
        ],
        [
          0.4149,
          0.8276
        ],
        [
          0.434,
          0.8244
        ],
        [
          0.4513,
          0.8203
        ],
        [
          0.4704,
          0.8153
        ],
        [
          0.4909,
          0.8076
        ],
        [
          0.5041,
          0.8057
        ],
        [
          0.515,
          0.8012
        ],
        [
          0.525,
          0.7971
        ],
        [
          0.5369,
          0.7939
        ],
        [
          0.5487,
          0.7894
        ],
        [
          0.5705,
          0.7793
        ],
        [
          0.5842,
          0.7734
        ],
        [
          0.5983,
          0.7693
        ],
        [
          0.6106,
          0.7648
        ],
        [
          0.6288,
          0.758
        ],
        [
          0.6529,
          0.748
        ],
        [
          0.6702,
          0.7416
        ],
        [
          0.6874,
          0.7338
        ],
        [
          0.707,
          0.7275
        ],
        [
          0.722,
          0.7229
        ],
        [
          0.7429,
          0.7111
        ],
        [
          0.7593,
          0.7056
        ],
        [
          0.7857,
          0.697
        ],
        [
          0.8085,
          0.687
        ],
        [
          0.8294,
          0.6783
        ],
        [
          0.8458,
          0.672
        ],
        [
          0.8731,
          0.6551
        ],
        [
          0.8908,
          0.6479
        ],
        [
          0.9049,
          0.6419
        ],
        [
          0.9208,
          0.6315
        ],
        [
          0.9409,
          0.6228
        ],
        [
          0.9518,
          0.6178
        ],
        [
          0.9704,
          0.6078
        ],
        [
          0.9891,
          0.5965
        ],
        [
          1.0,
          0.5919
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.29212247605321856,
        "difference": -0.001191461026675114,
        "worse": 0.43333333333333335,
        "kept_right": 0.2723198894106405,
        "kept_wrong": 0.019802586642578168,
        "handed_off": 0.7078775239467815
      },
      "usd_per_1000": 2.615658201415998
    },
    {
      "decision": 204,
      "frontier": 140,
      "frontier_accuracy": 0.9808917197452229,
      "in_sample": {
        "threshold": 0.9979663706569197,
        "answered": 0.08143767060964513,
        "accuracy": 0.9808917197452229
      },
      "curve": [
        [
          0.0,
          0.9809
        ],
        [
          0.01,
          0.9809
        ],
        [
          0.02,
          0.9809
        ],
        [
          0.03,
          0.9809
        ],
        [
          0.04,
          0.9809
        ],
        [
          0.05,
          0.9809
        ],
        [
          0.0601,
          0.9809
        ],
        [
          0.0705,
          0.9809
        ],
        [
          0.0805,
          0.9809
        ],
        [
          0.0905,
          0.9804
        ],
        [
          0.1005,
          0.98
        ],
        [
          0.1106,
          0.98
        ],
        [
          0.121,
          0.9795
        ],
        [
          0.131,
          0.9791
        ],
        [
          0.141,
          0.9791
        ],
        [
          0.151,
          0.9791
        ],
        [
          0.1611,
          0.9786
        ],
        [
          0.1715,
          0.9786
        ],
        [
          0.1815,
          0.9768
        ],
        [
          0.1915,
          0.9759
        ],
        [
          0.2015,
          0.9759
        ],
        [
          0.2116,
          0.975
        ],
        [
          0.222,
          0.975
        ],
        [
          0.232,
          0.9741
        ],
        [
          0.242,
          0.9732
        ],
        [
          0.252,
          0.9727
        ],
        [
          0.2621,
          0.9718
        ],
        [
          0.2725,
          0.9695
        ],
        [
          0.2825,
          0.9686
        ],
        [
          0.2925,
          0.9668
        ],
        [
          0.3025,
          0.965
        ],
        [
          0.3126,
          0.9627
        ],
        [
          0.323,
          0.9586
        ],
        [
          0.333,
          0.9577
        ],
        [
          0.343,
          0.9554
        ],
        [
          0.353,
          0.9527
        ],
        [
          0.3631,
          0.9481
        ],
        [
          0.3735,
          0.9463
        ],
        [
          0.3835,
          0.9431
        ],
        [
          0.3935,
          0.9404
        ],
        [
          0.4035,
          0.9368
        ],
        [
          0.4136,
          0.9336
        ],
        [
          0.424,
          0.9308
        ],
        [
          0.434,
          0.9272
        ],
        [
          0.444,
          0.9222
        ],
        [
          0.454,
          0.9181
        ],
        [
          0.4641,
          0.9145
        ],
        [
          0.4745,
          0.9095
        ],
        [
          0.4845,
          0.9058
        ],
        [
          0.4945,
          0.9004
        ],
        [
          0.5045,
          0.8949
        ],
        [
          0.5146,
          0.8908
        ],
        [
          0.525,
          0.8867
        ],
        [
          0.535,
          0.8799
        ],
        [
          0.545,
          0.8749
        ],
        [
          0.5551,
          0.8703
        ],
        [
          0.5655,
          0.8658
        ],
        [
          0.5755,
          0.8603
        ],
        [
          0.5855,
          0.8544
        ],
        [
          0.5955,
          0.8471
        ],
        [
          0.6056,
          0.8403
        ],
        [
          0.616,
          0.8348
        ],
        [
          0.626,
          0.8303
        ],
        [
          0.636,
          0.8248
        ],
        [
          0.646,
          0.8194
        ],
        [
          0.6561,
          0.813
        ],
        [
          0.6665,
          0.8053
        ],
        [
          0.6765,
          0.7989
        ],
        [
          0.6865,
          0.7934
        ],
        [
          0.6965,
          0.7866
        ],
        [
          0.7066,
          0.7798
        ],
        [
          0.717,
          0.7725
        ],
        [
          0.727,
          0.7643
        ],
        [
          0.737,
          0.7566
        ],
        [
          0.747,
          0.7502
        ],
        [
          0.7571,
          0.7434
        ],
        [
          0.7675,
          0.7366
        ],
        [
          0.7775,
          0.7302
        ],
        [
          0.7875,
          0.7243
        ],
        [
          0.7975,
          0.7193
        ],
        [
          0.8076,
          0.7125
        ],
        [
          0.818,
          0.707
        ],
        [
          0.828,
          0.7006
        ],
        [
          0.838,
          0.6956
        ],
        [
          0.848,
          0.6879
        ],
        [
          0.8581,
          0.6802
        ],
        [
          0.8685,
          0.6729
        ],
        [
          0.8785,
          0.6656
        ],
        [
          0.8885,
          0.6597
        ],
        [
          0.8985,
          0.6524
        ],
        [
          0.9086,
          0.647
        ],
        [
          0.919,
          0.6401
        ],
        [
          0.929,
          0.6315
        ],
        [
          0.939,
          0.626
        ],
        [
          0.949,
          0.6187
        ],
        [
          0.9591,
          0.6119
        ],
        [
          0.9695,
          0.6037
        ],
        [
          0.98,
          0.596
        ],
        [
          0.9904,
          0.5883
        ],
        [
          1.0,
          0.5828
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.09775663676008457,
        "difference": -0.0009882551940766809,
        "worse": 0.55,
        "kept_right": 0.09675929890887831,
        "kept_wrong": 0.0009973378512062381,
        "handed_off": 0.9022433632399156
      },
      "usd_per_1000": 2.1903312469433307
    },
    {
      "decision": 204,
      "frontier": 277,
      "frontier_accuracy": 0.9795268425841674,
      "in_sample": {
        "threshold": 0.9979663706569197,
        "answered": 0.08143767060964513,
        "accuracy": 0.9795268425841674
      },
      "curve": [
        [
          0.0,
          0.9795
        ],
        [
          0.01,
          0.9795
        ],
        [
          0.02,
          0.9795
        ],
        [
          0.03,
          0.9795
        ],
        [
          0.04,
          0.9795
        ],
        [
          0.05,
          0.9795
        ],
        [
          0.0601,
          0.9795
        ],
        [
          0.0705,
          0.9795
        ],
        [
          0.0805,
          0.9795
        ],
        [
          0.0905,
          0.9791
        ],
        [
          0.1005,
          0.9786
        ],
        [
          0.1106,
          0.9786
        ],
        [
          0.121,
          0.9782
        ],
        [
          0.131,
          0.9777
        ],
        [
          0.141,
          0.9777
        ],
        [
          0.151,
          0.9777
        ],
        [
          0.1611,
          0.9773
        ],
        [
          0.1715,
          0.9773
        ],
        [
          0.1815,
          0.9754
        ],
        [
          0.1915,
          0.9745
        ],
        [
          0.2015,
          0.9745
        ],
        [
          0.2116,
          0.9736
        ],
        [
          0.222,
          0.9736
        ],
        [
          0.232,
          0.9727
        ],
        [
          0.242,
          0.9718
        ],
        [
          0.252,
          0.9713
        ],
        [
          0.2621,
          0.97
        ],
        [
          0.2725,
          0.9677
        ],
        [
          0.2825,
          0.9668
        ],
        [
          0.2925,
          0.965
        ],
        [
          0.3025,
          0.9631
        ],
        [
          0.3126,
          0.9609
        ],
        [
          0.323,
          0.9568
        ],
        [
          0.333,
          0.9559
        ],
        [
          0.343,
          0.9536
        ],
        [
          0.353,
          0.9509
        ],
        [
          0.3631,
          0.9463
        ],
        [
          0.3735,
          0.9445
        ],
        [
          0.3835,
          0.9413
        ],
        [
          0.3935,
          0.939
        ],
        [
          0.4035,
          0.9359
        ],
        [
          0.4136,
          0.9331
        ],
        [
          0.424,
          0.9304
        ],
        [
          0.434,
          0.9258
        ],
        [
          0.444,
          0.9208
        ],
        [
          0.454,
          0.9167
        ],
        [
          0.4641,
          0.9131
        ],
        [
          0.4745,
          0.909
        ],
        [
          0.4845,
          0.9054
        ],
        [
          0.4945,
          0.8995
        ],
        [
          0.5045,
          0.894
        ],
        [
          0.5146,
          0.8899
        ],
        [
          0.525,
          0.8849
        ],
        [
          0.535,
          0.8781
        ],
        [
          0.545,
          0.8731
        ],
        [
          0.5551,
          0.8681
        ],
        [
          0.5655,
          0.864
        ],
        [
          0.5755,
          0.8585
        ],
        [
          0.5855,
          0.853
        ],
        [
          0.5955,
          0.8462
        ],
        [
          0.6056,
          0.8394
        ],
        [
          0.616,
          0.8339
        ],
        [
          0.626,
          0.8294
        ],
        [
          0.636,
          0.8244
        ],
        [
          0.646,
          0.8189
        ],
        [
          0.6561,
          0.8126
        ],
        [
          0.6665,
          0.8048
        ],
        [
          0.6765,
          0.7985
        ],
        [
          0.6865,
          0.7925
        ],
        [
          0.6965,
          0.7857
        ],
        [
          0.7066,
          0.7789
        ],
        [
          0.717,
          0.7716
        ],
        [
          0.727,
          0.7639
        ],
        [
          0.737,
          0.7566
        ],
        [
          0.747,
          0.7502
        ],
        [
          0.7571,
          0.7434
        ],
        [
          0.7675,
          0.7366
        ],
        [
          0.7775,
          0.7307
        ],
        [
          0.7875,
          0.7247
        ],
        [
          0.7975,
          0.7197
        ],
        [
          0.8076,
          0.7129
        ],
        [
          0.818,
          0.7066
        ],
        [
          0.828,
          0.7002
        ],
        [
          0.838,
          0.6952
        ],
        [
          0.848,
          0.6879
        ],
        [
          0.8581,
          0.6802
        ],
        [
          0.8685,
          0.6724
        ],
        [
          0.8785,
          0.6656
        ],
        [
          0.8885,
          0.6592
        ],
        [
          0.8985,
          0.652
        ],
        [
          0.9086,
          0.6465
        ],
        [
          0.919,
          0.6397
        ],
        [
          0.929,
          0.631
        ],
        [
          0.939,
          0.6256
        ],
        [
          0.949,
          0.6187
        ],
        [
          0.9591,
          0.6119
        ],
        [
          0.9695,
          0.6037
        ],
        [
          0.98,
          0.596
        ],
        [
          0.9904,
          0.5883
        ],
        [
          1.0,
          0.5828
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.09775663676008457,
        "difference": -0.0009882551940766809,
        "worse": 0.55,
        "kept_right": 0.09675929890887831,
        "kept_wrong": 0.0009973378512062381,
        "handed_off": 0.9022433632399156
      },
      "usd_per_1000": 5.480318262337235
    },
    {
      "decision": 204,
      "frontier": 276,
      "frontier_accuracy": 0.9777070063694268,
      "in_sample": {
        "threshold": 0.9979663706569197,
        "answered": 0.08143767060964513,
        "accuracy": 0.9777070063694268
      },
      "curve": [
        [
          0.0,
          0.9777
        ],
        [
          0.01,
          0.9777
        ],
        [
          0.02,
          0.9777
        ],
        [
          0.03,
          0.9777
        ],
        [
          0.04,
          0.9777
        ],
        [
          0.05,
          0.9777
        ],
        [
          0.0601,
          0.9777
        ],
        [
          0.0705,
          0.9777
        ],
        [
          0.0805,
          0.9777
        ],
        [
          0.0905,
          0.9773
        ],
        [
          0.1005,
          0.9768
        ],
        [
          0.1106,
          0.9768
        ],
        [
          0.121,
          0.9763
        ],
        [
          0.131,
          0.9759
        ],
        [
          0.141,
          0.9759
        ],
        [
          0.151,
          0.9759
        ],
        [
          0.1611,
          0.9754
        ],
        [
          0.1715,
          0.9754
        ],
        [
          0.1815,
          0.9736
        ],
        [
          0.1915,
          0.9727
        ],
        [
          0.2015,
          0.9732
        ],
        [
          0.2116,
          0.9722
        ],
        [
          0.222,
          0.9722
        ],
        [
          0.232,
          0.9713
        ],
        [
          0.242,
          0.9704
        ],
        [
          0.252,
          0.97
        ],
        [
          0.2621,
          0.9686
        ],
        [
          0.2725,
          0.9663
        ],
        [
          0.2825,
          0.9654
        ],
        [
          0.2925,
          0.9636
        ],
        [
          0.3025,
          0.9618
        ],
        [
          0.3126,
          0.96
        ],
        [
          0.323,
          0.9559
        ],
        [
          0.333,
          0.955
        ],
        [
          0.343,
          0.9527
        ],
        [
          0.353,
          0.9495
        ],
        [
          0.3631,
          0.9449
        ],
        [
          0.3735,
          0.9431
        ],
        [
          0.3835,
          0.9404
        ],
        [
          0.3935,
          0.9386
        ],
        [
          0.4035,
          0.9354
        ],
        [
          0.4136,
          0.9327
        ],
        [
          0.424,
          0.9299
        ],
        [
          0.434,
          0.9254
        ],
        [
          0.444,
          0.9204
        ],
        [
          0.454,
          0.9163
        ],
        [
          0.4641,
          0.9126
        ],
        [
          0.4745,
          0.9086
        ],
        [
          0.4845,
          0.9049
        ],
        [
          0.4945,
          0.899
        ],
        [
          0.5045,
          0.8935
        ],
        [
          0.5146,
          0.8894
        ],
        [
          0.525,
          0.8844
        ],
        [
          0.535,
          0.8776
        ],
        [
          0.545,
          0.8726
        ],
        [
          0.5551,
          0.8676
        ],
        [
          0.5655,
          0.864
        ],
        [
          0.5755,
          0.8585
        ],
        [
          0.5855,
          0.853
        ],
        [
          0.5955,
          0.8458
        ],
        [
          0.6056,
          0.8394
        ],
        [
          0.616,
          0.8339
        ],
        [
          0.626,
          0.8294
        ],
        [
          0.636,
          0.8248
        ],
        [
          0.646,
          0.8198
        ],
        [
          0.6561,
          0.8135
        ],
        [
          0.6665,
          0.8062
        ],
        [
          0.6765,
          0.7998
        ],
        [
          0.6865,
          0.7939
        ],
        [
          0.6965,
          0.7871
        ],
        [
          0.7066,
          0.7803
        ],
        [
          0.717,
          0.773
        ],
        [
          0.727,
          0.7652
        ],
        [
          0.737,
          0.758
        ],
        [
          0.747,
          0.7516
        ],
        [
          0.7571,
          0.7448
        ],
        [
          0.7675,
          0.7379
        ],
        [
          0.7775,
          0.7316
        ],
        [
          0.7875,
          0.7257
        ],
        [
          0.7975,
          0.7207
        ],
        [
          0.8076,
          0.7138
        ],
        [
          0.818,
          0.7075
        ],
        [
          0.828,
          0.7011
        ],
        [
          0.838,
          0.6961
        ],
        [
          0.848,
          0.6884
        ],
        [
          0.8581,
          0.6806
        ],
        [
          0.8685,
          0.6729
        ],
        [
          0.8785,
          0.6656
        ],
        [
          0.8885,
          0.6592
        ],
        [
          0.8985,
          0.652
        ],
        [
          0.9086,
          0.6465
        ],
        [
          0.919,
          0.6397
        ],
        [
          0.929,
          0.631
        ],
        [
          0.939,
          0.6256
        ],
        [
          0.949,
          0.6187
        ],
        [
          0.9591,
          0.6119
        ],
        [
          0.9695,
          0.6037
        ],
        [
          0.98,
          0.596
        ],
        [
          0.9904,
          0.5883
        ],
        [
          1.0,
          0.5828
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.09775663676008457,
        "difference": -0.0009882551940766809,
        "worse": 0.55,
        "kept_right": 0.09675929890887831,
        "kept_wrong": 0.0009973378512062381,
        "handed_off": 0.9022433632399156
      },
      "usd_per_1000": 1.471321791465331
    },
    {
      "decision": 204,
      "frontier": 48,
      "frontier_accuracy": 0.9758871701546861,
      "in_sample": {
        "threshold": 0.9979663706569197,
        "answered": 0.08143767060964513,
        "accuracy": 0.9758871701546861
      },
      "curve": [
        [
          0.0,
          0.9759
        ],
        [
          0.01,
          0.9759
        ],
        [
          0.02,
          0.9759
        ],
        [
          0.03,
          0.9759
        ],
        [
          0.04,
          0.9759
        ],
        [
          0.05,
          0.9759
        ],
        [
          0.0601,
          0.9759
        ],
        [
          0.0705,
          0.9759
        ],
        [
          0.0805,
          0.9759
        ],
        [
          0.0905,
          0.9754
        ],
        [
          0.1005,
          0.975
        ],
        [
          0.1106,
          0.975
        ],
        [
          0.121,
          0.9745
        ],
        [
          0.131,
          0.9741
        ],
        [
          0.141,
          0.9741
        ],
        [
          0.151,
          0.9741
        ],
        [
          0.1611,
          0.9736
        ],
        [
          0.1715,
          0.9736
        ],
        [
          0.1815,
          0.9722
        ],
        [
          0.1915,
          0.9713
        ],
        [
          0.2015,
          0.9713
        ],
        [
          0.2116,
          0.9704
        ],
        [
          0.222,
          0.9704
        ],
        [
          0.232,
          0.9695
        ],
        [
          0.242,
          0.9686
        ],
        [
          0.252,
          0.9682
        ],
        [
          0.2621,
          0.9668
        ],
        [
          0.2725,
          0.9645
        ],
        [
          0.2825,
          0.9636
        ],
        [
          0.2925,
          0.9618
        ],
        [
          0.3025,
          0.96
        ],
        [
          0.3126,
          0.9577
        ],
        [
          0.323,
          0.9536
        ],
        [
          0.333,
          0.9527
        ],
        [
          0.343,
          0.9504
        ],
        [
          0.353,
          0.9472
        ],
        [
          0.3631,
          0.9427
        ],
        [
          0.3735,
          0.9409
        ],
        [
          0.3835,
          0.9377
        ],
        [
          0.3935,
          0.9354
        ],
        [
          0.4035,
          0.9322
        ],
        [
          0.4136,
          0.9295
        ],
        [
          0.424,
          0.9268
        ],
        [
          0.434,
          0.9222
        ],
        [
          0.444,
          0.9172
        ],
        [
          0.454,
          0.9131
        ],
        [
          0.4641,
          0.9095
        ],
        [
          0.4745,
          0.9054
        ],
        [
          0.4845,
          0.9022
        ],
        [
          0.4945,
          0.8963
        ],
        [
          0.5045,
          0.8908
        ],
        [
          0.5146,
          0.8867
        ],
        [
          0.525,
          0.8817
        ],
        [
          0.535,
          0.8749
        ],
        [
          0.545,
          0.8699
        ],
        [
          0.5551,
          0.8649
        ],
        [
          0.5655,
          0.8612
        ],
        [
          0.5755,
          0.8558
        ],
        [
          0.5855,
          0.8503
        ],
        [
          0.5955,
          0.8435
        ],
        [
          0.6056,
          0.8371
        ],
        [
          0.616,
          0.8321
        ],
        [
          0.626,
          0.8276
        ],
        [
          0.636,
          0.8221
        ],
        [
          0.646,
          0.8167
        ],
        [
          0.6561,
          0.8103
        ],
        [
          0.6665,
          0.803
        ],
        [
          0.6765,
          0.7966
        ],
        [
          0.6865,
          0.7912
        ],
        [
          0.6965,
          0.7843
        ],
        [
          0.7066,
          0.7775
        ],
        [
          0.717,
          0.7707
        ],
        [
          0.727,
          0.763
        ],
        [
          0.737,
          0.7552
        ],
        [
          0.747,
          0.7489
        ],
        [
          0.7571,
          0.742
        ],
        [
          0.7675,
          0.7352
        ],
        [
          0.7775,
          0.7293
        ],
        [
          0.7875,
          0.7238
        ],
        [
          0.7975,
          0.7188
        ],
        [
          0.8076,
          0.7125
        ],
        [
          0.818,
          0.7061
        ],
        [
          0.828,
          0.6997
        ],
        [
          0.838,
          0.6947
        ],
        [
          0.848,
          0.687
        ],
        [
          0.8581,
          0.6793
        ],
        [
          0.8685,
          0.6715
        ],
        [
          0.8785,
          0.6647
        ],
        [
          0.8885,
          0.6588
        ],
        [
          0.8985,
          0.6515
        ],
        [
          0.9086,
          0.6456
        ],
        [
          0.919,
          0.6388
        ],
        [
          0.929,
          0.6301
        ],
        [
          0.939,
          0.6247
        ],
        [
          0.949,
          0.6183
        ],
        [
          0.9591,
          0.6115
        ],
        [
          0.9695,
          0.6033
        ],
        [
          0.98,
          0.596
        ],
        [
          0.9904,
          0.5883
        ],
        [
          1.0,
          0.5828
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.09899892657965793,
        "difference": -0.001079158800396828,
        "worse": 0.5533333333333333,
        "kept_right": 0.09789250879858347,
        "kept_wrong": 0.0011064177810744499,
        "handed_off": 0.9010010734203424
      },
      "usd_per_1000": 4.853499093318636
    },
    {
      "decision": 204,
      "frontier": 191,
      "frontier_accuracy": 0.9740673339399454,
      "in_sample": {
        "threshold": 0.9979663706569197,
        "answered": 0.08143767060964513,
        "accuracy": 0.9740673339399454
      },
      "curve": [
        [
          0.0,
          0.9741
        ],
        [
          0.01,
          0.9741
        ],
        [
          0.02,
          0.9741
        ],
        [
          0.03,
          0.9741
        ],
        [
          0.04,
          0.9741
        ],
        [
          0.05,
          0.9741
        ],
        [
          0.0601,
          0.9741
        ],
        [
          0.0705,
          0.9741
        ],
        [
          0.0805,
          0.9741
        ],
        [
          0.0905,
          0.9736
        ],
        [
          0.1005,
          0.9732
        ],
        [
          0.1106,
          0.9732
        ],
        [
          0.121,
          0.9727
        ],
        [
          0.131,
          0.9722
        ],
        [
          0.141,
          0.9722
        ],
        [
          0.151,
          0.9722
        ],
        [
          0.1611,
          0.9718
        ],
        [
          0.1715,
          0.9718
        ],
        [
          0.1815,
          0.97
        ],
        [
          0.1915,
          0.9691
        ],
        [
          0.2015,
          0.9695
        ],
        [
          0.2116,
          0.9686
        ],
        [
          0.222,
          0.9686
        ],
        [
          0.232,
          0.9677
        ],
        [
          0.242,
          0.9668
        ],
        [
          0.252,
          0.9668
        ],
        [
          0.2621,
          0.9654
        ],
        [
          0.2725,
          0.9631
        ],
        [
          0.2825,
          0.9622
        ],
        [
          0.2925,
          0.9604
        ],
        [
          0.3025,
          0.9586
        ],
        [
          0.3126,
          0.9568
        ],
        [
          0.323,
          0.9527
        ],
        [
          0.333,
          0.9518
        ],
        [
          0.343,
          0.9495
        ],
        [
          0.353,
          0.9463
        ],
        [
          0.3631,
          0.9418
        ],
        [
          0.3735,
          0.9399
        ],
        [
          0.3835,
          0.9368
        ],
        [
          0.3935,
          0.9345
        ],
        [
          0.4035,
          0.9318
        ],
        [
          0.4136,
          0.9286
        ],
        [
          0.424,
          0.9258
        ],
        [
          0.434,
          0.9213
        ],
        [
          0.444,
          0.9163
        ],
        [
          0.454,
          0.9122
        ],
        [
          0.4641,
          0.9086
        ],
        [
          0.4745,
          0.904
        ],
        [
          0.4845,
          0.9008
        ],
        [
          0.4945,
          0.8949
        ],
        [
          0.5045,
          0.8894
        ],
        [
          0.5146,
          0.8854
        ],
        [
          0.525,
          0.8803
        ],
        [
          0.535,
          0.8735
        ],
        [
          0.545,
          0.8685
        ],
        [
          0.5551,
          0.8635
        ],
        [
          0.5655,
          0.8599
        ],
        [
          0.5755,
          0.8544
        ],
        [
          0.5855,
          0.849
        ],
        [
          0.5955,
          0.8417
        ],
        [
          0.6056,
          0.8353
        ],
        [
          0.616,
          0.8303
        ],
        [
          0.626,
          0.8258
        ],
        [
          0.636,
          0.8207
        ],
        [
          0.646,
          0.8157
        ],
        [
          0.6561,
          0.8094
        ],
        [
          0.6665,
          0.8016
        ],
        [
          0.6765,
          0.7953
        ],
        [
          0.6865,
          0.7894
        ],
        [
          0.6965,
          0.7825
        ],
        [
          0.7066,
          0.7757
        ],
        [
          0.717,
          0.7693
        ],
        [
          0.727,
          0.7616
        ],
        [
          0.737,
          0.7548
        ],
        [
          0.747,
          0.7484
        ],
        [
          0.7571,
          0.7416
        ],
        [
          0.7675,
          0.7348
        ],
        [
          0.7775,
          0.7284
        ],
        [
          0.7875,
          0.7225
        ],
        [
          0.7975,
          0.7179
        ],
        [
          0.8076,
          0.7111
        ],
        [
          0.818,
          0.7047
        ],
        [
          0.828,
          0.6984
        ],
        [
          0.838,
          0.6934
        ],
        [
          0.848,
          0.6856
        ],
        [
          0.8581,
          0.6783
        ],
        [
          0.8685,
          0.6711
        ],
        [
          0.8785,
          0.6642
        ],
        [
          0.8885,
          0.6579
        ],
        [
          0.8985,
          0.6506
        ],
        [
          0.9086,
          0.6456
        ],
        [
          0.919,
          0.6392
        ],
        [
          0.929,
          0.6306
        ],
        [
          0.939,
          0.6251
        ],
        [
          0.949,
          0.6183
        ],
        [
          0.9591,
          0.6115
        ],
        [
          0.9695,
          0.6033
        ],
        [
          0.98,
          0.596
        ],
        [
          0.9904,
          0.5883
        ],
        [
          1.0,
          0.5828
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.0978443560583302,
        "difference": -0.0009912799974644605,
        "worse": 0.55,
        "kept_right": 0.09684096860034837,
        "kept_wrong": 0.0010033874579817978,
        "handed_off": 0.90215564394167
      },
      "usd_per_1000": 24.129274905269167
    },
    {
      "decision": 204,
      "frontier": 188,
      "frontier_accuracy": 0.9713375796178344,
      "in_sample": {
        "threshold": 0.9979663706569197,
        "answered": 0.08143767060964513,
        "accuracy": 0.9713375796178344
      },
      "curve": [
        [
          0.0,
          0.9713
        ],
        [
          0.01,
          0.9713
        ],
        [
          0.02,
          0.9713
        ],
        [
          0.03,
          0.9713
        ],
        [
          0.04,
          0.9713
        ],
        [
          0.05,
          0.9713
        ],
        [
          0.0601,
          0.9713
        ],
        [
          0.0705,
          0.9713
        ],
        [
          0.0805,
          0.9713
        ],
        [
          0.0905,
          0.9709
        ],
        [
          0.1005,
          0.9704
        ],
        [
          0.1106,
          0.9704
        ],
        [
          0.121,
          0.97
        ],
        [
          0.131,
          0.9695
        ],
        [
          0.141,
          0.9695
        ],
        [
          0.151,
          0.9695
        ],
        [
          0.1611,
          0.9691
        ],
        [
          0.1715,
          0.9691
        ],
        [
          0.1815,
          0.9672
        ],
        [
          0.1915,
          0.9663
        ],
        [
          0.2015,
          0.9663
        ],
        [
          0.2116,
          0.9654
        ],
        [
          0.222,
          0.9654
        ],
        [
          0.232,
          0.9645
        ],
        [
          0.242,
          0.9636
        ],
        [
          0.252,
          0.9631
        ],
        [
          0.2621,
          0.9622
        ],
        [
          0.2725,
          0.96
        ],
        [
          0.2825,
          0.9591
        ],
        [
          0.2925,
          0.9577
        ],
        [
          0.3025,
          0.9559
        ],
        [
          0.3126,
          0.954
        ],
        [
          0.323,
          0.9504
        ],
        [
          0.333,
          0.9495
        ],
        [
          0.343,
          0.9472
        ],
        [
          0.353,
          0.9449
        ],
        [
          0.3631,
          0.9404
        ],
        [
          0.3735,
          0.9386
        ],
        [
          0.3835,
          0.9354
        ],
        [
          0.3935,
          0.9327
        ],
        [
          0.4035,
          0.929
        ],
        [
          0.4136,
          0.9254
        ],
        [
          0.424,
          0.9227
        ],
        [
          0.434,
          0.919
        ],
        [
          0.444,
          0.914
        ],
        [
          0.454,
          0.9099
        ],
        [
          0.4641,
          0.9063
        ],
        [
          0.4745,
          0.9013
        ],
        [
          0.4845,
          0.8976
        ],
        [
          0.4945,
          0.8922
        ],
        [
          0.5045,
          0.8872
        ],
        [
          0.5146,
          0.8831
        ],
        [
          0.525,
          0.879
        ],
        [
          0.535,
          0.8722
        ],
        [
          0.545,
          0.8676
        ],
        [
          0.5551,
          0.8626
        ],
        [
          0.5655,
          0.8585
        ],
        [
          0.5755,
          0.853
        ],
        [
          0.5855,
          0.8471
        ],
        [
          0.5955,
          0.8403
        ],
        [
          0.6056,
          0.8348
        ],
        [
          0.616,
          0.8298
        ],
        [
          0.626,
          0.8258
        ],
        [
          0.636,
          0.8212
        ],
        [
          0.646,
          0.8157
        ],
        [
          0.6561,
          0.8094
        ],
        [
          0.6665,
          0.8016
        ],
        [
          0.6765,
          0.7953
        ],
        [
          0.6865,
          0.7903
        ],
        [
          0.6965,
          0.7834
        ],
        [
          0.7066,
          0.7766
        ],
        [
          0.717,
          0.7693
        ],
        [
          0.727,
          0.7611
        ],
        [
          0.737,
          0.7539
        ],
        [
          0.747,
          0.7475
        ],
        [
          0.7571,
          0.7411
        ],
        [
          0.7675,
          0.7348
        ],
        [
          0.7775,
          0.7279
        ],
        [
          0.7875,
          0.722
        ],
        [
          0.7975,
          0.717
        ],
        [
          0.8076,
          0.7111
        ],
        [
          0.818,
          0.7052
        ],
        [
          0.828,
          0.6988
        ],
        [
          0.838,
          0.6938
        ],
        [
          0.848,
          0.6861
        ],
        [
          0.8581,
          0.6783
        ],
        [
          0.8685,
          0.6711
        ],
        [
          0.8785,
          0.6642
        ],
        [
          0.8885,
          0.6583
        ],
        [
          0.8985,
          0.651
        ],
        [
          0.9086,
          0.6451
        ],
        [
          0.919,
          0.6392
        ],
        [
          0.929,
          0.6306
        ],
        [
          0.939,
          0.6251
        ],
        [
          0.949,
          0.6178
        ],
        [
          0.9591,
          0.6115
        ],
        [
          0.9695,
          0.6037
        ],
        [
          0.98,
          0.596
        ],
        [
          0.9904,
          0.5883
        ],
        [
          1.0,
          0.5828
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.09775663676008457,
        "difference": -0.0009882551940766809,
        "worse": 0.55,
        "kept_right": 0.09675929890887831,
        "kept_wrong": 0.0009973378512062381,
        "handed_off": 0.9022433632399156
      },
      "usd_per_1000": 0.876968668068197
    },
    {
      "decision": 204,
      "frontier": 190,
      "frontier_accuracy": 0.9672429481346679,
      "in_sample": {
        "threshold": 0.9979663706569197,
        "answered": 0.08143767060964513,
        "accuracy": 0.9672429481346679
      },
      "curve": [
        [
          0.0,
          0.9672
        ],
        [
          0.01,
          0.9672
        ],
        [
          0.02,
          0.9672
        ],
        [
          0.03,
          0.9672
        ],
        [
          0.04,
          0.9672
        ],
        [
          0.05,
          0.9672
        ],
        [
          0.0601,
          0.9672
        ],
        [
          0.0705,
          0.9672
        ],
        [
          0.0805,
          0.9672
        ],
        [
          0.0905,
          0.9668
        ],
        [
          0.1005,
          0.9663
        ],
        [
          0.1106,
          0.9663
        ],
        [
          0.121,
          0.9659
        ],
        [
          0.131,
          0.9654
        ],
        [
          0.141,
          0.9654
        ],
        [
          0.151,
          0.9654
        ],
        [
          0.1611,
          0.965
        ],
        [
          0.1715,
          0.965
        ],
        [
          0.1815,
          0.9631
        ],
        [
          0.1915,
          0.9622
        ],
        [
          0.2015,
          0.9622
        ],
        [
          0.2116,
          0.9613
        ],
        [
          0.222,
          0.9613
        ],
        [
          0.232,
          0.9604
        ],
        [
          0.242,
          0.9595
        ],
        [
          0.252,
          0.9595
        ],
        [
          0.2621,
          0.9581
        ],
        [
          0.2725,
          0.9563
        ],
        [
          0.2825,
          0.9554
        ],
        [
          0.2925,
          0.9536
        ],
        [
          0.3025,
          0.9518
        ],
        [
          0.3126,
          0.95
        ],
        [
          0.323,
          0.9459
        ],
        [
          0.333,
          0.9449
        ],
        [
          0.343,
          0.9427
        ],
        [
          0.353,
          0.9395
        ],
        [
          0.3631,
          0.9349
        ],
        [
          0.3735,
          0.9331
        ],
        [
          0.3835,
          0.9299
        ],
        [
          0.3935,
          0.9286
        ],
        [
          0.4035,
          0.9258
        ],
        [
          0.4136,
          0.9231
        ],
        [
          0.424,
          0.9204
        ],
        [
          0.434,
          0.9158
        ],
        [
          0.444,
          0.9108
        ],
        [
          0.454,
          0.9067
        ],
        [
          0.4641,
          0.9031
        ],
        [
          0.4745,
          0.8985
        ],
        [
          0.4845,
          0.8954
        ],
        [
          0.4945,
          0.8894
        ],
        [
          0.5045,
          0.884
        ],
        [
          0.5146,
          0.8799
        ],
        [
          0.525,
          0.8749
        ],
        [
          0.535,
          0.8681
        ],
        [
          0.545,
          0.8631
        ],
        [
          0.5551,
          0.8581
        ],
        [
          0.5655,
          0.8544
        ],
        [
          0.5755,
          0.8494
        ],
        [
          0.5855,
          0.8439
        ],
        [
          0.5955,
          0.8371
        ],
        [
          0.6056,
          0.8308
        ],
        [
          0.616,
          0.8253
        ],
        [
          0.626,
          0.8212
        ],
        [
          0.636,
          0.8167
        ],
        [
          0.646,
          0.8126
        ],
        [
          0.6561,
          0.8062
        ],
        [
          0.6665,
          0.7989
        ],
        [
          0.6765,
          0.7925
        ],
        [
          0.6865,
          0.7866
        ],
        [
          0.6965,
          0.7807
        ],
        [
          0.7066,
          0.7739
        ],
        [
          0.717,
          0.7671
        ],
        [
          0.727,
          0.7593
        ],
        [
          0.737,
          0.7525
        ],
        [
          0.747,
          0.7461
        ],
        [
          0.7571,
          0.7393
        ],
        [
          0.7675,
          0.7325
        ],
        [
          0.7775,
          0.7266
        ],
        [
          0.7875,
          0.7207
        ],
        [
          0.7975,
          0.7157
        ],
        [
          0.8076,
          0.7093
        ],
        [
          0.818,
          0.7034
        ],
        [
          0.828,
          0.697
        ],
        [
          0.838,
          0.692
        ],
        [
          0.848,
          0.6847
        ],
        [
          0.8581,
          0.6774
        ],
        [
          0.8685,
          0.6702
        ],
        [
          0.8785,
          0.6633
        ],
        [
          0.8885,
          0.6574
        ],
        [
          0.8985,
          0.6501
        ],
        [
          0.9086,
          0.6451
        ],
        [
          0.919,
          0.6388
        ],
        [
          0.929,
          0.6301
        ],
        [
          0.939,
          0.6247
        ],
        [
          0.949,
          0.6174
        ],
        [
          0.9591,
          0.6106
        ],
        [
          0.9695,
          0.6024
        ],
        [
          0.98,
          0.5951
        ],
        [
          0.9904,
          0.5878
        ],
        [
          1.0,
          0.5828
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.09775663676008457,
        "difference": -0.0009882551940766809,
        "worse": 0.55,
        "kept_right": 0.09675929890887831,
        "kept_wrong": 0.0009973378512062381,
        "handed_off": 0.9022433632399156
      },
      "usd_per_1000": 5.031870800491089
    },
    {
      "decision": 204,
      "frontier": 124,
      "frontier_accuracy": 0.9667879890809827,
      "in_sample": {
        "threshold": 0.9979663706569197,
        "answered": 0.08143767060964513,
        "accuracy": 0.9667879890809827
      },
      "curve": [
        [
          0.0,
          0.9668
        ],
        [
          0.01,
          0.9668
        ],
        [
          0.02,
          0.9668
        ],
        [
          0.03,
          0.9668
        ],
        [
          0.04,
          0.9668
        ],
        [
          0.05,
          0.9668
        ],
        [
          0.0601,
          0.9668
        ],
        [
          0.0705,
          0.9668
        ],
        [
          0.0805,
          0.9668
        ],
        [
          0.0905,
          0.9663
        ],
        [
          0.1005,
          0.9659
        ],
        [
          0.1106,
          0.9659
        ],
        [
          0.121,
          0.9654
        ],
        [
          0.131,
          0.965
        ],
        [
          0.141,
          0.965
        ],
        [
          0.151,
          0.965
        ],
        [
          0.1611,
          0.9645
        ],
        [
          0.1715,
          0.9645
        ],
        [
          0.1815,
          0.9631
        ],
        [
          0.1915,
          0.9622
        ],
        [
          0.2015,
          0.9622
        ],
        [
          0.2116,
          0.9613
        ],
        [
          0.222,
          0.9613
        ],
        [
          0.232,
          0.9604
        ],
        [
          0.242,
          0.9595
        ],
        [
          0.252,
          0.9595
        ],
        [
          0.2621,
          0.9581
        ],
        [
          0.2725,
          0.9559
        ],
        [
          0.2825,
          0.955
        ],
        [
          0.2925,
          0.9531
        ],
        [
          0.3025,
          0.9513
        ],
        [
          0.3126,
          0.949
        ],
        [
          0.323,
          0.9449
        ],
        [
          0.333,
          0.944
        ],
        [
          0.343,
          0.9418
        ],
        [
          0.353,
          0.939
        ],
        [
          0.3631,
          0.9345
        ],
        [
          0.3735,
          0.9327
        ],
        [
          0.3835,
          0.9299
        ],
        [
          0.3935,
          0.9281
        ],
        [
          0.4035,
          0.9249
        ],
        [
          0.4136,
          0.9222
        ],
        [
          0.424,
          0.9195
        ],
        [
          0.434,
          0.9149
        ],
        [
          0.444,
          0.9099
        ],
        [
          0.454,
          0.9058
        ],
        [
          0.4641,
          0.9022
        ],
        [
          0.4745,
          0.8981
        ],
        [
          0.4845,
          0.8949
        ],
        [
          0.4945,
          0.889
        ],
        [
          0.5045,
          0.8835
        ],
        [
          0.5146,
          0.8794
        ],
        [
          0.525,
          0.8749
        ],
        [
          0.535,
          0.8685
        ],
        [
          0.545,
          0.8635
        ],
        [
          0.5551,
          0.8585
        ],
        [
          0.5655,
          0.8549
        ],
        [
          0.5755,
          0.8494
        ],
        [
          0.5855,
          0.8444
        ],
        [
          0.5955,
          0.8376
        ],
        [
          0.6056,
          0.8312
        ],
        [
          0.616,
          0.8267
        ],
        [
          0.626,
          0.8221
        ],
        [
          0.636,
          0.8171
        ],
        [
          0.646,
          0.8126
        ],
        [
          0.6561,
          0.8062
        ],
        [
          0.6665,
          0.7989
        ],
        [
          0.6765,
          0.793
        ],
        [
          0.6865,
          0.7871
        ],
        [
          0.6965,
          0.7803
        ],
        [
          0.7066,
          0.7734
        ],
        [
          0.717,
          0.7671
        ],
        [
          0.727,
          0.7593
        ],
        [
          0.737,
          0.7525
        ],
        [
          0.747,
          0.7461
        ],
        [
          0.7571,
          0.7393
        ],
        [
          0.7675,
          0.7329
        ],
        [
          0.7775,
          0.7275
        ],
        [
          0.7875,
          0.7216
        ],
        [
          0.7975,
          0.7166
        ],
        [
          0.8076,
          0.7102
        ],
        [
          0.818,
          0.7043
        ],
        [
          0.828,
          0.6979
        ],
        [
          0.838,
          0.6934
        ],
        [
          0.848,
          0.6861
        ],
        [
          0.8581,
          0.6788
        ],
        [
          0.8685,
          0.672
        ],
        [
          0.8785,
          0.6647
        ],
        [
          0.8885,
          0.6583
        ],
        [
          0.8985,
          0.651
        ],
        [
          0.9086,
          0.6456
        ],
        [
          0.919,
          0.6392
        ],
        [
          0.929,
          0.6306
        ],
        [
          0.939,
          0.6251
        ],
        [
          0.949,
          0.6178
        ],
        [
          0.9591,
          0.611
        ],
        [
          0.9695,
          0.6028
        ],
        [
          0.98,
          0.5955
        ],
        [
          0.9904,
          0.5878
        ],
        [
          1.0,
          0.5828
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.09899892657965793,
        "difference": -0.001079158800396828,
        "worse": 0.5533333333333333,
        "kept_right": 0.09789250879858347,
        "kept_wrong": 0.0011064177810744499,
        "handed_off": 0.9010010734203424
      },
      "usd_per_1000": 1.0261788202808213
    },
    {
      "decision": 204,
      "frontier": 278,
      "frontier_accuracy": 0.9499545040946314,
      "in_sample": {
        "threshold": 0.9979663706569197,
        "answered": 0.08143767060964513,
        "accuracy": 0.9499545040946314
      },
      "curve": [
        [
          0.0,
          0.95
        ],
        [
          0.01,
          0.95
        ],
        [
          0.02,
          0.95
        ],
        [
          0.03,
          0.95
        ],
        [
          0.04,
          0.95
        ],
        [
          0.05,
          0.95
        ],
        [
          0.0601,
          0.95
        ],
        [
          0.0705,
          0.95
        ],
        [
          0.0805,
          0.95
        ],
        [
          0.0905,
          0.9495
        ],
        [
          0.1005,
          0.949
        ],
        [
          0.1106,
          0.949
        ],
        [
          0.121,
          0.9486
        ],
        [
          0.131,
          0.9481
        ],
        [
          0.141,
          0.9481
        ],
        [
          0.151,
          0.9481
        ],
        [
          0.1611,
          0.9481
        ],
        [
          0.1715,
          0.9481
        ],
        [
          0.1815,
          0.9468
        ],
        [
          0.1915,
          0.9463
        ],
        [
          0.2015,
          0.9463
        ],
        [
          0.2116,
          0.9454
        ],
        [
          0.222,
          0.9454
        ],
        [
          0.232,
          0.9449
        ],
        [
          0.242,
          0.944
        ],
        [
          0.252,
          0.944
        ],
        [
          0.2621,
          0.9427
        ],
        [
          0.2725,
          0.9404
        ],
        [
          0.2825,
          0.9395
        ],
        [
          0.2925,
          0.9386
        ],
        [
          0.3025,
          0.9368
        ],
        [
          0.3126,
          0.9349
        ],
        [
          0.323,
          0.9308
        ],
        [
          0.333,
          0.9299
        ],
        [
          0.343,
          0.9277
        ],
        [
          0.353,
          0.9254
        ],
        [
          0.3631,
          0.9208
        ],
        [
          0.3735,
          0.9195
        ],
        [
          0.3835,
          0.9167
        ],
        [
          0.3935,
          0.9145
        ],
        [
          0.4035,
          0.9108
        ],
        [
          0.4136,
          0.9081
        ],
        [
          0.424,
          0.9054
        ],
        [
          0.434,
          0.9017
        ],
        [
          0.444,
          0.8967
        ],
        [
          0.454,
          0.8926
        ],
        [
          0.4641,
          0.889
        ],
        [
          0.4745,
          0.8844
        ],
        [
          0.4845,
          0.8813
        ],
        [
          0.4945,
          0.8758
        ],
        [
          0.5045,
          0.8708
        ],
        [
          0.5146,
          0.8667
        ],
        [
          0.525,
          0.8635
        ],
        [
          0.535,
          0.8567
        ],
        [
          0.545,
          0.8526
        ],
        [
          0.5551,
          0.848
        ],
        [
          0.5655,
          0.8439
        ],
        [
          0.5755,
          0.8389
        ],
        [
          0.5855,
          0.8335
        ],
        [
          0.5955,
          0.8262
        ],
        [
          0.6056,
          0.8198
        ],
        [
          0.616,
          0.8157
        ],
        [
          0.626,
          0.8121
        ],
        [
          0.636,
          0.808
        ],
        [
          0.646,
          0.8035
        ],
        [
          0.6561,
          0.7971
        ],
        [
          0.6665,
          0.7894
        ],
        [
          0.6765,
          0.7834
        ],
        [
          0.6865,
          0.7784
        ],
        [
          0.6965,
          0.7716
        ],
        [
          0.7066,
          0.7648
        ],
        [
          0.717,
          0.758
        ],
        [
          0.727,
          0.7511
        ],
        [
          0.737,
          0.7439
        ],
        [
          0.747,
          0.7384
        ],
        [
          0.7571,
          0.7329
        ],
        [
          0.7675,
          0.7266
        ],
        [
          0.7775,
          0.7211
        ],
        [
          0.7875,
          0.7152
        ],
        [
          0.7975,
          0.7102
        ],
        [
          0.8076,
          0.7034
        ],
        [
          0.818,
          0.6984
        ],
        [
          0.828,
          0.6924
        ],
        [
          0.838,
          0.6874
        ],
        [
          0.848,
          0.6811
        ],
        [
          0.8581,
          0.6733
        ],
        [
          0.8685,
          0.667
        ],
        [
          0.8785,
          0.6601
        ],
        [
          0.8885,
          0.6551
        ],
        [
          0.8985,
          0.6483
        ],
        [
          0.9086,
          0.6438
        ],
        [
          0.919,
          0.6369
        ],
        [
          0.929,
          0.6292
        ],
        [
          0.939,
          0.6237
        ],
        [
          0.949,
          0.6174
        ],
        [
          0.9591,
          0.611
        ],
        [
          0.9695,
          0.6028
        ],
        [
          0.98,
          0.5955
        ],
        [
          0.9904,
          0.5883
        ],
        [
          1.0,
          0.5828
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.10086031477443073,
        "difference": -0.0011155637866979028,
        "worse": 0.5533333333333333,
        "kept_right": 0.09964779771455906,
        "kept_wrong": 0.0012125170598716714,
        "handed_off": 0.8991396852255694
      },
      "usd_per_1000": 5.860055501610109
    },
    {
      "decision": 204,
      "frontier": 275,
      "frontier_accuracy": 0.9426751592356688,
      "in_sample": {
        "threshold": 0.9979663706569197,
        "answered": 0.08143767060964513,
        "accuracy": 0.9426751592356688
      },
      "curve": [
        [
          0.0,
          0.9427
        ],
        [
          0.01,
          0.9427
        ],
        [
          0.02,
          0.9427
        ],
        [
          0.03,
          0.9427
        ],
        [
          0.04,
          0.9427
        ],
        [
          0.05,
          0.9427
        ],
        [
          0.0601,
          0.9427
        ],
        [
          0.0705,
          0.9427
        ],
        [
          0.0805,
          0.9427
        ],
        [
          0.0905,
          0.9422
        ],
        [
          0.1005,
          0.9418
        ],
        [
          0.1106,
          0.9418
        ],
        [
          0.121,
          0.9413
        ],
        [
          0.131,
          0.9409
        ],
        [
          0.141,
          0.9409
        ],
        [
          0.151,
          0.9409
        ],
        [
          0.1611,
          0.9404
        ],
        [
          0.1715,
          0.9404
        ],
        [
          0.1815,
          0.9386
        ],
        [
          0.1915,
          0.9377
        ],
        [
          0.2015,
          0.9381
        ],
        [
          0.2116,
          0.9381
        ],
        [
          0.222,
          0.9381
        ],
        [
          0.232,
          0.9372
        ],
        [
          0.242,
          0.9363
        ],
        [
          0.252,
          0.9363
        ],
        [
          0.2621,
          0.9349
        ],
        [
          0.2725,
          0.9327
        ],
        [
          0.2825,
          0.9318
        ],
        [
          0.2925,
          0.9304
        ],
        [
          0.3025,
          0.929
        ],
        [
          0.3126,
          0.9272
        ],
        [
          0.323,
          0.9231
        ],
        [
          0.333,
          0.9227
        ],
        [
          0.343,
          0.9204
        ],
        [
          0.353,
          0.9186
        ],
        [
          0.3631,
          0.914
        ],
        [
          0.3735,
          0.9126
        ],
        [
          0.3835,
          0.9099
        ],
        [
          0.3935,
          0.9076
        ],
        [
          0.4035,
          0.9045
        ],
        [
          0.4136,
          0.9017
        ],
        [
          0.424,
          0.899
        ],
        [
          0.434,
          0.8949
        ],
        [
          0.444,
          0.8904
        ],
        [
          0.454,
          0.8863
        ],
        [
          0.4641,
          0.8826
        ],
        [
          0.4745,
          0.8785
        ],
        [
          0.4845,
          0.8758
        ],
        [
          0.4945,
          0.8708
        ],
        [
          0.5045,
          0.8658
        ],
        [
          0.5146,
          0.8626
        ],
        [
          0.525,
          0.859
        ],
        [
          0.535,
          0.8526
        ],
        [
          0.545,
          0.848
        ],
        [
          0.5551,
          0.8435
        ],
        [
          0.5655,
          0.8399
        ],
        [
          0.5755,
          0.8358
        ],
        [
          0.5855,
          0.8308
        ],
        [
          0.5955,
          0.8239
        ],
        [
          0.6056,
          0.8176
        ],
        [
          0.616,
          0.8135
        ],
        [
          0.626,
          0.8103
        ],
        [
          0.636,
          0.8053
        ],
        [
          0.646,
          0.8012
        ],
        [
          0.6561,
          0.7948
        ],
        [
          0.6665,
          0.788
        ],
        [
          0.6765,
          0.7816
        ],
        [
          0.6865,
          0.7757
        ],
        [
          0.6965,
          0.7689
        ],
        [
          0.7066,
          0.763
        ],
        [
          0.717,
          0.7571
        ],
        [
          0.727,
          0.7507
        ],
        [
          0.737,
          0.7434
        ],
        [
          0.747,
          0.737
        ],
        [
          0.7571,
          0.7302
        ],
        [
          0.7675,
          0.7234
        ],
        [
          0.7775,
          0.7179
        ],
        [
          0.7875,
          0.7134
        ],
        [
          0.7975,
          0.7093
        ],
        [
          0.8076,
          0.7025
        ],
        [
          0.818,
          0.6965
        ],
        [
          0.828,
          0.6911
        ],
        [
          0.838,
          0.6861
        ],
        [
          0.848,
          0.6802
        ],
        [
          0.8581,
          0.6733
        ],
        [
          0.8685,
          0.667
        ],
        [
          0.8785,
          0.6601
        ],
        [
          0.8885,
          0.6538
        ],
        [
          0.8985,
          0.647
        ],
        [
          0.9086,
          0.6419
        ],
        [
          0.919,
          0.6351
        ],
        [
          0.929,
          0.6274
        ],
        [
          0.939,
          0.6219
        ],
        [
          0.949,
          0.6151
        ],
        [
          0.9591,
          0.6083
        ],
        [
          0.9695,
          0.601
        ],
        [
          0.98,
          0.5951
        ],
        [
          0.9904,
          0.5878
        ],
        [
          1.0,
          0.5828
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.0989252118027068,
        "difference": -0.0010609274703787216,
        "worse": 0.5533333333333333,
        "kept_right": 0.09783401155255647,
        "kept_wrong": 0.001091200250150307,
        "handed_off": 0.9010747881972937
      },
      "usd_per_1000": 0.47132634120669653
    },
    {
      "decision": 204,
      "frontier": 192,
      "frontier_accuracy": 0.9331210191082803,
      "in_sample": {
        "threshold": 0.9979663706569197,
        "answered": 0.08143767060964513,
        "accuracy": 0.9331210191082803
      },
      "curve": [
        [
          0.0,
          0.9331
        ],
        [
          0.01,
          0.9331
        ],
        [
          0.02,
          0.9331
        ],
        [
          0.03,
          0.9331
        ],
        [
          0.04,
          0.9331
        ],
        [
          0.05,
          0.9331
        ],
        [
          0.0601,
          0.9331
        ],
        [
          0.0705,
          0.9331
        ],
        [
          0.0805,
          0.9331
        ],
        [
          0.0905,
          0.9327
        ],
        [
          0.1005,
          0.9322
        ],
        [
          0.1106,
          0.9322
        ],
        [
          0.121,
          0.9318
        ],
        [
          0.131,
          0.9313
        ],
        [
          0.141,
          0.9313
        ],
        [
          0.151,
          0.9313
        ],
        [
          0.1611,
          0.9313
        ],
        [
          0.1715,
          0.9313
        ],
        [
          0.1815,
          0.9299
        ],
        [
          0.1915,
          0.9295
        ],
        [
          0.2015,
          0.9295
        ],
        [
          0.2116,
          0.9286
        ],
        [
          0.222,
          0.9286
        ],
        [
          0.232,
          0.9281
        ],
        [
          0.242,
          0.9272
        ],
        [
          0.252,
          0.9272
        ],
        [
          0.2621,
          0.9258
        ],
        [
          0.2725,
          0.924
        ],
        [
          0.2825,
          0.9231
        ],
        [
          0.2925,
          0.9213
        ],
        [
          0.3025,
          0.9204
        ],
        [
          0.3126,
          0.9186
        ],
        [
          0.323,
          0.9149
        ],
        [
          0.333,
          0.9145
        ],
        [
          0.343,
          0.9122
        ],
        [
          0.353,
          0.9099
        ],
        [
          0.3631,
          0.9054
        ],
        [
          0.3735,
          0.9054
        ],
        [
          0.3835,
          0.9022
        ],
        [
          0.3935,
          0.8995
        ],
        [
          0.4035,
          0.8963
        ],
        [
          0.4136,
          0.8935
        ],
        [
          0.424,
          0.8913
        ],
        [
          0.434,
          0.8872
        ],
        [
          0.444,
          0.8822
        ],
        [
          0.454,
          0.8785
        ],
        [
          0.4641,
          0.8749
        ],
        [
          0.4745,
          0.8708
        ],
        [
          0.4845,
          0.8685
        ],
        [
          0.4945,
          0.8635
        ],
        [
          0.5045,
          0.8594
        ],
        [
          0.5146,
          0.8558
        ],
        [
          0.525,
          0.8517
        ],
        [
          0.535,
          0.8453
        ],
        [
          0.545,
          0.8412
        ],
        [
          0.5551,
          0.8371
        ],
        [
          0.5655,
          0.8335
        ],
        [
          0.5755,
          0.8285
        ],
        [
          0.5855,
          0.8235
        ],
        [
          0.5955,
          0.8167
        ],
        [
          0.6056,
          0.8098
        ],
        [
          0.616,
          0.8053
        ],
        [
          0.626,
          0.8025
        ],
        [
          0.636,
          0.7985
        ],
        [
          0.646,
          0.7957
        ],
        [
          0.6561,
          0.7898
        ],
        [
          0.6665,
          0.7821
        ],
        [
          0.6765,
          0.7762
        ],
        [
          0.6865,
          0.7702
        ],
        [
          0.6965,
          0.7639
        ],
        [
          0.7066,
          0.7571
        ],
        [
          0.717,
          0.7511
        ],
        [
          0.727,
          0.7439
        ],
        [
          0.737,
          0.737
        ],
        [
          0.747,
          0.732
        ],
        [
          0.7571,
          0.7266
        ],
        [
          0.7675,
          0.7207
        ],
        [
          0.7775,
          0.7147
        ],
        [
          0.7875,
          0.7097
        ],
        [
          0.7975,
          0.7047
        ],
        [
          0.8076,
          0.6984
        ],
        [
          0.818,
          0.6924
        ],
        [
          0.828,
          0.687
        ],
        [
          0.838,
          0.6824
        ],
        [
          0.848,
          0.6761
        ],
        [
          0.8581,
          0.6688
        ],
        [
          0.8685,
          0.6624
        ],
        [
          0.8785,
          0.6556
        ],
        [
          0.8885,
          0.6506
        ],
        [
          0.8985,
          0.6442
        ],
        [
          0.9086,
          0.6392
        ],
        [
          0.919,
          0.6333
        ],
        [
          0.929,
          0.6251
        ],
        [
          0.939,
          0.6206
        ],
        [
          0.949,
          0.6142
        ],
        [
          0.9591,
          0.6078
        ],
        [
          0.9695,
          0.6005
        ],
        [
          0.98,
          0.5937
        ],
        [
          0.9904,
          0.5874
        ],
        [
          1.0,
          0.5828
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.10173595675279751,
        "difference": -0.0011701037366057,
        "worse": 0.5566666666666666,
        "kept_right": 0.10045677853089681,
        "kept_wrong": 0.0012791782219006812,
        "handed_off": 0.8982640432472025
      },
      "usd_per_1000": 10.436166569640093
    },
    {
      "decision": 204,
      "frontier": 123,
      "frontier_accuracy": 0.9326660600545951,
      "in_sample": {
        "threshold": 0.9979663706569197,
        "answered": 0.08143767060964513,
        "accuracy": 0.9326660600545951
      },
      "curve": [
        [
          0.0,
          0.9327
        ],
        [
          0.01,
          0.9327
        ],
        [
          0.02,
          0.9327
        ],
        [
          0.03,
          0.9327
        ],
        [
          0.04,
          0.9327
        ],
        [
          0.05,
          0.9327
        ],
        [
          0.0601,
          0.9327
        ],
        [
          0.0705,
          0.9327
        ],
        [
          0.0805,
          0.9327
        ],
        [
          0.0905,
          0.9322
        ],
        [
          0.1005,
          0.9318
        ],
        [
          0.1106,
          0.9318
        ],
        [
          0.121,
          0.9313
        ],
        [
          0.131,
          0.9308
        ],
        [
          0.141,
          0.9308
        ],
        [
          0.151,
          0.9308
        ],
        [
          0.1611,
          0.9304
        ],
        [
          0.1715,
          0.9304
        ],
        [
          0.1815,
          0.929
        ],
        [
          0.1915,
          0.9281
        ],
        [
          0.2015,
          0.9286
        ],
        [
          0.2116,
          0.9277
        ],
        [
          0.222,
          0.9277
        ],
        [
          0.232,
          0.9272
        ],
        [
          0.242,
          0.9263
        ],
        [
          0.252,
          0.9263
        ],
        [
          0.2621,
          0.9254
        ],
        [
          0.2725,
          0.9231
        ],
        [
          0.2825,
          0.9227
        ],
        [
          0.2925,
          0.9213
        ],
        [
          0.3025,
          0.9195
        ],
        [
          0.3126,
          0.9181
        ],
        [
          0.323,
          0.9145
        ],
        [
          0.333,
          0.9136
        ],
        [
          0.343,
          0.9113
        ],
        [
          0.353,
          0.909
        ],
        [
          0.3631,
          0.9049
        ],
        [
          0.3735,
          0.9026
        ],
        [
          0.3835,
          0.8999
        ],
        [
          0.3935,
          0.8972
        ],
        [
          0.4035,
          0.8935
        ],
        [
          0.4136,
          0.8904
        ],
        [
          0.424,
          0.8881
        ],
        [
          0.434,
          0.8844
        ],
        [
          0.444,
          0.8794
        ],
        [
          0.454,
          0.8753
        ],
        [
          0.4641,
          0.8717
        ],
        [
          0.4745,
          0.8672
        ],
        [
          0.4845,
          0.8644
        ],
        [
          0.4945,
          0.8599
        ],
        [
          0.5045,
          0.8544
        ],
        [
          0.5146,
          0.8508
        ],
        [
          0.525,
          0.8471
        ],
        [
          0.535,
          0.8403
        ],
        [
          0.545,
          0.8367
        ],
        [
          0.5551,
          0.8321
        ],
        [
          0.5655,
          0.828
        ],
        [
          0.5755,
          0.823
        ],
        [
          0.5855,
          0.8176
        ],
        [
          0.5955,
          0.8103
        ],
        [
          0.6056,
          0.8048
        ],
        [
          0.616,
          0.8012
        ],
        [
          0.626,
          0.7971
        ],
        [
          0.636,
          0.7921
        ],
        [
          0.646,
          0.7871
        ],
        [
          0.6561,
          0.7816
        ],
        [
          0.6665,
          0.7757
        ],
        [
          0.6765,
          0.7693
        ],
        [
          0.6865,
          0.7648
        ],
        [
          0.6965,
          0.758
        ],
        [
          0.7066,
          0.7511
        ],
        [
          0.717,
          0.7439
        ],
        [
          0.727,
          0.737
        ],
        [
          0.737,
          0.7307
        ],
        [
          0.747,
          0.7252
        ],
        [
          0.7571,
          0.7188
        ],
        [
          0.7675,
          0.7134
        ],
        [
          0.7775,
          0.7066
        ],
        [
          0.7875,
          0.702
        ],
        [
          0.7975,
          0.6984
        ],
        [
          0.8076,
          0.6934
        ],
        [
          0.818,
          0.6888
        ],
        [
          0.828,
          0.6843
        ],
        [
          0.838,
          0.6802
        ],
        [
          0.848,
          0.6729
        ],
        [
          0.8581,
          0.6656
        ],
        [
          0.8685,
          0.6592
        ],
        [
          0.8785,
          0.6533
        ],
        [
          0.8885,
          0.6488
        ],
        [
          0.8985,
          0.6424
        ],
        [
          0.9086,
          0.6374
        ],
        [
          0.919,
          0.6324
        ],
        [
          0.929,
          0.6251
        ],
        [
          0.939,
          0.6201
        ],
        [
          0.949,
          0.6137
        ],
        [
          0.9591,
          0.6074
        ],
        [
          0.9695,
          0.6005
        ],
        [
          0.98,
          0.5937
        ],
        [
          0.9904,
          0.5864
        ],
        [
          1.0,
          0.5828
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.09966543375669143,
        "difference": -0.0011276381492391534,
        "worse": 0.5566666666666666,
        "kept_right": 0.09849842091429595,
        "kept_wrong": 0.001167012842395464,
        "handed_off": 0.9003345662433088
      },
      "usd_per_1000": 0.24952687913799895
    },
    {
      "decision": 204,
      "frontier": 274,
      "frontier_accuracy": 0.9085532302092811,
      "in_sample": {
        "threshold": 0.9979663706569197,
        "answered": 0.08143767060964513,
        "accuracy": 0.9085532302092811
      },
      "curve": [
        [
          0.0,
          0.9086
        ],
        [
          0.01,
          0.9086
        ],
        [
          0.02,
          0.9086
        ],
        [
          0.03,
          0.9086
        ],
        [
          0.04,
          0.9086
        ],
        [
          0.05,
          0.9086
        ],
        [
          0.0601,
          0.9086
        ],
        [
          0.0705,
          0.9086
        ],
        [
          0.0805,
          0.9086
        ],
        [
          0.0905,
          0.9081
        ],
        [
          0.1005,
          0.9076
        ],
        [
          0.1106,
          0.9076
        ],
        [
          0.121,
          0.9072
        ],
        [
          0.131,
          0.9067
        ],
        [
          0.141,
          0.9067
        ],
        [
          0.151,
          0.9067
        ],
        [
          0.1611,
          0.9067
        ],
        [
          0.1715,
          0.9072
        ],
        [
          0.1815,
          0.9054
        ],
        [
          0.1915,
          0.9049
        ],
        [
          0.2015,
          0.9058
        ],
        [
          0.2116,
          0.9054
        ],
        [
          0.222,
          0.9054
        ],
        [
          0.232,
          0.9045
        ],
        [
          0.242,
          0.9035
        ],
        [
          0.252,
          0.9035
        ],
        [
          0.2621,
          0.9022
        ],
        [
          0.2725,
          0.9004
        ],
        [
          0.2825,
          0.8995
        ],
        [
          0.2925,
          0.8981
        ],
        [
          0.3025,
          0.8963
        ],
        [
          0.3126,
          0.8944
        ],
        [
          0.323,
          0.8908
        ],
        [
          0.333,
          0.8904
        ],
        [
          0.343,
          0.8881
        ],
        [
          0.353,
          0.8863
        ],
        [
          0.3631,
          0.8817
        ],
        [
          0.3735,
          0.8803
        ],
        [
          0.3835,
          0.8776
        ],
        [
          0.3935,
          0.8753
        ],
        [
          0.4035,
          0.8722
        ],
        [
          0.4136,
          0.8699
        ],
        [
          0.424,
          0.8676
        ],
        [
          0.434,
          0.8635
        ],
        [
          0.444,
          0.859
        ],
        [
          0.454,
          0.8553
        ],
        [
          0.4641,
          0.8517
        ],
        [
          0.4745,
          0.848
        ],
        [
          0.4845,
          0.8449
        ],
        [
          0.4945,
          0.8403
        ],
        [
          0.5045,
          0.8353
        ],
        [
          0.5146,
          0.8326
        ],
        [
          0.525,
          0.8285
        ],
        [
          0.535,
          0.8221
        ],
        [
          0.545,
          0.8189
        ],
        [
          0.5551,
          0.8139
        ],
        [
          0.5655,
          0.8103
        ],
        [
          0.5755,
          0.8062
        ],
        [
          0.5855,
          0.8016
        ],
        [
          0.5955,
          0.7953
        ],
        [
          0.6056,
          0.7884
        ],
        [
          0.616,
          0.7843
        ],
        [
          0.626,
          0.7816
        ],
        [
          0.636,
          0.7775
        ],
        [
          0.646,
          0.7743
        ],
        [
          0.6561,
          0.7684
        ],
        [
          0.6665,
          0.7625
        ],
        [
          0.6765,
          0.7566
        ],
        [
          0.6865,
          0.7511
        ],
        [
          0.6965,
          0.7443
        ],
        [
          0.7066,
          0.7379
        ],
        [
          0.717,
          0.7325
        ],
        [
          0.727,
          0.7261
        ],
        [
          0.737,
          0.7197
        ],
        [
          0.747,
          0.7152
        ],
        [
          0.7571,
          0.7088
        ],
        [
          0.7675,
          0.7029
        ],
        [
          0.7775,
          0.6975
        ],
        [
          0.7875,
          0.6934
        ],
        [
          0.7975,
          0.6888
        ],
        [
          0.8076,
          0.6843
        ],
        [
          0.818,
          0.6783
        ],
        [
          0.828,
          0.6733
        ],
        [
          0.838,
          0.6688
        ],
        [
          0.848,
          0.6633
        ],
        [
          0.8581,
          0.6565
        ],
        [
          0.8685,
          0.6497
        ],
        [
          0.8785,
          0.6433
        ],
        [
          0.8885,
          0.6374
        ],
        [
          0.8985,
          0.6324
        ],
        [
          0.9086,
          0.6283
        ],
        [
          0.919,
          0.6233
        ],
        [
          0.929,
          0.6169
        ],
        [
          0.939,
          0.6137
        ],
        [
          0.949,
          0.6083
        ],
        [
          0.9591,
          0.6024
        ],
        [
          0.9695,
          0.596
        ],
        [
          0.98,
          0.5919
        ],
        [
          0.9904,
          0.586
        ],
        [
          1.0,
          0.5828
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.11166342856288149,
        "difference": -0.001430889379704343,
        "worse": 0.5866666666666667,
        "kept_right": 0.10993242574148551,
        "kept_wrong": 0.0017310028213959568,
        "handed_off": 0.8883365714371185
      },
      "usd_per_1000": 0.38636109815312975
    },
    {
      "decision": 204,
      "frontier": 47,
      "frontier_accuracy": 0.9044585987261147,
      "in_sample": {
        "threshold": 0.9979663706569197,
        "answered": 0.08143767060964513,
        "accuracy": 0.9044585987261147
      },
      "curve": [
        [
          0.0,
          0.9045
        ],
        [
          0.01,
          0.9045
        ],
        [
          0.02,
          0.9045
        ],
        [
          0.03,
          0.9045
        ],
        [
          0.04,
          0.9045
        ],
        [
          0.05,
          0.9045
        ],
        [
          0.0601,
          0.9045
        ],
        [
          0.0705,
          0.9045
        ],
        [
          0.0805,
          0.9045
        ],
        [
          0.0905,
          0.904
        ],
        [
          0.1005,
          0.9035
        ],
        [
          0.1106,
          0.9035
        ],
        [
          0.121,
          0.9031
        ],
        [
          0.131,
          0.9031
        ],
        [
          0.141,
          0.9031
        ],
        [
          0.151,
          0.9031
        ],
        [
          0.1611,
          0.9031
        ],
        [
          0.1715,
          0.9031
        ],
        [
          0.1815,
          0.9017
        ],
        [
          0.1915,
          0.9013
        ],
        [
          0.2015,
          0.9013
        ],
        [
          0.2116,
          0.9004
        ],
        [
          0.222,
          0.9004
        ],
        [
          0.232,
          0.9004
        ],
        [
          0.242,
          0.8995
        ],
        [
          0.252,
          0.8995
        ],
        [
          0.2621,
          0.8981
        ],
        [
          0.2725,
          0.8963
        ],
        [
          0.2825,
          0.8958
        ],
        [
          0.2925,
          0.894
        ],
        [
          0.3025,
          0.8935
        ],
        [
          0.3126,
          0.8917
        ],
        [
          0.323,
          0.8876
        ],
        [
          0.333,
          0.8872
        ],
        [
          0.343,
          0.8854
        ],
        [
          0.353,
          0.8835
        ],
        [
          0.3631,
          0.879
        ],
        [
          0.3735,
          0.8781
        ],
        [
          0.3835,
          0.8753
        ],
        [
          0.3935,
          0.8731
        ],
        [
          0.4035,
          0.8703
        ],
        [
          0.4136,
          0.8685
        ],
        [
          0.424,
          0.8667
        ],
        [
          0.434,
          0.8631
        ],
        [
          0.444,
          0.8581
        ],
        [
          0.454,
          0.8549
        ],
        [
          0.4641,
          0.8512
        ],
        [
          0.4745,
          0.8462
        ],
        [
          0.4845,
          0.8439
        ],
        [
          0.4945,
          0.8389
        ],
        [
          0.5045,
          0.8348
        ],
        [
          0.5146,
          0.8321
        ],
        [
          0.525,
          0.8285
        ],
        [
          0.535,
          0.8221
        ],
        [
          0.545,
          0.818
        ],
        [
          0.5551,
          0.8139
        ],
        [
          0.5655,
          0.8103
        ],
        [
          0.5755,
          0.8057
        ],
        [
          0.5855,
          0.8012
        ],
        [
          0.5955,
          0.7944
        ],
        [
          0.6056,
          0.7884
        ],
        [
          0.616,
          0.7839
        ],
        [
          0.626,
          0.7807
        ],
        [
          0.636,
          0.7766
        ],
        [
          0.646,
          0.773
        ],
        [
          0.6561,
          0.7671
        ],
        [
          0.6665,
          0.7607
        ],
        [
          0.6765,
          0.7557
        ],
        [
          0.6865,
          0.7507
        ],
        [
          0.6965,
          0.7448
        ],
        [
          0.7066,
          0.7384
        ],
        [
          0.717,
          0.7325
        ],
        [
          0.727,
          0.7266
        ],
        [
          0.737,
          0.7193
        ],
        [
          0.747,
          0.7143
        ],
        [
          0.7571,
          0.7088
        ],
        [
          0.7675,
          0.7034
        ],
        [
          0.7775,
          0.6975
        ],
        [
          0.7875,
          0.6934
        ],
        [
          0.7975,
          0.6893
        ],
        [
          0.8076,
          0.6838
        ],
        [
          0.818,
          0.6779
        ],
        [
          0.828,
          0.6729
        ],
        [
          0.838,
          0.6702
        ],
        [
          0.848,
          0.6638
        ],
        [
          0.8581,
          0.6565
        ],
        [
          0.8685,
          0.6501
        ],
        [
          0.8785,
          0.6438
        ],
        [
          0.8885,
          0.6397
        ],
        [
          0.8985,
          0.6342
        ],
        [
          0.9086,
          0.6292
        ],
        [
          0.919,
          0.6247
        ],
        [
          0.929,
          0.6174
        ],
        [
          0.939,
          0.6128
        ],
        [
          0.949,
          0.6074
        ],
        [
          0.9591,
          0.6024
        ],
        [
          0.9695,
          0.5965
        ],
        [
          0.98,
          0.5905
        ],
        [
          0.9904,
          0.5855
        ],
        [
          1.0,
          0.5828
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.10965232495434482,
        "difference": -0.0014278256425887172,
        "worse": 0.5666666666666667,
        "kept_right": 0.10787593691721405,
        "kept_wrong": 0.0017763880371307725,
        "handed_off": 0.8903476750456552
      },
      "usd_per_1000": 4.397647231607655
    },
    {
      "decision": 204,
      "frontier": 193,
      "frontier_accuracy": 0.89171974522293,
      "in_sample": {
        "threshold": 0.993357948995032,
        "answered": 0.17925386715195632,
        "accuracy": 0.89171974522293
      },
      "curve": [
        [
          0.0,
          0.8917
        ],
        [
          0.01,
          0.8917
        ],
        [
          0.02,
          0.8917
        ],
        [
          0.03,
          0.8917
        ],
        [
          0.04,
          0.8917
        ],
        [
          0.05,
          0.8917
        ],
        [
          0.0601,
          0.8917
        ],
        [
          0.0705,
          0.8917
        ],
        [
          0.0805,
          0.8917
        ],
        [
          0.0905,
          0.8922
        ],
        [
          0.1005,
          0.8917
        ],
        [
          0.1106,
          0.8917
        ],
        [
          0.121,
          0.8913
        ],
        [
          0.131,
          0.8913
        ],
        [
          0.141,
          0.8913
        ],
        [
          0.151,
          0.8913
        ],
        [
          0.1611,
          0.8917
        ],
        [
          0.1715,
          0.8922
        ],
        [
          0.1815,
          0.8908
        ],
        [
          0.1915,
          0.8899
        ],
        [
          0.2015,
          0.8899
        ],
        [
          0.2116,
          0.889
        ],
        [
          0.222,
          0.8894
        ],
        [
          0.232,
          0.889
        ],
        [
          0.242,
          0.8881
        ],
        [
          0.252,
          0.8881
        ],
        [
          0.2621,
          0.8872
        ],
        [
          0.2725,
          0.8849
        ],
        [
          0.2825,
          0.8844
        ],
        [
          0.2925,
          0.8835
        ],
        [
          0.3025,
          0.8831
        ],
        [
          0.3126,
          0.8813
        ],
        [
          0.323,
          0.8785
        ],
        [
          0.333,
          0.8785
        ],
        [
          0.343,
          0.8772
        ],
        [
          0.353,
          0.8749
        ],
        [
          0.3631,
          0.8703
        ],
        [
          0.3735,
          0.869
        ],
        [
          0.3835,
          0.8658
        ],
        [
          0.3935,
          0.864
        ],
        [
          0.4035,
          0.8612
        ],
        [
          0.4136,
          0.8594
        ],
        [
          0.424,
          0.8576
        ],
        [
          0.434,
          0.854
        ],
        [
          0.444,
          0.8494
        ],
        [
          0.454,
          0.8467
        ],
        [
          0.4641,
          0.8435
        ],
        [
          0.4745,
          0.8394
        ],
        [
          0.4845,
          0.8371
        ],
        [
          0.4945,
          0.8339
        ],
        [
          0.5045,
          0.8298
        ],
        [
          0.5146,
          0.8267
        ],
        [
          0.525,
          0.823
        ],
        [
          0.535,
          0.8171
        ],
        [
          0.545,
          0.8135
        ],
        [
          0.5551,
          0.8094
        ],
        [
          0.5655,
          0.8062
        ],
        [
          0.5755,
          0.8012
        ],
        [
          0.5855,
          0.7957
        ],
        [
          0.5955,
          0.7898
        ],
        [
          0.6056,
          0.7834
        ],
        [
          0.616,
          0.7798
        ],
        [
          0.626,
          0.7766
        ],
        [
          0.636,
          0.7721
        ],
        [
          0.646,
          0.768
        ],
        [
          0.6561,
          0.7625
        ],
        [
          0.6665,
          0.7552
        ],
        [
          0.6765,
          0.7493
        ],
        [
          0.6865,
          0.7439
        ],
        [
          0.6965,
          0.737
        ],
        [
          0.7066,
          0.7316
        ],
        [
          0.717,
          0.7261
        ],
        [
          0.727,
          0.7184
        ],
        [
          0.737,
          0.7129
        ],
        [
          0.747,
          0.7088
        ],
        [
          0.7571,
          0.7047
        ],
        [
          0.7675,
          0.6993
        ],
        [
          0.7775,
          0.6943
        ],
        [
          0.7875,
          0.6906
        ],
        [
          0.7975,
          0.6874
        ],
        [
          0.8076,
          0.6811
        ],
        [
          0.818,
          0.6761
        ],
        [
          0.828,
          0.6711
        ],
        [
          0.838,
          0.6674
        ],
        [
          0.848,
          0.662
        ],
        [
          0.8581,
          0.6551
        ],
        [
          0.8685,
          0.6506
        ],
        [
          0.8785,
          0.6442
        ],
        [
          0.8885,
          0.6401
        ],
        [
          0.8985,
          0.6347
        ],
        [
          0.9086,
          0.631
        ],
        [
          0.919,
          0.6256
        ],
        [
          0.929,
          0.6192
        ],
        [
          0.939,
          0.6151
        ],
        [
          0.949,
          0.6092
        ],
        [
          0.9591,
          0.6037
        ],
        [
          0.9695,
          0.5969
        ],
        [
          0.98,
          0.5914
        ],
        [
          0.9904,
          0.5855
        ],
        [
          1.0,
          0.5828
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.183733584837344,
        "difference": -0.0018217514445887254,
        "worse": 0.53,
        "kept_right": 0.1785285302563714,
        "kept_wrong": 0.0052050545809726005,
        "handed_off": 0.8162664151626557
      },
      "usd_per_1000": 6.33180898209002
    },
    {
      "decision": 204,
      "frontier": 55,
      "frontier_accuracy": 0.8503184713375797,
      "in_sample": {
        "threshold": 0.9669039098934442,
        "answered": 0.3093721565059145,
        "accuracy": 0.8503184713375797
      },
      "curve": [
        [
          0.0,
          0.8503
        ],
        [
          0.01,
          0.8503
        ],
        [
          0.02,
          0.8503
        ],
        [
          0.03,
          0.8508
        ],
        [
          0.04,
          0.8508
        ],
        [
          0.05,
          0.8512
        ],
        [
          0.0601,
          0.8512
        ],
        [
          0.0705,
          0.8512
        ],
        [
          0.0805,
          0.8512
        ],
        [
          0.0905,
          0.8517
        ],
        [
          0.1005,
          0.8512
        ],
        [
          0.1106,
          0.8512
        ],
        [
          0.121,
          0.8512
        ],
        [
          0.131,
          0.8508
        ],
        [
          0.141,
          0.8508
        ],
        [
          0.151,
          0.8508
        ],
        [
          0.1611,
          0.8512
        ],
        [
          0.1715,
          0.8517
        ],
        [
          0.1815,
          0.8512
        ],
        [
          0.1915,
          0.8508
        ],
        [
          0.2015,
          0.8517
        ],
        [
          0.2116,
          0.8512
        ],
        [
          0.222,
          0.8521
        ],
        [
          0.232,
          0.8512
        ],
        [
          0.242,
          0.8517
        ],
        [
          0.252,
          0.8517
        ],
        [
          0.2621,
          0.8512
        ],
        [
          0.2725,
          0.8503
        ],
        [
          0.2825,
          0.8508
        ],
        [
          0.2925,
          0.8512
        ],
        [
          0.3025,
          0.8508
        ],
        [
          0.3126,
          0.849
        ],
        [
          0.323,
          0.8467
        ],
        [
          0.333,
          0.8471
        ],
        [
          0.343,
          0.8462
        ],
        [
          0.353,
          0.8444
        ],
        [
          0.3631,
          0.8417
        ],
        [
          0.3735,
          0.8421
        ],
        [
          0.3835,
          0.8408
        ],
        [
          0.3935,
          0.8403
        ],
        [
          0.4035,
          0.8371
        ],
        [
          0.4136,
          0.8348
        ],
        [
          0.424,
          0.8335
        ],
        [
          0.434,
          0.8298
        ],
        [
          0.444,
          0.8262
        ],
        [
          0.454,
          0.8244
        ],
        [
          0.4641,
          0.8212
        ],
        [
          0.4745,
          0.8189
        ],
        [
          0.4845,
          0.8167
        ],
        [
          0.4945,
          0.8144
        ],
        [
          0.5045,
          0.8116
        ],
        [
          0.5146,
          0.8076
        ],
        [
          0.525,
          0.8048
        ],
        [
          0.535,
          0.7998
        ],
        [
          0.545,
          0.7971
        ],
        [
          0.5551,
          0.7939
        ],
        [
          0.5655,
          0.7921
        ],
        [
          0.5755,
          0.7884
        ],
        [
          0.5855,
          0.7834
        ],
        [
          0.5955,
          0.7784
        ],
        [
          0.6056,
          0.773
        ],
        [
          0.616,
          0.7684
        ],
        [
          0.626,
          0.7666
        ],
        [
          0.636,
          0.763
        ],
        [
          0.646,
          0.7602
        ],
        [
          0.6561,
          0.7552
        ],
        [
          0.6665,
          0.7493
        ],
        [
          0.6765,
          0.7443
        ],
        [
          0.6865,
          0.7398
        ],
        [
          0.6965,
          0.7343
        ],
        [
          0.7066,
          0.7284
        ],
        [
          0.717,
          0.7229
        ],
        [
          0.727,
          0.7161
        ],
        [
          0.737,
          0.7102
        ],
        [
          0.747,
          0.7047
        ],
        [
          0.7571,
          0.7006
        ],
        [
          0.7675,
          0.6961
        ],
        [
          0.7775,
          0.6911
        ],
        [
          0.7875,
          0.6879
        ],
        [
          0.7975,
          0.6843
        ],
        [
          0.8076,
          0.6788
        ],
        [
          0.818,
          0.6752
        ],
        [
          0.828,
          0.6697
        ],
        [
          0.838,
          0.6661
        ],
        [
          0.848,
          0.6611
        ],
        [
          0.8581,
          0.6547
        ],
        [
          0.8685,
          0.6497
        ],
        [
          0.8785,
          0.6438
        ],
        [
          0.8885,
          0.6401
        ],
        [
          0.8985,
          0.6347
        ],
        [
          0.9086,
          0.631
        ],
        [
          0.919,
          0.626
        ],
        [
          0.929,
          0.6192
        ],
        [
          0.939,
          0.6151
        ],
        [
          0.949,
          0.6101
        ],
        [
          0.9591,
          0.6042
        ],
        [
          0.9695,
          0.5983
        ],
        [
          0.98,
          0.5919
        ],
        [
          0.9904,
          0.5851
        ],
        [
          1.0,
          0.5828
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.2976269797644378,
        "difference": -0.0030198812610218503,
        "worse": 0.5,
        "kept_right": 0.2768567729442918,
        "kept_wrong": 0.020770206820146082,
        "handed_off": 0.7023730202355619
      },
      "usd_per_1000": 2.574416815631526
    },
    {
      "decision": 49,
      "frontier": 140,
      "frontier_accuracy": 0.9808917197452229,
      "in_sample": {
        "threshold": 1.0,
        "answered": 0.11010009099181074,
        "accuracy": 0.9808917197452229
      },
      "curve": [
        [
          0.0,
          0.9809
        ],
        [
          0.1101,
          0.9809
        ],
        [
          0.1419,
          0.9804
        ],
        [
          0.1629,
          0.98
        ],
        [
          0.1761,
          0.98
        ],
        [
          0.1884,
          0.9791
        ],
        [
          0.2002,
          0.9777
        ],
        [
          0.2152,
          0.9745
        ],
        [
          0.2261,
          0.9722
        ],
        [
          0.2429,
          0.9718
        ],
        [
          0.2611,
          0.9704
        ],
        [
          0.2734,
          0.9682
        ],
        [
          0.2839,
          0.9668
        ],
        [
          0.2957,
          0.9654
        ],
        [
          0.3062,
          0.9641
        ],
        [
          0.3198,
          0.96
        ],
        [
          0.3344,
          0.9563
        ],
        [
          0.3512,
          0.9509
        ],
        [
          0.3658,
          0.9463
        ],
        [
          0.379,
          0.9431
        ],
        [
          0.3908,
          0.9399
        ],
        [
          0.4026,
          0.9354
        ],
        [
          0.4136,
          0.9313
        ],
        [
          0.4304,
          0.9258
        ],
        [
          0.4459,
          0.9199
        ],
        [
          0.4572,
          0.9163
        ],
        [
          0.4695,
          0.9113
        ],
        [
          0.4868,
          0.9031
        ],
        [
          0.4977,
          0.8981
        ],
        [
          0.5159,
          0.8904
        ],
        [
          0.5309,
          0.8831
        ],
        [
          0.546,
          0.8772
        ],
        [
          0.5596,
          0.8681
        ],
        [
          0.5732,
          0.8594
        ],
        [
          0.5883,
          0.8517
        ],
        [
          0.601,
          0.8417
        ],
        [
          0.6151,
          0.8326
        ],
        [
          0.6278,
          0.8248
        ],
        [
          0.6497,
          0.8116
        ],
        [
          0.6624,
          0.8057
        ],
        [
          0.6802,
          0.7948
        ],
        [
          0.6965,
          0.7834
        ],
        [
          0.7102,
          0.7734
        ],
        [
          0.7311,
          0.7593
        ],
        [
          0.748,
          0.748
        ],
        [
          0.763,
          0.7357
        ],
        [
          0.7825,
          0.7257
        ],
        [
          0.8025,
          0.7116
        ],
        [
          0.8212,
          0.7006
        ],
        [
          0.8412,
          0.6852
        ],
        [
          0.8603,
          0.672
        ],
        [
          0.8735,
          0.6638
        ],
        [
          0.8972,
          0.6479
        ],
        [
          0.9081,
          0.6406
        ],
        [
          0.9217,
          0.6315
        ],
        [
          0.9354,
          0.6224
        ],
        [
          0.9468,
          0.6156
        ],
        [
          0.96,
          0.6056
        ],
        [
          0.9754,
          0.5937
        ],
        [
          0.9927,
          0.581
        ],
        [
          1.0,
          0.5751
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.13674055815862077,
        "difference": -0.0008245879478292797,
        "worse": 0.5166666666666667,
        "kept_right": 0.13591597021079144,
        "kept_wrong": 0.0008245879478292797,
        "handed_off": 0.8632594418413796
      },
      "usd_per_1000": 2.11716205802467
    },
    {
      "decision": 49,
      "frontier": 277,
      "frontier_accuracy": 0.9795268425841674,
      "in_sample": {
        "threshold": 1.0,
        "answered": 0.11010009099181074,
        "accuracy": 0.9795268425841674
      },
      "curve": [
        [
          0.0,
          0.9795
        ],
        [
          0.1101,
          0.9795
        ],
        [
          0.1419,
          0.9791
        ],
        [
          0.1629,
          0.9786
        ],
        [
          0.1761,
          0.9791
        ],
        [
          0.1884,
          0.9782
        ],
        [
          0.2002,
          0.9768
        ],
        [
          0.2152,
          0.9732
        ],
        [
          0.2261,
          0.9704
        ],
        [
          0.2429,
          0.97
        ],
        [
          0.2611,
          0.9682
        ],
        [
          0.2734,
          0.9659
        ],
        [
          0.2839,
          0.9654
        ],
        [
          0.2957,
          0.9641
        ],
        [
          0.3062,
          0.9627
        ],
        [
          0.3198,
          0.9586
        ],
        [
          0.3344,
          0.955
        ],
        [
          0.3512,
          0.95
        ],
        [
          0.3658,
          0.9459
        ],
        [
          0.379,
          0.9422
        ],
        [
          0.3908,
          0.9386
        ],
        [
          0.4026,
          0.9345
        ],
        [
          0.4136,
          0.9299
        ],
        [
          0.4304,
          0.9245
        ],
        [
          0.4459,
          0.9186
        ],
        [
          0.4572,
          0.9149
        ],
        [
          0.4695,
          0.9104
        ],
        [
          0.4868,
          0.9026
        ],
        [
          0.4977,
          0.8976
        ],
        [
          0.5159,
          0.8894
        ],
        [
          0.5309,
          0.8822
        ],
        [
          0.546,
          0.8763
        ],
        [
          0.5596,
          0.8676
        ],
        [
          0.5732,
          0.859
        ],
        [
          0.5883,
          0.8517
        ],
        [
          0.601,
          0.8417
        ],
        [
          0.6151,
          0.8326
        ],
        [
          0.6278,
          0.8248
        ],
        [
          0.6497,
          0.8116
        ],
        [
          0.6624,
          0.8057
        ],
        [
          0.6802,
          0.7948
        ],
        [
          0.6965,
          0.7834
        ],
        [
          0.7102,
          0.773
        ],
        [
          0.7311,
          0.7593
        ],
        [
          0.748,
          0.748
        ],
        [
          0.763,
          0.7357
        ],
        [
          0.7825,
          0.7257
        ],
        [
          0.8025,
          0.7116
        ],
        [
          0.8212,
          0.7006
        ],
        [
          0.8412,
          0.6843
        ],
        [
          0.8603,
          0.6711
        ],
        [
          0.8735,
          0.6633
        ],
        [
          0.8972,
          0.6474
        ],
        [
          0.9081,
          0.6401
        ],
        [
          0.9217,
          0.6319
        ],
        [
          0.9354,
          0.6224
        ],
        [
          0.9468,
          0.6156
        ],
        [
          0.96,
          0.6056
        ],
        [
          0.9754,
          0.5937
        ],
        [
          0.9927,
          0.581
        ],
        [
          1.0,
          0.5751
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.15321201129213904,
        "difference": -0.0010853708939105374,
        "worse": 0.6766666666666666,
        "kept_right": 0.15199020220864382,
        "kept_wrong": 0.0012218090834952727,
        "handed_off": 0.8467879887078618
      },
      "usd_per_1000": 5.1650875738945174
    },
    {
      "decision": 49,
      "frontier": 276,
      "frontier_accuracy": 0.9777070063694268,
      "in_sample": {
        "threshold": 1.0,
        "answered": 0.11010009099181074,
        "accuracy": 0.9777070063694268
      },
      "curve": [
        [
          0.0,
          0.9777
        ],
        [
          0.1101,
          0.9777
        ],
        [
          0.1419,
          0.9773
        ],
        [
          0.1629,
          0.9768
        ],
        [
          0.1761,
          0.9773
        ],
        [
          0.1884,
          0.9763
        ],
        [
          0.2002,
          0.975
        ],
        [
          0.2152,
          0.9713
        ],
        [
          0.2261,
          0.9686
        ],
        [
          0.2429,
          0.9682
        ],
        [
          0.2611,
          0.9668
        ],
        [
          0.2734,
          0.9645
        ],
        [
          0.2839,
          0.9641
        ],
        [
          0.2957,
          0.9627
        ],
        [
          0.3062,
          0.9613
        ],
        [
          0.3198,
          0.9577
        ],
        [
          0.3344,
          0.954
        ],
        [
          0.3512,
          0.9486
        ],
        [
          0.3658,
          0.9445
        ],
        [
          0.379,
          0.9409
        ],
        [
          0.3908,
          0.9372
        ],
        [
          0.4026,
          0.9336
        ],
        [
          0.4136,
          0.929
        ],
        [
          0.4304,
          0.924
        ],
        [
          0.4459,
          0.9181
        ],
        [
          0.4572,
          0.9145
        ],
        [
          0.4695,
          0.9104
        ],
        [
          0.4868,
          0.9031
        ],
        [
          0.4977,
          0.8981
        ],
        [
          0.5159,
          0.8899
        ],
        [
          0.5309,
          0.8826
        ],
        [
          0.546,
          0.8767
        ],
        [
          0.5596,
          0.8681
        ],
        [
          0.5732,
          0.8594
        ],
        [
          0.5883,
          0.8517
        ],
        [
          0.601,
          0.8421
        ],
        [
          0.6151,
          0.833
        ],
        [
          0.6278,
          0.8253
        ],
        [
          0.6497,
          0.8121
        ],
        [
          0.6624,
          0.8062
        ],
        [
          0.6802,
          0.7953
        ],
        [
          0.6965,
          0.7839
        ],
        [
          0.7102,
          0.7739
        ],
        [
          0.7311,
          0.7602
        ],
        [
          0.748,
          0.7489
        ],
        [
          0.763,
          0.7366
        ],
        [
          0.7825,
          0.7266
        ],
        [
          0.8025,
          0.7125
        ],
        [
          0.8212,
          0.7015
        ],
        [
          0.8412,
          0.6852
        ],
        [
          0.8603,
          0.672
        ],
        [
          0.8735,
          0.6642
        ],
        [
          0.8972,
          0.6483
        ],
        [
          0.9081,
          0.641
        ],
        [
          0.9217,
          0.6324
        ],
        [
          0.9354,
          0.6228
        ],
        [
          0.9468,
          0.6156
        ],
        [
          0.96,
          0.6056
        ],
        [
          0.9754,
          0.5937
        ],
        [
          0.9927,
          0.581
        ],
        [
          1.0,
          0.5751
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.15321201129213904,
        "difference": -0.0010853708939105374,
        "worse": 0.6766666666666666,
        "kept_right": 0.15199020220864382,
        "kept_wrong": 0.0012218090834952727,
        "handed_off": 0.8467879887078618
      },
      "usd_per_1000": 1.4024995664681557
    },
    {
      "decision": 49,
      "frontier": 48,
      "frontier_accuracy": 0.9758871701546861,
      "in_sample": {
        "threshold": 1.0,
        "answered": 0.11010009099181074,
        "accuracy": 0.9758871701546861
      },
      "curve": [
        [
          0.0,
          0.9759
        ],
        [
          0.1101,
          0.9759
        ],
        [
          0.1419,
          0.9754
        ],
        [
          0.1629,
          0.975
        ],
        [
          0.1761,
          0.9754
        ],
        [
          0.1884,
          0.9745
        ],
        [
          0.2002,
          0.9732
        ],
        [
          0.2152,
          0.9695
        ],
        [
          0.2261,
          0.9668
        ],
        [
          0.2429,
          0.9663
        ],
        [
          0.2611,
          0.9645
        ],
        [
          0.2734,
          0.9622
        ],
        [
          0.2839,
          0.9618
        ],
        [
          0.2957,
          0.9604
        ],
        [
          0.3062,
          0.9591
        ],
        [
          0.3198,
          0.955
        ],
        [
          0.3344,
          0.9513
        ],
        [
          0.3512,
          0.9459
        ],
        [
          0.3658,
          0.9418
        ],
        [
          0.379,
          0.9381
        ],
        [
          0.3908,
          0.9345
        ],
        [
          0.4026,
          0.9299
        ],
        [
          0.4136,
          0.9254
        ],
        [
          0.4304,
          0.9199
        ],
        [
          0.4459,
          0.914
        ],
        [
          0.4572,
          0.9104
        ],
        [
          0.4695,
          0.9063
        ],
        [
          0.4868,
          0.8985
        ],
        [
          0.4977,
          0.8935
        ],
        [
          0.5159,
          0.8854
        ],
        [
          0.5309,
          0.8781
        ],
        [
          0.546,
          0.8722
        ],
        [
          0.5596,
          0.8635
        ],
        [
          0.5732,
          0.8553
        ],
        [
          0.5883,
          0.848
        ],
        [
          0.601,
          0.8385
        ],
        [
          0.6151,
          0.8294
        ],
        [
          0.6278,
          0.8217
        ],
        [
          0.6497,
          0.8085
        ],
        [
          0.6624,
          0.803
        ],
        [
          0.6802,
          0.7921
        ],
        [
          0.6965,
          0.7816
        ],
        [
          0.7102,
          0.7721
        ],
        [
          0.7311,
          0.7584
        ],
        [
          0.748,
          0.747
        ],
        [
          0.763,
          0.7352
        ],
        [
          0.7825,
          0.7252
        ],
        [
          0.8025,
          0.7116
        ],
        [
          0.8212,
          0.7006
        ],
        [
          0.8412,
          0.6843
        ],
        [
          0.8603,
          0.6711
        ],
        [
          0.8735,
          0.6633
        ],
        [
          0.8972,
          0.6479
        ],
        [
          0.9081,
          0.6406
        ],
        [
          0.9217,
          0.6319
        ],
        [
          0.9354,
          0.6219
        ],
        [
          0.9468,
          0.6151
        ],
        [
          0.96,
          0.6051
        ],
        [
          0.9754,
          0.5937
        ],
        [
          0.9927,
          0.581
        ],
        [
          1.0,
          0.5751
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.15321201129213904,
        "difference": -0.0010853708939105374,
        "worse": 0.6766666666666666,
        "kept_right": 0.15199020220864382,
        "kept_wrong": 0.0012218090834952727,
        "handed_off": 0.8467879887078618
      },
      "usd_per_1000": 4.5830657969334805
    },
    {
      "decision": 49,
      "frontier": 191,
      "frontier_accuracy": 0.9740673339399454,
      "in_sample": {
        "threshold": 1.0,
        "answered": 0.11010009099181074,
        "accuracy": 0.9740673339399454
      },
      "curve": [
        [
          0.0,
          0.9741
        ],
        [
          0.1101,
          0.9741
        ],
        [
          0.1419,
          0.9736
        ],
        [
          0.1629,
          0.9732
        ],
        [
          0.1761,
          0.9732
        ],
        [
          0.1884,
          0.9727
        ],
        [
          0.2002,
          0.9713
        ],
        [
          0.2152,
          0.9677
        ],
        [
          0.2261,
          0.9659
        ],
        [
          0.2429,
          0.965
        ],
        [
          0.2611,
          0.9631
        ],
        [
          0.2734,
          0.9609
        ],
        [
          0.2839,
          0.96
        ],
        [
          0.2957,
          0.9586
        ],
        [
          0.3062,
          0.9572
        ],
        [
          0.3198,
          0.9536
        ],
        [
          0.3344,
          0.95
        ],
        [
          0.3512,
          0.944
        ],
        [
          0.3658,
          0.9399
        ],
        [
          0.379,
          0.9363
        ],
        [
          0.3908,
          0.9327
        ],
        [
          0.4026,
          0.9286
        ],
        [
          0.4136,
          0.924
        ],
        [
          0.4304,
          0.9199
        ],
        [
          0.4459,
          0.914
        ],
        [
          0.4572,
          0.9104
        ],
        [
          0.4695,
          0.9058
        ],
        [
          0.4868,
          0.8981
        ],
        [
          0.4977,
          0.8931
        ],
        [
          0.5159,
          0.8844
        ],
        [
          0.5309,
          0.8772
        ],
        [
          0.546,
          0.8717
        ],
        [
          0.5596,
          0.8635
        ],
        [
          0.5732,
          0.8549
        ],
        [
          0.5883,
          0.8476
        ],
        [
          0.601,
          0.838
        ],
        [
          0.6151,
          0.8289
        ],
        [
          0.6278,
          0.8212
        ],
        [
          0.6497,
          0.808
        ],
        [
          0.6624,
          0.8025
        ],
        [
          0.6802,
          0.7921
        ],
        [
          0.6965,
          0.7807
        ],
        [
          0.7102,
          0.7707
        ],
        [
          0.7311,
          0.7571
        ],
        [
          0.748,
          0.7452
        ],
        [
          0.763,
          0.7329
        ],
        [
          0.7825,
          0.7229
        ],
        [
          0.8025,
          0.7093
        ],
        [
          0.8212,
          0.6984
        ],
        [
          0.8412,
          0.682
        ],
        [
          0.8603,
          0.6688
        ],
        [
          0.8735,
          0.6615
        ],
        [
          0.8972,
          0.6456
        ],
        [
          0.9081,
          0.6383
        ],
        [
          0.9217,
          0.6301
        ],
        [
          0.9354,
          0.6206
        ],
        [
          0.9468,
          0.6142
        ],
        [
          0.96,
          0.6042
        ],
        [
          0.9754,
          0.5933
        ],
        [
          0.9927,
          0.581
        ],
        [
          1.0,
          0.5751
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.13752589265612974,
        "difference": -0.000839772591215025,
        "worse": 0.5166666666666667,
        "kept_right": 0.13661942300244823,
        "kept_wrong": 0.0009064696536814844,
        "handed_off": 0.8624741073438703
      },
      "usd_per_1000": 23.089418978040406
    },
    {
      "decision": 49,
      "frontier": 188,
      "frontier_accuracy": 0.9713375796178344,
      "in_sample": {
        "threshold": 1.0,
        "answered": 0.11010009099181074,
        "accuracy": 0.9713375796178344
      },
      "curve": [
        [
          0.0,
          0.9713
        ],
        [
          0.1101,
          0.9713
        ],
        [
          0.1419,
          0.9709
        ],
        [
          0.1629,
          0.9704
        ],
        [
          0.1761,
          0.9704
        ],
        [
          0.1884,
          0.9695
        ],
        [
          0.2002,
          0.9686
        ],
        [
          0.2152,
          0.9654
        ],
        [
          0.2261,
          0.9631
        ],
        [
          0.2429,
          0.9622
        ],
        [
          0.2611,
          0.9613
        ],
        [
          0.2734,
          0.9591
        ],
        [
          0.2839,
          0.9577
        ],
        [
          0.2957,
          0.9563
        ],
        [
          0.3062,
          0.9554
        ],
        [
          0.3198,
          0.9518
        ],
        [
          0.3344,
          0.9481
        ],
        [
          0.3512,
          0.9427
        ],
        [
          0.3658,
          0.9386
        ],
        [
          0.379,
          0.9354
        ],
        [
          0.3908,
          0.9322
        ],
        [
          0.4026,
          0.9286
        ],
        [
          0.4136,
          0.9245
        ],
        [
          0.4304,
          0.919
        ],
        [
          0.4459,
          0.9131
        ],
        [
          0.4572,
          0.9095
        ],
        [
          0.4695,
          0.9049
        ],
        [
          0.4868,
          0.8967
        ],
        [
          0.4977,
          0.8917
        ],
        [
          0.5159,
          0.884
        ],
        [
          0.5309,
          0.8767
        ],
        [
          0.546,
          0.8712
        ],
        [
          0.5596,
          0.8621
        ],
        [
          0.5732,
          0.854
        ],
        [
          0.5883,
          0.8458
        ],
        [
          0.601,
          0.8362
        ],
        [
          0.6151,
          0.8271
        ],
        [
          0.6278,
          0.8194
        ],
        [
          0.6497,
          0.8062
        ],
        [
          0.6624,
          0.8007
        ],
        [
          0.6802,
          0.7907
        ],
        [
          0.6965,
          0.7793
        ],
        [
          0.7102,
          0.7698
        ],
        [
          0.7311,
          0.7571
        ],
        [
          0.748,
          0.7457
        ],
        [
          0.763,
          0.7338
        ],
        [
          0.7825,
          0.7247
        ],
        [
          0.8025,
          0.7106
        ],
        [
          0.8212,
          0.6997
        ],
        [
          0.8412,
          0.6843
        ],
        [
          0.8603,
          0.6711
        ],
        [
          0.8735,
          0.6629
        ],
        [
          0.8972,
          0.647
        ],
        [
          0.9081,
          0.6397
        ],
        [
          0.9217,
          0.631
        ],
        [
          0.9354,
          0.621
        ],
        [
          0.9468,
          0.6146
        ],
        [
          0.96,
          0.6046
        ],
        [
          0.9754,
          0.5933
        ],
        [
          0.9927,
          0.581
        ],
        [
          1.0,
          0.5751
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1378654494898357,
        "difference": -0.0009519738813806581,
        "worse": 0.52,
        "kept_right": 0.136886180782368,
        "kept_wrong": 0.0009792687074675925,
        "handed_off": 0.8621345505101646
      },
      "usd_per_1000": 0.8594631788859058
    },
    {
      "decision": 49,
      "frontier": 190,
      "frontier_accuracy": 0.9672429481346679,
      "in_sample": {
        "threshold": 0.97,
        "answered": 0.17606915377616014,
        "accuracy": 0.9672429481346679
      },
      "curve": [
        [
          0.0,
          0.9672
        ],
        [
          0.1101,
          0.9672
        ],
        [
          0.1419,
          0.9668
        ],
        [
          0.1629,
          0.9668
        ],
        [
          0.1761,
          0.9672
        ],
        [
          0.1884,
          0.9668
        ],
        [
          0.2002,
          0.9654
        ],
        [
          0.2152,
          0.9618
        ],
        [
          0.2261,
          0.9591
        ],
        [
          0.2429,
          0.9586
        ],
        [
          0.2611,
          0.9568
        ],
        [
          0.2734,
          0.9545
        ],
        [
          0.2839,
          0.9536
        ],
        [
          0.2957,
          0.9522
        ],
        [
          0.3062,
          0.9509
        ],
        [
          0.3198,
          0.9472
        ],
        [
          0.3344,
          0.944
        ],
        [
          0.3512,
          0.9395
        ],
        [
          0.3658,
          0.9354
        ],
        [
          0.379,
          0.9318
        ],
        [
          0.3908,
          0.9281
        ],
        [
          0.4026,
          0.9245
        ],
        [
          0.4136,
          0.9199
        ],
        [
          0.4304,
          0.9149
        ],
        [
          0.4459,
          0.909
        ],
        [
          0.4572,
          0.9054
        ],
        [
          0.4695,
          0.9013
        ],
        [
          0.4868,
          0.8935
        ],
        [
          0.4977,
          0.8885
        ],
        [
          0.5159,
          0.8803
        ],
        [
          0.5309,
          0.8731
        ],
        [
          0.546,
          0.8685
        ],
        [
          0.5596,
          0.8603
        ],
        [
          0.5732,
          0.8517
        ],
        [
          0.5883,
          0.8444
        ],
        [
          0.601,
          0.8344
        ],
        [
          0.6151,
          0.8253
        ],
        [
          0.6278,
          0.8176
        ],
        [
          0.6497,
          0.8044
        ],
        [
          0.6624,
          0.7985
        ],
        [
          0.6802,
          0.7875
        ],
        [
          0.6965,
          0.7766
        ],
        [
          0.7102,
          0.7671
        ],
        [
          0.7311,
          0.7534
        ],
        [
          0.748,
          0.7416
        ],
        [
          0.763,
          0.7298
        ],
        [
          0.7825,
          0.7197
        ],
        [
          0.8025,
          0.7066
        ],
        [
          0.8212,
          0.6956
        ],
        [
          0.8412,
          0.6793
        ],
        [
          0.8603,
          0.6665
        ],
        [
          0.8735,
          0.6592
        ],
        [
          0.8972,
          0.6433
        ],
        [
          0.9081,
          0.636
        ],
        [
          0.9217,
          0.6288
        ],
        [
          0.9354,
          0.6192
        ],
        [
          0.9468,
          0.6137
        ],
        [
          0.96,
          0.6042
        ],
        [
          0.9754,
          0.5933
        ],
        [
          0.9927,
          0.581
        ],
        [
          1.0,
          0.5751
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1691725392866281,
        "difference": -0.001082376531538575,
        "worse": 0.5866666666666667,
        "kept_right": 0.16730191085002497,
        "kept_wrong": 0.001870628436603163,
        "handed_off": 0.8308274607133729
      },
      "usd_per_1000": 4.655326927737563
    },
    {
      "decision": 49,
      "frontier": 124,
      "frontier_accuracy": 0.9667879890809827,
      "in_sample": {
        "threshold": 1.0,
        "answered": 0.11010009099181074,
        "accuracy": 0.9667879890809827
      },
      "curve": [
        [
          0.0,
          0.9668
        ],
        [
          0.1101,
          0.9668
        ],
        [
          0.1419,
          0.9663
        ],
        [
          0.1629,
          0.9659
        ],
        [
          0.1761,
          0.9663
        ],
        [
          0.1884,
          0.9654
        ],
        [
          0.2002,
          0.9641
        ],
        [
          0.2152,
          0.9604
        ],
        [
          0.2261,
          0.9581
        ],
        [
          0.2429,
          0.9577
        ],
        [
          0.2611,
          0.9559
        ],
        [
          0.2734,
          0.9536
        ],
        [
          0.2839,
          0.9531
        ],
        [
          0.2957,
          0.9518
        ],
        [
          0.3062,
          0.9504
        ],
        [
          0.3198,
          0.9463
        ],
        [
          0.3344,
          0.9427
        ],
        [
          0.3512,
          0.9381
        ],
        [
          0.3658,
          0.934
        ],
        [
          0.379,
          0.9304
        ],
        [
          0.3908,
          0.9268
        ],
        [
          0.4026,
          0.9227
        ],
        [
          0.4136,
          0.9181
        ],
        [
          0.4304,
          0.9136
        ],
        [
          0.4459,
          0.9076
        ],
        [
          0.4572,
          0.904
        ],
        [
          0.4695,
          0.8995
        ],
        [
          0.4868,
          0.8922
        ],
        [
          0.4977,
          0.8872
        ],
        [
          0.5159,
          0.8794
        ],
        [
          0.5309,
          0.8726
        ],
        [
          0.546,
          0.8667
        ],
        [
          0.5596,
          0.8581
        ],
        [
          0.5732,
          0.8499
        ],
        [
          0.5883,
          0.8426
        ],
        [
          0.601,
          0.833
        ],
        [
          0.6151,
          0.8239
        ],
        [
          0.6278,
          0.8167
        ],
        [
          0.6497,
          0.8039
        ],
        [
          0.6624,
          0.798
        ],
        [
          0.6802,
          0.7875
        ],
        [
          0.6965,
          0.7762
        ],
        [
          0.7102,
          0.7666
        ],
        [
          0.7311,
          0.7543
        ],
        [
          0.748,
          0.7434
        ],
        [
          0.763,
          0.7316
        ],
        [
          0.7825,
          0.7216
        ],
        [
          0.8025,
          0.7084
        ],
        [
          0.8212,
          0.6975
        ],
        [
          0.8412,
          0.6811
        ],
        [
          0.8603,
          0.6683
        ],
        [
          0.8735,
          0.6606
        ],
        [
          0.8972,
          0.6447
        ],
        [
          0.9081,
          0.6379
        ],
        [
          0.9217,
          0.6301
        ],
        [
          0.9354,
          0.6206
        ],
        [
          0.9468,
          0.6142
        ],
        [
          0.96,
          0.6042
        ],
        [
          0.9754,
          0.5933
        ],
        [
          0.9927,
          0.581
        ],
        [
          1.0,
          0.5751
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.15321201129213904,
        "difference": -0.0010853708939105374,
        "worse": 0.6766666666666666,
        "kept_right": 0.15199020220864382,
        "kept_wrong": 0.0012218090834952727,
        "handed_off": 0.8467879887078618
      },
      "usd_per_1000": 0.9860347480764181
    },
    {
      "decision": 49,
      "frontier": 278,
      "frontier_accuracy": 0.9499545040946314,
      "in_sample": {
        "threshold": 1.0,
        "answered": 0.11010009099181074,
        "accuracy": 0.9499545040946314
      },
      "curve": [
        [
          0.0,
          0.95
        ],
        [
          0.1101,
          0.95
        ],
        [
          0.1419,
          0.9495
        ],
        [
          0.1629,
          0.9495
        ],
        [
          0.1761,
          0.9495
        ],
        [
          0.1884,
          0.949
        ],
        [
          0.2002,
          0.9477
        ],
        [
          0.2152,
          0.9449
        ],
        [
          0.2261,
          0.9422
        ],
        [
          0.2429,
          0.9422
        ],
        [
          0.2611,
          0.9409
        ],
        [
          0.2734,
          0.939
        ],
        [
          0.2839,
          0.9377
        ],
        [
          0.2957,
          0.9368
        ],
        [
          0.3062,
          0.9354
        ],
        [
          0.3198,
          0.9322
        ],
        [
          0.3344,
          0.929
        ],
        [
          0.3512,
          0.9236
        ],
        [
          0.3658,
          0.9199
        ],
        [
          0.379,
          0.9172
        ],
        [
          0.3908,
          0.9136
        ],
        [
          0.4026,
          0.9099
        ],
        [
          0.4136,
          0.9058
        ],
        [
          0.4304,
          0.9013
        ],
        [
          0.4459,
          0.8958
        ],
        [
          0.4572,
          0.8922
        ],
        [
          0.4695,
          0.8881
        ],
        [
          0.4868,
          0.8808
        ],
        [
          0.4977,
          0.8758
        ],
        [
          0.5159,
          0.869
        ],
        [
          0.5309,
          0.8621
        ],
        [
          0.546,
          0.8562
        ],
        [
          0.5596,
          0.848
        ],
        [
          0.5732,
          0.8408
        ],
        [
          0.5883,
          0.8344
        ],
        [
          0.601,
          0.8248
        ],
        [
          0.6151,
          0.8157
        ],
        [
          0.6278,
          0.808
        ],
        [
          0.6497,
          0.7966
        ],
        [
          0.6624,
          0.7907
        ],
        [
          0.6802,
          0.7807
        ],
        [
          0.6965,
          0.7707
        ],
        [
          0.7102,
          0.7607
        ],
        [
          0.7311,
          0.747
        ],
        [
          0.748,
          0.7366
        ],
        [
          0.763,
          0.7252
        ],
        [
          0.7825,
          0.7152
        ],
        [
          0.8025,
          0.7015
        ],
        [
          0.8212,
          0.6929
        ],
        [
          0.8412,
          0.677
        ],
        [
          0.8603,
          0.6642
        ],
        [
          0.8735,
          0.6565
        ],
        [
          0.8972,
          0.6415
        ],
        [
          0.9081,
          0.6347
        ],
        [
          0.9217,
          0.6269
        ],
        [
          0.9354,
          0.6178
        ],
        [
          0.9468,
          0.6124
        ],
        [
          0.96,
          0.6024
        ],
        [
          0.9754,
          0.5919
        ],
        [
          0.9927,
          0.5801
        ],
        [
          1.0,
          0.5751
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.14946049900204264,
        "difference": -0.0008669270187978878,
        "worse": 0.52,
        "kept_right": 0.14819043095101836,
        "kept_wrong": 0.0012700680510243487,
        "handed_off": 0.8505395009979579
      },
      "usd_per_1000": 5.564862283090555
    },
    {
      "decision": 49,
      "frontier": 275,
      "frontier_accuracy": 0.9426751592356688,
      "in_sample": {
        "threshold": 1.0,
        "answered": 0.11010009099181074,
        "accuracy": 0.9426751592356688
      },
      "curve": [
        [
          0.0,
          0.9427
        ],
        [
          0.1101,
          0.9427
        ],
        [
          0.1419,
          0.9422
        ],
        [
          0.1629,
          0.9418
        ],
        [
          0.1761,
          0.9422
        ],
        [
          0.1884,
          0.9418
        ],
        [
          0.2002,
          0.9404
        ],
        [
          0.2152,
          0.9372
        ],
        [
          0.2261,
          0.9349
        ],
        [
          0.2429,
          0.9345
        ],
        [
          0.2611,
          0.9331
        ],
        [
          0.2734,
          0.9308
        ],
        [
          0.2839,
          0.9299
        ],
        [
          0.2957,
          0.929
        ],
        [
          0.3062,
          0.9277
        ],
        [
          0.3198,
          0.9236
        ],
        [
          0.3344,
          0.9199
        ],
        [
          0.3512,
          0.9145
        ],
        [
          0.3658,
          0.9104
        ],
        [
          0.379,
          0.9072
        ],
        [
          0.3908,
          0.9035
        ],
        [
          0.4026,
          0.8999
        ],
        [
          0.4136,
          0.8958
        ],
        [
          0.4304,
          0.8917
        ],
        [
          0.4459,
          0.8858
        ],
        [
          0.4572,
          0.8822
        ],
        [
          0.4695,
          0.8781
        ],
        [
          0.4868,
          0.8722
        ],
        [
          0.4977,
          0.8672
        ],
        [
          0.5159,
          0.8599
        ],
        [
          0.5309,
          0.8535
        ],
        [
          0.546,
          0.8494
        ],
        [
          0.5596,
          0.8408
        ],
        [
          0.5732,
          0.833
        ],
        [
          0.5883,
          0.8267
        ],
        [
          0.601,
          0.8171
        ],
        [
          0.6151,
          0.808
        ],
        [
          0.6278,
          0.8003
        ],
        [
          0.6497,
          0.788
        ],
        [
          0.6624,
          0.783
        ],
        [
          0.6802,
          0.773
        ],
        [
          0.6965,
          0.763
        ],
        [
          0.7102,
          0.7534
        ],
        [
          0.7311,
          0.7416
        ],
        [
          0.748,
          0.732
        ],
        [
          0.763,
          0.7216
        ],
        [
          0.7825,
          0.7134
        ],
        [
          0.8025,
          0.6997
        ],
        [
          0.8212,
          0.6897
        ],
        [
          0.8412,
          0.6742
        ],
        [
          0.8603,
          0.662
        ],
        [
          0.8735,
          0.6556
        ],
        [
          0.8972,
          0.6406
        ],
        [
          0.9081,
          0.6342
        ],
        [
          0.9217,
          0.6269
        ],
        [
          0.9354,
          0.6174
        ],
        [
          0.9468,
          0.6119
        ],
        [
          0.96,
          0.6028
        ],
        [
          0.9754,
          0.5924
        ],
        [
          0.9927,
          0.5805
        ],
        [
          1.0,
          0.5751
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.15538225380205312,
        "difference": -0.0011368695967637708,
        "worse": 0.6766666666666666,
        "kept_right": 0.15393615515902978,
        "kept_wrong": 0.0014460986430233432,
        "handed_off": 0.8446177461979474
      },
      "usd_per_1000": 0.4634153868167525
    },
    {
      "decision": 49,
      "frontier": 192,
      "frontier_accuracy": 0.9331210191082803,
      "in_sample": {
        "threshold": 0.97,
        "answered": 0.17606915377616014,
        "accuracy": 0.9331210191082803
      },
      "curve": [
        [
          0.0,
          0.9331
        ],
        [
          0.1101,
          0.9331
        ],
        [
          0.1419,
          0.9327
        ],
        [
          0.1629,
          0.9327
        ],
        [
          0.1761,
          0.9331
        ],
        [
          0.1884,
          0.9322
        ],
        [
          0.2002,
          0.9308
        ],
        [
          0.2152,
          0.9277
        ],
        [
          0.2261,
          0.9263
        ],
        [
          0.2429,
          0.9258
        ],
        [
          0.2611,
          0.9245
        ],
        [
          0.2734,
          0.9227
        ],
        [
          0.2839,
          0.9217
        ],
        [
          0.2957,
          0.9213
        ],
        [
          0.3062,
          0.9199
        ],
        [
          0.3198,
          0.9167
        ],
        [
          0.3344,
          0.914
        ],
        [
          0.3512,
          0.909
        ],
        [
          0.3658,
          0.9054
        ],
        [
          0.379,
          0.9017
        ],
        [
          0.3908,
          0.8981
        ],
        [
          0.4026,
          0.8944
        ],
        [
          0.4136,
          0.8904
        ],
        [
          0.4304,
          0.8858
        ],
        [
          0.4459,
          0.8808
        ],
        [
          0.4572,
          0.8772
        ],
        [
          0.4695,
          0.8735
        ],
        [
          0.4868,
          0.8667
        ],
        [
          0.4977,
          0.8617
        ],
        [
          0.5159,
          0.8544
        ],
        [
          0.5309,
          0.8476
        ],
        [
          0.546,
          0.8435
        ],
        [
          0.5596,
          0.8362
        ],
        [
          0.5732,
          0.8289
        ],
        [
          0.5883,
          0.8226
        ],
        [
          0.601,
          0.813
        ],
        [
          0.6151,
          0.8039
        ],
        [
          0.6278,
          0.7966
        ],
        [
          0.6497,
          0.7843
        ],
        [
          0.6624,
          0.7798
        ],
        [
          0.6802,
          0.7698
        ],
        [
          0.6965,
          0.7607
        ],
        [
          0.7102,
          0.7507
        ],
        [
          0.7311,
          0.7375
        ],
        [
          0.748,
          0.7275
        ],
        [
          0.763,
          0.7166
        ],
        [
          0.7825,
          0.7093
        ],
        [
          0.8025,
          0.6956
        ],
        [
          0.8212,
          0.6865
        ],
        [
          0.8412,
          0.6711
        ],
        [
          0.8603,
          0.6592
        ],
        [
          0.8735,
          0.6515
        ],
        [
          0.8972,
          0.6383
        ],
        [
          0.9081,
          0.6319
        ],
        [
          0.9217,
          0.6247
        ],
        [
          0.9354,
          0.6156
        ],
        [
          0.9468,
          0.611
        ],
        [
          0.96,
          0.6024
        ],
        [
          0.9754,
          0.5919
        ],
        [
          0.9927,
          0.581
        ],
        [
          1.0,
          0.5751
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.16712840411167365,
        "difference": -0.0008458002847201594,
        "worse": 0.38,
        "kept_right": 0.16562776813070584,
        "kept_wrong": 0.0015006359809678874,
        "handed_off": 0.8328715958883276
      },
      "usd_per_1000": 9.698125618402347
    },
    {
      "decision": 49,
      "frontier": 123,
      "frontier_accuracy": 0.9326660600545951,
      "in_sample": {
        "threshold": 1.0,
        "answered": 0.11010009099181074,
        "accuracy": 0.9326660600545951
      },
      "curve": [
        [
          0.0,
          0.9327
        ],
        [
          0.1101,
          0.9327
        ],
        [
          0.1419,
          0.9322
        ],
        [
          0.1629,
          0.9322
        ],
        [
          0.1761,
          0.9322
        ],
        [
          0.1884,
          0.9313
        ],
        [
          0.2002,
          0.9304
        ],
        [
          0.2152,
          0.9277
        ],
        [
          0.2261,
          0.9258
        ],
        [
          0.2429,
          0.9249
        ],
        [
          0.2611,
          0.9236
        ],
        [
          0.2734,
          0.9208
        ],
        [
          0.2839,
          0.9195
        ],
        [
          0.2957,
          0.9186
        ],
        [
          0.3062,
          0.9172
        ],
        [
          0.3198,
          0.9136
        ],
        [
          0.3344,
          0.9108
        ],
        [
          0.3512,
          0.9058
        ],
        [
          0.3658,
          0.9013
        ],
        [
          0.379,
          0.8981
        ],
        [
          0.3908,
          0.8949
        ],
        [
          0.4026,
          0.8913
        ],
        [
          0.4136,
          0.8881
        ],
        [
          0.4304,
          0.884
        ],
        [
          0.4459,
          0.8785
        ],
        [
          0.4572,
          0.8749
        ],
        [
          0.4695,
          0.8703
        ],
        [
          0.4868,
          0.8631
        ],
        [
          0.4977,
          0.8585
        ],
        [
          0.5159,
          0.8508
        ],
        [
          0.5309,
          0.8444
        ],
        [
          0.546,
          0.8417
        ],
        [
          0.5596,
          0.8335
        ],
        [
          0.5732,
          0.8267
        ],
        [
          0.5883,
          0.8194
        ],
        [
          0.601,
          0.8103
        ],
        [
          0.6151,
          0.8021
        ],
        [
          0.6278,
          0.7948
        ],
        [
          0.6497,
          0.7843
        ],
        [
          0.6624,
          0.7798
        ],
        [
          0.6802,
          0.7693
        ],
        [
          0.6965,
          0.7602
        ],
        [
          0.7102,
          0.7507
        ],
        [
          0.7311,
          0.7384
        ],
        [
          0.748,
          0.7284
        ],
        [
          0.763,
          0.7166
        ],
        [
          0.7825,
          0.7079
        ],
        [
          0.8025,
          0.6938
        ],
        [
          0.8212,
          0.6843
        ],
        [
          0.8412,
          0.6706
        ],
        [
          0.8603,
          0.6597
        ],
        [
          0.8735,
          0.6515
        ],
        [
          0.8972,
          0.6379
        ],
        [
          0.9081,
          0.6315
        ],
        [
          0.9217,
          0.6237
        ],
        [
          0.9354,
          0.6156
        ],
        [
          0.9468,
          0.6087
        ],
        [
          0.96,
          0.5996
        ],
        [
          0.9754,
          0.5901
        ],
        [
          0.9927,
          0.5796
        ],
        [
          1.0,
          0.5751
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.15538324926139335,
        "difference": -0.0012854703333944344,
        "worse": 0.6733333333333333,
        "kept_right": 0.1538856106207444,
        "kept_wrong": 0.0014976386406489763,
        "handed_off": 0.844616750738607
      },
      "usd_per_1000": 0.2556989073269443
    },
    {
      "decision": 49,
      "frontier": 274,
      "frontier_accuracy": 0.9085532302092811,
      "in_sample": {
        "threshold": 0.97,
        "answered": 0.17606915377616014,
        "accuracy": 0.9085532302092811
      },
      "curve": [
        [
          0.0,
          0.9086
        ],
        [
          0.1101,
          0.9086
        ],
        [
          0.1419,
          0.9081
        ],
        [
          0.1629,
          0.9076
        ],
        [
          0.1761,
          0.9086
        ],
        [
          0.1884,
          0.9081
        ],
        [
          0.2002,
          0.9067
        ],
        [
          0.2152,
          0.9035
        ],
        [
          0.2261,
          0.9022
        ],
        [
          0.2429,
          0.9013
        ],
        [
          0.2611,
          0.9004
        ],
        [
          0.2734,
          0.8985
        ],
        [
          0.2839,
          0.8976
        ],
        [
          0.2957,
          0.8967
        ],
        [
          0.3062,
          0.8963
        ],
        [
          0.3198,
          0.8935
        ],
        [
          0.3344,
          0.8899
        ],
        [
          0.3512,
          0.8863
        ],
        [
          0.3658,
          0.8822
        ],
        [
          0.379,
          0.879
        ],
        [
          0.3908,
          0.8753
        ],
        [
          0.4026,
          0.8726
        ],
        [
          0.4136,
          0.8703
        ],
        [
          0.4304,
          0.8667
        ],
        [
          0.4459,
          0.8612
        ],
        [
          0.4572,
          0.8576
        ],
        [
          0.4695,
          0.8549
        ],
        [
          0.4868,
          0.848
        ],
        [
          0.4977,
          0.843
        ],
        [
          0.5159,
          0.8362
        ],
        [
          0.5309,
          0.8321
        ],
        [
          0.546,
          0.8289
        ],
        [
          0.5596,
          0.8207
        ],
        [
          0.5732,
          0.8144
        ],
        [
          0.5883,
          0.8089
        ],
        [
          0.601,
          0.7989
        ],
        [
          0.6151,
          0.7912
        ],
        [
          0.6278,
          0.7843
        ],
        [
          0.6497,
          0.7721
        ],
        [
          0.6624,
          0.7671
        ],
        [
          0.6802,
          0.7575
        ],
        [
          0.6965,
          0.7475
        ],
        [
          0.7102,
          0.7389
        ],
        [
          0.7311,
          0.7284
        ],
        [
          0.748,
          0.7193
        ],
        [
          0.763,
          0.7088
        ],
        [
          0.7825,
          0.7015
        ],
        [
          0.8025,
          0.6888
        ],
        [
          0.8212,
          0.6806
        ],
        [
          0.8412,
          0.6647
        ],
        [
          0.8603,
          0.6533
        ],
        [
          0.8735,
          0.6474
        ],
        [
          0.8972,
          0.6347
        ],
        [
          0.9081,
          0.6283
        ],
        [
          0.9217,
          0.6219
        ],
        [
          0.9354,
          0.6133
        ],
        [
          0.9468,
          0.6078
        ],
        [
          0.96,
          0.5992
        ],
        [
          0.9754,
          0.5914
        ],
        [
          0.9927,
          0.5801
        ],
        [
          1.0,
          0.5751
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.16802208144384867,
        "difference": -0.0011823022686761142,
        "worse": 0.5766666666666667,
        "kept_right": 0.1660363702003133,
        "kept_wrong": 0.001985711243535407,
        "handed_off": 0.8319779185561516
      },
      "usd_per_1000": 0.38347540455332085
    },
    {
      "decision": 49,
      "frontier": 47,
      "frontier_accuracy": 0.9044585987261147,
      "in_sample": {
        "threshold": 0.97,
        "answered": 0.17606915377616014,
        "accuracy": 0.9049135577797999
      },
      "curve": [
        [
          0.0,
          0.9045
        ],
        [
          0.1101,
          0.9045
        ],
        [
          0.1419,
          0.9045
        ],
        [
          0.1629,
          0.9045
        ],
        [
          0.1761,
          0.9049
        ],
        [
          0.1884,
          0.904
        ],
        [
          0.2002,
          0.9026
        ],
        [
          0.2152,
          0.8999
        ],
        [
          0.2261,
          0.899
        ],
        [
          0.2429,
          0.8985
        ],
        [
          0.2611,
          0.8972
        ],
        [
          0.2734,
          0.8944
        ],
        [
          0.2839,
          0.8935
        ],
        [
          0.2957,
          0.8922
        ],
        [
          0.3062,
          0.8913
        ],
        [
          0.3198,
          0.8881
        ],
        [
          0.3344,
          0.8858
        ],
        [
          0.3512,
          0.8813
        ],
        [
          0.3658,
          0.8785
        ],
        [
          0.379,
          0.8758
        ],
        [
          0.3908,
          0.8726
        ],
        [
          0.4026,
          0.869
        ],
        [
          0.4136,
          0.8658
        ],
        [
          0.4304,
          0.8621
        ],
        [
          0.4459,
          0.8576
        ],
        [
          0.4572,
          0.854
        ],
        [
          0.4695,
          0.8503
        ],
        [
          0.4868,
          0.8449
        ],
        [
          0.4977,
          0.8412
        ],
        [
          0.5159,
          0.8348
        ],
        [
          0.5309,
          0.828
        ],
        [
          0.546,
          0.8235
        ],
        [
          0.5596,
          0.8171
        ],
        [
          0.5732,
          0.8094
        ],
        [
          0.5883,
          0.8025
        ],
        [
          0.601,
          0.7939
        ],
        [
          0.6151,
          0.7853
        ],
        [
          0.6278,
          0.7793
        ],
        [
          0.6497,
          0.7684
        ],
        [
          0.6624,
          0.7639
        ],
        [
          0.6802,
          0.7539
        ],
        [
          0.6965,
          0.7461
        ],
        [
          0.7102,
          0.7366
        ],
        [
          0.7311,
          0.7257
        ],
        [
          0.748,
          0.7175
        ],
        [
          0.763,
          0.7056
        ],
        [
          0.7825,
          0.6965
        ],
        [
          0.8025,
          0.6843
        ],
        [
          0.8212,
          0.6793
        ],
        [
          0.8412,
          0.6652
        ],
        [
          0.8603,
          0.6524
        ],
        [
          0.8735,
          0.647
        ],
        [
          0.8972,
          0.6347
        ],
        [
          0.9081,
          0.6288
        ],
        [
          0.9217,
          0.6215
        ],
        [
          0.9354,
          0.6119
        ],
        [
          0.9468,
          0.6069
        ],
        [
          0.96,
          0.5978
        ],
        [
          0.9754,
          0.5892
        ],
        [
          0.9927,
          0.5792
        ],
        [
          1.0,
          0.5751
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.17750560672775909,
        "difference": -0.0009095305885676967,
        "worse": 0.52,
        "kept_right": 0.17540768036177964,
        "kept_wrong": 0.0020979263659795006,
        "handed_off": 0.8224943932722417
      },
      "usd_per_1000": 4.0842277308861235
    },
    {
      "decision": 49,
      "frontier": 193,
      "frontier_accuracy": 0.89171974522293,
      "in_sample": {
        "threshold": 1.0,
        "answered": 0.11010009099181074,
        "accuracy": 0.89171974522293
      },
      "curve": [
        [
          0.0,
          0.8917
        ],
        [
          0.1101,
          0.8917
        ],
        [
          0.1419,
          0.8913
        ],
        [
          0.1629,
          0.8908
        ],
        [
          0.1761,
          0.8913
        ],
        [
          0.1884,
          0.8908
        ],
        [
          0.2002,
          0.8894
        ],
        [
          0.2152,
          0.8863
        ],
        [
          0.2261,
          0.8849
        ],
        [
          0.2429,
          0.8858
        ],
        [
          0.2611,
          0.8854
        ],
        [
          0.2734,
          0.8831
        ],
        [
          0.2839,
          0.8826
        ],
        [
          0.2957,
          0.8822
        ],
        [
          0.3062,
          0.8817
        ],
        [
          0.3198,
          0.8794
        ],
        [
          0.3344,
          0.8776
        ],
        [
          0.3512,
          0.8731
        ],
        [
          0.3658,
          0.8699
        ],
        [
          0.379,
          0.8667
        ],
        [
          0.3908,
          0.864
        ],
        [
          0.4026,
          0.8603
        ],
        [
          0.4136,
          0.8571
        ],
        [
          0.4304,
          0.8526
        ],
        [
          0.4459,
          0.8476
        ],
        [
          0.4572,
          0.8444
        ],
        [
          0.4695,
          0.8408
        ],
        [
          0.4868,
          0.8339
        ],
        [
          0.4977,
          0.8289
        ],
        [
          0.5159,
          0.823
        ],
        [
          0.5309,
          0.818
        ],
        [
          0.546,
          0.8139
        ],
        [
          0.5596,
          0.8057
        ],
        [
          0.5732,
          0.7989
        ],
        [
          0.5883,
          0.7934
        ],
        [
          0.601,
          0.7853
        ],
        [
          0.6151,
          0.7766
        ],
        [
          0.6278,
          0.7693
        ],
        [
          0.6497,
          0.7593
        ],
        [
          0.6624,
          0.7557
        ],
        [
          0.6802,
          0.747
        ],
        [
          0.6965,
          0.7407
        ],
        [
          0.7102,
          0.7311
        ],
        [
          0.7311,
          0.7202
        ],
        [
          0.748,
          0.712
        ],
        [
          0.763,
          0.7015
        ],
        [
          0.7825,
          0.697
        ],
        [
          0.8025,
          0.6843
        ],
        [
          0.8212,
          0.6756
        ],
        [
          0.8412,
          0.6611
        ],
        [
          0.8603,
          0.6497
        ],
        [
          0.8735,
          0.6433
        ],
        [
          0.8972,
          0.631
        ],
        [
          0.9081,
          0.6251
        ],
        [
          0.9217,
          0.6187
        ],
        [
          0.9354,
          0.6124
        ],
        [
          0.9468,
          0.6078
        ],
        [
          0.96,
          0.5987
        ],
        [
          0.9754,
          0.5901
        ],
        [
          0.9927,
          0.5801
        ],
        [
          1.0,
          0.5751
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1551649630051554,
        "difference": -0.0011369827077115802,
        "worse": 0.6766666666666666,
        "kept_right": 0.15372782280374842,
        "kept_wrong": 0.0014371402014068959,
        "handed_off": 0.8448350369948453
      },
      "usd_per_1000": 6.574283889253264
    },
    {
      "decision": 49,
      "frontier": 55,
      "frontier_accuracy": 0.8503184713375797,
      "in_sample": {
        "threshold": 0.8099999999999999,
        "answered": 0.30846223839854414,
        "accuracy": 0.8507734303912647
      },
      "curve": [
        [
          0.0,
          0.8503
        ],
        [
          0.1101,
          0.8517
        ],
        [
          0.1419,
          0.8521
        ],
        [
          0.1629,
          0.853
        ],
        [
          0.1761,
          0.8535
        ],
        [
          0.1884,
          0.8544
        ],
        [
          0.2002,
          0.8535
        ],
        [
          0.2152,
          0.8521
        ],
        [
          0.2261,
          0.8517
        ],
        [
          0.2429,
          0.8517
        ],
        [
          0.2611,
          0.8526
        ],
        [
          0.2734,
          0.8517
        ],
        [
          0.2839,
          0.8512
        ],
        [
          0.2957,
          0.8503
        ],
        [
          0.3062,
          0.8508
        ],
        [
          0.3198,
          0.8485
        ],
        [
          0.3344,
          0.8467
        ],
        [
          0.3512,
          0.8426
        ],
        [
          0.3658,
          0.8399
        ],
        [
          0.379,
          0.8371
        ],
        [
          0.3908,
          0.8344
        ],
        [
          0.4026,
          0.8312
        ],
        [
          0.4136,
          0.8298
        ],
        [
          0.4304,
          0.8253
        ],
        [
          0.4459,
          0.8212
        ],
        [
          0.4572,
          0.8198
        ],
        [
          0.4695,
          0.8157
        ],
        [
          0.4868,
          0.8116
        ],
        [
          0.4977,
          0.8071
        ],
        [
          0.5159,
          0.803
        ],
        [
          0.5309,
          0.7971
        ],
        [
          0.546,
          0.7948
        ],
        [
          0.5596,
          0.7875
        ],
        [
          0.5732,
          0.7825
        ],
        [
          0.5883,
          0.7771
        ],
        [
          0.601,
          0.7689
        ],
        [
          0.6151,
          0.7607
        ],
        [
          0.6278,
          0.7543
        ],
        [
          0.6497,
          0.7452
        ],
        [
          0.6624,
          0.742
        ],
        [
          0.6802,
          0.7352
        ],
        [
          0.6965,
          0.7288
        ],
        [
          0.7102,
          0.7216
        ],
        [
          0.7311,
          0.7111
        ],
        [
          0.748,
          0.7038
        ],
        [
          0.763,
          0.6943
        ],
        [
          0.7825,
          0.6879
        ],
        [
          0.8025,
          0.677
        ],
        [
          0.8212,
          0.6697
        ],
        [
          0.8412,
          0.6574
        ],
        [
          0.8603,
          0.6465
        ],
        [
          0.8735,
          0.641
        ],
        [
          0.8972,
          0.6288
        ],
        [
          0.9081,
          0.6228
        ],
        [
          0.9217,
          0.6178
        ],
        [
          0.9354,
          0.6096
        ],
        [
          0.9468,
          0.6051
        ],
        [
          0.96,
          0.5974
        ],
        [
          0.9754,
          0.5878
        ],
        [
          0.9927,
          0.5792
        ],
        [
          1.0,
          0.5751
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.2937219147285595,
        "difference": -0.0011431171588492603,
        "worse": 0.4666666666666667,
        "kept_right": 0.2720090736365594,
        "kept_wrong": 0.02171284109200011,
        "handed_off": 0.7062780852714414
      },
      "usd_per_1000": 2.6098241646824185
    },
    {
      "decision": 125,
      "frontier": 140,
      "frontier_accuracy": 0.9808917197452229,
      "in_sample": {
        "threshold": 0.9705,
        "answered": 0.062329390354868064,
        "accuracy": 0.9808917197452229
      },
      "curve": [
        [
          0.0,
          0.9809
        ],
        [
          0.01,
          0.9809
        ],
        [
          0.02,
          0.9809
        ],
        [
          0.03,
          0.9809
        ],
        [
          0.04,
          0.9809
        ],
        [
          0.05,
          0.9809
        ],
        [
          0.0601,
          0.9809
        ],
        [
          0.071,
          0.9804
        ],
        [
          0.0823,
          0.9804
        ],
        [
          0.0924,
          0.9804
        ],
        [
          0.1033,
          0.98
        ],
        [
          0.1137,
          0.98
        ],
        [
          0.1242,
          0.98
        ],
        [
          0.1342,
          0.9795
        ],
        [
          0.1442,
          0.9791
        ],
        [
          0.1551,
          0.9791
        ],
        [
          0.1656,
          0.9786
        ],
        [
          0.1756,
          0.9777
        ],
        [
          0.1856,
          0.9773
        ],
        [
          0.1956,
          0.9768
        ],
        [
          0.2056,
          0.9763
        ],
        [
          0.2157,
          0.9759
        ],
        [
          0.2266,
          0.9741
        ],
        [
          0.237,
          0.9736
        ],
        [
          0.247,
          0.9732
        ],
        [
          0.2571,
          0.9691
        ],
        [
          0.2675,
          0.9668
        ],
        [
          0.2775,
          0.9627
        ],
        [
          0.2875,
          0.9609
        ],
        [
          0.2975,
          0.9586
        ],
        [
          0.3076,
          0.9563
        ],
        [
          0.318,
          0.9536
        ],
        [
          0.328,
          0.949
        ],
        [
          0.338,
          0.9472
        ],
        [
          0.348,
          0.9436
        ],
        [
          0.3581,
          0.9404
        ],
        [
          0.3685,
          0.9363
        ],
        [
          0.3785,
          0.9318
        ],
        [
          0.3885,
          0.9281
        ],
        [
          0.3985,
          0.924
        ],
        [
          0.4086,
          0.9199
        ],
        [
          0.419,
          0.9145
        ],
        [
          0.429,
          0.9117
        ],
        [
          0.439,
          0.9054
        ],
        [
          0.449,
          0.9026
        ],
        [
          0.4591,
          0.8981
        ],
        [
          0.4695,
          0.8931
        ],
        [
          0.4795,
          0.8881
        ],
        [
          0.49,
          0.8849
        ],
        [
          0.5005,
          0.8781
        ],
        [
          0.5114,
          0.8712
        ],
        [
          0.5218,
          0.8649
        ],
        [
          0.5318,
          0.8617
        ],
        [
          0.5428,
          0.8558
        ],
        [
          0.5532,
          0.8503
        ],
        [
          0.5637,
          0.8458
        ],
        [
          0.5737,
          0.8426
        ],
        [
          0.5842,
          0.838
        ],
        [
          0.5946,
          0.833
        ],
        [
          0.6046,
          0.8262
        ],
        [
          0.6146,
          0.8217
        ],
        [
          0.6247,
          0.818
        ],
        [
          0.6351,
          0.8112
        ],
        [
          0.6451,
          0.8053
        ],
        [
          0.6551,
          0.798
        ],
        [
          0.6652,
          0.7921
        ],
        [
          0.6756,
          0.7862
        ],
        [
          0.6856,
          0.7803
        ],
        [
          0.6956,
          0.7743
        ],
        [
          0.7056,
          0.7693
        ],
        [
          0.7157,
          0.763
        ],
        [
          0.7261,
          0.7571
        ],
        [
          0.7361,
          0.7489
        ],
        [
          0.7461,
          0.7434
        ],
        [
          0.7561,
          0.7375
        ],
        [
          0.7662,
          0.7329
        ],
        [
          0.7771,
          0.7257
        ],
        [
          0.7875,
          0.7188
        ],
        [
          0.7975,
          0.7116
        ],
        [
          0.8076,
          0.7043
        ],
        [
          0.818,
          0.6984
        ],
        [
          0.828,
          0.6906
        ],
        [
          0.838,
          0.6852
        ],
        [
          0.848,
          0.6783
        ],
        [
          0.8585,
          0.6711
        ],
        [
          0.869,
          0.662
        ],
        [
          0.8794,
          0.6561
        ],
        [
          0.8894,
          0.6497
        ],
        [
          0.8995,
          0.6424
        ],
        [
          0.9099,
          0.6356
        ],
        [
          0.9199,
          0.6288
        ],
        [
          0.9299,
          0.621
        ],
        [
          0.9399,
          0.6142
        ],
        [
          0.95,
          0.6078
        ],
        [
          0.9604,
          0.5996
        ],
        [
          0.9704,
          0.5937
        ],
        [
          0.9804,
          0.5851
        ],
        [
          0.9904,
          0.5778
        ],
        [
          1.0,
          0.5728
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.09468632563085014,
        "difference": -0.0010671587299893497,
        "worse": 0.5266666666666666,
        "kept_right": 0.09357670401066172,
        "kept_wrong": 0.001109621620188422,
        "handed_off": 0.9053136743691497
      },
      "usd_per_1000": 2.299732522555417
    },
    {
      "decision": 125,
      "frontier": 277,
      "frontier_accuracy": 0.9795268425841674,
      "in_sample": {
        "threshold": 0.9705,
        "answered": 0.062329390354868064,
        "accuracy": 0.9795268425841674
      },
      "curve": [
        [
          0.0,
          0.9795
        ],
        [
          0.01,
          0.9795
        ],
        [
          0.02,
          0.9795
        ],
        [
          0.03,
          0.9795
        ],
        [
          0.04,
          0.9795
        ],
        [
          0.05,
          0.9795
        ],
        [
          0.0601,
          0.9795
        ],
        [
          0.071,
          0.9791
        ],
        [
          0.0823,
          0.9791
        ],
        [
          0.0924,
          0.9791
        ],
        [
          0.1033,
          0.9791
        ],
        [
          0.1137,
          0.9791
        ],
        [
          0.1242,
          0.9791
        ],
        [
          0.1342,
          0.9786
        ],
        [
          0.1442,
          0.9782
        ],
        [
          0.1551,
          0.9782
        ],
        [
          0.1656,
          0.9777
        ],
        [
          0.1756,
          0.9768
        ],
        [
          0.1856,
          0.9763
        ],
        [
          0.1956,
          0.9759
        ],
        [
          0.2056,
          0.9754
        ],
        [
          0.2157,
          0.975
        ],
        [
          0.2266,
          0.9732
        ],
        [
          0.237,
          0.9732
        ],
        [
          0.247,
          0.9727
        ],
        [
          0.2571,
          0.9686
        ],
        [
          0.2675,
          0.9663
        ],
        [
          0.2775,
          0.9622
        ],
        [
          0.2875,
          0.96
        ],
        [
          0.2975,
          0.9577
        ],
        [
          0.3076,
          0.9554
        ],
        [
          0.318,
          0.9531
        ],
        [
          0.328,
          0.949
        ],
        [
          0.338,
          0.9472
        ],
        [
          0.348,
          0.9436
        ],
        [
          0.3581,
          0.9404
        ],
        [
          0.3685,
          0.9363
        ],
        [
          0.3785,
          0.9318
        ],
        [
          0.3885,
          0.9281
        ],
        [
          0.3985,
          0.924
        ],
        [
          0.4086,
          0.9199
        ],
        [
          0.419,
          0.9149
        ],
        [
          0.429,
          0.9122
        ],
        [
          0.439,
          0.9058
        ],
        [
          0.449,
          0.9026
        ],
        [
          0.4591,
          0.8981
        ],
        [
          0.4695,
          0.8931
        ],
        [
          0.4795,
          0.8881
        ],
        [
          0.49,
          0.8844
        ],
        [
          0.5005,
          0.8776
        ],
        [
          0.5114,
          0.8708
        ],
        [
          0.5218,
          0.8644
        ],
        [
          0.5318,
          0.8612
        ],
        [
          0.5428,
          0.8553
        ],
        [
          0.5532,
          0.8494
        ],
        [
          0.5637,
          0.8449
        ],
        [
          0.5737,
          0.8421
        ],
        [
          0.5842,
          0.8376
        ],
        [
          0.5946,
          0.8326
        ],
        [
          0.6046,
          0.8253
        ],
        [
          0.6146,
          0.8207
        ],
        [
          0.6247,
          0.8171
        ],
        [
          0.6351,
          0.8103
        ],
        [
          0.6451,
          0.8044
        ],
        [
          0.6551,
          0.7971
        ],
        [
          0.6652,
          0.7907
        ],
        [
          0.6756,
          0.7848
        ],
        [
          0.6856,
          0.7784
        ],
        [
          0.6956,
          0.7721
        ],
        [
          0.7056,
          0.7671
        ],
        [
          0.7157,
          0.7607
        ],
        [
          0.7261,
          0.7543
        ],
        [
          0.7361,
          0.7466
        ],
        [
          0.7461,
          0.7407
        ],
        [
          0.7561,
          0.7348
        ],
        [
          0.7662,
          0.7302
        ],
        [
          0.7771,
          0.7229
        ],
        [
          0.7875,
          0.7166
        ],
        [
          0.7975,
          0.7097
        ],
        [
          0.8076,
          0.7029
        ],
        [
          0.818,
          0.697
        ],
        [
          0.828,
          0.6893
        ],
        [
          0.838,
          0.6838
        ],
        [
          0.848,
          0.677
        ],
        [
          0.8585,
          0.6697
        ],
        [
          0.869,
          0.6606
        ],
        [
          0.8794,
          0.6547
        ],
        [
          0.8894,
          0.6483
        ],
        [
          0.8995,
          0.641
        ],
        [
          0.9099,
          0.6342
        ],
        [
          0.9199,
          0.6274
        ],
        [
          0.9299,
          0.6197
        ],
        [
          0.9399,
          0.6128
        ],
        [
          0.95,
          0.6065
        ],
        [
          0.9604,
          0.5983
        ],
        [
          0.9704,
          0.5928
        ],
        [
          0.9804,
          0.5851
        ],
        [
          0.9904,
          0.5778
        ],
        [
          1.0,
          0.5728
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.11173405941522305,
        "difference": -0.0013855064192252276,
        "worse": 0.6233333333333333,
        "kept_right": 0.11011811460580662,
        "kept_wrong": 0.0016159448094163562,
        "handed_off": 0.8882659405847768
      },
      "usd_per_1000": 5.4975113120158765
    },
    {
      "decision": 125,
      "frontier": 276,
      "frontier_accuracy": 0.9777070063694268,
      "in_sample": {
        "threshold": 0.9705,
        "answered": 0.062329390354868064,
        "accuracy": 0.9777070063694268
      },
      "curve": [
        [
          0.0,
          0.9777
        ],
        [
          0.01,
          0.9777
        ],
        [
          0.02,
          0.9777
        ],
        [
          0.03,
          0.9777
        ],
        [
          0.04,
          0.9777
        ],
        [
          0.05,
          0.9777
        ],
        [
          0.0601,
          0.9777
        ],
        [
          0.071,
          0.9773
        ],
        [
          0.0823,
          0.9773
        ],
        [
          0.0924,
          0.9773
        ],
        [
          0.1033,
          0.9773
        ],
        [
          0.1137,
          0.9773
        ],
        [
          0.1242,
          0.9773
        ],
        [
          0.1342,
          0.9768
        ],
        [
          0.1442,
          0.9763
        ],
        [
          0.1551,
          0.9763
        ],
        [
          0.1656,
          0.9759
        ],
        [
          0.1756,
          0.975
        ],
        [
          0.1856,
          0.9745
        ],
        [
          0.1956,
          0.9741
        ],
        [
          0.2056,
          0.9741
        ],
        [
          0.2157,
          0.9736
        ],
        [
          0.2266,
          0.9718
        ],
        [
          0.237,
          0.9722
        ],
        [
          0.247,
          0.9718
        ],
        [
          0.2571,
          0.9677
        ],
        [
          0.2675,
          0.9654
        ],
        [
          0.2775,
          0.9613
        ],
        [
          0.2875,
          0.9591
        ],
        [
          0.2975,
          0.9568
        ],
        [
          0.3076,
          0.9545
        ],
        [
          0.318,
          0.9522
        ],
        [
          0.328,
          0.9486
        ],
        [
          0.338,
          0.9468
        ],
        [
          0.348,
          0.944
        ],
        [
          0.3581,
          0.9409
        ],
        [
          0.3685,
          0.9372
        ],
        [
          0.3785,
          0.9327
        ],
        [
          0.3885,
          0.929
        ],
        [
          0.3985,
          0.9249
        ],
        [
          0.4086,
          0.9208
        ],
        [
          0.419,
          0.9158
        ],
        [
          0.429,
          0.9131
        ],
        [
          0.439,
          0.9067
        ],
        [
          0.449,
          0.9035
        ],
        [
          0.4591,
          0.899
        ],
        [
          0.4695,
          0.894
        ],
        [
          0.4795,
          0.889
        ],
        [
          0.49,
          0.8854
        ],
        [
          0.5005,
          0.8785
        ],
        [
          0.5114,
          0.8722
        ],
        [
          0.5218,
          0.8658
        ],
        [
          0.5318,
          0.8626
        ],
        [
          0.5428,
          0.8567
        ],
        [
          0.5532,
          0.8508
        ],
        [
          0.5637,
          0.8467
        ],
        [
          0.5737,
          0.8439
        ],
        [
          0.5842,
          0.8394
        ],
        [
          0.5946,
          0.8344
        ],
        [
          0.6046,
          0.8271
        ],
        [
          0.6146,
          0.8226
        ],
        [
          0.6247,
          0.8189
        ],
        [
          0.6351,
          0.8121
        ],
        [
          0.6451,
          0.8062
        ],
        [
          0.6551,
          0.7989
        ],
        [
          0.6652,
          0.7925
        ],
        [
          0.6756,
          0.7866
        ],
        [
          0.6856,
          0.7803
        ],
        [
          0.6956,
          0.7739
        ],
        [
          0.7056,
          0.7689
        ],
        [
          0.7157,
          0.7625
        ],
        [
          0.7261,
          0.7561
        ],
        [
          0.7361,
          0.7484
        ],
        [
          0.7461,
          0.7425
        ],
        [
          0.7561,
          0.7366
        ],
        [
          0.7662,
          0.732
        ],
        [
          0.7771,
          0.7247
        ],
        [
          0.7875,
          0.7179
        ],
        [
          0.7975,
          0.7106
        ],
        [
          0.8076,
          0.7034
        ],
        [
          0.818,
          0.6975
        ],
        [
          0.828,
          0.6897
        ],
        [
          0.838,
          0.6843
        ],
        [
          0.848,
          0.6774
        ],
        [
          0.8585,
          0.6702
        ],
        [
          0.869,
          0.6611
        ],
        [
          0.8794,
          0.6551
        ],
        [
          0.8894,
          0.6488
        ],
        [
          0.8995,
          0.6415
        ],
        [
          0.9099,
          0.6347
        ],
        [
          0.9199,
          0.6278
        ],
        [
          0.9299,
          0.6201
        ],
        [
          0.9399,
          0.6133
        ],
        [
          0.95,
          0.6069
        ],
        [
          0.9604,
          0.5987
        ],
        [
          0.9704,
          0.5933
        ],
        [
          0.9804,
          0.5851
        ],
        [
          0.9904,
          0.5778
        ],
        [
          1.0,
          0.5728
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.11275868945071582,
        "difference": -0.0014643109674032594,
        "worse": 0.6233333333333333,
        "kept_right": 0.11102757362421636,
        "kept_wrong": 0.0017311158264993758,
        "handed_off": 0.887241310549284
      },
      "usd_per_1000": 1.548959485268632
    },
    {
      "decision": 125,
      "frontier": 48,
      "frontier_accuracy": 0.9758871701546861,
      "in_sample": {
        "threshold": 0.9705,
        "answered": 0.062329390354868064,
        "accuracy": 0.9758871701546861
      },
      "curve": [
        [
          0.0,
          0.9759
        ],
        [
          0.01,
          0.9759
        ],
        [
          0.02,
          0.9759
        ],
        [
          0.03,
          0.9759
        ],
        [
          0.04,
          0.9759
        ],
        [
          0.05,
          0.9759
        ],
        [
          0.0601,
          0.9759
        ],
        [
          0.071,
          0.9754
        ],
        [
          0.0823,
          0.9754
        ],
        [
          0.0924,
          0.9754
        ],
        [
          0.1033,
          0.9754
        ],
        [
          0.1137,
          0.9754
        ],
        [
          0.1242,
          0.9754
        ],
        [
          0.1342,
          0.975
        ],
        [
          0.1442,
          0.9745
        ],
        [
          0.1551,
          0.9745
        ],
        [
          0.1656,
          0.9741
        ],
        [
          0.1756,
          0.9732
        ],
        [
          0.1856,
          0.9727
        ],
        [
          0.1956,
          0.9722
        ],
        [
          0.2056,
          0.9718
        ],
        [
          0.2157,
          0.9713
        ],
        [
          0.2266,
          0.9695
        ],
        [
          0.237,
          0.97
        ],
        [
          0.247,
          0.9695
        ],
        [
          0.2571,
          0.9654
        ],
        [
          0.2675,
          0.9631
        ],
        [
          0.2775,
          0.9591
        ],
        [
          0.2875,
          0.9568
        ],
        [
          0.2975,
          0.955
        ],
        [
          0.3076,
          0.9527
        ],
        [
          0.318,
          0.9504
        ],
        [
          0.328,
          0.9459
        ],
        [
          0.338,
          0.944
        ],
        [
          0.348,
          0.9404
        ],
        [
          0.3581,
          0.9372
        ],
        [
          0.3685,
          0.9331
        ],
        [
          0.3785,
          0.9286
        ],
        [
          0.3885,
          0.9249
        ],
        [
          0.3985,
          0.9208
        ],
        [
          0.4086,
          0.9167
        ],
        [
          0.419,
          0.9122
        ],
        [
          0.429,
          0.9095
        ],
        [
          0.439,
          0.9031
        ],
        [
          0.449,
          0.8999
        ],
        [
          0.4591,
          0.8954
        ],
        [
          0.4695,
          0.8904
        ],
        [
          0.4795,
          0.8854
        ],
        [
          0.49,
          0.8817
        ],
        [
          0.5005,
          0.8749
        ],
        [
          0.5114,
          0.8685
        ],
        [
          0.5218,
          0.8621
        ],
        [
          0.5318,
          0.8594
        ],
        [
          0.5428,
          0.8535
        ],
        [
          0.5532,
          0.848
        ],
        [
          0.5637,
          0.8439
        ],
        [
          0.5737,
          0.8412
        ],
        [
          0.5842,
          0.8362
        ],
        [
          0.5946,
          0.8312
        ],
        [
          0.6046,
          0.8239
        ],
        [
          0.6146,
          0.8194
        ],
        [
          0.6247,
          0.8157
        ],
        [
          0.6351,
          0.8089
        ],
        [
          0.6451,
          0.803
        ],
        [
          0.6551,
          0.7957
        ],
        [
          0.6652,
          0.7894
        ],
        [
          0.6756,
          0.7839
        ],
        [
          0.6856,
          0.778
        ],
        [
          0.6956,
          0.7716
        ],
        [
          0.7056,
          0.7666
        ],
        [
          0.7157,
          0.7602
        ],
        [
          0.7261,
          0.7539
        ],
        [
          0.7361,
          0.7461
        ],
        [
          0.7461,
          0.7402
        ],
        [
          0.7561,
          0.7343
        ],
        [
          0.7662,
          0.7298
        ],
        [
          0.7771,
          0.7225
        ],
        [
          0.7875,
          0.7161
        ],
        [
          0.7975,
          0.7088
        ],
        [
          0.8076,
          0.702
        ],
        [
          0.818,
          0.6961
        ],
        [
          0.828,
          0.6884
        ],
        [
          0.838,
          0.6829
        ],
        [
          0.848,
          0.6761
        ],
        [
          0.8585,
          0.6688
        ],
        [
          0.869,
          0.6597
        ],
        [
          0.8794,
          0.6538
        ],
        [
          0.8894,
          0.6479
        ],
        [
          0.8995,
          0.6406
        ],
        [
          0.9099,
          0.6338
        ],
        [
          0.9199,
          0.6269
        ],
        [
          0.9299,
          0.6192
        ],
        [
          0.9399,
          0.6124
        ],
        [
          0.95,
          0.606
        ],
        [
          0.9604,
          0.5983
        ],
        [
          0.9704,
          0.5928
        ],
        [
          0.9804,
          0.5851
        ],
        [
          0.9904,
          0.5778
        ],
        [
          1.0,
          0.5728
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.11173405941522305,
        "difference": -0.0013855064192252276,
        "worse": 0.6233333333333333,
        "kept_right": 0.11011811460580662,
        "kept_wrong": 0.0016159448094163562,
        "handed_off": 0.8882659405847768
      },
      "usd_per_1000": 4.88698054392833
    },
    {
      "decision": 125,
      "frontier": 191,
      "frontier_accuracy": 0.9740673339399454,
      "in_sample": {
        "threshold": 0.9705,
        "answered": 0.062329390354868064,
        "accuracy": 0.9740673339399454
      },
      "curve": [
        [
          0.0,
          0.9741
        ],
        [
          0.01,
          0.9741
        ],
        [
          0.02,
          0.9741
        ],
        [
          0.03,
          0.9741
        ],
        [
          0.04,
          0.9741
        ],
        [
          0.05,
          0.9741
        ],
        [
          0.0601,
          0.9741
        ],
        [
          0.071,
          0.9736
        ],
        [
          0.0823,
          0.9736
        ],
        [
          0.0924,
          0.9736
        ],
        [
          0.1033,
          0.9732
        ],
        [
          0.1137,
          0.9732
        ],
        [
          0.1242,
          0.9732
        ],
        [
          0.1342,
          0.9727
        ],
        [
          0.1442,
          0.9722
        ],
        [
          0.1551,
          0.9722
        ],
        [
          0.1656,
          0.9718
        ],
        [
          0.1756,
          0.9709
        ],
        [
          0.1856,
          0.9704
        ],
        [
          0.1956,
          0.97
        ],
        [
          0.2056,
          0.9695
        ],
        [
          0.2157,
          0.9691
        ],
        [
          0.2266,
          0.9672
        ],
        [
          0.237,
          0.9677
        ],
        [
          0.247,
          0.9677
        ],
        [
          0.2571,
          0.9641
        ],
        [
          0.2675,
          0.9618
        ],
        [
          0.2775,
          0.9577
        ],
        [
          0.2875,
          0.955
        ],
        [
          0.2975,
          0.9527
        ],
        [
          0.3076,
          0.9504
        ],
        [
          0.318,
          0.9477
        ],
        [
          0.328,
          0.9436
        ],
        [
          0.338,
          0.9418
        ],
        [
          0.348,
          0.9381
        ],
        [
          0.3581,
          0.9349
        ],
        [
          0.3685,
          0.9313
        ],
        [
          0.3785,
          0.9268
        ],
        [
          0.3885,
          0.9231
        ],
        [
          0.3985,
          0.919
        ],
        [
          0.4086,
          0.9149
        ],
        [
          0.419,
          0.9095
        ],
        [
          0.429,
          0.9067
        ],
        [
          0.439,
          0.9004
        ],
        [
          0.449,
          0.8972
        ],
        [
          0.4591,
          0.8926
        ],
        [
          0.4695,
          0.8876
        ],
        [
          0.4795,
          0.8826
        ],
        [
          0.49,
          0.879
        ],
        [
          0.5005,
          0.8726
        ],
        [
          0.5114,
          0.8658
        ],
        [
          0.5218,
          0.8594
        ],
        [
          0.5318,
          0.8567
        ],
        [
          0.5428,
          0.8508
        ],
        [
          0.5532,
          0.8453
        ],
        [
          0.5637,
          0.8417
        ],
        [
          0.5737,
          0.8389
        ],
        [
          0.5842,
          0.8339
        ],
        [
          0.5946,
          0.8289
        ],
        [
          0.6046,
          0.8221
        ],
        [
          0.6146,
          0.8176
        ],
        [
          0.6247,
          0.8139
        ],
        [
          0.6351,
          0.8071
        ],
        [
          0.6451,
          0.8016
        ],
        [
          0.6551,
          0.7944
        ],
        [
          0.6652,
          0.788
        ],
        [
          0.6756,
          0.783
        ],
        [
          0.6856,
          0.7766
        ],
        [
          0.6956,
          0.7702
        ],
        [
          0.7056,
          0.7657
        ],
        [
          0.7157,
          0.7593
        ],
        [
          0.7261,
          0.7534
        ],
        [
          0.7361,
          0.7457
        ],
        [
          0.7461,
          0.7398
        ],
        [
          0.7561,
          0.7338
        ],
        [
          0.7662,
          0.7293
        ],
        [
          0.7771,
          0.722
        ],
        [
          0.7875,
          0.7157
        ],
        [
          0.7975,
          0.7088
        ],
        [
          0.8076,
          0.7025
        ],
        [
          0.818,
          0.6965
        ],
        [
          0.828,
          0.6888
        ],
        [
          0.838,
          0.6838
        ],
        [
          0.848,
          0.6774
        ],
        [
          0.8585,
          0.6697
        ],
        [
          0.869,
          0.6606
        ],
        [
          0.8794,
          0.6547
        ],
        [
          0.8894,
          0.6488
        ],
        [
          0.8995,
          0.6415
        ],
        [
          0.9099,
          0.6347
        ],
        [
          0.9199,
          0.6278
        ],
        [
          0.9299,
          0.6201
        ],
        [
          0.9399,
          0.6133
        ],
        [
          0.95,
          0.6069
        ],
        [
          0.9604,
          0.5987
        ],
        [
          0.9704,
          0.5933
        ],
        [
          0.9804,
          0.5851
        ],
        [
          0.9904,
          0.5778
        ],
        [
          1.0,
          0.5728
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.09468632563085014,
        "difference": -0.0010671587299893497,
        "worse": 0.5266666666666666,
        "kept_right": 0.09357670401066172,
        "kept_wrong": 0.001109621620188422,
        "handed_off": 0.9053136743691497
      },
      "usd_per_1000": 24.315687239324877
    },
    {
      "decision": 125,
      "frontier": 188,
      "frontier_accuracy": 0.9713375796178344,
      "in_sample": {
        "threshold": 0.9575,
        "answered": 0.12829845313921748,
        "accuracy": 0.9713375796178344
      },
      "curve": [
        [
          0.0,
          0.9713
        ],
        [
          0.01,
          0.9713
        ],
        [
          0.02,
          0.9713
        ],
        [
          0.03,
          0.9713
        ],
        [
          0.04,
          0.9713
        ],
        [
          0.05,
          0.9718
        ],
        [
          0.0601,
          0.9718
        ],
        [
          0.071,
          0.9713
        ],
        [
          0.0823,
          0.9713
        ],
        [
          0.0924,
          0.9713
        ],
        [
          0.1033,
          0.9709
        ],
        [
          0.1137,
          0.9713
        ],
        [
          0.1242,
          0.9713
        ],
        [
          0.1342,
          0.9709
        ],
        [
          0.1442,
          0.9704
        ],
        [
          0.1551,
          0.9704
        ],
        [
          0.1656,
          0.97
        ],
        [
          0.1756,
          0.9691
        ],
        [
          0.1856,
          0.9686
        ],
        [
          0.1956,
          0.9682
        ],
        [
          0.2056,
          0.9677
        ],
        [
          0.2157,
          0.9672
        ],
        [
          0.2266,
          0.9654
        ],
        [
          0.237,
          0.965
        ],
        [
          0.247,
          0.9645
        ],
        [
          0.2571,
          0.9609
        ],
        [
          0.2675,
          0.9586
        ],
        [
          0.2775,
          0.955
        ],
        [
          0.2875,
          0.9527
        ],
        [
          0.2975,
          0.95
        ],
        [
          0.3076,
          0.9477
        ],
        [
          0.318,
          0.9454
        ],
        [
          0.328,
          0.9418
        ],
        [
          0.338,
          0.9399
        ],
        [
          0.348,
          0.9363
        ],
        [
          0.3581,
          0.9331
        ],
        [
          0.3685,
          0.9295
        ],
        [
          0.3785,
          0.9254
        ],
        [
          0.3885,
          0.9217
        ],
        [
          0.3985,
          0.9177
        ],
        [
          0.4086,
          0.9136
        ],
        [
          0.419,
          0.9086
        ],
        [
          0.429,
          0.9058
        ],
        [
          0.439,
          0.8995
        ],
        [
          0.449,
          0.8967
        ],
        [
          0.4591,
          0.8926
        ],
        [
          0.4695,
          0.8876
        ],
        [
          0.4795,
          0.8826
        ],
        [
          0.49,
          0.879
        ],
        [
          0.5005,
          0.8722
        ],
        [
          0.5114,
          0.8653
        ],
        [
          0.5218,
          0.859
        ],
        [
          0.5318,
          0.8562
        ],
        [
          0.5428,
          0.8508
        ],
        [
          0.5532,
          0.8453
        ],
        [
          0.5637,
          0.8403
        ],
        [
          0.5737,
          0.8371
        ],
        [
          0.5842,
          0.8326
        ],
        [
          0.5946,
          0.8276
        ],
        [
          0.6046,
          0.8207
        ],
        [
          0.6146,
          0.8162
        ],
        [
          0.6247,
          0.813
        ],
        [
          0.6351,
          0.8062
        ],
        [
          0.6451,
          0.8003
        ],
        [
          0.6551,
          0.7934
        ],
        [
          0.6652,
          0.7875
        ],
        [
          0.6756,
          0.7816
        ],
        [
          0.6856,
          0.7753
        ],
        [
          0.6956,
          0.7693
        ],
        [
          0.7056,
          0.7643
        ],
        [
          0.7157,
          0.758
        ],
        [
          0.7261,
          0.752
        ],
        [
          0.7361,
          0.7448
        ],
        [
          0.7461,
          0.7398
        ],
        [
          0.7561,
          0.7338
        ],
        [
          0.7662,
          0.7293
        ],
        [
          0.7771,
          0.722
        ],
        [
          0.7875,
          0.7152
        ],
        [
          0.7975,
          0.7079
        ],
        [
          0.8076,
          0.7011
        ],
        [
          0.818,
          0.6952
        ],
        [
          0.828,
          0.6879
        ],
        [
          0.838,
          0.6829
        ],
        [
          0.848,
          0.6761
        ],
        [
          0.8585,
          0.6692
        ],
        [
          0.869,
          0.6606
        ],
        [
          0.8794,
          0.6547
        ],
        [
          0.8894,
          0.6492
        ],
        [
          0.8995,
          0.6419
        ],
        [
          0.9099,
          0.6351
        ],
        [
          0.9199,
          0.6288
        ],
        [
          0.9299,
          0.621
        ],
        [
          0.9399,
          0.6142
        ],
        [
          0.95,
          0.6078
        ],
        [
          0.9604,
          0.5996
        ],
        [
          0.9704,
          0.5937
        ],
        [
          0.9804,
          0.5851
        ],
        [
          0.9904,
          0.5778
        ],
        [
          1.0,
          0.5728
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.13081879945014233,
        "difference": -0.0012158147034941455,
        "worse": 0.5166666666666667,
        "kept_right": 0.1286449694100308,
        "kept_wrong": 0.002173830040111483,
        "handed_off": 0.8691812005498576
      },
      "usd_per_1000": 0.9470891398627393
    },
    {
      "decision": 125,
      "frontier": 190,
      "frontier_accuracy": 0.9672429481346679,
      "in_sample": {
        "threshold": 0.9535,
        "answered": 0.14194722474977253,
        "accuracy": 0.9672429481346679
      },
      "curve": [
        [
          0.0,
          0.9672
        ],
        [
          0.01,
          0.9672
        ],
        [
          0.02,
          0.9672
        ],
        [
          0.03,
          0.9672
        ],
        [
          0.04,
          0.9672
        ],
        [
          0.05,
          0.9672
        ],
        [
          0.0601,
          0.9672
        ],
        [
          0.071,
          0.9668
        ],
        [
          0.0823,
          0.9668
        ],
        [
          0.0924,
          0.9672
        ],
        [
          0.1033,
          0.9668
        ],
        [
          0.1137,
          0.9672
        ],
        [
          0.1242,
          0.9672
        ],
        [
          0.1342,
          0.9668
        ],
        [
          0.1442,
          0.9668
        ],
        [
          0.1551,
          0.9668
        ],
        [
          0.1656,
          0.9663
        ],
        [
          0.1756,
          0.9654
        ],
        [
          0.1856,
          0.965
        ],
        [
          0.1956,
          0.9645
        ],
        [
          0.2056,
          0.9641
        ],
        [
          0.2157,
          0.9636
        ],
        [
          0.2266,
          0.9618
        ],
        [
          0.237,
          0.9622
        ],
        [
          0.247,
          0.9618
        ],
        [
          0.2571,
          0.9581
        ],
        [
          0.2675,
          0.9559
        ],
        [
          0.2775,
          0.9513
        ],
        [
          0.2875,
          0.949
        ],
        [
          0.2975,
          0.9472
        ],
        [
          0.3076,
          0.9449
        ],
        [
          0.318,
          0.9427
        ],
        [
          0.328,
          0.939
        ],
        [
          0.338,
          0.9372
        ],
        [
          0.348,
          0.934
        ],
        [
          0.3581,
          0.9308
        ],
        [
          0.3685,
          0.9272
        ],
        [
          0.3785,
          0.9227
        ],
        [
          0.3885,
          0.919
        ],
        [
          0.3985,
          0.9149
        ],
        [
          0.4086,
          0.9108
        ],
        [
          0.419,
          0.9063
        ],
        [
          0.429,
          0.9035
        ],
        [
          0.439,
          0.8972
        ],
        [
          0.449,
          0.894
        ],
        [
          0.4591,
          0.8894
        ],
        [
          0.4695,
          0.8844
        ],
        [
          0.4795,
          0.8794
        ],
        [
          0.49,
          0.8758
        ],
        [
          0.5005,
          0.8694
        ],
        [
          0.5114,
          0.8631
        ],
        [
          0.5218,
          0.8567
        ],
        [
          0.5318,
          0.854
        ],
        [
          0.5428,
          0.848
        ],
        [
          0.5532,
          0.8426
        ],
        [
          0.5637,
          0.8385
        ],
        [
          0.5737,
          0.8358
        ],
        [
          0.5842,
          0.8312
        ],
        [
          0.5946,
          0.8267
        ],
        [
          0.6046,
          0.8198
        ],
        [
          0.6146,
          0.8153
        ],
        [
          0.6247,
          0.8116
        ],
        [
          0.6351,
          0.8048
        ],
        [
          0.6451,
          0.7989
        ],
        [
          0.6551,
          0.7916
        ],
        [
          0.6652,
          0.7853
        ],
        [
          0.6756,
          0.7803
        ],
        [
          0.6856,
          0.7739
        ],
        [
          0.6956,
          0.768
        ],
        [
          0.7056,
          0.7634
        ],
        [
          0.7157,
          0.7571
        ],
        [
          0.7261,
          0.7511
        ],
        [
          0.7361,
          0.7434
        ],
        [
          0.7461,
          0.7375
        ],
        [
          0.7561,
          0.7316
        ],
        [
          0.7662,
          0.7275
        ],
        [
          0.7771,
          0.7207
        ],
        [
          0.7875,
          0.7143
        ],
        [
          0.7975,
          0.7075
        ],
        [
          0.8076,
          0.7011
        ],
        [
          0.818,
          0.6952
        ],
        [
          0.828,
          0.6884
        ],
        [
          0.838,
          0.6838
        ],
        [
          0.848,
          0.6774
        ],
        [
          0.8585,
          0.6697
        ],
        [
          0.869,
          0.6606
        ],
        [
          0.8794,
          0.6547
        ],
        [
          0.8894,
          0.6483
        ],
        [
          0.8995,
          0.641
        ],
        [
          0.9099,
          0.6342
        ],
        [
          0.9199,
          0.6274
        ],
        [
          0.9299,
          0.6197
        ],
        [
          0.9399,
          0.6128
        ],
        [
          0.95,
          0.6065
        ],
        [
          0.9604,
          0.5987
        ],
        [
          0.9704,
          0.5933
        ],
        [
          0.9804,
          0.5851
        ],
        [
          0.9904,
          0.5778
        ],
        [
          1.0,
          0.5728
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1433683888884556,
        "difference": -0.0015158316086532765,
        "worse": 0.5433333333333333,
        "kept_right": 0.14063983081818338,
        "kept_wrong": 0.0027285580702723663,
        "handed_off": 0.8566316111115448
      },
      "usd_per_1000": 4.879854767603545
    },
    {
      "decision": 125,
      "frontier": 124,
      "frontier_accuracy": 0.9667879890809827,
      "in_sample": {
        "threshold": 0.9705,
        "answered": 0.062329390354868064,
        "accuracy": 0.9667879890809827
      },
      "curve": [
        [
          0.0,
          0.9668
        ],
        [
          0.01,
          0.9668
        ],
        [
          0.02,
          0.9668
        ],
        [
          0.03,
          0.9668
        ],
        [
          0.04,
          0.9668
        ],
        [
          0.05,
          0.9668
        ],
        [
          0.0601,
          0.9668
        ],
        [
          0.071,
          0.9663
        ],
        [
          0.0823,
          0.9663
        ],
        [
          0.0924,
          0.9663
        ],
        [
          0.1033,
          0.9663
        ],
        [
          0.1137,
          0.9663
        ],
        [
          0.1242,
          0.9663
        ],
        [
          0.1342,
          0.9659
        ],
        [
          0.1442,
          0.9654
        ],
        [
          0.1551,
          0.9654
        ],
        [
          0.1656,
          0.965
        ],
        [
          0.1756,
          0.9641
        ],
        [
          0.1856,
          0.9636
        ],
        [
          0.1956,
          0.9631
        ],
        [
          0.2056,
          0.9627
        ],
        [
          0.2157,
          0.9622
        ],
        [
          0.2266,
          0.9604
        ],
        [
          0.237,
          0.9609
        ],
        [
          0.247,
          0.9604
        ],
        [
          0.2571,
          0.9563
        ],
        [
          0.2675,
          0.954
        ],
        [
          0.2775,
          0.9504
        ],
        [
          0.2875,
          0.9481
        ],
        [
          0.2975,
          0.9459
        ],
        [
          0.3076,
          0.9436
        ],
        [
          0.318,
          0.9413
        ],
        [
          0.328,
          0.9372
        ],
        [
          0.338,
          0.9354
        ],
        [
          0.348,
          0.9327
        ],
        [
          0.3581,
          0.9295
        ],
        [
          0.3685,
          0.9254
        ],
        [
          0.3785,
          0.9208
        ],
        [
          0.3885,
          0.9172
        ],
        [
          0.3985,
          0.9131
        ],
        [
          0.4086,
          0.909
        ],
        [
          0.419,
          0.904
        ],
        [
          0.429,
          0.9013
        ],
        [
          0.439,
          0.8954
        ],
        [
          0.449,
          0.8922
        ],
        [
          0.4591,
          0.8876
        ],
        [
          0.4695,
          0.8826
        ],
        [
          0.4795,
          0.8776
        ],
        [
          0.49,
          0.8744
        ],
        [
          0.5005,
          0.8681
        ],
        [
          0.5114,
          0.8617
        ],
        [
          0.5218,
          0.8553
        ],
        [
          0.5318,
          0.8526
        ],
        [
          0.5428,
          0.8467
        ],
        [
          0.5532,
          0.8412
        ],
        [
          0.5637,
          0.8376
        ],
        [
          0.5737,
          0.8353
        ],
        [
          0.5842,
          0.8308
        ],
        [
          0.5946,
          0.8258
        ],
        [
          0.6046,
          0.8189
        ],
        [
          0.6146,
          0.8144
        ],
        [
          0.6247,
          0.8107
        ],
        [
          0.6351,
          0.8039
        ],
        [
          0.6451,
          0.798
        ],
        [
          0.6551,
          0.7907
        ],
        [
          0.6652,
          0.7843
        ],
        [
          0.6756,
          0.7793
        ],
        [
          0.6856,
          0.7734
        ],
        [
          0.6956,
          0.7671
        ],
        [
          0.7056,
          0.7621
        ],
        [
          0.7157,
          0.7557
        ],
        [
          0.7261,
          0.7493
        ],
        [
          0.7361,
          0.742
        ],
        [
          0.7461,
          0.7361
        ],
        [
          0.7561,
          0.7307
        ],
        [
          0.7662,
          0.7261
        ],
        [
          0.7771,
          0.7193
        ],
        [
          0.7875,
          0.7129
        ],
        [
          0.7975,
          0.7061
        ],
        [
          0.8076,
          0.6988
        ],
        [
          0.818,
          0.6929
        ],
        [
          0.828,
          0.6865
        ],
        [
          0.838,
          0.6811
        ],
        [
          0.848,
          0.6752
        ],
        [
          0.8585,
          0.6683
        ],
        [
          0.869,
          0.6592
        ],
        [
          0.8794,
          0.6533
        ],
        [
          0.8894,
          0.647
        ],
        [
          0.8995,
          0.6397
        ],
        [
          0.9099,
          0.6328
        ],
        [
          0.9199,
          0.6269
        ],
        [
          0.9299,
          0.6192
        ],
        [
          0.9399,
          0.6124
        ],
        [
          0.95,
          0.606
        ],
        [
          0.9604,
          0.5978
        ],
        [
          0.9704,
          0.5924
        ],
        [
          0.9804,
          0.5846
        ],
        [
          0.9904,
          0.5773
        ],
        [
          1.0,
          0.5728
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.11173405941522305,
        "difference": -0.0013855064192252276,
        "worse": 0.6233333333333333,
        "kept_right": 0.11011811460580662,
        "kept_wrong": 0.0016159448094163562,
        "handed_off": 0.8882659405847768
      },
      "usd_per_1000": 1.1137572453773037
    },
    {
      "decision": 125,
      "frontier": 278,
      "frontier_accuracy": 0.9499545040946314,
      "in_sample": {
        "threshold": 0.9705,
        "answered": 0.062329390354868064,
        "accuracy": 0.9499545040946314
      },
      "curve": [
        [
          0.0,
          0.95
        ],
        [
          0.01,
          0.95
        ],
        [
          0.02,
          0.95
        ],
        [
          0.03,
          0.95
        ],
        [
          0.04,
          0.95
        ],
        [
          0.05,
          0.95
        ],
        [
          0.0601,
          0.95
        ],
        [
          0.071,
          0.9495
        ],
        [
          0.0823,
          0.9495
        ],
        [
          0.0924,
          0.9495
        ],
        [
          0.1033,
          0.949
        ],
        [
          0.1137,
          0.949
        ],
        [
          0.1242,
          0.949
        ],
        [
          0.1342,
          0.9486
        ],
        [
          0.1442,
          0.9481
        ],
        [
          0.1551,
          0.9481
        ],
        [
          0.1656,
          0.9477
        ],
        [
          0.1756,
          0.9468
        ],
        [
          0.1856,
          0.9463
        ],
        [
          0.1956,
          0.9459
        ],
        [
          0.2056,
          0.9454
        ],
        [
          0.2157,
          0.9449
        ],
        [
          0.2266,
          0.9431
        ],
        [
          0.237,
          0.9431
        ],
        [
          0.247,
          0.9436
        ],
        [
          0.2571,
          0.9399
        ],
        [
          0.2675,
          0.9381
        ],
        [
          0.2775,
          0.9354
        ],
        [
          0.2875,
          0.9336
        ],
        [
          0.2975,
          0.9313
        ],
        [
          0.3076,
          0.9299
        ],
        [
          0.318,
          0.9277
        ],
        [
          0.328,
          0.924
        ],
        [
          0.338,
          0.9222
        ],
        [
          0.348,
          0.9195
        ],
        [
          0.3581,
          0.9163
        ],
        [
          0.3685,
          0.9131
        ],
        [
          0.3785,
          0.909
        ],
        [
          0.3885,
          0.9063
        ],
        [
          0.3985,
          0.9026
        ],
        [
          0.4086,
          0.8985
        ],
        [
          0.419,
          0.894
        ],
        [
          0.429,
          0.8917
        ],
        [
          0.439,
          0.8858
        ],
        [
          0.449,
          0.8826
        ],
        [
          0.4591,
          0.8785
        ],
        [
          0.4695,
          0.8744
        ],
        [
          0.4795,
          0.8694
        ],
        [
          0.49,
          0.8662
        ],
        [
          0.5005,
          0.8603
        ],
        [
          0.5114,
          0.8535
        ],
        [
          0.5218,
          0.8476
        ],
        [
          0.5318,
          0.8453
        ],
        [
          0.5428,
          0.8399
        ],
        [
          0.5532,
          0.8348
        ],
        [
          0.5637,
          0.8317
        ],
        [
          0.5737,
          0.8289
        ],
        [
          0.5842,
          0.8244
        ],
        [
          0.5946,
          0.8194
        ],
        [
          0.6046,
          0.8121
        ],
        [
          0.6146,
          0.8076
        ],
        [
          0.6247,
          0.8039
        ],
        [
          0.6351,
          0.798
        ],
        [
          0.6451,
          0.7925
        ],
        [
          0.6551,
          0.7857
        ],
        [
          0.6652,
          0.7798
        ],
        [
          0.6756,
          0.7748
        ],
        [
          0.6856,
          0.7693
        ],
        [
          0.6956,
          0.7634
        ],
        [
          0.7056,
          0.7584
        ],
        [
          0.7157,
          0.7525
        ],
        [
          0.7261,
          0.7475
        ],
        [
          0.7361,
          0.7402
        ],
        [
          0.7461,
          0.7343
        ],
        [
          0.7561,
          0.7284
        ],
        [
          0.7662,
          0.7243
        ],
        [
          0.7771,
          0.717
        ],
        [
          0.7875,
          0.7106
        ],
        [
          0.7975,
          0.7038
        ],
        [
          0.8076,
          0.697
        ],
        [
          0.818,
          0.6911
        ],
        [
          0.828,
          0.6838
        ],
        [
          0.838,
          0.6783
        ],
        [
          0.848,
          0.6729
        ],
        [
          0.8585,
          0.6661
        ],
        [
          0.869,
          0.6574
        ],
        [
          0.8794,
          0.652
        ],
        [
          0.8894,
          0.646
        ],
        [
          0.8995,
          0.6392
        ],
        [
          0.9099,
          0.6328
        ],
        [
          0.9199,
          0.6265
        ],
        [
          0.9299,
          0.6187
        ],
        [
          0.9399,
          0.6124
        ],
        [
          0.95,
          0.606
        ],
        [
          0.9604,
          0.5983
        ],
        [
          0.9704,
          0.5924
        ],
        [
          0.9804,
          0.5851
        ],
        [
          0.9904,
          0.5778
        ],
        [
          1.0,
          0.5728
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.09468632563085014,
        "difference": -0.0010671587299893497,
        "worse": 0.5266666666666666,
        "kept_right": 0.09357670401066172,
        "kept_wrong": 0.001109621620188422,
        "handed_off": 0.9053136743691497
      },
      "usd_per_1000": 6.002214800754111
    },
    {
      "decision": 125,
      "frontier": 275,
      "frontier_accuracy": 0.9426751592356688,
      "in_sample": {
        "threshold": 0.9705,
        "answered": 0.062329390354868064,
        "accuracy": 0.9426751592356688
      },
      "curve": [
        [
          0.0,
          0.9427
        ],
        [
          0.01,
          0.9427
        ],
        [
          0.02,
          0.9427
        ],
        [
          0.03,
          0.9427
        ],
        [
          0.04,
          0.9427
        ],
        [
          0.05,
          0.9427
        ],
        [
          0.0601,
          0.9427
        ],
        [
          0.071,
          0.9422
        ],
        [
          0.0823,
          0.9422
        ],
        [
          0.0924,
          0.9422
        ],
        [
          0.1033,
          0.9418
        ],
        [
          0.1137,
          0.9418
        ],
        [
          0.1242,
          0.9418
        ],
        [
          0.1342,
          0.9413
        ],
        [
          0.1442,
          0.9409
        ],
        [
          0.1551,
          0.9409
        ],
        [
          0.1656,
          0.9404
        ],
        [
          0.1756,
          0.9395
        ],
        [
          0.1856,
          0.939
        ],
        [
          0.1956,
          0.9386
        ],
        [
          0.2056,
          0.9381
        ],
        [
          0.2157,
          0.9377
        ],
        [
          0.2266,
          0.9359
        ],
        [
          0.237,
          0.9363
        ],
        [
          0.247,
          0.9363
        ],
        [
          0.2571,
          0.9322
        ],
        [
          0.2675,
          0.9299
        ],
        [
          0.2775,
          0.9268
        ],
        [
          0.2875,
          0.9245
        ],
        [
          0.2975,
          0.9231
        ],
        [
          0.3076,
          0.9213
        ],
        [
          0.318,
          0.9195
        ],
        [
          0.328,
          0.9158
        ],
        [
          0.338,
          0.914
        ],
        [
          0.348,
          0.9108
        ],
        [
          0.3581,
          0.9076
        ],
        [
          0.3685,
          0.9045
        ],
        [
          0.3785,
          0.8999
        ],
        [
          0.3885,
          0.8963
        ],
        [
          0.3985,
          0.8926
        ],
        [
          0.4086,
          0.889
        ],
        [
          0.419,
          0.8844
        ],
        [
          0.429,
          0.8826
        ],
        [
          0.439,
          0.8772
        ],
        [
          0.449,
          0.874
        ],
        [
          0.4591,
          0.8694
        ],
        [
          0.4695,
          0.8653
        ],
        [
          0.4795,
          0.8603
        ],
        [
          0.49,
          0.8571
        ],
        [
          0.5005,
          0.8508
        ],
        [
          0.5114,
          0.8453
        ],
        [
          0.5218,
          0.8389
        ],
        [
          0.5318,
          0.8371
        ],
        [
          0.5428,
          0.8321
        ],
        [
          0.5532,
          0.8262
        ],
        [
          0.5637,
          0.8226
        ],
        [
          0.5737,
          0.8203
        ],
        [
          0.5842,
          0.8162
        ],
        [
          0.5946,
          0.8116
        ],
        [
          0.6046,
          0.8066
        ],
        [
          0.6146,
          0.8025
        ],
        [
          0.6247,
          0.7989
        ],
        [
          0.6351,
          0.7925
        ],
        [
          0.6451,
          0.7875
        ],
        [
          0.6551,
          0.7807
        ],
        [
          0.6652,
          0.7753
        ],
        [
          0.6756,
          0.7702
        ],
        [
          0.6856,
          0.7643
        ],
        [
          0.6956,
          0.7584
        ],
        [
          0.7056,
          0.7539
        ],
        [
          0.7157,
          0.748
        ],
        [
          0.7261,
          0.7434
        ],
        [
          0.7361,
          0.7357
        ],
        [
          0.7461,
          0.7298
        ],
        [
          0.7561,
          0.7243
        ],
        [
          0.7662,
          0.7202
        ],
        [
          0.7771,
          0.7129
        ],
        [
          0.7875,
          0.7075
        ],
        [
          0.7975,
          0.7011
        ],
        [
          0.8076,
          0.6947
        ],
        [
          0.818,
          0.6888
        ],
        [
          0.828,
          0.682
        ],
        [
          0.838,
          0.677
        ],
        [
          0.848,
          0.6711
        ],
        [
          0.8585,
          0.6638
        ],
        [
          0.869,
          0.6551
        ],
        [
          0.8794,
          0.6497
        ],
        [
          0.8894,
          0.6438
        ],
        [
          0.8995,
          0.6374
        ],
        [
          0.9099,
          0.6315
        ],
        [
          0.9199,
          0.6251
        ],
        [
          0.9299,
          0.6174
        ],
        [
          0.9399,
          0.6106
        ],
        [
          0.95,
          0.6046
        ],
        [
          0.9604,
          0.5978
        ],
        [
          0.9704,
          0.5933
        ],
        [
          0.9804,
          0.5846
        ],
        [
          0.9904,
          0.5769
        ],
        [
          1.0,
          0.5728
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.09468632563085014,
        "difference": -0.0010671587299893497,
        "worse": 0.5266666666666666,
        "kept_right": 0.09357670401066172,
        "kept_wrong": 0.001109621620188422,
        "handed_off": 0.9053136743691497
      },
      "usd_per_1000": 0.5754811803203238
    },
    {
      "decision": 125,
      "frontier": 192,
      "frontier_accuracy": 0.9331210191082803,
      "in_sample": {
        "threshold": 0.9705,
        "answered": 0.062329390354868064,
        "accuracy": 0.9331210191082803
      },
      "curve": [
        [
          0.0,
          0.9331
        ],
        [
          0.01,
          0.9331
        ],
        [
          0.02,
          0.9331
        ],
        [
          0.03,
          0.9331
        ],
        [
          0.04,
          0.9331
        ],
        [
          0.05,
          0.9331
        ],
        [
          0.0601,
          0.9331
        ],
        [
          0.071,
          0.9327
        ],
        [
          0.0823,
          0.9327
        ],
        [
          0.0924,
          0.9327
        ],
        [
          0.1033,
          0.9322
        ],
        [
          0.1137,
          0.9322
        ],
        [
          0.1242,
          0.9322
        ],
        [
          0.1342,
          0.9318
        ],
        [
          0.1442,
          0.9318
        ],
        [
          0.1551,
          0.9318
        ],
        [
          0.1656,
          0.9313
        ],
        [
          0.1756,
          0.9304
        ],
        [
          0.1856,
          0.9299
        ],
        [
          0.1956,
          0.9295
        ],
        [
          0.2056,
          0.929
        ],
        [
          0.2157,
          0.9286
        ],
        [
          0.2266,
          0.9272
        ],
        [
          0.237,
          0.9277
        ],
        [
          0.247,
          0.9277
        ],
        [
          0.2571,
          0.924
        ],
        [
          0.2675,
          0.9217
        ],
        [
          0.2775,
          0.9181
        ],
        [
          0.2875,
          0.9154
        ],
        [
          0.2975,
          0.9126
        ],
        [
          0.3076,
          0.9113
        ],
        [
          0.318,
          0.9108
        ],
        [
          0.328,
          0.9076
        ],
        [
          0.338,
          0.9063
        ],
        [
          0.348,
          0.9035
        ],
        [
          0.3581,
          0.9013
        ],
        [
          0.3685,
          0.899
        ],
        [
          0.3785,
          0.8949
        ],
        [
          0.3885,
          0.8922
        ],
        [
          0.3985,
          0.889
        ],
        [
          0.4086,
          0.8854
        ],
        [
          0.419,
          0.8817
        ],
        [
          0.429,
          0.8794
        ],
        [
          0.439,
          0.8744
        ],
        [
          0.449,
          0.8712
        ],
        [
          0.4591,
          0.8667
        ],
        [
          0.4695,
          0.8621
        ],
        [
          0.4795,
          0.8571
        ],
        [
          0.49,
          0.854
        ],
        [
          0.5005,
          0.8485
        ],
        [
          0.5114,
          0.8417
        ],
        [
          0.5218,
          0.8353
        ],
        [
          0.5318,
          0.8335
        ],
        [
          0.5428,
          0.828
        ],
        [
          0.5532,
          0.8239
        ],
        [
          0.5637,
          0.8207
        ],
        [
          0.5737,
          0.8185
        ],
        [
          0.5842,
          0.8139
        ],
        [
          0.5946,
          0.8103
        ],
        [
          0.6046,
          0.8035
        ],
        [
          0.6146,
          0.7989
        ],
        [
          0.6247,
          0.7962
        ],
        [
          0.6351,
          0.7898
        ],
        [
          0.6451,
          0.7843
        ],
        [
          0.6551,
          0.7771
        ],
        [
          0.6652,
          0.7716
        ],
        [
          0.6756,
          0.7662
        ],
        [
          0.6856,
          0.7611
        ],
        [
          0.6956,
          0.7552
        ],
        [
          0.7056,
          0.7516
        ],
        [
          0.7157,
          0.7461
        ],
        [
          0.7261,
          0.7407
        ],
        [
          0.7361,
          0.7329
        ],
        [
          0.7461,
          0.727
        ],
        [
          0.7561,
          0.7216
        ],
        [
          0.7662,
          0.7175
        ],
        [
          0.7771,
          0.7106
        ],
        [
          0.7875,
          0.7047
        ],
        [
          0.7975,
          0.6979
        ],
        [
          0.8076,
          0.692
        ],
        [
          0.818,
          0.6861
        ],
        [
          0.828,
          0.6797
        ],
        [
          0.838,
          0.6761
        ],
        [
          0.848,
          0.6706
        ],
        [
          0.8585,
          0.6638
        ],
        [
          0.869,
          0.6556
        ],
        [
          0.8794,
          0.6497
        ],
        [
          0.8894,
          0.6438
        ],
        [
          0.8995,
          0.6365
        ],
        [
          0.9099,
          0.6301
        ],
        [
          0.9199,
          0.6242
        ],
        [
          0.9299,
          0.6174
        ],
        [
          0.9399,
          0.611
        ],
        [
          0.95,
          0.6046
        ],
        [
          0.9604,
          0.5965
        ],
        [
          0.9704,
          0.591
        ],
        [
          0.9804,
          0.5846
        ],
        [
          0.9904,
          0.5773
        ],
        [
          1.0,
          0.5728
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.10932450787696604,
        "difference": -0.0014491869909601044,
        "worse": 0.6,
        "kept_right": 0.10766000917479107,
        "kept_wrong": 0.0016644987021750212,
        "handed_off": 0.8906754921230343
      },
      "usd_per_1000": 10.450040665839065
    },
    {
      "decision": 125,
      "frontier": 123,
      "frontier_accuracy": 0.9326660600545951,
      "in_sample": {
        "threshold": 0.9705,
        "answered": 0.062329390354868064,
        "accuracy": 0.9326660600545951
      },
      "curve": [
        [
          0.0,
          0.9327
        ],
        [
          0.01,
          0.9327
        ],
        [
          0.02,
          0.9327
        ],
        [
          0.03,
          0.9327
        ],
        [
          0.04,
          0.9327
        ],
        [
          0.05,
          0.9327
        ],
        [
          0.0601,
          0.9327
        ],
        [
          0.071,
          0.9322
        ],
        [
          0.0823,
          0.9322
        ],
        [
          0.0924,
          0.9322
        ],
        [
          0.1033,
          0.9318
        ],
        [
          0.1137,
          0.9318
        ],
        [
          0.1242,
          0.9318
        ],
        [
          0.1342,
          0.9318
        ],
        [
          0.1442,
          0.9313
        ],
        [
          0.1551,
          0.9313
        ],
        [
          0.1656,
          0.9308
        ],
        [
          0.1756,
          0.9299
        ],
        [
          0.1856,
          0.9295
        ],
        [
          0.1956,
          0.929
        ],
        [
          0.2056,
          0.9286
        ],
        [
          0.2157,
          0.9286
        ],
        [
          0.2266,
          0.9268
        ],
        [
          0.237,
          0.9263
        ],
        [
          0.247,
          0.9263
        ],
        [
          0.2571,
          0.9222
        ],
        [
          0.2675,
          0.9199
        ],
        [
          0.2775,
          0.9163
        ],
        [
          0.2875,
          0.914
        ],
        [
          0.2975,
          0.9122
        ],
        [
          0.3076,
          0.9099
        ],
        [
          0.318,
          0.9072
        ],
        [
          0.328,
          0.904
        ],
        [
          0.338,
          0.9022
        ],
        [
          0.348,
          0.899
        ],
        [
          0.3581,
          0.8963
        ],
        [
          0.3685,
          0.8926
        ],
        [
          0.3785,
          0.8885
        ],
        [
          0.3885,
          0.8849
        ],
        [
          0.3985,
          0.8808
        ],
        [
          0.4086,
          0.8767
        ],
        [
          0.419,
          0.8717
        ],
        [
          0.429,
          0.8703
        ],
        [
          0.439,
          0.8644
        ],
        [
          0.449,
          0.8617
        ],
        [
          0.4591,
          0.8576
        ],
        [
          0.4695,
          0.8526
        ],
        [
          0.4795,
          0.848
        ],
        [
          0.49,
          0.8449
        ],
        [
          0.5005,
          0.8385
        ],
        [
          0.5114,
          0.8321
        ],
        [
          0.5218,
          0.8271
        ],
        [
          0.5318,
          0.8248
        ],
        [
          0.5428,
          0.8194
        ],
        [
          0.5532,
          0.8144
        ],
        [
          0.5637,
          0.8098
        ],
        [
          0.5737,
          0.8071
        ],
        [
          0.5842,
          0.8025
        ],
        [
          0.5946,
          0.798
        ],
        [
          0.6046,
          0.7912
        ],
        [
          0.6146,
          0.7866
        ],
        [
          0.6247,
          0.7834
        ],
        [
          0.6351,
          0.778
        ],
        [
          0.6451,
          0.7734
        ],
        [
          0.6551,
          0.7671
        ],
        [
          0.6652,
          0.7621
        ],
        [
          0.6756,
          0.7575
        ],
        [
          0.6856,
          0.7525
        ],
        [
          0.6956,
          0.7466
        ],
        [
          0.7056,
          0.742
        ],
        [
          0.7157,
          0.7357
        ],
        [
          0.7261,
          0.7302
        ],
        [
          0.7361,
          0.7234
        ],
        [
          0.7461,
          0.7193
        ],
        [
          0.7561,
          0.7143
        ],
        [
          0.7662,
          0.7106
        ],
        [
          0.7771,
          0.7038
        ],
        [
          0.7875,
          0.6988
        ],
        [
          0.7975,
          0.6929
        ],
        [
          0.8076,
          0.6865
        ],
        [
          0.818,
          0.6806
        ],
        [
          0.828,
          0.6742
        ],
        [
          0.838,
          0.6692
        ],
        [
          0.848,
          0.6638
        ],
        [
          0.8585,
          0.6579
        ],
        [
          0.869,
          0.6497
        ],
        [
          0.8794,
          0.6442
        ],
        [
          0.8894,
          0.6383
        ],
        [
          0.8995,
          0.6315
        ],
        [
          0.9099,
          0.6256
        ],
        [
          0.9199,
          0.6197
        ],
        [
          0.9299,
          0.6128
        ],
        [
          0.9399,
          0.606
        ],
        [
          0.95,
          0.601
        ],
        [
          0.9604,
          0.5951
        ],
        [
          0.9704,
          0.5914
        ],
        [
          0.9804,
          0.5833
        ],
        [
          0.9904,
          0.576
        ],
        [
          1.0,
          0.5728
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.10764981801200536,
        "difference": -0.0013704266653445532,
        "worse": 0.5933333333333334,
        "kept_right": 0.10609136065475014,
        "kept_wrong": 0.0015584573572551845,
        "handed_off": 0.892350181987995
      },
      "usd_per_1000": 0.3493562780078061
    },
    {
      "decision": 125,
      "frontier": 274,
      "frontier_accuracy": 0.9085532302092811,
      "in_sample": {
        "threshold": 0.9705,
        "answered": 0.062329390354868064,
        "accuracy": 0.9085532302092811
      },
      "curve": [
        [
          0.0,
          0.9086
        ],
        [
          0.01,
          0.9086
        ],
        [
          0.02,
          0.9086
        ],
        [
          0.03,
          0.9086
        ],
        [
          0.04,
          0.9086
        ],
        [
          0.05,
          0.9086
        ],
        [
          0.0601,
          0.9086
        ],
        [
          0.071,
          0.9081
        ],
        [
          0.0823,
          0.9081
        ],
        [
          0.0924,
          0.9081
        ],
        [
          0.1033,
          0.9076
        ],
        [
          0.1137,
          0.9076
        ],
        [
          0.1242,
          0.9076
        ],
        [
          0.1342,
          0.9072
        ],
        [
          0.1442,
          0.9067
        ],
        [
          0.1551,
          0.9067
        ],
        [
          0.1656,
          0.9067
        ],
        [
          0.1756,
          0.9058
        ],
        [
          0.1856,
          0.9054
        ],
        [
          0.1956,
          0.9049
        ],
        [
          0.2056,
          0.9049
        ],
        [
          0.2157,
          0.9045
        ],
        [
          0.2266,
          0.9026
        ],
        [
          0.237,
          0.9031
        ],
        [
          0.247,
          0.9035
        ],
        [
          0.2571,
          0.8999
        ],
        [
          0.2675,
          0.8976
        ],
        [
          0.2775,
          0.894
        ],
        [
          0.2875,
          0.8917
        ],
        [
          0.2975,
          0.8904
        ],
        [
          0.3076,
          0.8885
        ],
        [
          0.318,
          0.8863
        ],
        [
          0.328,
          0.8835
        ],
        [
          0.338,
          0.8817
        ],
        [
          0.348,
          0.8794
        ],
        [
          0.3581,
          0.8763
        ],
        [
          0.3685,
          0.8731
        ],
        [
          0.3785,
          0.8694
        ],
        [
          0.3885,
          0.8658
        ],
        [
          0.3985,
          0.8621
        ],
        [
          0.4086,
          0.859
        ],
        [
          0.419,
          0.8553
        ],
        [
          0.429,
          0.8535
        ],
        [
          0.439,
          0.848
        ],
        [
          0.449,
          0.8449
        ],
        [
          0.4591,
          0.8403
        ],
        [
          0.4695,
          0.8362
        ],
        [
          0.4795,
          0.8312
        ],
        [
          0.49,
          0.828
        ],
        [
          0.5005,
          0.8217
        ],
        [
          0.5114,
          0.8167
        ],
        [
          0.5218,
          0.8107
        ],
        [
          0.5318,
          0.8098
        ],
        [
          0.5428,
          0.8057
        ],
        [
          0.5532,
          0.8012
        ],
        [
          0.5637,
          0.798
        ],
        [
          0.5737,
          0.7957
        ],
        [
          0.5842,
          0.7921
        ],
        [
          0.5946,
          0.7875
        ],
        [
          0.6046,
          0.7825
        ],
        [
          0.6146,
          0.7775
        ],
        [
          0.6247,
          0.7748
        ],
        [
          0.6351,
          0.7698
        ],
        [
          0.6451,
          0.7662
        ],
        [
          0.6551,
          0.7593
        ],
        [
          0.6652,
          0.7543
        ],
        [
          0.6756,
          0.7507
        ],
        [
          0.6856,
          0.7452
        ],
        [
          0.6956,
          0.7402
        ],
        [
          0.7056,
          0.7357
        ],
        [
          0.7157,
          0.7302
        ],
        [
          0.7261,
          0.7261
        ],
        [
          0.7361,
          0.7197
        ],
        [
          0.7461,
          0.7143
        ],
        [
          0.7561,
          0.7093
        ],
        [
          0.7662,
          0.7056
        ],
        [
          0.7771,
          0.6984
        ],
        [
          0.7875,
          0.6924
        ],
        [
          0.7975,
          0.6856
        ],
        [
          0.8076,
          0.6806
        ],
        [
          0.818,
          0.6761
        ],
        [
          0.828,
          0.6697
        ],
        [
          0.838,
          0.667
        ],
        [
          0.848,
          0.6611
        ],
        [
          0.8585,
          0.6547
        ],
        [
          0.869,
          0.6479
        ],
        [
          0.8794,
          0.6429
        ],
        [
          0.8894,
          0.6379
        ],
        [
          0.8995,
          0.6315
        ],
        [
          0.9099,
          0.6251
        ],
        [
          0.9199,
          0.6187
        ],
        [
          0.9299,
          0.6119
        ],
        [
          0.9399,
          0.6056
        ],
        [
          0.95,
          0.6001
        ],
        [
          0.9604,
          0.5937
        ],
        [
          0.9704,
          0.5896
        ],
        [
          0.9804,
          0.5823
        ],
        [
          0.9904,
          0.5755
        ],
        [
          1.0,
          0.5728
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.09771111696344882,
        "difference": -0.0012156326843597248,
        "worse": 0.53,
        "kept_right": 0.09632266304147657,
        "kept_wrong": 0.0013884539219722687,
        "handed_off": 0.902288883036551
      },
      "usd_per_1000": 0.4942820869200857
    },
    {
      "decision": 125,
      "frontier": 47,
      "frontier_accuracy": 0.9044585987261147,
      "in_sample": {
        "threshold": 0.9705,
        "answered": 0.062329390354868064,
        "accuracy": 0.9044585987261147
      },
      "curve": [
        [
          0.0,
          0.9045
        ],
        [
          0.01,
          0.9045
        ],
        [
          0.02,
          0.9045
        ],
        [
          0.03,
          0.9045
        ],
        [
          0.04,
          0.9045
        ],
        [
          0.05,
          0.9045
        ],
        [
          0.0601,
          0.9045
        ],
        [
          0.071,
          0.904
        ],
        [
          0.0823,
          0.904
        ],
        [
          0.0924,
          0.904
        ],
        [
          0.1033,
          0.9035
        ],
        [
          0.1137,
          0.9035
        ],
        [
          0.1242,
          0.9035
        ],
        [
          0.1342,
          0.904
        ],
        [
          0.1442,
          0.9035
        ],
        [
          0.1551,
          0.9035
        ],
        [
          0.1656,
          0.9031
        ],
        [
          0.1756,
          0.9022
        ],
        [
          0.1856,
          0.9017
        ],
        [
          0.1956,
          0.9013
        ],
        [
          0.2056,
          0.9008
        ],
        [
          0.2157,
          0.8999
        ],
        [
          0.2266,
          0.8985
        ],
        [
          0.237,
          0.8985
        ],
        [
          0.247,
          0.899
        ],
        [
          0.2571,
          0.8949
        ],
        [
          0.2675,
          0.8926
        ],
        [
          0.2775,
          0.8913
        ],
        [
          0.2875,
          0.889
        ],
        [
          0.2975,
          0.8876
        ],
        [
          0.3076,
          0.8872
        ],
        [
          0.318,
          0.8858
        ],
        [
          0.328,
          0.8822
        ],
        [
          0.338,
          0.8813
        ],
        [
          0.348,
          0.8776
        ],
        [
          0.3581,
          0.8753
        ],
        [
          0.3685,
          0.8726
        ],
        [
          0.3785,
          0.869
        ],
        [
          0.3885,
          0.8658
        ],
        [
          0.3985,
          0.8626
        ],
        [
          0.4086,
          0.8585
        ],
        [
          0.419,
          0.8544
        ],
        [
          0.429,
          0.8521
        ],
        [
          0.439,
          0.8467
        ],
        [
          0.449,
          0.8439
        ],
        [
          0.4591,
          0.8394
        ],
        [
          0.4695,
          0.8362
        ],
        [
          0.4795,
          0.8317
        ],
        [
          0.49,
          0.8285
        ],
        [
          0.5005,
          0.8239
        ],
        [
          0.5114,
          0.8176
        ],
        [
          0.5218,
          0.813
        ],
        [
          0.5318,
          0.8112
        ],
        [
          0.5428,
          0.8062
        ],
        [
          0.5532,
          0.8021
        ],
        [
          0.5637,
          0.798
        ],
        [
          0.5737,
          0.7962
        ],
        [
          0.5842,
          0.7916
        ],
        [
          0.5946,
          0.7866
        ],
        [
          0.6046,
          0.7793
        ],
        [
          0.6146,
          0.7748
        ],
        [
          0.6247,
          0.7721
        ],
        [
          0.6351,
          0.7671
        ],
        [
          0.6451,
          0.7625
        ],
        [
          0.6551,
          0.7552
        ],
        [
          0.6652,
          0.7493
        ],
        [
          0.6756,
          0.7448
        ],
        [
          0.6856,
          0.7407
        ],
        [
          0.6956,
          0.7348
        ],
        [
          0.7056,
          0.7311
        ],
        [
          0.7157,
          0.7261
        ],
        [
          0.7261,
          0.7216
        ],
        [
          0.7361,
          0.7143
        ],
        [
          0.7461,
          0.7093
        ],
        [
          0.7561,
          0.7047
        ],
        [
          0.7662,
          0.7015
        ],
        [
          0.7771,
          0.6952
        ],
        [
          0.7875,
          0.6902
        ],
        [
          0.7975,
          0.6843
        ],
        [
          0.8076,
          0.6774
        ],
        [
          0.818,
          0.6715
        ],
        [
          0.828,
          0.6656
        ],
        [
          0.838,
          0.6615
        ],
        [
          0.848,
          0.6574
        ],
        [
          0.8585,
          0.651
        ],
        [
          0.869,
          0.6429
        ],
        [
          0.8794,
          0.6397
        ],
        [
          0.8894,
          0.6347
        ],
        [
          0.8995,
          0.6278
        ],
        [
          0.9099,
          0.6233
        ],
        [
          0.9199,
          0.6183
        ],
        [
          0.9299,
          0.6119
        ],
        [
          0.9399,
          0.6051
        ],
        [
          0.95,
          0.5996
        ],
        [
          0.9604,
          0.5933
        ],
        [
          0.9704,
          0.5896
        ],
        [
          0.9804,
          0.5823
        ],
        [
          0.9904,
          0.5764
        ],
        [
          1.0,
          0.5728
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.11954698131492317,
        "difference": -0.0013792553365715033,
        "worse": 0.64,
        "kept_right": 0.11757352948083859,
        "kept_wrong": 0.001973451834084465,
        "handed_off": 0.8804530186850763
      },
      "usd_per_1000": 4.450834654001147
    },
    {
      "decision": 125,
      "frontier": 193,
      "frontier_accuracy": 0.89171974522293,
      "in_sample": {
        "threshold": 0.9705,
        "answered": 0.062329390354868064,
        "accuracy": 0.89171974522293
      },
      "curve": [
        [
          0.0,
          0.8917
        ],
        [
          0.01,
          0.8917
        ],
        [
          0.02,
          0.8917
        ],
        [
          0.03,
          0.8917
        ],
        [
          0.04,
          0.8917
        ],
        [
          0.05,
          0.8917
        ],
        [
          0.0601,
          0.8917
        ],
        [
          0.071,
          0.8913
        ],
        [
          0.0823,
          0.8913
        ],
        [
          0.0924,
          0.8913
        ],
        [
          0.1033,
          0.8913
        ],
        [
          0.1137,
          0.8913
        ],
        [
          0.1242,
          0.8913
        ],
        [
          0.1342,
          0.8913
        ],
        [
          0.1442,
          0.8908
        ],
        [
          0.1551,
          0.8908
        ],
        [
          0.1656,
          0.8908
        ],
        [
          0.1756,
          0.8899
        ],
        [
          0.1856,
          0.8894
        ],
        [
          0.1956,
          0.8894
        ],
        [
          0.2056,
          0.889
        ],
        [
          0.2157,
          0.8881
        ],
        [
          0.2266,
          0.8867
        ],
        [
          0.237,
          0.8872
        ],
        [
          0.247,
          0.8872
        ],
        [
          0.2571,
          0.884
        ],
        [
          0.2675,
          0.8817
        ],
        [
          0.2775,
          0.8785
        ],
        [
          0.2875,
          0.8767
        ],
        [
          0.2975,
          0.8753
        ],
        [
          0.3076,
          0.8735
        ],
        [
          0.318,
          0.8726
        ],
        [
          0.328,
          0.8699
        ],
        [
          0.338,
          0.8681
        ],
        [
          0.348,
          0.8662
        ],
        [
          0.3581,
          0.864
        ],
        [
          0.3685,
          0.8621
        ],
        [
          0.3785,
          0.8581
        ],
        [
          0.3885,
          0.8549
        ],
        [
          0.3985,
          0.8512
        ],
        [
          0.4086,
          0.848
        ],
        [
          0.419,
          0.8435
        ],
        [
          0.429,
          0.8417
        ],
        [
          0.439,
          0.8371
        ],
        [
          0.449,
          0.8344
        ],
        [
          0.4591,
          0.8303
        ],
        [
          0.4695,
          0.8271
        ],
        [
          0.4795,
          0.8226
        ],
        [
          0.49,
          0.8203
        ],
        [
          0.5005,
          0.8162
        ],
        [
          0.5114,
          0.8121
        ],
        [
          0.5218,
          0.8071
        ],
        [
          0.5318,
          0.8048
        ],
        [
          0.5428,
          0.7989
        ],
        [
          0.5532,
          0.7953
        ],
        [
          0.5637,
          0.7925
        ],
        [
          0.5737,
          0.7907
        ],
        [
          0.5842,
          0.7871
        ],
        [
          0.5946,
          0.783
        ],
        [
          0.6046,
          0.7771
        ],
        [
          0.6146,
          0.773
        ],
        [
          0.6247,
          0.7707
        ],
        [
          0.6351,
          0.7652
        ],
        [
          0.6451,
          0.7611
        ],
        [
          0.6551,
          0.7543
        ],
        [
          0.6652,
          0.7493
        ],
        [
          0.6756,
          0.7443
        ],
        [
          0.6856,
          0.7402
        ],
        [
          0.6956,
          0.7348
        ],
        [
          0.7056,
          0.7307
        ],
        [
          0.7157,
          0.7257
        ],
        [
          0.7261,
          0.7216
        ],
        [
          0.7361,
          0.7143
        ],
        [
          0.7461,
          0.7093
        ],
        [
          0.7561,
          0.7047
        ],
        [
          0.7662,
          0.7015
        ],
        [
          0.7771,
          0.6961
        ],
        [
          0.7875,
          0.692
        ],
        [
          0.7975,
          0.6852
        ],
        [
          0.8076,
          0.6806
        ],
        [
          0.818,
          0.6747
        ],
        [
          0.828,
          0.6683
        ],
        [
          0.838,
          0.6652
        ],
        [
          0.848,
          0.6611
        ],
        [
          0.8585,
          0.6542
        ],
        [
          0.869,
          0.647
        ],
        [
          0.8794,
          0.6415
        ],
        [
          0.8894,
          0.636
        ],
        [
          0.8995,
          0.6292
        ],
        [
          0.9099,
          0.6242
        ],
        [
          0.9199,
          0.6192
        ],
        [
          0.9299,
          0.6128
        ],
        [
          0.9399,
          0.6065
        ],
        [
          0.95,
          0.6005
        ],
        [
          0.9604,
          0.5942
        ],
        [
          0.9704,
          0.5905
        ],
        [
          0.9804,
          0.5837
        ],
        [
          0.9904,
          0.5773
        ],
        [
          1.0,
          0.5728
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1356289140651219,
        "difference": -0.0016885810448841192,
        "worse": 0.73,
        "kept_right": 0.13284886299793044,
        "kept_wrong": 0.002780051067191389,
        "handed_off": 0.864371085934878
      },
      "usd_per_1000": 6.806478124131604
    },
    {
      "decision": 125,
      "frontier": 55,
      "frontier_accuracy": 0.8503184713375797,
      "in_sample": {
        "threshold": 0.9083,
        "answered": 0.24977252047315743,
        "accuracy": 0.8503184713375797
      },
      "curve": [
        [
          0.0,
          0.8503
        ],
        [
          0.01,
          0.8503
        ],
        [
          0.02,
          0.8503
        ],
        [
          0.03,
          0.8508
        ],
        [
          0.04,
          0.8508
        ],
        [
          0.05,
          0.8508
        ],
        [
          0.0601,
          0.8512
        ],
        [
          0.071,
          0.8508
        ],
        [
          0.0823,
          0.8512
        ],
        [
          0.0924,
          0.8517
        ],
        [
          0.1033,
          0.8517
        ],
        [
          0.1137,
          0.8517
        ],
        [
          0.1242,
          0.8517
        ],
        [
          0.1342,
          0.8512
        ],
        [
          0.1442,
          0.8517
        ],
        [
          0.1551,
          0.8517
        ],
        [
          0.1656,
          0.8512
        ],
        [
          0.1756,
          0.8503
        ],
        [
          0.1856,
          0.8508
        ],
        [
          0.1956,
          0.8508
        ],
        [
          0.2056,
          0.8503
        ],
        [
          0.2157,
          0.8499
        ],
        [
          0.2266,
          0.8485
        ],
        [
          0.237,
          0.8499
        ],
        [
          0.247,
          0.8508
        ],
        [
          0.2571,
          0.848
        ],
        [
          0.2675,
          0.8476
        ],
        [
          0.2775,
          0.8449
        ],
        [
          0.2875,
          0.8449
        ],
        [
          0.2975,
          0.8439
        ],
        [
          0.3076,
          0.8435
        ],
        [
          0.318,
          0.8426
        ],
        [
          0.328,
          0.8403
        ],
        [
          0.338,
          0.8394
        ],
        [
          0.348,
          0.8385
        ],
        [
          0.3581,
          0.8371
        ],
        [
          0.3685,
          0.8353
        ],
        [
          0.3785,
          0.833
        ],
        [
          0.3885,
          0.8321
        ],
        [
          0.3985,
          0.8289
        ],
        [
          0.4086,
          0.8262
        ],
        [
          0.419,
          0.8226
        ],
        [
          0.429,
          0.8217
        ],
        [
          0.439,
          0.818
        ],
        [
          0.449,
          0.8157
        ],
        [
          0.4591,
          0.8116
        ],
        [
          0.4695,
          0.8094
        ],
        [
          0.4795,
          0.8057
        ],
        [
          0.49,
          0.8039
        ],
        [
          0.5005,
          0.7994
        ],
        [
          0.5114,
          0.7957
        ],
        [
          0.5218,
          0.7907
        ],
        [
          0.5318,
          0.7889
        ],
        [
          0.5428,
          0.783
        ],
        [
          0.5532,
          0.7798
        ],
        [
          0.5637,
          0.7766
        ],
        [
          0.5737,
          0.7743
        ],
        [
          0.5842,
          0.7707
        ],
        [
          0.5946,
          0.7666
        ],
        [
          0.6046,
          0.7616
        ],
        [
          0.6146,
          0.758
        ],
        [
          0.6247,
          0.7552
        ],
        [
          0.6351,
          0.7507
        ],
        [
          0.6451,
          0.7461
        ],
        [
          0.6551,
          0.7407
        ],
        [
          0.6652,
          0.7357
        ],
        [
          0.6756,
          0.7316
        ],
        [
          0.6856,
          0.7288
        ],
        [
          0.6956,
          0.7229
        ],
        [
          0.7056,
          0.7202
        ],
        [
          0.7157,
          0.7161
        ],
        [
          0.7261,
          0.712
        ],
        [
          0.7361,
          0.7038
        ],
        [
          0.7461,
          0.7015
        ],
        [
          0.7561,
          0.697
        ],
        [
          0.7662,
          0.6943
        ],
        [
          0.7771,
          0.6897
        ],
        [
          0.7875,
          0.6838
        ],
        [
          0.7975,
          0.6783
        ],
        [
          0.8076,
          0.6724
        ],
        [
          0.818,
          0.667
        ],
        [
          0.828,
          0.6606
        ],
        [
          0.838,
          0.6565
        ],
        [
          0.848,
          0.6529
        ],
        [
          0.8585,
          0.646
        ],
        [
          0.869,
          0.6383
        ],
        [
          0.8794,
          0.6338
        ],
        [
          0.8894,
          0.6297
        ],
        [
          0.8995,
          0.6237
        ],
        [
          0.9099,
          0.6183
        ],
        [
          0.9199,
          0.6142
        ],
        [
          0.9299,
          0.6078
        ],
        [
          0.9399,
          0.6024
        ],
        [
          0.95,
          0.5974
        ],
        [
          0.9604,
          0.591
        ],
        [
          0.9704,
          0.5874
        ],
        [
          0.9804,
          0.5814
        ],
        [
          0.9904,
          0.576
        ],
        [
          1.0,
          0.5728
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.22837585420252102,
        "difference": -0.0018651054497957368,
        "worse": 0.4866666666666667,
        "kept_right": 0.21671816254259274,
        "kept_wrong": 0.011657691659928299,
        "handed_off": 0.7716241457974795
      },
      "usd_per_1000": 2.9294574082401503
    },
    {
      "decision": 52,
      "frontier": 140,
      "frontier_accuracy": 0.9808917197452229,
      "in_sample": {
        "threshold": 0.9868238910207704,
        "answered": 0.14194722474977253,
        "accuracy": 0.9808917197452229
      },
      "curve": [
        [
          0.0,
          0.9809
        ],
        [
          0.01,
          0.9809
        ],
        [
          0.02,
          0.9809
        ],
        [
          0.03,
          0.9809
        ],
        [
          0.04,
          0.9809
        ],
        [
          0.05,
          0.9809
        ],
        [
          0.0601,
          0.9809
        ],
        [
          0.0705,
          0.9809
        ],
        [
          0.0805,
          0.9809
        ],
        [
          0.0905,
          0.9809
        ],
        [
          0.1005,
          0.9809
        ],
        [
          0.1106,
          0.9809
        ],
        [
          0.121,
          0.9809
        ],
        [
          0.131,
          0.9809
        ],
        [
          0.141,
          0.9809
        ],
        [
          0.151,
          0.9804
        ],
        [
          0.1611,
          0.98
        ],
        [
          0.1715,
          0.9791
        ],
        [
          0.1815,
          0.9782
        ],
        [
          0.1915,
          0.9782
        ],
        [
          0.2015,
          0.9773
        ],
        [
          0.2116,
          0.9759
        ],
        [
          0.222,
          0.9745
        ],
        [
          0.232,
          0.9732
        ],
        [
          0.242,
          0.9718
        ],
        [
          0.252,
          0.97
        ],
        [
          0.2621,
          0.9686
        ],
        [
          0.2725,
          0.9677
        ],
        [
          0.2825,
          0.9641
        ],
        [
          0.2925,
          0.9622
        ],
        [
          0.3025,
          0.9586
        ],
        [
          0.3126,
          0.9559
        ],
        [
          0.323,
          0.9536
        ],
        [
          0.333,
          0.9504
        ],
        [
          0.343,
          0.9463
        ],
        [
          0.353,
          0.9436
        ],
        [
          0.3631,
          0.9395
        ],
        [
          0.3735,
          0.9354
        ],
        [
          0.3835,
          0.9318
        ],
        [
          0.3935,
          0.9268
        ],
        [
          0.4035,
          0.9227
        ],
        [
          0.4136,
          0.9199
        ],
        [
          0.424,
          0.9158
        ],
        [
          0.434,
          0.9113
        ],
        [
          0.444,
          0.9058
        ],
        [
          0.454,
          0.9013
        ],
        [
          0.4641,
          0.8958
        ],
        [
          0.4745,
          0.8913
        ],
        [
          0.4845,
          0.8872
        ],
        [
          0.4945,
          0.8831
        ],
        [
          0.5045,
          0.879
        ],
        [
          0.5146,
          0.8749
        ],
        [
          0.525,
          0.8708
        ],
        [
          0.535,
          0.8681
        ],
        [
          0.545,
          0.8617
        ],
        [
          0.5551,
          0.8558
        ],
        [
          0.5655,
          0.8508
        ],
        [
          0.5755,
          0.8449
        ],
        [
          0.5855,
          0.8385
        ],
        [
          0.5955,
          0.8339
        ],
        [
          0.6056,
          0.8294
        ],
        [
          0.616,
          0.8239
        ],
        [
          0.626,
          0.8189
        ],
        [
          0.636,
          0.8116
        ],
        [
          0.646,
          0.8053
        ],
        [
          0.6561,
          0.8003
        ],
        [
          0.6665,
          0.7925
        ],
        [
          0.6765,
          0.7866
        ],
        [
          0.6865,
          0.7807
        ],
        [
          0.6965,
          0.7753
        ],
        [
          0.7066,
          0.7693
        ],
        [
          0.717,
          0.7621
        ],
        [
          0.727,
          0.7557
        ],
        [
          0.737,
          0.7489
        ],
        [
          0.747,
          0.742
        ],
        [
          0.7571,
          0.7366
        ],
        [
          0.7675,
          0.7307
        ],
        [
          0.7775,
          0.7252
        ],
        [
          0.7875,
          0.7184
        ],
        [
          0.7975,
          0.7134
        ],
        [
          0.8076,
          0.7075
        ],
        [
          0.818,
          0.6993
        ],
        [
          0.828,
          0.6938
        ],
        [
          0.838,
          0.6856
        ],
        [
          0.848,
          0.677
        ],
        [
          0.8581,
          0.6683
        ],
        [
          0.8685,
          0.6611
        ],
        [
          0.8785,
          0.6533
        ],
        [
          0.8885,
          0.647
        ],
        [
          0.8985,
          0.6401
        ],
        [
          0.9086,
          0.6338
        ],
        [
          0.919,
          0.6256
        ],
        [
          0.929,
          0.6197
        ],
        [
          0.939,
          0.6119
        ],
        [
          0.949,
          0.6056
        ],
        [
          0.9591,
          0.5996
        ],
        [
          0.9695,
          0.5928
        ],
        [
          0.9795,
          0.586
        ],
        [
          0.9895,
          0.5773
        ],
        [
          0.9995,
          0.5705
        ],
        [
          1.0,
          0.5701
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1546471742543022,
        "difference": -0.0009885032587207867,
        "worse": 0.53,
        "kept_right": 0.1531555822625856,
        "kept_wrong": 0.001491591991716564,
        "handed_off": 0.8453528257456979
      },
      "usd_per_1000": 2.0524283634210803
    },
    {
      "decision": 52,
      "frontier": 277,
      "frontier_accuracy": 0.9795268425841674,
      "in_sample": {
        "threshold": 0.9956824052064029,
        "answered": 0.0818926296633303,
        "accuracy": 0.9795268425841674
      },
      "curve": [
        [
          0.0,
          0.9795
        ],
        [
          0.01,
          0.9795
        ],
        [
          0.02,
          0.9795
        ],
        [
          0.03,
          0.9795
        ],
        [
          0.04,
          0.9795
        ],
        [
          0.05,
          0.9795
        ],
        [
          0.0601,
          0.9795
        ],
        [
          0.0705,
          0.9795
        ],
        [
          0.0805,
          0.9795
        ],
        [
          0.0905,
          0.9791
        ],
        [
          0.1005,
          0.9791
        ],
        [
          0.1106,
          0.9791
        ],
        [
          0.121,
          0.9791
        ],
        [
          0.131,
          0.9791
        ],
        [
          0.141,
          0.9791
        ],
        [
          0.151,
          0.9791
        ],
        [
          0.1611,
          0.9786
        ],
        [
          0.1715,
          0.9777
        ],
        [
          0.1815,
          0.9768
        ],
        [
          0.1915,
          0.9763
        ],
        [
          0.2015,
          0.9754
        ],
        [
          0.2116,
          0.9741
        ],
        [
          0.222,
          0.9727
        ],
        [
          0.232,
          0.9718
        ],
        [
          0.242,
          0.9704
        ],
        [
          0.252,
          0.9686
        ],
        [
          0.2621,
          0.9672
        ],
        [
          0.2725,
          0.9663
        ],
        [
          0.2825,
          0.9627
        ],
        [
          0.2925,
          0.9609
        ],
        [
          0.3025,
          0.9572
        ],
        [
          0.3126,
          0.9545
        ],
        [
          0.323,
          0.9522
        ],
        [
          0.333,
          0.949
        ],
        [
          0.343,
          0.9445
        ],
        [
          0.353,
          0.9418
        ],
        [
          0.3631,
          0.9377
        ],
        [
          0.3735,
          0.9336
        ],
        [
          0.3835,
          0.9299
        ],
        [
          0.3935,
          0.9249
        ],
        [
          0.4035,
          0.9208
        ],
        [
          0.4136,
          0.9181
        ],
        [
          0.424,
          0.914
        ],
        [
          0.434,
          0.9095
        ],
        [
          0.444,
          0.9035
        ],
        [
          0.454,
          0.899
        ],
        [
          0.4641,
          0.8935
        ],
        [
          0.4745,
          0.8894
        ],
        [
          0.4845,
          0.8854
        ],
        [
          0.4945,
          0.8813
        ],
        [
          0.5045,
          0.8776
        ],
        [
          0.5146,
          0.8735
        ],
        [
          0.525,
          0.8694
        ],
        [
          0.535,
          0.8662
        ],
        [
          0.545,
          0.8599
        ],
        [
          0.5551,
          0.854
        ],
        [
          0.5655,
          0.849
        ],
        [
          0.5755,
          0.843
        ],
        [
          0.5855,
          0.8362
        ],
        [
          0.5955,
          0.8317
        ],
        [
          0.6056,
          0.8271
        ],
        [
          0.616,
          0.8221
        ],
        [
          0.626,
          0.8167
        ],
        [
          0.636,
          0.8103
        ],
        [
          0.646,
          0.8039
        ],
        [
          0.6561,
          0.7989
        ],
        [
          0.6665,
          0.7912
        ],
        [
          0.6765,
          0.7848
        ],
        [
          0.6865,
          0.7789
        ],
        [
          0.6965,
          0.7734
        ],
        [
          0.7066,
          0.7675
        ],
        [
          0.717,
          0.7602
        ],
        [
          0.727,
          0.7543
        ],
        [
          0.737,
          0.748
        ],
        [
          0.747,
          0.7411
        ],
        [
          0.7571,
          0.7357
        ],
        [
          0.7675,
          0.7288
        ],
        [
          0.7775,
          0.7234
        ],
        [
          0.7875,
          0.7166
        ],
        [
          0.7975,
          0.7116
        ],
        [
          0.8076,
          0.7056
        ],
        [
          0.818,
          0.6979
        ],
        [
          0.828,
          0.6924
        ],
        [
          0.838,
          0.6843
        ],
        [
          0.848,
          0.6756
        ],
        [
          0.8581,
          0.667
        ],
        [
          0.8685,
          0.6601
        ],
        [
          0.8785,
          0.6524
        ],
        [
          0.8885,
          0.6456
        ],
        [
          0.8985,
          0.6388
        ],
        [
          0.9086,
          0.6324
        ],
        [
          0.919,
          0.6242
        ],
        [
          0.929,
          0.6178
        ],
        [
          0.939,
          0.6106
        ],
        [
          0.949,
          0.6042
        ],
        [
          0.9591,
          0.5992
        ],
        [
          0.9695,
          0.5924
        ],
        [
          0.9795,
          0.5855
        ],
        [
          0.9895,
          0.5773
        ],
        [
          0.9995,
          0.5705
        ],
        [
          1.0,
          0.5701
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1267090104527231,
        "difference": -0.0009940176361617119,
        "worse": 0.5266666666666666,
        "kept_right": 0.1254815848248912,
        "kept_wrong": 0.0012274256278319768,
        "handed_off": 0.8732909895472769
      },
      "usd_per_1000": 5.304427241490972
    },
    {
      "decision": 52,
      "frontier": 276,
      "frontier_accuracy": 0.9777070063694268,
      "in_sample": {
        "threshold": 0.9820072119216988,
        "answered": 0.16014558689717925,
        "accuracy": 0.9777070063694268
      },
      "curve": [
        [
          0.0,
          0.9777
        ],
        [
          0.01,
          0.9777
        ],
        [
          0.02,
          0.9777
        ],
        [
          0.03,
          0.9777
        ],
        [
          0.04,
          0.9777
        ],
        [
          0.05,
          0.9777
        ],
        [
          0.0601,
          0.9777
        ],
        [
          0.0705,
          0.9777
        ],
        [
          0.0805,
          0.9777
        ],
        [
          0.0905,
          0.9773
        ],
        [
          0.1005,
          0.9773
        ],
        [
          0.1106,
          0.9773
        ],
        [
          0.121,
          0.9773
        ],
        [
          0.131,
          0.9773
        ],
        [
          0.141,
          0.9777
        ],
        [
          0.151,
          0.9777
        ],
        [
          0.1611,
          0.9773
        ],
        [
          0.1715,
          0.9763
        ],
        [
          0.1815,
          0.9754
        ],
        [
          0.1915,
          0.975
        ],
        [
          0.2015,
          0.9741
        ],
        [
          0.2116,
          0.9727
        ],
        [
          0.222,
          0.9713
        ],
        [
          0.232,
          0.9704
        ],
        [
          0.242,
          0.9691
        ],
        [
          0.252,
          0.9677
        ],
        [
          0.2621,
          0.9663
        ],
        [
          0.2725,
          0.9654
        ],
        [
          0.2825,
          0.9618
        ],
        [
          0.2925,
          0.9604
        ],
        [
          0.3025,
          0.9568
        ],
        [
          0.3126,
          0.954
        ],
        [
          0.323,
          0.9518
        ],
        [
          0.333,
          0.9486
        ],
        [
          0.343,
          0.9445
        ],
        [
          0.353,
          0.9418
        ],
        [
          0.3631,
          0.9377
        ],
        [
          0.3735,
          0.934
        ],
        [
          0.3835,
          0.9304
        ],
        [
          0.3935,
          0.9254
        ],
        [
          0.4035,
          0.9213
        ],
        [
          0.4136,
          0.9186
        ],
        [
          0.424,
          0.9145
        ],
        [
          0.434,
          0.9099
        ],
        [
          0.444,
          0.904
        ],
        [
          0.454,
          0.8995
        ],
        [
          0.4641,
          0.894
        ],
        [
          0.4745,
          0.8894
        ],
        [
          0.4845,
          0.8854
        ],
        [
          0.4945,
          0.8813
        ],
        [
          0.5045,
          0.8776
        ],
        [
          0.5146,
          0.8735
        ],
        [
          0.525,
          0.8694
        ],
        [
          0.535,
          0.8662
        ],
        [
          0.545,
          0.8599
        ],
        [
          0.5551,
          0.854
        ],
        [
          0.5655,
          0.849
        ],
        [
          0.5755,
          0.843
        ],
        [
          0.5855,
          0.8362
        ],
        [
          0.5955,
          0.8317
        ],
        [
          0.6056,
          0.8271
        ],
        [
          0.616,
          0.8226
        ],
        [
          0.626,
          0.8171
        ],
        [
          0.636,
          0.8107
        ],
        [
          0.646,
          0.8044
        ],
        [
          0.6561,
          0.7994
        ],
        [
          0.6665,
          0.7916
        ],
        [
          0.6765,
          0.7853
        ],
        [
          0.6865,
          0.7793
        ],
        [
          0.6965,
          0.7739
        ],
        [
          0.7066,
          0.768
        ],
        [
          0.717,
          0.7607
        ],
        [
          0.727,
          0.7548
        ],
        [
          0.737,
          0.7484
        ],
        [
          0.747,
          0.7416
        ],
        [
          0.7571,
          0.7361
        ],
        [
          0.7675,
          0.7293
        ],
        [
          0.7775,
          0.7238
        ],
        [
          0.7875,
          0.717
        ],
        [
          0.7975,
          0.712
        ],
        [
          0.8076,
          0.7066
        ],
        [
          0.818,
          0.6984
        ],
        [
          0.828,
          0.6929
        ],
        [
          0.838,
          0.6847
        ],
        [
          0.848,
          0.6761
        ],
        [
          0.8581,
          0.6674
        ],
        [
          0.8685,
          0.6601
        ],
        [
          0.8785,
          0.6524
        ],
        [
          0.8885,
          0.6456
        ],
        [
          0.8985,
          0.6392
        ],
        [
          0.9086,
          0.6328
        ],
        [
          0.919,
          0.6247
        ],
        [
          0.929,
          0.6183
        ],
        [
          0.939,
          0.6106
        ],
        [
          0.949,
          0.6042
        ],
        [
          0.9591,
          0.5992
        ],
        [
          0.9695,
          0.5924
        ],
        [
          0.9795,
          0.5855
        ],
        [
          0.9895,
          0.5773
        ],
        [
          0.9995,
          0.5705
        ],
        [
          1.0,
          0.5701
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1493936767086399,
        "difference": -0.0011516625778592105,
        "worse": 0.5,
        "kept_right": 0.14759331075659327,
        "kept_wrong": 0.0018003659520467571,
        "handed_off": 0.8506063232913608
      },
      "usd_per_1000": 1.3872778883198673
    },
    {
      "decision": 52,
      "frontier": 48,
      "frontier_accuracy": 0.9758871701546861,
      "in_sample": {
        "threshold": 0.9868238910207704,
        "answered": 0.14194722474977253,
        "accuracy": 0.9758871701546861
      },
      "curve": [
        [
          0.0,
          0.9759
        ],
        [
          0.01,
          0.9759
        ],
        [
          0.02,
          0.9759
        ],
        [
          0.03,
          0.9759
        ],
        [
          0.04,
          0.9759
        ],
        [
          0.05,
          0.9759
        ],
        [
          0.0601,
          0.9759
        ],
        [
          0.0705,
          0.9759
        ],
        [
          0.0805,
          0.9759
        ],
        [
          0.0905,
          0.9754
        ],
        [
          0.1005,
          0.9754
        ],
        [
          0.1106,
          0.9754
        ],
        [
          0.121,
          0.9754
        ],
        [
          0.131,
          0.9754
        ],
        [
          0.141,
          0.9759
        ],
        [
          0.151,
          0.9754
        ],
        [
          0.1611,
          0.975
        ],
        [
          0.1715,
          0.9741
        ],
        [
          0.1815,
          0.9732
        ],
        [
          0.1915,
          0.9727
        ],
        [
          0.2015,
          0.9718
        ],
        [
          0.2116,
          0.9704
        ],
        [
          0.222,
          0.9691
        ],
        [
          0.232,
          0.9686
        ],
        [
          0.242,
          0.9672
        ],
        [
          0.252,
          0.9654
        ],
        [
          0.2621,
          0.9641
        ],
        [
          0.2725,
          0.9631
        ],
        [
          0.2825,
          0.9595
        ],
        [
          0.2925,
          0.9577
        ],
        [
          0.3025,
          0.9536
        ],
        [
          0.3126,
          0.9509
        ],
        [
          0.323,
          0.9486
        ],
        [
          0.333,
          0.9454
        ],
        [
          0.343,
          0.9409
        ],
        [
          0.353,
          0.9381
        ],
        [
          0.3631,
          0.934
        ],
        [
          0.3735,
          0.9304
        ],
        [
          0.3835,
          0.9268
        ],
        [
          0.3935,
          0.9217
        ],
        [
          0.4035,
          0.9177
        ],
        [
          0.4136,
          0.9149
        ],
        [
          0.424,
          0.9108
        ],
        [
          0.434,
          0.9063
        ],
        [
          0.444,
          0.9004
        ],
        [
          0.454,
          0.8958
        ],
        [
          0.4641,
          0.8904
        ],
        [
          0.4745,
          0.8863
        ],
        [
          0.4845,
          0.8822
        ],
        [
          0.4945,
          0.8781
        ],
        [
          0.5045,
          0.8744
        ],
        [
          0.5146,
          0.8703
        ],
        [
          0.525,
          0.8662
        ],
        [
          0.535,
          0.8631
        ],
        [
          0.545,
          0.8567
        ],
        [
          0.5551,
          0.8508
        ],
        [
          0.5655,
          0.8458
        ],
        [
          0.5755,
          0.8399
        ],
        [
          0.5855,
          0.8335
        ],
        [
          0.5955,
          0.8289
        ],
        [
          0.6056,
          0.8248
        ],
        [
          0.616,
          0.8198
        ],
        [
          0.626,
          0.8148
        ],
        [
          0.636,
          0.808
        ],
        [
          0.646,
          0.8016
        ],
        [
          0.6561,
          0.7971
        ],
        [
          0.6665,
          0.7894
        ],
        [
          0.6765,
          0.783
        ],
        [
          0.6865,
          0.7771
        ],
        [
          0.6965,
          0.7716
        ],
        [
          0.7066,
          0.7657
        ],
        [
          0.717,
          0.7584
        ],
        [
          0.727,
          0.753
        ],
        [
          0.737,
          0.7466
        ],
        [
          0.747,
          0.7398
        ],
        [
          0.7571,
          0.7343
        ],
        [
          0.7675,
          0.7275
        ],
        [
          0.7775,
          0.7225
        ],
        [
          0.7875,
          0.7157
        ],
        [
          0.7975,
          0.7106
        ],
        [
          0.8076,
          0.7052
        ],
        [
          0.818,
          0.697
        ],
        [
          0.828,
          0.692
        ],
        [
          0.838,
          0.6838
        ],
        [
          0.848,
          0.6752
        ],
        [
          0.8581,
          0.6665
        ],
        [
          0.8685,
          0.6597
        ],
        [
          0.8785,
          0.6524
        ],
        [
          0.8885,
          0.6456
        ],
        [
          0.8985,
          0.6392
        ],
        [
          0.9086,
          0.6328
        ],
        [
          0.919,
          0.6247
        ],
        [
          0.929,
          0.6183
        ],
        [
          0.939,
          0.6115
        ],
        [
          0.949,
          0.6051
        ],
        [
          0.9591,
          0.5996
        ],
        [
          0.9695,
          0.5928
        ],
        [
          0.9795,
          0.586
        ],
        [
          0.9895,
          0.5773
        ],
        [
          0.9995,
          0.5705
        ],
        [
          1.0,
          0.5701
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.14262291203062552,
        "difference": -0.0012366600188863115,
        "worse": 0.55,
        "kept_right": 0.14111955199166254,
        "kept_wrong": 0.0015033600389630709,
        "handed_off": 0.8573770879693745
      },
      "usd_per_1000": 4.6186006254992895
    },
    {
      "decision": 52,
      "frontier": 191,
      "frontier_accuracy": 0.9740673339399454,
      "in_sample": {
        "threshold": 0.9868238910207704,
        "answered": 0.14194722474977253,
        "accuracy": 0.9740673339399454
      },
      "curve": [
        [
          0.0,
          0.9741
        ],
        [
          0.01,
          0.9741
        ],
        [
          0.02,
          0.9741
        ],
        [
          0.03,
          0.9741
        ],
        [
          0.04,
          0.9741
        ],
        [
          0.05,
          0.9741
        ],
        [
          0.0601,
          0.9741
        ],
        [
          0.0705,
          0.9741
        ],
        [
          0.0805,
          0.9741
        ],
        [
          0.0905,
          0.9736
        ],
        [
          0.1005,
          0.9736
        ],
        [
          0.1106,
          0.9736
        ],
        [
          0.121,
          0.9736
        ],
        [
          0.131,
          0.9736
        ],
        [
          0.141,
          0.9741
        ],
        [
          0.151,
          0.9736
        ],
        [
          0.1611,
          0.9732
        ],
        [
          0.1715,
          0.9722
        ],
        [
          0.1815,
          0.9713
        ],
        [
          0.1915,
          0.9709
        ],
        [
          0.2015,
          0.97
        ],
        [
          0.2116,
          0.9686
        ],
        [
          0.222,
          0.9672
        ],
        [
          0.232,
          0.9659
        ],
        [
          0.242,
          0.9645
        ],
        [
          0.252,
          0.9631
        ],
        [
          0.2621,
          0.9618
        ],
        [
          0.2725,
          0.9609
        ],
        [
          0.2825,
          0.9572
        ],
        [
          0.2925,
          0.9559
        ],
        [
          0.3025,
          0.9522
        ],
        [
          0.3126,
          0.9495
        ],
        [
          0.323,
          0.9472
        ],
        [
          0.333,
          0.9445
        ],
        [
          0.343,
          0.9399
        ],
        [
          0.353,
          0.9372
        ],
        [
          0.3631,
          0.9331
        ],
        [
          0.3735,
          0.929
        ],
        [
          0.3835,
          0.9254
        ],
        [
          0.3935,
          0.9199
        ],
        [
          0.4035,
          0.9158
        ],
        [
          0.4136,
          0.9131
        ],
        [
          0.424,
          0.909
        ],
        [
          0.434,
          0.9045
        ],
        [
          0.444,
          0.8985
        ],
        [
          0.454,
          0.894
        ],
        [
          0.4641,
          0.8885
        ],
        [
          0.4745,
          0.884
        ],
        [
          0.4845,
          0.8799
        ],
        [
          0.4945,
          0.8758
        ],
        [
          0.5045,
          0.8722
        ],
        [
          0.5146,
          0.8694
        ],
        [
          0.525,
          0.8653
        ],
        [
          0.535,
          0.8621
        ],
        [
          0.545,
          0.8558
        ],
        [
          0.5551,
          0.8503
        ],
        [
          0.5655,
          0.8453
        ],
        [
          0.5755,
          0.8394
        ],
        [
          0.5855,
          0.8335
        ],
        [
          0.5955,
          0.8294
        ],
        [
          0.6056,
          0.8253
        ],
        [
          0.616,
          0.8203
        ],
        [
          0.626,
          0.8153
        ],
        [
          0.636,
          0.808
        ],
        [
          0.646,
          0.8016
        ],
        [
          0.6561,
          0.7971
        ],
        [
          0.6665,
          0.7894
        ],
        [
          0.6765,
          0.783
        ],
        [
          0.6865,
          0.7771
        ],
        [
          0.6965,
          0.7716
        ],
        [
          0.7066,
          0.7657
        ],
        [
          0.717,
          0.7584
        ],
        [
          0.727,
          0.7525
        ],
        [
          0.737,
          0.7461
        ],
        [
          0.747,
          0.7393
        ],
        [
          0.7571,
          0.7338
        ],
        [
          0.7675,
          0.7279
        ],
        [
          0.7775,
          0.7225
        ],
        [
          0.7875,
          0.7157
        ],
        [
          0.7975,
          0.7111
        ],
        [
          0.8076,
          0.7052
        ],
        [
          0.818,
          0.697
        ],
        [
          0.828,
          0.692
        ],
        [
          0.838,
          0.6838
        ],
        [
          0.848,
          0.6752
        ],
        [
          0.8581,
          0.6665
        ],
        [
          0.8685,
          0.6597
        ],
        [
          0.8785,
          0.6524
        ],
        [
          0.8885,
          0.6456
        ],
        [
          0.8985,
          0.6392
        ],
        [
          0.9086,
          0.6328
        ],
        [
          0.919,
          0.6247
        ],
        [
          0.929,
          0.6183
        ],
        [
          0.939,
          0.6106
        ],
        [
          0.949,
          0.6042
        ],
        [
          0.9591,
          0.5992
        ],
        [
          0.9695,
          0.5924
        ],
        [
          0.9795,
          0.5855
        ],
        [
          0.9895,
          0.5773
        ],
        [
          0.9995,
          0.5705
        ],
        [
          1.0,
          0.5701
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.14262291203062552,
        "difference": -0.0012366600188863115,
        "worse": 0.55,
        "kept_right": 0.14111955199166254,
        "kept_wrong": 0.0015033600389630709,
        "handed_off": 0.8573770879693745
      },
      "usd_per_1000": 22.93172068006737
    },
    {
      "decision": 52,
      "frontier": 188,
      "frontier_accuracy": 0.9713375796178344,
      "in_sample": {
        "threshold": 0.9820072119216988,
        "answered": 0.16014558689717925,
        "accuracy": 0.9713375796178344
      },
      "curve": [
        [
          0.0,
          0.9713
        ],
        [
          0.01,
          0.9713
        ],
        [
          0.02,
          0.9713
        ],
        [
          0.03,
          0.9713
        ],
        [
          0.04,
          0.9713
        ],
        [
          0.05,
          0.9713
        ],
        [
          0.0601,
          0.9713
        ],
        [
          0.0705,
          0.9713
        ],
        [
          0.0805,
          0.9713
        ],
        [
          0.0905,
          0.9713
        ],
        [
          0.1005,
          0.9713
        ],
        [
          0.1106,
          0.9713
        ],
        [
          0.121,
          0.9713
        ],
        [
          0.131,
          0.9713
        ],
        [
          0.141,
          0.9713
        ],
        [
          0.151,
          0.9713
        ],
        [
          0.1611,
          0.9709
        ],
        [
          0.1715,
          0.9704
        ],
        [
          0.1815,
          0.9695
        ],
        [
          0.1915,
          0.9695
        ],
        [
          0.2015,
          0.9686
        ],
        [
          0.2116,
          0.9672
        ],
        [
          0.222,
          0.9659
        ],
        [
          0.232,
          0.9645
        ],
        [
          0.242,
          0.9631
        ],
        [
          0.252,
          0.9618
        ],
        [
          0.2621,
          0.9604
        ],
        [
          0.2725,
          0.9595
        ],
        [
          0.2825,
          0.9559
        ],
        [
          0.2925,
          0.954
        ],
        [
          0.3025,
          0.9504
        ],
        [
          0.3126,
          0.9477
        ],
        [
          0.323,
          0.9454
        ],
        [
          0.333,
          0.9422
        ],
        [
          0.343,
          0.9381
        ],
        [
          0.353,
          0.9354
        ],
        [
          0.3631,
          0.9313
        ],
        [
          0.3735,
          0.9272
        ],
        [
          0.3835,
          0.9236
        ],
        [
          0.3935,
          0.9181
        ],
        [
          0.4035,
          0.9145
        ],
        [
          0.4136,
          0.9117
        ],
        [
          0.424,
          0.9072
        ],
        [
          0.434,
          0.9031
        ],
        [
          0.444,
          0.8976
        ],
        [
          0.454,
          0.8931
        ],
        [
          0.4641,
          0.8872
        ],
        [
          0.4745,
          0.8826
        ],
        [
          0.4845,
          0.8794
        ],
        [
          0.4945,
          0.8753
        ],
        [
          0.5045,
          0.8712
        ],
        [
          0.5146,
          0.8676
        ],
        [
          0.525,
          0.8635
        ],
        [
          0.535,
          0.8603
        ],
        [
          0.545,
          0.8544
        ],
        [
          0.5551,
          0.8485
        ],
        [
          0.5655,
          0.8435
        ],
        [
          0.5755,
          0.838
        ],
        [
          0.5855,
          0.8317
        ],
        [
          0.5955,
          0.8271
        ],
        [
          0.6056,
          0.823
        ],
        [
          0.616,
          0.8185
        ],
        [
          0.626,
          0.8139
        ],
        [
          0.636,
          0.8076
        ],
        [
          0.646,
          0.8012
        ],
        [
          0.6561,
          0.7962
        ],
        [
          0.6665,
          0.7884
        ],
        [
          0.6765,
          0.7825
        ],
        [
          0.6865,
          0.7766
        ],
        [
          0.6965,
          0.7712
        ],
        [
          0.7066,
          0.7652
        ],
        [
          0.717,
          0.7584
        ],
        [
          0.727,
          0.752
        ],
        [
          0.737,
          0.7452
        ],
        [
          0.747,
          0.7384
        ],
        [
          0.7571,
          0.7329
        ],
        [
          0.7675,
          0.727
        ],
        [
          0.7775,
          0.7216
        ],
        [
          0.7875,
          0.7147
        ],
        [
          0.7975,
          0.7097
        ],
        [
          0.8076,
          0.7043
        ],
        [
          0.818,
          0.6961
        ],
        [
          0.828,
          0.6906
        ],
        [
          0.838,
          0.6824
        ],
        [
          0.848,
          0.6738
        ],
        [
          0.8581,
          0.6652
        ],
        [
          0.8685,
          0.6583
        ],
        [
          0.8785,
          0.6506
        ],
        [
          0.8885,
          0.6447
        ],
        [
          0.8985,
          0.6388
        ],
        [
          0.9086,
          0.6324
        ],
        [
          0.919,
          0.6242
        ],
        [
          0.929,
          0.6187
        ],
        [
          0.939,
          0.6115
        ],
        [
          0.949,
          0.6051
        ],
        [
          0.9591,
          0.5992
        ],
        [
          0.9695,
          0.5924
        ],
        [
          0.9795,
          0.5855
        ],
        [
          0.9895,
          0.5769
        ],
        [
          0.9995,
          0.5705
        ],
        [
          1.0,
          0.5701
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.17033337321981892,
        "difference": -0.0011823215704077782,
        "worse": 0.6433333333333333,
        "kept_right": 0.16798091488550118,
        "kept_wrong": 0.0023524583343178837,
        "handed_off": 0.8296666267801814
      },
      "usd_per_1000": 0.8067665449265485
    },
    {
      "decision": 52,
      "frontier": 190,
      "frontier_accuracy": 0.9672429481346679,
      "in_sample": {
        "threshold": 0.9786261274873826,
        "answered": 0.17060964513193813,
        "accuracy": 0.9672429481346679
      },
      "curve": [
        [
          0.0,
          0.9672
        ],
        [
          0.01,
          0.9672
        ],
        [
          0.02,
          0.9672
        ],
        [
          0.03,
          0.9672
        ],
        [
          0.04,
          0.9672
        ],
        [
          0.05,
          0.9672
        ],
        [
          0.0601,
          0.9672
        ],
        [
          0.0705,
          0.9672
        ],
        [
          0.0805,
          0.9672
        ],
        [
          0.0905,
          0.9668
        ],
        [
          0.1005,
          0.9668
        ],
        [
          0.1106,
          0.9668
        ],
        [
          0.121,
          0.9668
        ],
        [
          0.131,
          0.9668
        ],
        [
          0.141,
          0.9672
        ],
        [
          0.151,
          0.9672
        ],
        [
          0.1611,
          0.9668
        ],
        [
          0.1715,
          0.9668
        ],
        [
          0.1815,
          0.9659
        ],
        [
          0.1915,
          0.9654
        ],
        [
          0.2015,
          0.9645
        ],
        [
          0.2116,
          0.9631
        ],
        [
          0.222,
          0.9618
        ],
        [
          0.232,
          0.9609
        ],
        [
          0.242,
          0.9595
        ],
        [
          0.252,
          0.9581
        ],
        [
          0.2621,
          0.9568
        ],
        [
          0.2725,
          0.9559
        ],
        [
          0.2825,
          0.9522
        ],
        [
          0.2925,
          0.9509
        ],
        [
          0.3025,
          0.9468
        ],
        [
          0.3126,
          0.944
        ],
        [
          0.323,
          0.9418
        ],
        [
          0.333,
          0.939
        ],
        [
          0.343,
          0.9349
        ],
        [
          0.353,
          0.9322
        ],
        [
          0.3631,
          0.9281
        ],
        [
          0.3735,
          0.924
        ],
        [
          0.3835,
          0.9204
        ],
        [
          0.3935,
          0.9154
        ],
        [
          0.4035,
          0.9113
        ],
        [
          0.4136,
          0.9086
        ],
        [
          0.424,
          0.9045
        ],
        [
          0.434,
          0.9004
        ],
        [
          0.444,
          0.8944
        ],
        [
          0.454,
          0.8899
        ],
        [
          0.4641,
          0.8849
        ],
        [
          0.4745,
          0.8808
        ],
        [
          0.4845,
          0.8767
        ],
        [
          0.4945,
          0.8726
        ],
        [
          0.5045,
          0.869
        ],
        [
          0.5146,
          0.8658
        ],
        [
          0.525,
          0.8617
        ],
        [
          0.535,
          0.8585
        ],
        [
          0.545,
          0.8521
        ],
        [
          0.5551,
          0.8462
        ],
        [
          0.5655,
          0.8417
        ],
        [
          0.5755,
          0.8358
        ],
        [
          0.5855,
          0.8294
        ],
        [
          0.5955,
          0.8248
        ],
        [
          0.6056,
          0.8203
        ],
        [
          0.616,
          0.8153
        ],
        [
          0.626,
          0.8103
        ],
        [
          0.636,
          0.8039
        ],
        [
          0.646,
          0.7975
        ],
        [
          0.6561,
          0.793
        ],
        [
          0.6665,
          0.7853
        ],
        [
          0.6765,
          0.7793
        ],
        [
          0.6865,
          0.7734
        ],
        [
          0.6965,
          0.768
        ],
        [
          0.7066,
          0.7621
        ],
        [
          0.717,
          0.7548
        ],
        [
          0.727,
          0.7493
        ],
        [
          0.737,
          0.7429
        ],
        [
          0.747,
          0.7361
        ],
        [
          0.7571,
          0.7311
        ],
        [
          0.7675,
          0.7252
        ],
        [
          0.7775,
          0.7197
        ],
        [
          0.7875,
          0.7129
        ],
        [
          0.7975,
          0.7084
        ],
        [
          0.8076,
          0.7029
        ],
        [
          0.818,
          0.6952
        ],
        [
          0.828,
          0.6902
        ],
        [
          0.838,
          0.682
        ],
        [
          0.848,
          0.6733
        ],
        [
          0.8581,
          0.6647
        ],
        [
          0.8685,
          0.6583
        ],
        [
          0.8785,
          0.6515
        ],
        [
          0.8885,
          0.6447
        ],
        [
          0.8985,
          0.6383
        ],
        [
          0.9086,
          0.6324
        ],
        [
          0.919,
          0.6242
        ],
        [
          0.929,
          0.6183
        ],
        [
          0.939,
          0.6106
        ],
        [
          0.949,
          0.6042
        ],
        [
          0.9591,
          0.5987
        ],
        [
          0.9695,
          0.5924
        ],
        [
          0.9795,
          0.5855
        ],
        [
          0.9895,
          0.5773
        ],
        [
          0.9995,
          0.5705
        ],
        [
          1.0,
          0.5701
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1654143278215642,
        "difference": -0.0014275088594452384,
        "worse": 0.5233333333333333,
        "kept_right": 0.16265290499507673,
        "kept_wrong": 0.0027614228264874504,
        "handed_off": 0.8345856721784363
      },
      "usd_per_1000": 4.654838856765338
    },
    {
      "decision": 52,
      "frontier": 124,
      "frontier_accuracy": 0.9667879890809827,
      "in_sample": {
        "threshold": 0.9820072119216988,
        "answered": 0.16014558689717925,
        "accuracy": 0.9667879890809827
      },
      "curve": [
        [
          0.0,
          0.9668
        ],
        [
          0.01,
          0.9668
        ],
        [
          0.02,
          0.9668
        ],
        [
          0.03,
          0.9668
        ],
        [
          0.04,
          0.9668
        ],
        [
          0.05,
          0.9668
        ],
        [
          0.0601,
          0.9668
        ],
        [
          0.0705,
          0.9668
        ],
        [
          0.0805,
          0.9668
        ],
        [
          0.0905,
          0.9663
        ],
        [
          0.1005,
          0.9663
        ],
        [
          0.1106,
          0.9663
        ],
        [
          0.121,
          0.9663
        ],
        [
          0.131,
          0.9663
        ],
        [
          0.141,
          0.9668
        ],
        [
          0.151,
          0.9668
        ],
        [
          0.1611,
          0.9663
        ],
        [
          0.1715,
          0.9654
        ],
        [
          0.1815,
          0.9645
        ],
        [
          0.1915,
          0.9641
        ],
        [
          0.2015,
          0.9631
        ],
        [
          0.2116,
          0.9618
        ],
        [
          0.222,
          0.9604
        ],
        [
          0.232,
          0.96
        ],
        [
          0.242,
          0.9586
        ],
        [
          0.252,
          0.9568
        ],
        [
          0.2621,
          0.9554
        ],
        [
          0.2725,
          0.9545
        ],
        [
          0.2825,
          0.9509
        ],
        [
          0.2925,
          0.949
        ],
        [
          0.3025,
          0.9459
        ],
        [
          0.3126,
          0.9431
        ],
        [
          0.323,
          0.9409
        ],
        [
          0.333,
          0.9381
        ],
        [
          0.343,
          0.934
        ],
        [
          0.353,
          0.9313
        ],
        [
          0.3631,
          0.9272
        ],
        [
          0.3735,
          0.9236
        ],
        [
          0.3835,
          0.9199
        ],
        [
          0.3935,
          0.9149
        ],
        [
          0.4035,
          0.9108
        ],
        [
          0.4136,
          0.9081
        ],
        [
          0.424,
          0.9045
        ],
        [
          0.434,
          0.8999
        ],
        [
          0.444,
          0.894
        ],
        [
          0.454,
          0.8894
        ],
        [
          0.4641,
          0.884
        ],
        [
          0.4745,
          0.8799
        ],
        [
          0.4845,
          0.8758
        ],
        [
          0.4945,
          0.8717
        ],
        [
          0.5045,
          0.8681
        ],
        [
          0.5146,
          0.8644
        ],
        [
          0.525,
          0.8608
        ],
        [
          0.535,
          0.8581
        ],
        [
          0.545,
          0.8517
        ],
        [
          0.5551,
          0.8462
        ],
        [
          0.5655,
          0.8417
        ],
        [
          0.5755,
          0.8358
        ],
        [
          0.5855,
          0.8289
        ],
        [
          0.5955,
          0.8244
        ],
        [
          0.6056,
          0.8198
        ],
        [
          0.616,
          0.8148
        ],
        [
          0.626,
          0.8098
        ],
        [
          0.636,
          0.803
        ],
        [
          0.646,
          0.7966
        ],
        [
          0.6561,
          0.7925
        ],
        [
          0.6665,
          0.7853
        ],
        [
          0.6765,
          0.7789
        ],
        [
          0.6865,
          0.773
        ],
        [
          0.6965,
          0.7675
        ],
        [
          0.7066,
          0.7616
        ],
        [
          0.717,
          0.7543
        ],
        [
          0.727,
          0.7484
        ],
        [
          0.737,
          0.742
        ],
        [
          0.747,
          0.7352
        ],
        [
          0.7571,
          0.7298
        ],
        [
          0.7675,
          0.7238
        ],
        [
          0.7775,
          0.7184
        ],
        [
          0.7875,
          0.712
        ],
        [
          0.7975,
          0.7075
        ],
        [
          0.8076,
          0.702
        ],
        [
          0.818,
          0.6943
        ],
        [
          0.828,
          0.6893
        ],
        [
          0.838,
          0.6811
        ],
        [
          0.848,
          0.6724
        ],
        [
          0.8581,
          0.6638
        ],
        [
          0.8685,
          0.6574
        ],
        [
          0.8785,
          0.6501
        ],
        [
          0.8885,
          0.6433
        ],
        [
          0.8985,
          0.6379
        ],
        [
          0.9086,
          0.6315
        ],
        [
          0.919,
          0.6237
        ],
        [
          0.929,
          0.6174
        ],
        [
          0.939,
          0.6101
        ],
        [
          0.949,
          0.6042
        ],
        [
          0.9591,
          0.5983
        ],
        [
          0.9695,
          0.5919
        ],
        [
          0.9795,
          0.5851
        ],
        [
          0.9895,
          0.5769
        ],
        [
          0.9995,
          0.5705
        ],
        [
          1.0,
          0.5701
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1493936767086399,
        "difference": -0.0011516625778592105,
        "worse": 0.5,
        "kept_right": 0.14759331075659327,
        "kept_wrong": 0.0018003659520467571,
        "handed_off": 0.8506063232913608
      },
      "usd_per_1000": 0.9689351476617756
    },
    {
      "decision": 52,
      "frontier": 278,
      "frontier_accuracy": 0.9499545040946314,
      "in_sample": {
        "threshold": 0.979579633823867,
        "answered": 0.16742493175614195,
        "accuracy": 0.9499545040946314
      },
      "curve": [
        [
          0.0,
          0.95
        ],
        [
          0.01,
          0.95
        ],
        [
          0.02,
          0.95
        ],
        [
          0.03,
          0.95
        ],
        [
          0.04,
          0.95
        ],
        [
          0.05,
          0.95
        ],
        [
          0.0601,
          0.95
        ],
        [
          0.0705,
          0.9504
        ],
        [
          0.0805,
          0.9504
        ],
        [
          0.0905,
          0.9504
        ],
        [
          0.1005,
          0.9504
        ],
        [
          0.1106,
          0.9504
        ],
        [
          0.121,
          0.9504
        ],
        [
          0.131,
          0.9504
        ],
        [
          0.141,
          0.9504
        ],
        [
          0.151,
          0.9504
        ],
        [
          0.1611,
          0.95
        ],
        [
          0.1715,
          0.949
        ],
        [
          0.1815,
          0.9481
        ],
        [
          0.1915,
          0.9477
        ],
        [
          0.2015,
          0.9472
        ],
        [
          0.2116,
          0.9468
        ],
        [
          0.222,
          0.9454
        ],
        [
          0.232,
          0.9445
        ],
        [
          0.242,
          0.9436
        ],
        [
          0.252,
          0.9422
        ],
        [
          0.2621,
          0.9409
        ],
        [
          0.2725,
          0.9399
        ],
        [
          0.2825,
          0.9363
        ],
        [
          0.2925,
          0.9345
        ],
        [
          0.3025,
          0.9313
        ],
        [
          0.3126,
          0.9286
        ],
        [
          0.323,
          0.9263
        ],
        [
          0.333,
          0.9236
        ],
        [
          0.343,
          0.9195
        ],
        [
          0.353,
          0.9167
        ],
        [
          0.3631,
          0.9126
        ],
        [
          0.3735,
          0.909
        ],
        [
          0.3835,
          0.9058
        ],
        [
          0.3935,
          0.9008
        ],
        [
          0.4035,
          0.8972
        ],
        [
          0.4136,
          0.8949
        ],
        [
          0.424,
          0.8913
        ],
        [
          0.434,
          0.8867
        ],
        [
          0.444,
          0.8813
        ],
        [
          0.454,
          0.8772
        ],
        [
          0.4641,
          0.8717
        ],
        [
          0.4745,
          0.8672
        ],
        [
          0.4845,
          0.8631
        ],
        [
          0.4945,
          0.859
        ],
        [
          0.5045,
          0.8553
        ],
        [
          0.5146,
          0.8521
        ],
        [
          0.525,
          0.848
        ],
        [
          0.535,
          0.8444
        ],
        [
          0.545,
          0.838
        ],
        [
          0.5551,
          0.8321
        ],
        [
          0.5655,
          0.828
        ],
        [
          0.5755,
          0.8226
        ],
        [
          0.5855,
          0.8162
        ],
        [
          0.5955,
          0.8126
        ],
        [
          0.6056,
          0.8089
        ],
        [
          0.616,
          0.8039
        ],
        [
          0.626,
          0.7989
        ],
        [
          0.636,
          0.793
        ],
        [
          0.646,
          0.7866
        ],
        [
          0.6561,
          0.7821
        ],
        [
          0.6665,
          0.7753
        ],
        [
          0.6765,
          0.7693
        ],
        [
          0.6865,
          0.7634
        ],
        [
          0.6965,
          0.7589
        ],
        [
          0.7066,
          0.7534
        ],
        [
          0.717,
          0.748
        ],
        [
          0.727,
          0.742
        ],
        [
          0.737,
          0.7361
        ],
        [
          0.747,
          0.7302
        ],
        [
          0.7571,
          0.7261
        ],
        [
          0.7675,
          0.7202
        ],
        [
          0.7775,
          0.7147
        ],
        [
          0.7875,
          0.7084
        ],
        [
          0.7975,
          0.7038
        ],
        [
          0.8076,
          0.6988
        ],
        [
          0.818,
          0.6911
        ],
        [
          0.828,
          0.6861
        ],
        [
          0.838,
          0.6783
        ],
        [
          0.848,
          0.6697
        ],
        [
          0.8581,
          0.6615
        ],
        [
          0.8685,
          0.6556
        ],
        [
          0.8785,
          0.6492
        ],
        [
          0.8885,
          0.6424
        ],
        [
          0.8985,
          0.636
        ],
        [
          0.9086,
          0.6297
        ],
        [
          0.919,
          0.6215
        ],
        [
          0.929,
          0.616
        ],
        [
          0.939,
          0.6087
        ],
        [
          0.949,
          0.6024
        ],
        [
          0.9591,
          0.5983
        ],
        [
          0.9695,
          0.5914
        ],
        [
          0.9795,
          0.5846
        ],
        [
          0.9895,
          0.576
        ],
        [
          0.9995,
          0.5705
        ],
        [
          1.0,
          0.5701
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.17148280022451534,
        "difference": -0.0008944176959152624,
        "worse": 0.5066666666666667,
        "kept_right": 0.16909994780262724,
        "kept_wrong": 0.0023828524218882834,
        "handed_off": 0.8285171997754855
      },
      "usd_per_1000": 5.400107206794919
    },
    {
      "decision": 52,
      "frontier": 275,
      "frontier_accuracy": 0.9426751592356688,
      "in_sample": {
        "threshold": 0.979579633823867,
        "answered": 0.16742493175614195,
        "accuracy": 0.9426751592356688
      },
      "curve": [
        [
          0.0,
          0.9427
        ],
        [
          0.01,
          0.9427
        ],
        [
          0.02,
          0.9427
        ],
        [
          0.03,
          0.9427
        ],
        [
          0.04,
          0.9427
        ],
        [
          0.05,
          0.9427
        ],
        [
          0.0601,
          0.9427
        ],
        [
          0.0705,
          0.9427
        ],
        [
          0.0805,
          0.9427
        ],
        [
          0.0905,
          0.9427
        ],
        [
          0.1005,
          0.9427
        ],
        [
          0.1106,
          0.9427
        ],
        [
          0.121,
          0.9427
        ],
        [
          0.131,
          0.9427
        ],
        [
          0.141,
          0.9431
        ],
        [
          0.151,
          0.9431
        ],
        [
          0.1611,
          0.9427
        ],
        [
          0.1715,
          0.9418
        ],
        [
          0.1815,
          0.9409
        ],
        [
          0.1915,
          0.9404
        ],
        [
          0.2015,
          0.9399
        ],
        [
          0.2116,
          0.9395
        ],
        [
          0.222,
          0.9381
        ],
        [
          0.232,
          0.9372
        ],
        [
          0.242,
          0.9359
        ],
        [
          0.252,
          0.9345
        ],
        [
          0.2621,
          0.9331
        ],
        [
          0.2725,
          0.9322
        ],
        [
          0.2825,
          0.929
        ],
        [
          0.2925,
          0.9272
        ],
        [
          0.3025,
          0.924
        ],
        [
          0.3126,
          0.9213
        ],
        [
          0.323,
          0.919
        ],
        [
          0.333,
          0.9167
        ],
        [
          0.343,
          0.9126
        ],
        [
          0.353,
          0.9099
        ],
        [
          0.3631,
          0.9063
        ],
        [
          0.3735,
          0.9026
        ],
        [
          0.3835,
          0.8995
        ],
        [
          0.3935,
          0.8949
        ],
        [
          0.4035,
          0.8908
        ],
        [
          0.4136,
          0.8885
        ],
        [
          0.424,
          0.8849
        ],
        [
          0.434,
          0.8808
        ],
        [
          0.444,
          0.8749
        ],
        [
          0.454,
          0.8703
        ],
        [
          0.4641,
          0.8653
        ],
        [
          0.4745,
          0.8608
        ],
        [
          0.4845,
          0.8567
        ],
        [
          0.4945,
          0.8535
        ],
        [
          0.5045,
          0.8499
        ],
        [
          0.5146,
          0.8467
        ],
        [
          0.525,
          0.8426
        ],
        [
          0.535,
          0.8394
        ],
        [
          0.545,
          0.8335
        ],
        [
          0.5551,
          0.828
        ],
        [
          0.5655,
          0.8239
        ],
        [
          0.5755,
          0.8185
        ],
        [
          0.5855,
          0.813
        ],
        [
          0.5955,
          0.8098
        ],
        [
          0.6056,
          0.8066
        ],
        [
          0.616,
          0.8016
        ],
        [
          0.626,
          0.7962
        ],
        [
          0.636,
          0.7898
        ],
        [
          0.646,
          0.7834
        ],
        [
          0.6561,
          0.7789
        ],
        [
          0.6665,
          0.7721
        ],
        [
          0.6765,
          0.7666
        ],
        [
          0.6865,
          0.7616
        ],
        [
          0.6965,
          0.7561
        ],
        [
          0.7066,
          0.7507
        ],
        [
          0.717,
          0.7443
        ],
        [
          0.727,
          0.7384
        ],
        [
          0.737,
          0.7325
        ],
        [
          0.747,
          0.7257
        ],
        [
          0.7571,
          0.7207
        ],
        [
          0.7675,
          0.7147
        ],
        [
          0.7775,
          0.7097
        ],
        [
          0.7875,
          0.7038
        ],
        [
          0.7975,
          0.6993
        ],
        [
          0.8076,
          0.6947
        ],
        [
          0.818,
          0.6879
        ],
        [
          0.828,
          0.6829
        ],
        [
          0.838,
          0.6752
        ],
        [
          0.848,
          0.6674
        ],
        [
          0.8581,
          0.6592
        ],
        [
          0.8685,
          0.6529
        ],
        [
          0.8785,
          0.646
        ],
        [
          0.8885,
          0.6397
        ],
        [
          0.8985,
          0.6338
        ],
        [
          0.9086,
          0.6274
        ],
        [
          0.919,
          0.6197
        ],
        [
          0.929,
          0.6137
        ],
        [
          0.939,
          0.6074
        ],
        [
          0.949,
          0.6019
        ],
        [
          0.9591,
          0.5974
        ],
        [
          0.9695,
          0.5905
        ],
        [
          0.9795,
          0.5837
        ],
        [
          0.9895,
          0.576
        ],
        [
          0.9995,
          0.5705
        ],
        [
          1.0,
          0.5701
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.17125447065482663,
        "difference": -0.0008791117298272636,
        "worse": 0.5266666666666666,
        "kept_right": 0.16887174515076506,
        "kept_wrong": 0.0023827255040617776,
        "handed_off": 0.8287455293451736
      },
      "usd_per_1000": 0.4338331788445211
    },
    {
      "decision": 52,
      "frontier": 192,
      "frontier_accuracy": 0.9331210191082803,
      "in_sample": {
        "threshold": 0.9820072119216988,
        "answered": 0.16014558689717925,
        "accuracy": 0.9331210191082803
      },
      "curve": [
        [
          0.0,
          0.9331
        ],
        [
          0.01,
          0.9331
        ],
        [
          0.02,
          0.9331
        ],
        [
          0.03,
          0.9331
        ],
        [
          0.04,
          0.9331
        ],
        [
          0.05,
          0.9331
        ],
        [
          0.0601,
          0.9331
        ],
        [
          0.0705,
          0.9331
        ],
        [
          0.0805,
          0.9331
        ],
        [
          0.0905,
          0.9327
        ],
        [
          0.1005,
          0.9327
        ],
        [
          0.1106,
          0.9327
        ],
        [
          0.121,
          0.9327
        ],
        [
          0.131,
          0.9327
        ],
        [
          0.141,
          0.9331
        ],
        [
          0.151,
          0.9331
        ],
        [
          0.1611,
          0.9327
        ],
        [
          0.1715,
          0.9318
        ],
        [
          0.1815,
          0.9308
        ],
        [
          0.1915,
          0.9304
        ],
        [
          0.2015,
          0.9295
        ],
        [
          0.2116,
          0.929
        ],
        [
          0.222,
          0.9277
        ],
        [
          0.232,
          0.9272
        ],
        [
          0.242,
          0.9258
        ],
        [
          0.252,
          0.9245
        ],
        [
          0.2621,
          0.9231
        ],
        [
          0.2725,
          0.9222
        ],
        [
          0.2825,
          0.919
        ],
        [
          0.2925,
          0.9177
        ],
        [
          0.3025,
          0.914
        ],
        [
          0.3126,
          0.9117
        ],
        [
          0.323,
          0.9095
        ],
        [
          0.333,
          0.9067
        ],
        [
          0.343,
          0.9026
        ],
        [
          0.353,
          0.9008
        ],
        [
          0.3631,
          0.8967
        ],
        [
          0.3735,
          0.8931
        ],
        [
          0.3835,
          0.8899
        ],
        [
          0.3935,
          0.8844
        ],
        [
          0.4035,
          0.8813
        ],
        [
          0.4136,
          0.8794
        ],
        [
          0.424,
          0.8758
        ],
        [
          0.434,
          0.8717
        ],
        [
          0.444,
          0.8667
        ],
        [
          0.454,
          0.8631
        ],
        [
          0.4641,
          0.8581
        ],
        [
          0.4745,
          0.8549
        ],
        [
          0.4845,
          0.8508
        ],
        [
          0.4945,
          0.8467
        ],
        [
          0.5045,
          0.843
        ],
        [
          0.5146,
          0.8403
        ],
        [
          0.525,
          0.8367
        ],
        [
          0.535,
          0.8339
        ],
        [
          0.545,
          0.828
        ],
        [
          0.5551,
          0.823
        ],
        [
          0.5655,
          0.8194
        ],
        [
          0.5755,
          0.8135
        ],
        [
          0.5855,
          0.808
        ],
        [
          0.5955,
          0.8057
        ],
        [
          0.6056,
          0.8021
        ],
        [
          0.616,
          0.7971
        ],
        [
          0.626,
          0.7916
        ],
        [
          0.636,
          0.7853
        ],
        [
          0.646,
          0.7789
        ],
        [
          0.6561,
          0.7739
        ],
        [
          0.6665,
          0.7675
        ],
        [
          0.6765,
          0.7621
        ],
        [
          0.6865,
          0.7561
        ],
        [
          0.6965,
          0.752
        ],
        [
          0.7066,
          0.7475
        ],
        [
          0.717,
          0.7425
        ],
        [
          0.727,
          0.7379
        ],
        [
          0.737,
          0.7316
        ],
        [
          0.747,
          0.7261
        ],
        [
          0.7571,
          0.7211
        ],
        [
          0.7675,
          0.7157
        ],
        [
          0.7775,
          0.7106
        ],
        [
          0.7875,
          0.7052
        ],
        [
          0.7975,
          0.7002
        ],
        [
          0.8076,
          0.6947
        ],
        [
          0.818,
          0.6879
        ],
        [
          0.828,
          0.6829
        ],
        [
          0.838,
          0.6747
        ],
        [
          0.848,
          0.667
        ],
        [
          0.8581,
          0.6583
        ],
        [
          0.8685,
          0.6529
        ],
        [
          0.8785,
          0.646
        ],
        [
          0.8885,
          0.6397
        ],
        [
          0.8985,
          0.6333
        ],
        [
          0.9086,
          0.6274
        ],
        [
          0.919,
          0.6197
        ],
        [
          0.929,
          0.6142
        ],
        [
          0.939,
          0.6074
        ],
        [
          0.949,
          0.6015
        ],
        [
          0.9591,
          0.5969
        ],
        [
          0.9695,
          0.5901
        ],
        [
          0.9795,
          0.5833
        ],
        [
          0.9895,
          0.5755
        ],
        [
          0.9995,
          0.5705
        ],
        [
          1.0,
          0.5701
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1496027676177308,
        "difference": -0.0011849959111925438,
        "worse": 0.5,
        "kept_right": 0.14776906833235084,
        "kept_wrong": 0.0018336992853800905,
        "handed_off": 0.8503972323822698
      },
      "usd_per_1000": 9.880174897279337
    },
    {
      "decision": 52,
      "frontier": 123,
      "frontier_accuracy": 0.9326660600545951,
      "in_sample": {
        "threshold": 0.9786261274873826,
        "answered": 0.17060964513193813,
        "accuracy": 0.9326660600545951
      },
      "curve": [
        [
          0.0,
          0.9327
        ],
        [
          0.01,
          0.9327
        ],
        [
          0.02,
          0.9327
        ],
        [
          0.03,
          0.9327
        ],
        [
          0.04,
          0.9327
        ],
        [
          0.05,
          0.9327
        ],
        [
          0.0601,
          0.9327
        ],
        [
          0.0705,
          0.9327
        ],
        [
          0.0805,
          0.9327
        ],
        [
          0.0905,
          0.9327
        ],
        [
          0.1005,
          0.9327
        ],
        [
          0.1106,
          0.9327
        ],
        [
          0.121,
          0.9327
        ],
        [
          0.131,
          0.9327
        ],
        [
          0.141,
          0.9327
        ],
        [
          0.151,
          0.9327
        ],
        [
          0.1611,
          0.9331
        ],
        [
          0.1715,
          0.9322
        ],
        [
          0.1815,
          0.9313
        ],
        [
          0.1915,
          0.9313
        ],
        [
          0.2015,
          0.9304
        ],
        [
          0.2116,
          0.929
        ],
        [
          0.222,
          0.9277
        ],
        [
          0.232,
          0.9268
        ],
        [
          0.242,
          0.9258
        ],
        [
          0.252,
          0.924
        ],
        [
          0.2621,
          0.9231
        ],
        [
          0.2725,
          0.9217
        ],
        [
          0.2825,
          0.9186
        ],
        [
          0.2925,
          0.9167
        ],
        [
          0.3025,
          0.9131
        ],
        [
          0.3126,
          0.9104
        ],
        [
          0.323,
          0.9086
        ],
        [
          0.333,
          0.9063
        ],
        [
          0.343,
          0.9026
        ],
        [
          0.353,
          0.8999
        ],
        [
          0.3631,
          0.8963
        ],
        [
          0.3735,
          0.8935
        ],
        [
          0.3835,
          0.8899
        ],
        [
          0.3935,
          0.8844
        ],
        [
          0.4035,
          0.8813
        ],
        [
          0.4136,
          0.8785
        ],
        [
          0.424,
          0.8749
        ],
        [
          0.434,
          0.8703
        ],
        [
          0.444,
          0.8653
        ],
        [
          0.454,
          0.8608
        ],
        [
          0.4641,
          0.8562
        ],
        [
          0.4745,
          0.8521
        ],
        [
          0.4845,
          0.848
        ],
        [
          0.4945,
          0.8439
        ],
        [
          0.5045,
          0.8403
        ],
        [
          0.5146,
          0.8371
        ],
        [
          0.525,
          0.8335
        ],
        [
          0.535,
          0.8308
        ],
        [
          0.545,
          0.8244
        ],
        [
          0.5551,
          0.8198
        ],
        [
          0.5655,
          0.8153
        ],
        [
          0.5755,
          0.8103
        ],
        [
          0.5855,
          0.8057
        ],
        [
          0.5955,
          0.8021
        ],
        [
          0.6056,
          0.7985
        ],
        [
          0.616,
          0.7944
        ],
        [
          0.626,
          0.7894
        ],
        [
          0.636,
          0.7834
        ],
        [
          0.646,
          0.778
        ],
        [
          0.6561,
          0.773
        ],
        [
          0.6665,
          0.7662
        ],
        [
          0.6765,
          0.7611
        ],
        [
          0.6865,
          0.7552
        ],
        [
          0.6965,
          0.7498
        ],
        [
          0.7066,
          0.7448
        ],
        [
          0.717,
          0.7375
        ],
        [
          0.727,
          0.7316
        ],
        [
          0.737,
          0.7261
        ],
        [
          0.747,
          0.7193
        ],
        [
          0.7571,
          0.7147
        ],
        [
          0.7675,
          0.7088
        ],
        [
          0.7775,
          0.7047
        ],
        [
          0.7875,
          0.6979
        ],
        [
          0.7975,
          0.6938
        ],
        [
          0.8076,
          0.6893
        ],
        [
          0.818,
          0.6811
        ],
        [
          0.828,
          0.6756
        ],
        [
          0.838,
          0.6679
        ],
        [
          0.848,
          0.6601
        ],
        [
          0.8581,
          0.6529
        ],
        [
          0.8685,
          0.6479
        ],
        [
          0.8785,
          0.641
        ],
        [
          0.8885,
          0.6351
        ],
        [
          0.8985,
          0.6297
        ],
        [
          0.9086,
          0.6237
        ],
        [
          0.919,
          0.6174
        ],
        [
          0.929,
          0.6119
        ],
        [
          0.939,
          0.6046
        ],
        [
          0.949,
          0.5987
        ],
        [
          0.9591,
          0.5937
        ],
        [
          0.9695,
          0.5883
        ],
        [
          0.9795,
          0.5837
        ],
        [
          0.9895,
          0.5764
        ],
        [
          0.9995,
          0.5705
        ],
        [
          1.0,
          0.5701
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.17814289898082805,
        "difference": -0.0009943407461621382,
        "worse": 0.53,
        "kept_right": 0.17504465422362833,
        "kept_wrong": 0.0030982447571998345,
        "handed_off": 0.8218571010191718
      },
      "usd_per_1000": 0.22817028794709585
    },
    {
      "decision": 52,
      "frontier": 274,
      "frontier_accuracy": 0.9085532302092811,
      "in_sample": {
        "threshold": 0.9786261274873826,
        "answered": 0.17060964513193813,
        "accuracy": 0.9085532302092811
      },
      "curve": [
        [
          0.0,
          0.9086
        ],
        [
          0.01,
          0.9086
        ],
        [
          0.02,
          0.9086
        ],
        [
          0.03,
          0.9086
        ],
        [
          0.04,
          0.9086
        ],
        [
          0.05,
          0.909
        ],
        [
          0.0601,
          0.909
        ],
        [
          0.0705,
          0.909
        ],
        [
          0.0805,
          0.909
        ],
        [
          0.0905,
          0.909
        ],
        [
          0.1005,
          0.909
        ],
        [
          0.1106,
          0.909
        ],
        [
          0.121,
          0.909
        ],
        [
          0.131,
          0.909
        ],
        [
          0.141,
          0.9095
        ],
        [
          0.151,
          0.9095
        ],
        [
          0.1611,
          0.909
        ],
        [
          0.1715,
          0.9081
        ],
        [
          0.1815,
          0.9072
        ],
        [
          0.1915,
          0.9067
        ],
        [
          0.2015,
          0.9063
        ],
        [
          0.2116,
          0.9058
        ],
        [
          0.222,
          0.9049
        ],
        [
          0.232,
          0.904
        ],
        [
          0.242,
          0.9026
        ],
        [
          0.252,
          0.9022
        ],
        [
          0.2621,
          0.9008
        ],
        [
          0.2725,
          0.8995
        ],
        [
          0.2825,
          0.8967
        ],
        [
          0.2925,
          0.8954
        ],
        [
          0.3025,
          0.8922
        ],
        [
          0.3126,
          0.8894
        ],
        [
          0.323,
          0.8876
        ],
        [
          0.333,
          0.8854
        ],
        [
          0.343,
          0.8817
        ],
        [
          0.353,
          0.8794
        ],
        [
          0.3631,
          0.8758
        ],
        [
          0.3735,
          0.8722
        ],
        [
          0.3835,
          0.8694
        ],
        [
          0.3935,
          0.8649
        ],
        [
          0.4035,
          0.8612
        ],
        [
          0.4136,
          0.8594
        ],
        [
          0.424,
          0.8558
        ],
        [
          0.434,
          0.8521
        ],
        [
          0.444,
          0.8476
        ],
        [
          0.454,
          0.843
        ],
        [
          0.4641,
          0.838
        ],
        [
          0.4745,
          0.8339
        ],
        [
          0.4845,
          0.8303
        ],
        [
          0.4945,
          0.8267
        ],
        [
          0.5045,
          0.8235
        ],
        [
          0.5146,
          0.8207
        ],
        [
          0.525,
          0.8171
        ],
        [
          0.535,
          0.8139
        ],
        [
          0.545,
          0.808
        ],
        [
          0.5551,
          0.803
        ],
        [
          0.5655,
          0.7998
        ],
        [
          0.5755,
          0.7948
        ],
        [
          0.5855,
          0.7898
        ],
        [
          0.5955,
          0.7866
        ],
        [
          0.6056,
          0.783
        ],
        [
          0.616,
          0.7789
        ],
        [
          0.626,
          0.7739
        ],
        [
          0.636,
          0.768
        ],
        [
          0.646,
          0.7621
        ],
        [
          0.6561,
          0.7575
        ],
        [
          0.6665,
          0.7525
        ],
        [
          0.6765,
          0.747
        ],
        [
          0.6865,
          0.742
        ],
        [
          0.6965,
          0.7375
        ],
        [
          0.7066,
          0.7334
        ],
        [
          0.717,
          0.7275
        ],
        [
          0.727,
          0.722
        ],
        [
          0.737,
          0.7161
        ],
        [
          0.747,
          0.7102
        ],
        [
          0.7571,
          0.7061
        ],
        [
          0.7675,
          0.7011
        ],
        [
          0.7775,
          0.697
        ],
        [
          0.7875,
          0.6906
        ],
        [
          0.7975,
          0.6865
        ],
        [
          0.8076,
          0.6829
        ],
        [
          0.818,
          0.6761
        ],
        [
          0.828,
          0.6715
        ],
        [
          0.838,
          0.6642
        ],
        [
          0.848,
          0.6574
        ],
        [
          0.8581,
          0.6492
        ],
        [
          0.8685,
          0.6447
        ],
        [
          0.8785,
          0.6383
        ],
        [
          0.8885,
          0.6315
        ],
        [
          0.8985,
          0.626
        ],
        [
          0.9086,
          0.6206
        ],
        [
          0.919,
          0.6137
        ],
        [
          0.929,
          0.6078
        ],
        [
          0.939,
          0.6015
        ],
        [
          0.949,
          0.5965
        ],
        [
          0.9591,
          0.5928
        ],
        [
          0.9695,
          0.5874
        ],
        [
          0.9795,
          0.5814
        ],
        [
          0.9895,
          0.5746
        ],
        [
          0.9995,
          0.5705
        ],
        [
          1.0,
          0.5701
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.17720302371793883,
        "difference": -0.0008882498948260875,
        "worse": 0.49,
        "kept_right": 0.17421394968183335,
        "kept_wrong": 0.002989074036105609,
        "handed_off": 0.822796976282061
      },
      "usd_per_1000": 0.3581463246036163
    },
    {
      "decision": 52,
      "frontier": 47,
      "frontier_accuracy": 0.9044585987261147,
      "in_sample": {
        "threshold": 0.9820072119216988,
        "answered": 0.16014558689717925,
        "accuracy": 0.9044585987261147
      },
      "curve": [
        [
          0.0,
          0.9045
        ],
        [
          0.01,
          0.9045
        ],
        [
          0.02,
          0.9045
        ],
        [
          0.03,
          0.9045
        ],
        [
          0.04,
          0.9045
        ],
        [
          0.05,
          0.9045
        ],
        [
          0.0601,
          0.9045
        ],
        [
          0.0705,
          0.9045
        ],
        [
          0.0805,
          0.9045
        ],
        [
          0.0905,
          0.904
        ],
        [
          0.1005,
          0.904
        ],
        [
          0.1106,
          0.904
        ],
        [
          0.121,
          0.904
        ],
        [
          0.131,
          0.904
        ],
        [
          0.141,
          0.904
        ],
        [
          0.151,
          0.9045
        ],
        [
          0.1611,
          0.904
        ],
        [
          0.1715,
          0.9031
        ],
        [
          0.1815,
          0.9022
        ],
        [
          0.1915,
          0.9022
        ],
        [
          0.2015,
          0.9013
        ],
        [
          0.2116,
          0.9013
        ],
        [
          0.222,
          0.9004
        ],
        [
          0.232,
          0.8999
        ],
        [
          0.242,
          0.8995
        ],
        [
          0.252,
          0.8981
        ],
        [
          0.2621,
          0.8967
        ],
        [
          0.2725,
          0.8963
        ],
        [
          0.2825,
          0.8935
        ],
        [
          0.2925,
          0.8917
        ],
        [
          0.3025,
          0.8885
        ],
        [
          0.3126,
          0.8858
        ],
        [
          0.323,
          0.884
        ],
        [
          0.333,
          0.8817
        ],
        [
          0.343,
          0.8781
        ],
        [
          0.353,
          0.8763
        ],
        [
          0.3631,
          0.8726
        ],
        [
          0.3735,
          0.8699
        ],
        [
          0.3835,
          0.8672
        ],
        [
          0.3935,
          0.8621
        ],
        [
          0.4035,
          0.859
        ],
        [
          0.4136,
          0.8567
        ],
        [
          0.424,
          0.853
        ],
        [
          0.434,
          0.8485
        ],
        [
          0.444,
          0.8439
        ],
        [
          0.454,
          0.8408
        ],
        [
          0.4641,
          0.8358
        ],
        [
          0.4745,
          0.8317
        ],
        [
          0.4845,
          0.8276
        ],
        [
          0.4945,
          0.8239
        ],
        [
          0.5045,
          0.8217
        ],
        [
          0.5146,
          0.818
        ],
        [
          0.525,
          0.8144
        ],
        [
          0.535,
          0.8126
        ],
        [
          0.545,
          0.8066
        ],
        [
          0.5551,
          0.803
        ],
        [
          0.5655,
          0.7989
        ],
        [
          0.5755,
          0.7944
        ],
        [
          0.5855,
          0.7898
        ],
        [
          0.5955,
          0.7853
        ],
        [
          0.6056,
          0.7816
        ],
        [
          0.616,
          0.7766
        ],
        [
          0.626,
          0.7721
        ],
        [
          0.636,
          0.7657
        ],
        [
          0.646,
          0.7602
        ],
        [
          0.6561,
          0.7557
        ],
        [
          0.6665,
          0.7489
        ],
        [
          0.6765,
          0.7439
        ],
        [
          0.6865,
          0.7379
        ],
        [
          0.6965,
          0.7338
        ],
        [
          0.7066,
          0.7293
        ],
        [
          0.717,
          0.7243
        ],
        [
          0.727,
          0.7193
        ],
        [
          0.737,
          0.7157
        ],
        [
          0.747,
          0.7102
        ],
        [
          0.7571,
          0.7075
        ],
        [
          0.7675,
          0.7011
        ],
        [
          0.7775,
          0.697
        ],
        [
          0.7875,
          0.6906
        ],
        [
          0.7975,
          0.6852
        ],
        [
          0.8076,
          0.6806
        ],
        [
          0.818,
          0.6742
        ],
        [
          0.828,
          0.6702
        ],
        [
          0.838,
          0.6624
        ],
        [
          0.848,
          0.6547
        ],
        [
          0.8581,
          0.6479
        ],
        [
          0.8685,
          0.6429
        ],
        [
          0.8785,
          0.6365
        ],
        [
          0.8885,
          0.6301
        ],
        [
          0.8985,
          0.6242
        ],
        [
          0.9086,
          0.6192
        ],
        [
          0.919,
          0.6115
        ],
        [
          0.929,
          0.6074
        ],
        [
          0.939,
          0.601
        ],
        [
          0.949,
          0.596
        ],
        [
          0.9591,
          0.5928
        ],
        [
          0.9695,
          0.5878
        ],
        [
          0.9795,
          0.5828
        ],
        [
          0.9895,
          0.5755
        ],
        [
          0.9995,
          0.5705
        ],
        [
          1.0,
          0.5701
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.15147141951842374,
        "difference": -0.0012669356566811217,
        "worse": 0.5366666666666666,
        "kept_right": 0.1494254138076976,
        "kept_wrong": 0.002046005710726281,
        "handed_off": 0.8485285804815764
      },
      "usd_per_1000": 4.191175786168616
    },
    {
      "decision": 52,
      "frontier": 193,
      "frontier_accuracy": 0.89171974522293,
      "in_sample": {
        "threshold": 0.9820072119216988,
        "answered": 0.16014558689717925,
        "accuracy": 0.89171974522293
      },
      "curve": [
        [
          0.0,
          0.8917
        ],
        [
          0.01,
          0.8917
        ],
        [
          0.02,
          0.8917
        ],
        [
          0.03,
          0.8917
        ],
        [
          0.04,
          0.8917
        ],
        [
          0.05,
          0.8917
        ],
        [
          0.0601,
          0.8917
        ],
        [
          0.0705,
          0.8917
        ],
        [
          0.0805,
          0.8917
        ],
        [
          0.0905,
          0.8917
        ],
        [
          0.1005,
          0.8917
        ],
        [
          0.1106,
          0.8917
        ],
        [
          0.121,
          0.8917
        ],
        [
          0.131,
          0.8917
        ],
        [
          0.141,
          0.8917
        ],
        [
          0.151,
          0.8917
        ],
        [
          0.1611,
          0.8913
        ],
        [
          0.1715,
          0.8904
        ],
        [
          0.1815,
          0.8894
        ],
        [
          0.1915,
          0.889
        ],
        [
          0.2015,
          0.8885
        ],
        [
          0.2116,
          0.889
        ],
        [
          0.222,
          0.889
        ],
        [
          0.232,
          0.8885
        ],
        [
          0.242,
          0.8881
        ],
        [
          0.252,
          0.8876
        ],
        [
          0.2621,
          0.8867
        ],
        [
          0.2725,
          0.8872
        ],
        [
          0.2825,
          0.884
        ],
        [
          0.2925,
          0.8835
        ],
        [
          0.3025,
          0.8799
        ],
        [
          0.3126,
          0.8776
        ],
        [
          0.323,
          0.8753
        ],
        [
          0.333,
          0.8726
        ],
        [
          0.343,
          0.8699
        ],
        [
          0.353,
          0.8681
        ],
        [
          0.3631,
          0.8644
        ],
        [
          0.3735,
          0.8608
        ],
        [
          0.3835,
          0.8581
        ],
        [
          0.3935,
          0.854
        ],
        [
          0.4035,
          0.8508
        ],
        [
          0.4136,
          0.849
        ],
        [
          0.424,
          0.8449
        ],
        [
          0.434,
          0.8408
        ],
        [
          0.444,
          0.8358
        ],
        [
          0.454,
          0.833
        ],
        [
          0.4641,
          0.8276
        ],
        [
          0.4745,
          0.8244
        ],
        [
          0.4845,
          0.8203
        ],
        [
          0.4945,
          0.8167
        ],
        [
          0.5045,
          0.813
        ],
        [
          0.5146,
          0.8094
        ],
        [
          0.525,
          0.8062
        ],
        [
          0.535,
          0.8039
        ],
        [
          0.545,
          0.7985
        ],
        [
          0.5551,
          0.7948
        ],
        [
          0.5655,
          0.7907
        ],
        [
          0.5755,
          0.7866
        ],
        [
          0.5855,
          0.7807
        ],
        [
          0.5955,
          0.7793
        ],
        [
          0.6056,
          0.7753
        ],
        [
          0.616,
          0.7716
        ],
        [
          0.626,
          0.7666
        ],
        [
          0.636,
          0.7602
        ],
        [
          0.646,
          0.7543
        ],
        [
          0.6561,
          0.7507
        ],
        [
          0.6665,
          0.7443
        ],
        [
          0.6765,
          0.7402
        ],
        [
          0.6865,
          0.7352
        ],
        [
          0.6965,
          0.7311
        ],
        [
          0.7066,
          0.7284
        ],
        [
          0.717,
          0.7234
        ],
        [
          0.727,
          0.7188
        ],
        [
          0.737,
          0.7129
        ],
        [
          0.747,
          0.7075
        ],
        [
          0.7571,
          0.7038
        ],
        [
          0.7675,
          0.6984
        ],
        [
          0.7775,
          0.6934
        ],
        [
          0.7875,
          0.6874
        ],
        [
          0.7975,
          0.6829
        ],
        [
          0.8076,
          0.6788
        ],
        [
          0.818,
          0.6724
        ],
        [
          0.828,
          0.6683
        ],
        [
          0.838,
          0.662
        ],
        [
          0.848,
          0.6547
        ],
        [
          0.8581,
          0.6474
        ],
        [
          0.8685,
          0.6415
        ],
        [
          0.8785,
          0.636
        ],
        [
          0.8885,
          0.6301
        ],
        [
          0.8985,
          0.6242
        ],
        [
          0.9086,
          0.6197
        ],
        [
          0.919,
          0.6124
        ],
        [
          0.929,
          0.6078
        ],
        [
          0.939,
          0.6015
        ],
        [
          0.949,
          0.5955
        ],
        [
          0.9591,
          0.5928
        ],
        [
          0.9695,
          0.5874
        ],
        [
          0.9795,
          0.5823
        ],
        [
          0.9895,
          0.5746
        ],
        [
          0.9995,
          0.5705
        ],
        [
          1.0,
          0.5701
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1787142655065475,
        "difference": -0.002015859843513009,
        "worse": 0.5833333333333334,
        "kept_right": 0.1748856510219132,
        "kept_wrong": 0.003828614484634503,
        "handed_off": 0.8212857344934528
      },
      "usd_per_1000": 6.370417805914358
    },
    {
      "decision": 52,
      "frontier": 55,
      "frontier_accuracy": 0.8503184713375797,
      "in_sample": {
        "threshold": 0.8895799348397195,
        "answered": 0.2966333030027298,
        "accuracy": 0.8503184713375797
      },
      "curve": [
        [
          0.0,
          0.8503
        ],
        [
          0.01,
          0.8503
        ],
        [
          0.02,
          0.8503
        ],
        [
          0.03,
          0.8503
        ],
        [
          0.04,
          0.8503
        ],
        [
          0.05,
          0.8503
        ],
        [
          0.0601,
          0.8508
        ],
        [
          0.0705,
          0.8508
        ],
        [
          0.0805,
          0.8508
        ],
        [
          0.0905,
          0.8512
        ],
        [
          0.1005,
          0.8512
        ],
        [
          0.1106,
          0.8512
        ],
        [
          0.121,
          0.8512
        ],
        [
          0.131,
          0.8517
        ],
        [
          0.141,
          0.8521
        ],
        [
          0.151,
          0.853
        ],
        [
          0.1611,
          0.8535
        ],
        [
          0.1715,
          0.8526
        ],
        [
          0.1815,
          0.853
        ],
        [
          0.1915,
          0.853
        ],
        [
          0.2015,
          0.854
        ],
        [
          0.2116,
          0.8535
        ],
        [
          0.222,
          0.854
        ],
        [
          0.232,
          0.853
        ],
        [
          0.242,
          0.8526
        ],
        [
          0.252,
          0.8535
        ],
        [
          0.2621,
          0.853
        ],
        [
          0.2725,
          0.853
        ],
        [
          0.2825,
          0.8512
        ],
        [
          0.2925,
          0.8508
        ],
        [
          0.3025,
          0.848
        ],
        [
          0.3126,
          0.8471
        ],
        [
          0.323,
          0.8449
        ],
        [
          0.333,
          0.8435
        ],
        [
          0.343,
          0.8417
        ],
        [
          0.353,
          0.8399
        ],
        [
          0.3631,
          0.8367
        ],
        [
          0.3735,
          0.8353
        ],
        [
          0.3835,
          0.8335
        ],
        [
          0.3935,
          0.8289
        ],
        [
          0.4035,
          0.8262
        ],
        [
          0.4136,
          0.8253
        ],
        [
          0.424,
          0.8212
        ],
        [
          0.434,
          0.8189
        ],
        [
          0.444,
          0.8144
        ],
        [
          0.454,
          0.8112
        ],
        [
          0.4641,
          0.8066
        ],
        [
          0.4745,
          0.8039
        ],
        [
          0.4845,
          0.8003
        ],
        [
          0.4945,
          0.7975
        ],
        [
          0.5045,
          0.7944
        ],
        [
          0.5146,
          0.7903
        ],
        [
          0.525,
          0.7871
        ],
        [
          0.535,
          0.7848
        ],
        [
          0.545,
          0.7807
        ],
        [
          0.5551,
          0.7775
        ],
        [
          0.5655,
          0.773
        ],
        [
          0.5755,
          0.7689
        ],
        [
          0.5855,
          0.7643
        ],
        [
          0.5955,
          0.7625
        ],
        [
          0.6056,
          0.7607
        ],
        [
          0.616,
          0.7561
        ],
        [
          0.626,
          0.752
        ],
        [
          0.636,
          0.7452
        ],
        [
          0.646,
          0.7398
        ],
        [
          0.6561,
          0.737
        ],
        [
          0.6665,
          0.7329
        ],
        [
          0.6765,
          0.7288
        ],
        [
          0.6865,
          0.7238
        ],
        [
          0.6965,
          0.7188
        ],
        [
          0.7066,
          0.7161
        ],
        [
          0.717,
          0.712
        ],
        [
          0.727,
          0.7084
        ],
        [
          0.737,
          0.7043
        ],
        [
          0.747,
          0.7006
        ],
        [
          0.7571,
          0.6975
        ],
        [
          0.7675,
          0.692
        ],
        [
          0.7775,
          0.687
        ],
        [
          0.7875,
          0.6811
        ],
        [
          0.7975,
          0.6783
        ],
        [
          0.8076,
          0.6747
        ],
        [
          0.818,
          0.6692
        ],
        [
          0.828,
          0.6642
        ],
        [
          0.838,
          0.6579
        ],
        [
          0.848,
          0.6501
        ],
        [
          0.8581,
          0.6415
        ],
        [
          0.8685,
          0.6369
        ],
        [
          0.8785,
          0.6324
        ],
        [
          0.8885,
          0.6278
        ],
        [
          0.8985,
          0.6228
        ],
        [
          0.9086,
          0.6178
        ],
        [
          0.919,
          0.6124
        ],
        [
          0.929,
          0.6083
        ],
        [
          0.939,
          0.601
        ],
        [
          0.949,
          0.5951
        ],
        [
          0.9591,
          0.5937
        ],
        [
          0.9695,
          0.5878
        ],
        [
          0.9795,
          0.5819
        ],
        [
          0.9895,
          0.5755
        ],
        [
          0.9995,
          0.5705
        ],
        [
          1.0,
          0.5701
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.2925166650887415,
        "difference": -0.001770473496498138,
        "worse": 0.47,
        "kept_right": 0.2689882951151185,
        "kept_wrong": 0.02352836997362297,
        "handed_off": 0.7074833349112586
      },
      "usd_per_1000": 2.592812805975164
    },
    {
      "decision": 334,
      "frontier": 140,
      "frontier_accuracy": 0.9808917197452229,
      "in_sample": {
        "threshold": 0.9027,
        "answered": 0.12192902638762511,
        "accuracy": 0.9808917197452229
      },
      "curve": [
        [
          0.0,
          0.9809
        ],
        [
          0.01,
          0.9809
        ],
        [
          0.0205,
          0.9809
        ],
        [
          0.0314,
          0.9809
        ],
        [
          0.0414,
          0.9809
        ],
        [
          0.0514,
          0.9809
        ],
        [
          0.0614,
          0.9809
        ],
        [
          0.0714,
          0.9809
        ],
        [
          0.0814,
          0.9809
        ],
        [
          0.0914,
          0.9809
        ],
        [
          0.1019,
          0.9809
        ],
        [
          0.1119,
          0.9809
        ],
        [
          0.1219,
          0.9809
        ],
        [
          0.1319,
          0.9804
        ],
        [
          0.1419,
          0.9804
        ],
        [
          0.152,
          0.9804
        ],
        [
          0.1624,
          0.9795
        ],
        [
          0.1724,
          0.9786
        ],
        [
          0.1824,
          0.9786
        ],
        [
          0.1924,
          0.9786
        ],
        [
          0.2025,
          0.9773
        ],
        [
          0.2129,
          0.9768
        ],
        [
          0.2229,
          0.9754
        ],
        [
          0.2329,
          0.9745
        ],
        [
          0.2429,
          0.9713
        ],
        [
          0.253,
          0.9695
        ],
        [
          0.2634,
          0.9672
        ],
        [
          0.2734,
          0.9654
        ],
        [
          0.2834,
          0.9622
        ],
        [
          0.2934,
          0.9586
        ],
        [
          0.3035,
          0.9563
        ],
        [
          0.3139,
          0.954
        ],
        [
          0.3239,
          0.9518
        ],
        [
          0.3339,
          0.949
        ],
        [
          0.3439,
          0.9459
        ],
        [
          0.354,
          0.9404
        ],
        [
          0.3644,
          0.9368
        ],
        [
          0.3744,
          0.9322
        ],
        [
          0.3844,
          0.9299
        ],
        [
          0.3944,
          0.9254
        ],
        [
          0.4049,
          0.9195
        ],
        [
          0.4149,
          0.9167
        ],
        [
          0.4249,
          0.9145
        ],
        [
          0.4349,
          0.9108
        ],
        [
          0.4449,
          0.9054
        ],
        [
          0.455,
          0.9008
        ],
        [
          0.4654,
          0.8963
        ],
        [
          0.4759,
          0.8908
        ],
        [
          0.4864,
          0.8872
        ],
        [
          0.4968,
          0.8835
        ],
        [
          0.5068,
          0.8785
        ],
        [
          0.5177,
          0.8744
        ],
        [
          0.5278,
          0.8699
        ],
        [
          0.5382,
          0.864
        ],
        [
          0.5487,
          0.8581
        ],
        [
          0.5591,
          0.8517
        ],
        [
          0.5692,
          0.848
        ],
        [
          0.5796,
          0.8421
        ],
        [
          0.5896,
          0.8385
        ],
        [
          0.5996,
          0.8326
        ],
        [
          0.6096,
          0.8267
        ],
        [
          0.6197,
          0.8198
        ],
        [
          0.6301,
          0.8116
        ],
        [
          0.6401,
          0.8057
        ],
        [
          0.6501,
          0.7998
        ],
        [
          0.6601,
          0.7939
        ],
        [
          0.6706,
          0.7862
        ],
        [
          0.6806,
          0.7793
        ],
        [
          0.6906,
          0.7712
        ],
        [
          0.7006,
          0.7662
        ],
        [
          0.7106,
          0.7602
        ],
        [
          0.7207,
          0.7552
        ],
        [
          0.7311,
          0.7493
        ],
        [
          0.7416,
          0.7416
        ],
        [
          0.752,
          0.7352
        ],
        [
          0.7621,
          0.7288
        ],
        [
          0.7725,
          0.7207
        ],
        [
          0.7825,
          0.7134
        ],
        [
          0.7925,
          0.7061
        ],
        [
          0.803,
          0.6984
        ],
        [
          0.813,
          0.6915
        ],
        [
          0.8235,
          0.6843
        ],
        [
          0.8348,
          0.6774
        ],
        [
          0.8453,
          0.6697
        ],
        [
          0.8562,
          0.6629
        ],
        [
          0.8662,
          0.657
        ],
        [
          0.8763,
          0.6497
        ],
        [
          0.8867,
          0.6433
        ],
        [
          0.8967,
          0.636
        ],
        [
          0.9067,
          0.6278
        ],
        [
          0.9172,
          0.621
        ],
        [
          0.9272,
          0.6128
        ],
        [
          0.9372,
          0.6056
        ],
        [
          0.9472,
          0.5978
        ],
        [
          0.9572,
          0.5919
        ],
        [
          0.9672,
          0.5833
        ],
        [
          0.9777,
          0.5746
        ],
        [
          0.9877,
          0.5678
        ],
        [
          0.9977,
          0.5591
        ],
        [
          1.0,
          0.5573
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.14268033534271854,
        "difference": -0.000818491852530247,
        "worse": 0.5233333333333333,
        "kept_right": 0.14160415988588124,
        "kept_wrong": 0.0010761754568372902,
        "handed_off": 0.8573196646572815
      },
      "usd_per_1000": 2.093098960905127
    },
    {
      "decision": 334,
      "frontier": 277,
      "frontier_accuracy": 0.9795268425841674,
      "in_sample": {
        "threshold": 0.9027,
        "answered": 0.12192902638762511,
        "accuracy": 0.9795268425841674
      },
      "curve": [
        [
          0.0,
          0.9795
        ],
        [
          0.01,
          0.9795
        ],
        [
          0.0205,
          0.9795
        ],
        [
          0.0314,
          0.9795
        ],
        [
          0.0414,
          0.9795
        ],
        [
          0.0514,
          0.9795
        ],
        [
          0.0614,
          0.9795
        ],
        [
          0.0714,
          0.9795
        ],
        [
          0.0814,
          0.9795
        ],
        [
          0.0914,
          0.9795
        ],
        [
          0.1019,
          0.9795
        ],
        [
          0.1119,
          0.9795
        ],
        [
          0.1219,
          0.9795
        ],
        [
          0.1319,
          0.9786
        ],
        [
          0.1419,
          0.9786
        ],
        [
          0.152,
          0.9786
        ],
        [
          0.1624,
          0.9777
        ],
        [
          0.1724,
          0.9768
        ],
        [
          0.1824,
          0.9768
        ],
        [
          0.1924,
          0.9768
        ],
        [
          0.2025,
          0.9754
        ],
        [
          0.2129,
          0.975
        ],
        [
          0.2229,
          0.9736
        ],
        [
          0.2329,
          0.9722
        ],
        [
          0.2429,
          0.9695
        ],
        [
          0.253,
          0.9677
        ],
        [
          0.2634,
          0.9654
        ],
        [
          0.2734,
          0.9636
        ],
        [
          0.2834,
          0.9609
        ],
        [
          0.2934,
          0.9572
        ],
        [
          0.3035,
          0.9545
        ],
        [
          0.3139,
          0.9522
        ],
        [
          0.3239,
          0.95
        ],
        [
          0.3339,
          0.9472
        ],
        [
          0.3439,
          0.944
        ],
        [
          0.354,
          0.9386
        ],
        [
          0.3644,
          0.9349
        ],
        [
          0.3744,
          0.9308
        ],
        [
          0.3844,
          0.929
        ],
        [
          0.3944,
          0.924
        ],
        [
          0.4049,
          0.9181
        ],
        [
          0.4149,
          0.9149
        ],
        [
          0.4249,
          0.9126
        ],
        [
          0.4349,
          0.909
        ],
        [
          0.4449,
          0.9035
        ],
        [
          0.455,
          0.899
        ],
        [
          0.4654,
          0.8944
        ],
        [
          0.4759,
          0.889
        ],
        [
          0.4864,
          0.8854
        ],
        [
          0.4968,
          0.8813
        ],
        [
          0.5068,
          0.8763
        ],
        [
          0.5177,
          0.8722
        ],
        [
          0.5278,
          0.8676
        ],
        [
          0.5382,
          0.8617
        ],
        [
          0.5487,
          0.8558
        ],
        [
          0.5591,
          0.8494
        ],
        [
          0.5692,
          0.8458
        ],
        [
          0.5796,
          0.8399
        ],
        [
          0.5896,
          0.8362
        ],
        [
          0.5996,
          0.8303
        ],
        [
          0.6096,
          0.8244
        ],
        [
          0.6197,
          0.8176
        ],
        [
          0.6301,
          0.8094
        ],
        [
          0.6401,
          0.8039
        ],
        [
          0.6501,
          0.7975
        ],
        [
          0.6601,
          0.7921
        ],
        [
          0.6706,
          0.7834
        ],
        [
          0.6806,
          0.7766
        ],
        [
          0.6906,
          0.7684
        ],
        [
          0.7006,
          0.7634
        ],
        [
          0.7106,
          0.7575
        ],
        [
          0.7207,
          0.7525
        ],
        [
          0.7311,
          0.747
        ],
        [
          0.7416,
          0.7393
        ],
        [
          0.752,
          0.7329
        ],
        [
          0.7621,
          0.7266
        ],
        [
          0.7725,
          0.7184
        ],
        [
          0.7825,
          0.7116
        ],
        [
          0.7925,
          0.7043
        ],
        [
          0.803,
          0.6965
        ],
        [
          0.813,
          0.6897
        ],
        [
          0.8235,
          0.6824
        ],
        [
          0.8348,
          0.6756
        ],
        [
          0.8453,
          0.6679
        ],
        [
          0.8562,
          0.6611
        ],
        [
          0.8662,
          0.6551
        ],
        [
          0.8763,
          0.6488
        ],
        [
          0.8867,
          0.6429
        ],
        [
          0.8967,
          0.6356
        ],
        [
          0.9067,
          0.6274
        ],
        [
          0.9172,
          0.621
        ],
        [
          0.9272,
          0.6128
        ],
        [
          0.9372,
          0.6056
        ],
        [
          0.9472,
          0.5978
        ],
        [
          0.9572,
          0.5914
        ],
        [
          0.9672,
          0.5833
        ],
        [
          0.9777,
          0.5746
        ],
        [
          0.9877,
          0.5678
        ],
        [
          0.9977,
          0.5591
        ],
        [
          1.0,
          0.5573
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.13427167640525406,
        "difference": -0.000921530397299035,
        "worse": 0.5233333333333333,
        "kept_right": 0.13335014600795506,
        "kept_wrong": 0.000921530397299035,
        "handed_off": 0.8657283235947463
      },
      "usd_per_1000": 5.270277054465094
    },
    {
      "decision": 334,
      "frontier": 276,
      "frontier_accuracy": 0.9777070063694268,
      "in_sample": {
        "threshold": 0.9027,
        "answered": 0.12192902638762511,
        "accuracy": 0.9777070063694268
      },
      "curve": [
        [
          0.0,
          0.9777
        ],
        [
          0.01,
          0.9777
        ],
        [
          0.0205,
          0.9777
        ],
        [
          0.0314,
          0.9777
        ],
        [
          0.0414,
          0.9777
        ],
        [
          0.0514,
          0.9777
        ],
        [
          0.0614,
          0.9777
        ],
        [
          0.0714,
          0.9777
        ],
        [
          0.0814,
          0.9777
        ],
        [
          0.0914,
          0.9777
        ],
        [
          0.1019,
          0.9777
        ],
        [
          0.1119,
          0.9777
        ],
        [
          0.1219,
          0.9777
        ],
        [
          0.1319,
          0.9768
        ],
        [
          0.1419,
          0.9768
        ],
        [
          0.152,
          0.9768
        ],
        [
          0.1624,
          0.9759
        ],
        [
          0.1724,
          0.975
        ],
        [
          0.1824,
          0.975
        ],
        [
          0.1924,
          0.975
        ],
        [
          0.2025,
          0.9736
        ],
        [
          0.2129,
          0.9736
        ],
        [
          0.2229,
          0.9722
        ],
        [
          0.2329,
          0.9709
        ],
        [
          0.2429,
          0.9682
        ],
        [
          0.253,
          0.9668
        ],
        [
          0.2634,
          0.965
        ],
        [
          0.2734,
          0.9631
        ],
        [
          0.2834,
          0.9609
        ],
        [
          0.2934,
          0.9577
        ],
        [
          0.3035,
          0.955
        ],
        [
          0.3139,
          0.9531
        ],
        [
          0.3239,
          0.9509
        ],
        [
          0.3339,
          0.9481
        ],
        [
          0.3439,
          0.9449
        ],
        [
          0.354,
          0.9395
        ],
        [
          0.3644,
          0.9359
        ],
        [
          0.3744,
          0.9318
        ],
        [
          0.3844,
          0.9299
        ],
        [
          0.3944,
          0.9249
        ],
        [
          0.4049,
          0.919
        ],
        [
          0.4149,
          0.9158
        ],
        [
          0.4249,
          0.9136
        ],
        [
          0.4349,
          0.9099
        ],
        [
          0.4449,
          0.9045
        ],
        [
          0.455,
          0.8999
        ],
        [
          0.4654,
          0.8954
        ],
        [
          0.4759,
          0.8899
        ],
        [
          0.4864,
          0.8863
        ],
        [
          0.4968,
          0.8822
        ],
        [
          0.5068,
          0.8772
        ],
        [
          0.5177,
          0.8731
        ],
        [
          0.5278,
          0.8685
        ],
        [
          0.5382,
          0.8626
        ],
        [
          0.5487,
          0.8567
        ],
        [
          0.5591,
          0.8503
        ],
        [
          0.5692,
          0.8467
        ],
        [
          0.5796,
          0.8408
        ],
        [
          0.5896,
          0.8371
        ],
        [
          0.5996,
          0.8312
        ],
        [
          0.6096,
          0.8253
        ],
        [
          0.6197,
          0.8185
        ],
        [
          0.6301,
          0.8103
        ],
        [
          0.6401,
          0.8048
        ],
        [
          0.6501,
          0.7985
        ],
        [
          0.6601,
          0.7925
        ],
        [
          0.6706,
          0.7839
        ],
        [
          0.6806,
          0.7771
        ],
        [
          0.6906,
          0.7689
        ],
        [
          0.7006,
          0.7639
        ],
        [
          0.7106,
          0.758
        ],
        [
          0.7207,
          0.753
        ],
        [
          0.7311,
          0.747
        ],
        [
          0.7416,
          0.7393
        ],
        [
          0.752,
          0.7334
        ],
        [
          0.7621,
          0.727
        ],
        [
          0.7725,
          0.7188
        ],
        [
          0.7825,
          0.7116
        ],
        [
          0.7925,
          0.7043
        ],
        [
          0.803,
          0.6965
        ],
        [
          0.813,
          0.6897
        ],
        [
          0.8235,
          0.6824
        ],
        [
          0.8348,
          0.6756
        ],
        [
          0.8453,
          0.6679
        ],
        [
          0.8562,
          0.6611
        ],
        [
          0.8662,
          0.6551
        ],
        [
          0.8763,
          0.6488
        ],
        [
          0.8867,
          0.6429
        ],
        [
          0.8967,
          0.6356
        ],
        [
          0.9067,
          0.6278
        ],
        [
          0.9172,
          0.6215
        ],
        [
          0.9272,
          0.6133
        ],
        [
          0.9372,
          0.6056
        ],
        [
          0.9472,
          0.5978
        ],
        [
          0.9572,
          0.5914
        ],
        [
          0.9672,
          0.5833
        ],
        [
          0.9777,
          0.5746
        ],
        [
          0.9877,
          0.5678
        ],
        [
          0.9977,
          0.5591
        ],
        [
          1.0,
          0.5573
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1345958890947189,
        "difference": -0.0009427397962176734,
        "worse": 0.5233333333333333,
        "kept_right": 0.1336501244951135,
        "kept_wrong": 0.0009457645996054533,
        "handed_off": 0.8654041109052814
      },
      "usd_per_1000": 1.4230042936019045
    },
    {
      "decision": 334,
      "frontier": 48,
      "frontier_accuracy": 0.9758871701546861,
      "in_sample": {
        "threshold": 0.9027,
        "answered": 0.12192902638762511,
        "accuracy": 0.9758871701546861
      },
      "curve": [
        [
          0.0,
          0.9759
        ],
        [
          0.01,
          0.9759
        ],
        [
          0.0205,
          0.9759
        ],
        [
          0.0314,
          0.9759
        ],
        [
          0.0414,
          0.9759
        ],
        [
          0.0514,
          0.9759
        ],
        [
          0.0614,
          0.9759
        ],
        [
          0.0714,
          0.9759
        ],
        [
          0.0814,
          0.9759
        ],
        [
          0.0914,
          0.9759
        ],
        [
          0.1019,
          0.9759
        ],
        [
          0.1119,
          0.9759
        ],
        [
          0.1219,
          0.9759
        ],
        [
          0.1319,
          0.975
        ],
        [
          0.1419,
          0.975
        ],
        [
          0.152,
          0.975
        ],
        [
          0.1624,
          0.9741
        ],
        [
          0.1724,
          0.9732
        ],
        [
          0.1824,
          0.9732
        ],
        [
          0.1924,
          0.9732
        ],
        [
          0.2025,
          0.9718
        ],
        [
          0.2129,
          0.9718
        ],
        [
          0.2229,
          0.9704
        ],
        [
          0.2329,
          0.9691
        ],
        [
          0.2429,
          0.9659
        ],
        [
          0.253,
          0.9641
        ],
        [
          0.2634,
          0.9618
        ],
        [
          0.2734,
          0.96
        ],
        [
          0.2834,
          0.9577
        ],
        [
          0.2934,
          0.954
        ],
        [
          0.3035,
          0.9513
        ],
        [
          0.3139,
          0.9495
        ],
        [
          0.3239,
          0.9472
        ],
        [
          0.3339,
          0.9445
        ],
        [
          0.3439,
          0.9413
        ],
        [
          0.354,
          0.9359
        ],
        [
          0.3644,
          0.9322
        ],
        [
          0.3744,
          0.9281
        ],
        [
          0.3844,
          0.9263
        ],
        [
          0.3944,
          0.9217
        ],
        [
          0.4049,
          0.9158
        ],
        [
          0.4149,
          0.9126
        ],
        [
          0.4249,
          0.9104
        ],
        [
          0.4349,
          0.9067
        ],
        [
          0.4449,
          0.9013
        ],
        [
          0.455,
          0.8967
        ],
        [
          0.4654,
          0.8922
        ],
        [
          0.4759,
          0.8867
        ],
        [
          0.4864,
          0.8831
        ],
        [
          0.4968,
          0.879
        ],
        [
          0.5068,
          0.874
        ],
        [
          0.5177,
          0.8699
        ],
        [
          0.5278,
          0.8653
        ],
        [
          0.5382,
          0.8594
        ],
        [
          0.5487,
          0.8535
        ],
        [
          0.5591,
          0.8471
        ],
        [
          0.5692,
          0.8435
        ],
        [
          0.5796,
          0.8376
        ],
        [
          0.5896,
          0.8335
        ],
        [
          0.5996,
          0.8276
        ],
        [
          0.6096,
          0.8212
        ],
        [
          0.6197,
          0.8144
        ],
        [
          0.6301,
          0.8066
        ],
        [
          0.6401,
          0.8012
        ],
        [
          0.6501,
          0.7948
        ],
        [
          0.6601,
          0.7894
        ],
        [
          0.6706,
          0.7816
        ],
        [
          0.6806,
          0.7748
        ],
        [
          0.6906,
          0.7666
        ],
        [
          0.7006,
          0.7616
        ],
        [
          0.7106,
          0.7557
        ],
        [
          0.7207,
          0.7507
        ],
        [
          0.7311,
          0.7457
        ],
        [
          0.7416,
          0.7384
        ],
        [
          0.752,
          0.732
        ],
        [
          0.7621,
          0.7257
        ],
        [
          0.7725,
          0.7175
        ],
        [
          0.7825,
          0.7106
        ],
        [
          0.7925,
          0.7034
        ],
        [
          0.803,
          0.6956
        ],
        [
          0.813,
          0.6893
        ],
        [
          0.8235,
          0.682
        ],
        [
          0.8348,
          0.6752
        ],
        [
          0.8453,
          0.6674
        ],
        [
          0.8562,
          0.6606
        ],
        [
          0.8662,
          0.6547
        ],
        [
          0.8763,
          0.6483
        ],
        [
          0.8867,
          0.6424
        ],
        [
          0.8967,
          0.6356
        ],
        [
          0.9067,
          0.6283
        ],
        [
          0.9172,
          0.6219
        ],
        [
          0.9272,
          0.6137
        ],
        [
          0.9372,
          0.606
        ],
        [
          0.9472,
          0.5983
        ],
        [
          0.9572,
          0.5919
        ],
        [
          0.9672,
          0.5833
        ],
        [
          0.9777,
          0.5746
        ],
        [
          0.9877,
          0.5678
        ],
        [
          0.9977,
          0.5591
        ],
        [
          1.0,
          0.5573
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1345958890947189,
        "difference": -0.0009427397962176734,
        "worse": 0.5233333333333333,
        "kept_right": 0.1336501244951135,
        "kept_wrong": 0.0009457645996054533,
        "handed_off": 0.8654041109052814
      },
      "usd_per_1000": 4.673493350551436
    },
    {
      "decision": 334,
      "frontier": 191,
      "frontier_accuracy": 0.9740673339399454,
      "in_sample": {
        "threshold": 0.9027,
        "answered": 0.12192902638762511,
        "accuracy": 0.9740673339399454
      },
      "curve": [
        [
          0.0,
          0.9741
        ],
        [
          0.01,
          0.9741
        ],
        [
          0.0205,
          0.9741
        ],
        [
          0.0314,
          0.9741
        ],
        [
          0.0414,
          0.9741
        ],
        [
          0.0514,
          0.9741
        ],
        [
          0.0614,
          0.9741
        ],
        [
          0.0714,
          0.9741
        ],
        [
          0.0814,
          0.9741
        ],
        [
          0.0914,
          0.9741
        ],
        [
          0.1019,
          0.9741
        ],
        [
          0.1119,
          0.9741
        ],
        [
          0.1219,
          0.9741
        ],
        [
          0.1319,
          0.9732
        ],
        [
          0.1419,
          0.9732
        ],
        [
          0.152,
          0.9732
        ],
        [
          0.1624,
          0.9722
        ],
        [
          0.1724,
          0.9713
        ],
        [
          0.1824,
          0.9713
        ],
        [
          0.1924,
          0.9713
        ],
        [
          0.2025,
          0.97
        ],
        [
          0.2129,
          0.97
        ],
        [
          0.2229,
          0.9686
        ],
        [
          0.2329,
          0.9672
        ],
        [
          0.2429,
          0.9641
        ],
        [
          0.253,
          0.9627
        ],
        [
          0.2634,
          0.9604
        ],
        [
          0.2734,
          0.9586
        ],
        [
          0.2834,
          0.9554
        ],
        [
          0.2934,
          0.9518
        ],
        [
          0.3035,
          0.949
        ],
        [
          0.3139,
          0.9472
        ],
        [
          0.3239,
          0.9449
        ],
        [
          0.3339,
          0.9422
        ],
        [
          0.3439,
          0.939
        ],
        [
          0.354,
          0.9336
        ],
        [
          0.3644,
          0.9299
        ],
        [
          0.3744,
          0.9254
        ],
        [
          0.3844,
          0.9236
        ],
        [
          0.3944,
          0.9186
        ],
        [
          0.4049,
          0.9131
        ],
        [
          0.4149,
          0.9104
        ],
        [
          0.4249,
          0.9081
        ],
        [
          0.4349,
          0.9049
        ],
        [
          0.4449,
          0.8999
        ],
        [
          0.455,
          0.8954
        ],
        [
          0.4654,
          0.8908
        ],
        [
          0.4759,
          0.8854
        ],
        [
          0.4864,
          0.8817
        ],
        [
          0.4968,
          0.8776
        ],
        [
          0.5068,
          0.8726
        ],
        [
          0.5177,
          0.8685
        ],
        [
          0.5278,
          0.864
        ],
        [
          0.5382,
          0.8581
        ],
        [
          0.5487,
          0.8526
        ],
        [
          0.5591,
          0.8462
        ],
        [
          0.5692,
          0.8426
        ],
        [
          0.5796,
          0.8371
        ],
        [
          0.5896,
          0.8335
        ],
        [
          0.5996,
          0.8276
        ],
        [
          0.6096,
          0.8212
        ],
        [
          0.6197,
          0.8144
        ],
        [
          0.6301,
          0.8066
        ],
        [
          0.6401,
          0.8007
        ],
        [
          0.6501,
          0.7944
        ],
        [
          0.6601,
          0.7889
        ],
        [
          0.6706,
          0.7807
        ],
        [
          0.6806,
          0.7739
        ],
        [
          0.6906,
          0.7657
        ],
        [
          0.7006,
          0.7607
        ],
        [
          0.7106,
          0.7548
        ],
        [
          0.7207,
          0.7502
        ],
        [
          0.7311,
          0.7448
        ],
        [
          0.7416,
          0.7375
        ],
        [
          0.752,
          0.7311
        ],
        [
          0.7621,
          0.7247
        ],
        [
          0.7725,
          0.7166
        ],
        [
          0.7825,
          0.7097
        ],
        [
          0.7925,
          0.7025
        ],
        [
          0.803,
          0.6952
        ],
        [
          0.813,
          0.6884
        ],
        [
          0.8235,
          0.6806
        ],
        [
          0.8348,
          0.6738
        ],
        [
          0.8453,
          0.6661
        ],
        [
          0.8562,
          0.6592
        ],
        [
          0.8662,
          0.6533
        ],
        [
          0.8763,
          0.647
        ],
        [
          0.8867,
          0.6415
        ],
        [
          0.8967,
          0.6342
        ],
        [
          0.9067,
          0.6269
        ],
        [
          0.9172,
          0.6206
        ],
        [
          0.9272,
          0.6124
        ],
        [
          0.9372,
          0.6046
        ],
        [
          0.9472,
          0.5969
        ],
        [
          0.9572,
          0.591
        ],
        [
          0.9672,
          0.5828
        ],
        [
          0.9777,
          0.5742
        ],
        [
          0.9877,
          0.5678
        ],
        [
          0.9977,
          0.5591
        ],
        [
          1.0,
          0.5573
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1345958890947189,
        "difference": -0.0009427397962176734,
        "worse": 0.5233333333333333,
        "kept_right": 0.1336501244951135,
        "kept_wrong": 0.0009457645996054533,
        "handed_off": 0.8654041109052814
      },
      "usd_per_1000": 23.158066359448153
    },
    {
      "decision": 334,
      "frontier": 188,
      "frontier_accuracy": 0.9713375796178344,
      "in_sample": {
        "threshold": 0.9027,
        "answered": 0.12192902638762511,
        "accuracy": 0.9713375796178344
      },
      "curve": [
        [
          0.0,
          0.9713
        ],
        [
          0.01,
          0.9713
        ],
        [
          0.0205,
          0.9713
        ],
        [
          0.0314,
          0.9713
        ],
        [
          0.0414,
          0.9713
        ],
        [
          0.0514,
          0.9713
        ],
        [
          0.0614,
          0.9713
        ],
        [
          0.0714,
          0.9713
        ],
        [
          0.0814,
          0.9713
        ],
        [
          0.0914,
          0.9713
        ],
        [
          0.1019,
          0.9713
        ],
        [
          0.1119,
          0.9713
        ],
        [
          0.1219,
          0.9713
        ],
        [
          0.1319,
          0.9709
        ],
        [
          0.1419,
          0.9709
        ],
        [
          0.152,
          0.9709
        ],
        [
          0.1624,
          0.97
        ],
        [
          0.1724,
          0.9695
        ],
        [
          0.1824,
          0.97
        ],
        [
          0.1924,
          0.97
        ],
        [
          0.2025,
          0.9686
        ],
        [
          0.2129,
          0.9682
        ],
        [
          0.2229,
          0.9668
        ],
        [
          0.2329,
          0.9663
        ],
        [
          0.2429,
          0.9636
        ],
        [
          0.253,
          0.9618
        ],
        [
          0.2634,
          0.9595
        ],
        [
          0.2734,
          0.9577
        ],
        [
          0.2834,
          0.9545
        ],
        [
          0.2934,
          0.9509
        ],
        [
          0.3035,
          0.9486
        ],
        [
          0.3139,
          0.9468
        ],
        [
          0.3239,
          0.9445
        ],
        [
          0.3339,
          0.9418
        ],
        [
          0.3439,
          0.9386
        ],
        [
          0.354,
          0.9331
        ],
        [
          0.3644,
          0.929
        ],
        [
          0.3744,
          0.9249
        ],
        [
          0.3844,
          0.9227
        ],
        [
          0.3944,
          0.9181
        ],
        [
          0.4049,
          0.9126
        ],
        [
          0.4149,
          0.9099
        ],
        [
          0.4249,
          0.9076
        ],
        [
          0.4349,
          0.904
        ],
        [
          0.4449,
          0.8985
        ],
        [
          0.455,
          0.894
        ],
        [
          0.4654,
          0.8894
        ],
        [
          0.4759,
          0.884
        ],
        [
          0.4864,
          0.8799
        ],
        [
          0.4968,
          0.8763
        ],
        [
          0.5068,
          0.8712
        ],
        [
          0.5177,
          0.8672
        ],
        [
          0.5278,
          0.8626
        ],
        [
          0.5382,
          0.8567
        ],
        [
          0.5487,
          0.8517
        ],
        [
          0.5591,
          0.8458
        ],
        [
          0.5692,
          0.8421
        ],
        [
          0.5796,
          0.8362
        ],
        [
          0.5896,
          0.8326
        ],
        [
          0.5996,
          0.8267
        ],
        [
          0.6096,
          0.8207
        ],
        [
          0.6197,
          0.8139
        ],
        [
          0.6301,
          0.8062
        ],
        [
          0.6401,
          0.8016
        ],
        [
          0.6501,
          0.7953
        ],
        [
          0.6601,
          0.7894
        ],
        [
          0.6706,
          0.7816
        ],
        [
          0.6806,
          0.7748
        ],
        [
          0.6906,
          0.7666
        ],
        [
          0.7006,
          0.7616
        ],
        [
          0.7106,
          0.7557
        ],
        [
          0.7207,
          0.7507
        ],
        [
          0.7311,
          0.7448
        ],
        [
          0.7416,
          0.737
        ],
        [
          0.752,
          0.7307
        ],
        [
          0.7621,
          0.7243
        ],
        [
          0.7725,
          0.7161
        ],
        [
          0.7825,
          0.7093
        ],
        [
          0.7925,
          0.7025
        ],
        [
          0.803,
          0.6947
        ],
        [
          0.813,
          0.6879
        ],
        [
          0.8235,
          0.6806
        ],
        [
          0.8348,
          0.6738
        ],
        [
          0.8453,
          0.6665
        ],
        [
          0.8562,
          0.6597
        ],
        [
          0.8662,
          0.6538
        ],
        [
          0.8763,
          0.6479
        ],
        [
          0.8867,
          0.6415
        ],
        [
          0.8967,
          0.6342
        ],
        [
          0.9067,
          0.6269
        ],
        [
          0.9172,
          0.6201
        ],
        [
          0.9272,
          0.6119
        ],
        [
          0.9372,
          0.6046
        ],
        [
          0.9472,
          0.5969
        ],
        [
          0.9572,
          0.591
        ],
        [
          0.9672,
          0.5828
        ],
        [
          0.9777,
          0.5751
        ],
        [
          0.9877,
          0.5678
        ],
        [
          0.9977,
          0.5591
        ],
        [
          1.0,
          0.5573
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.15090798496734684,
        "difference": -0.0013490384656589254,
        "worse": 0.5633333333333334,
        "kept_right": 0.1491557639889398,
        "kept_wrong": 0.0017522209784070941,
        "handed_off": 0.8490920150326533
      },
      "usd_per_1000": 0.8372033173488439
    },
    {
      "decision": 334,
      "frontier": 190,
      "frontier_accuracy": 0.9672429481346679,
      "in_sample": {
        "threshold": 0.9027,
        "answered": 0.12192902638762511,
        "accuracy": 0.9672429481346679
      },
      "curve": [
        [
          0.0,
          0.9672
        ],
        [
          0.01,
          0.9672
        ],
        [
          0.0205,
          0.9672
        ],
        [
          0.0314,
          0.9672
        ],
        [
          0.0414,
          0.9672
        ],
        [
          0.0514,
          0.9672
        ],
        [
          0.0614,
          0.9672
        ],
        [
          0.0714,
          0.9672
        ],
        [
          0.0814,
          0.9672
        ],
        [
          0.0914,
          0.9672
        ],
        [
          0.1019,
          0.9672
        ],
        [
          0.1119,
          0.9672
        ],
        [
          0.1219,
          0.9672
        ],
        [
          0.1319,
          0.9663
        ],
        [
          0.1419,
          0.9663
        ],
        [
          0.152,
          0.9663
        ],
        [
          0.1624,
          0.9654
        ],
        [
          0.1724,
          0.965
        ],
        [
          0.1824,
          0.965
        ],
        [
          0.1924,
          0.965
        ],
        [
          0.2025,
          0.9636
        ],
        [
          0.2129,
          0.9641
        ],
        [
          0.2229,
          0.9627
        ],
        [
          0.2329,
          0.9613
        ],
        [
          0.2429,
          0.9586
        ],
        [
          0.253,
          0.9572
        ],
        [
          0.2634,
          0.955
        ],
        [
          0.2734,
          0.9531
        ],
        [
          0.2834,
          0.9509
        ],
        [
          0.2934,
          0.9477
        ],
        [
          0.3035,
          0.9449
        ],
        [
          0.3139,
          0.9431
        ],
        [
          0.3239,
          0.9409
        ],
        [
          0.3339,
          0.9381
        ],
        [
          0.3439,
          0.9349
        ],
        [
          0.354,
          0.9299
        ],
        [
          0.3644,
          0.9263
        ],
        [
          0.3744,
          0.9217
        ],
        [
          0.3844,
          0.9199
        ],
        [
          0.3944,
          0.9149
        ],
        [
          0.4049,
          0.9095
        ],
        [
          0.4149,
          0.9063
        ],
        [
          0.4249,
          0.904
        ],
        [
          0.4349,
          0.9004
        ],
        [
          0.4449,
          0.8954
        ],
        [
          0.455,
          0.8908
        ],
        [
          0.4654,
          0.8863
        ],
        [
          0.4759,
          0.8808
        ],
        [
          0.4864,
          0.8772
        ],
        [
          0.4968,
          0.8731
        ],
        [
          0.5068,
          0.8681
        ],
        [
          0.5177,
          0.864
        ],
        [
          0.5278,
          0.8594
        ],
        [
          0.5382,
          0.8535
        ],
        [
          0.5487,
          0.8476
        ],
        [
          0.5591,
          0.8412
        ],
        [
          0.5692,
          0.8376
        ],
        [
          0.5796,
          0.8321
        ],
        [
          0.5896,
          0.828
        ],
        [
          0.5996,
          0.8221
        ],
        [
          0.6096,
          0.8162
        ],
        [
          0.6197,
          0.8094
        ],
        [
          0.6301,
          0.8012
        ],
        [
          0.6401,
          0.7957
        ],
        [
          0.6501,
          0.7894
        ],
        [
          0.6601,
          0.7843
        ],
        [
          0.6706,
          0.7762
        ],
        [
          0.6806,
          0.7693
        ],
        [
          0.6906,
          0.7611
        ],
        [
          0.7006,
          0.7561
        ],
        [
          0.7106,
          0.7502
        ],
        [
          0.7207,
          0.7461
        ],
        [
          0.7311,
          0.7416
        ],
        [
          0.7416,
          0.7348
        ],
        [
          0.752,
          0.7288
        ],
        [
          0.7621,
          0.7225
        ],
        [
          0.7725,
          0.7143
        ],
        [
          0.7825,
          0.7075
        ],
        [
          0.7925,
          0.7002
        ],
        [
          0.803,
          0.6924
        ],
        [
          0.813,
          0.6856
        ],
        [
          0.8235,
          0.6783
        ],
        [
          0.8348,
          0.6715
        ],
        [
          0.8453,
          0.6638
        ],
        [
          0.8562,
          0.6574
        ],
        [
          0.8662,
          0.6515
        ],
        [
          0.8763,
          0.6451
        ],
        [
          0.8867,
          0.6401
        ],
        [
          0.8967,
          0.6328
        ],
        [
          0.9067,
          0.626
        ],
        [
          0.9172,
          0.6201
        ],
        [
          0.9272,
          0.6119
        ],
        [
          0.9372,
          0.6046
        ],
        [
          0.9472,
          0.5969
        ],
        [
          0.9572,
          0.591
        ],
        [
          0.9672,
          0.5828
        ],
        [
          0.9777,
          0.5742
        ],
        [
          0.9877,
          0.5678
        ],
        [
          0.9977,
          0.5591
        ],
        [
          1.0,
          0.5573
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1401421104351868,
        "difference": -0.0013185255214798774,
        "worse": 0.5333333333333333,
        "kept_right": 0.13877508619338008,
        "kept_wrong": 0.0013670242418067373,
        "handed_off": 0.8598578895648136
      },
      "usd_per_1000": 4.807289259524731
    },
    {
      "decision": 334,
      "frontier": 124,
      "frontier_accuracy": 0.9667879890809827,
      "in_sample": {
        "threshold": 0.9027,
        "answered": 0.12192902638762511,
        "accuracy": 0.9667879890809827
      },
      "curve": [
        [
          0.0,
          0.9668
        ],
        [
          0.01,
          0.9668
        ],
        [
          0.0205,
          0.9668
        ],
        [
          0.0314,
          0.9668
        ],
        [
          0.0414,
          0.9668
        ],
        [
          0.0514,
          0.9668
        ],
        [
          0.0614,
          0.9668
        ],
        [
          0.0714,
          0.9668
        ],
        [
          0.0814,
          0.9668
        ],
        [
          0.0914,
          0.9668
        ],
        [
          0.1019,
          0.9668
        ],
        [
          0.1119,
          0.9668
        ],
        [
          0.1219,
          0.9668
        ],
        [
          0.1319,
          0.9659
        ],
        [
          0.1419,
          0.9659
        ],
        [
          0.152,
          0.9659
        ],
        [
          0.1624,
          0.965
        ],
        [
          0.1724,
          0.9641
        ],
        [
          0.1824,
          0.9641
        ],
        [
          0.1924,
          0.9641
        ],
        [
          0.2025,
          0.9627
        ],
        [
          0.2129,
          0.9627
        ],
        [
          0.2229,
          0.9613
        ],
        [
          0.2329,
          0.96
        ],
        [
          0.2429,
          0.9572
        ],
        [
          0.253,
          0.9554
        ],
        [
          0.2634,
          0.9531
        ],
        [
          0.2734,
          0.9513
        ],
        [
          0.2834,
          0.949
        ],
        [
          0.2934,
          0.9459
        ],
        [
          0.3035,
          0.9431
        ],
        [
          0.3139,
          0.9409
        ],
        [
          0.3239,
          0.9386
        ],
        [
          0.3339,
          0.9359
        ],
        [
          0.3439,
          0.9327
        ],
        [
          0.354,
          0.9272
        ],
        [
          0.3644,
          0.9236
        ],
        [
          0.3744,
          0.9195
        ],
        [
          0.3844,
          0.9177
        ],
        [
          0.3944,
          0.9136
        ],
        [
          0.4049,
          0.9081
        ],
        [
          0.4149,
          0.9049
        ],
        [
          0.4249,
          0.9026
        ],
        [
          0.4349,
          0.8995
        ],
        [
          0.4449,
          0.8944
        ],
        [
          0.455,
          0.8899
        ],
        [
          0.4654,
          0.8854
        ],
        [
          0.4759,
          0.8799
        ],
        [
          0.4864,
          0.8763
        ],
        [
          0.4968,
          0.8722
        ],
        [
          0.5068,
          0.8672
        ],
        [
          0.5177,
          0.8635
        ],
        [
          0.5278,
          0.859
        ],
        [
          0.5382,
          0.853
        ],
        [
          0.5487,
          0.8476
        ],
        [
          0.5591,
          0.8412
        ],
        [
          0.5692,
          0.8376
        ],
        [
          0.5796,
          0.8317
        ],
        [
          0.5896,
          0.8285
        ],
        [
          0.5996,
          0.8226
        ],
        [
          0.6096,
          0.8167
        ],
        [
          0.6197,
          0.8098
        ],
        [
          0.6301,
          0.8016
        ],
        [
          0.6401,
          0.7957
        ],
        [
          0.6501,
          0.7898
        ],
        [
          0.6601,
          0.7843
        ],
        [
          0.6706,
          0.7762
        ],
        [
          0.6806,
          0.7693
        ],
        [
          0.6906,
          0.7611
        ],
        [
          0.7006,
          0.7561
        ],
        [
          0.7106,
          0.7502
        ],
        [
          0.7207,
          0.7457
        ],
        [
          0.7311,
          0.7402
        ],
        [
          0.7416,
          0.7334
        ],
        [
          0.752,
          0.7275
        ],
        [
          0.7621,
          0.7211
        ],
        [
          0.7725,
          0.7129
        ],
        [
          0.7825,
          0.7061
        ],
        [
          0.7925,
          0.6993
        ],
        [
          0.803,
          0.6915
        ],
        [
          0.813,
          0.6847
        ],
        [
          0.8235,
          0.6774
        ],
        [
          0.8348,
          0.6706
        ],
        [
          0.8453,
          0.6629
        ],
        [
          0.8562,
          0.6561
        ],
        [
          0.8662,
          0.6501
        ],
        [
          0.8763,
          0.6451
        ],
        [
          0.8867,
          0.6392
        ],
        [
          0.8967,
          0.6324
        ],
        [
          0.9067,
          0.6256
        ],
        [
          0.9172,
          0.6192
        ],
        [
          0.9272,
          0.6115
        ],
        [
          0.9372,
          0.6042
        ],
        [
          0.9472,
          0.5969
        ],
        [
          0.9572,
          0.5914
        ],
        [
          0.9672,
          0.5833
        ],
        [
          0.9777,
          0.5746
        ],
        [
          0.9877,
          0.5678
        ],
        [
          0.9977,
          0.5591
        ],
        [
          1.0,
          0.5573
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1345958890947189,
        "difference": -0.0009427397962176734,
        "worse": 0.5233333333333333,
        "kept_right": 0.1336501244951135,
        "kept_wrong": 0.0009457645996054533,
        "handed_off": 0.8654041109052814
      },
      "usd_per_1000": 0.9973837478575184
    },
    {
      "decision": 334,
      "frontier": 278,
      "frontier_accuracy": 0.9499545040946314,
      "in_sample": {
        "threshold": 0.9027,
        "answered": 0.12192902638762511,
        "accuracy": 0.9499545040946314
      },
      "curve": [
        [
          0.0,
          0.95
        ],
        [
          0.01,
          0.95
        ],
        [
          0.0205,
          0.95
        ],
        [
          0.0314,
          0.95
        ],
        [
          0.0414,
          0.95
        ],
        [
          0.0514,
          0.95
        ],
        [
          0.0614,
          0.95
        ],
        [
          0.0714,
          0.95
        ],
        [
          0.0814,
          0.95
        ],
        [
          0.0914,
          0.95
        ],
        [
          0.1019,
          0.95
        ],
        [
          0.1119,
          0.95
        ],
        [
          0.1219,
          0.95
        ],
        [
          0.1319,
          0.949
        ],
        [
          0.1419,
          0.949
        ],
        [
          0.152,
          0.949
        ],
        [
          0.1624,
          0.9481
        ],
        [
          0.1724,
          0.9477
        ],
        [
          0.1824,
          0.9477
        ],
        [
          0.1924,
          0.9477
        ],
        [
          0.2025,
          0.9468
        ],
        [
          0.2129,
          0.9463
        ],
        [
          0.2229,
          0.9449
        ],
        [
          0.2329,
          0.9445
        ],
        [
          0.2429,
          0.9418
        ],
        [
          0.253,
          0.9404
        ],
        [
          0.2634,
          0.9381
        ],
        [
          0.2734,
          0.9368
        ],
        [
          0.2834,
          0.9336
        ],
        [
          0.2934,
          0.9304
        ],
        [
          0.3035,
          0.9281
        ],
        [
          0.3139,
          0.9263
        ],
        [
          0.3239,
          0.924
        ],
        [
          0.3339,
          0.9213
        ],
        [
          0.3439,
          0.9181
        ],
        [
          0.354,
          0.9131
        ],
        [
          0.3644,
          0.9095
        ],
        [
          0.3744,
          0.9054
        ],
        [
          0.3844,
          0.9035
        ],
        [
          0.3944,
          0.8995
        ],
        [
          0.4049,
          0.8944
        ],
        [
          0.4149,
          0.8922
        ],
        [
          0.4249,
          0.8899
        ],
        [
          0.4349,
          0.8872
        ],
        [
          0.4449,
          0.8826
        ],
        [
          0.455,
          0.8781
        ],
        [
          0.4654,
          0.8744
        ],
        [
          0.4759,
          0.869
        ],
        [
          0.4864,
          0.8653
        ],
        [
          0.4968,
          0.8617
        ],
        [
          0.5068,
          0.8567
        ],
        [
          0.5177,
          0.853
        ],
        [
          0.5278,
          0.8494
        ],
        [
          0.5382,
          0.8439
        ],
        [
          0.5487,
          0.8389
        ],
        [
          0.5591,
          0.8326
        ],
        [
          0.5692,
          0.8289
        ],
        [
          0.5796,
          0.823
        ],
        [
          0.5896,
          0.8203
        ],
        [
          0.5996,
          0.8144
        ],
        [
          0.6096,
          0.8089
        ],
        [
          0.6197,
          0.803
        ],
        [
          0.6301,
          0.7953
        ],
        [
          0.6401,
          0.7903
        ],
        [
          0.6501,
          0.7848
        ],
        [
          0.6601,
          0.7793
        ],
        [
          0.6706,
          0.7716
        ],
        [
          0.6806,
          0.7648
        ],
        [
          0.6906,
          0.7566
        ],
        [
          0.7006,
          0.752
        ],
        [
          0.7106,
          0.7466
        ],
        [
          0.7207,
          0.742
        ],
        [
          0.7311,
          0.737
        ],
        [
          0.7416,
          0.7298
        ],
        [
          0.752,
          0.7238
        ],
        [
          0.7621,
          0.7175
        ],
        [
          0.7725,
          0.7097
        ],
        [
          0.7825,
          0.7034
        ],
        [
          0.7925,
          0.6961
        ],
        [
          0.803,
          0.6888
        ],
        [
          0.813,
          0.6824
        ],
        [
          0.8235,
          0.6752
        ],
        [
          0.8348,
          0.6688
        ],
        [
          0.8453,
          0.662
        ],
        [
          0.8562,
          0.6547
        ],
        [
          0.8662,
          0.6488
        ],
        [
          0.8763,
          0.6429
        ],
        [
          0.8867,
          0.6379
        ],
        [
          0.8967,
          0.6306
        ],
        [
          0.9067,
          0.6251
        ],
        [
          0.9172,
          0.6187
        ],
        [
          0.9272,
          0.611
        ],
        [
          0.9372,
          0.6042
        ],
        [
          0.9472,
          0.5969
        ],
        [
          0.9572,
          0.591
        ],
        [
          0.9672,
          0.5828
        ],
        [
          0.9777,
          0.5742
        ],
        [
          0.9877,
          0.5678
        ],
        [
          0.9977,
          0.5591
        ],
        [
          1.0,
          0.5573
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.13936470028343467,
        "difference": -0.0012186439617988215,
        "worse": 0.5566666666666666,
        "kept_right": 0.13809148870313465,
        "kept_wrong": 0.0012732115803000053,
        "handed_off": 0.8606352997165658
      },
      "usd_per_1000": 5.620879919923626
    },
    {
      "decision": 334,
      "frontier": 275,
      "frontier_accuracy": 0.9426751592356688,
      "in_sample": {
        "threshold": 0.9027,
        "answered": 0.12192902638762511,
        "accuracy": 0.9426751592356688
      },
      "curve": [
        [
          0.0,
          0.9427
        ],
        [
          0.01,
          0.9427
        ],
        [
          0.0205,
          0.9427
        ],
        [
          0.0314,
          0.9427
        ],
        [
          0.0414,
          0.9427
        ],
        [
          0.0514,
          0.9427
        ],
        [
          0.0614,
          0.9427
        ],
        [
          0.0714,
          0.9427
        ],
        [
          0.0814,
          0.9427
        ],
        [
          0.0914,
          0.9427
        ],
        [
          0.1019,
          0.9427
        ],
        [
          0.1119,
          0.9427
        ],
        [
          0.1219,
          0.9427
        ],
        [
          0.1319,
          0.9418
        ],
        [
          0.1419,
          0.9418
        ],
        [
          0.152,
          0.9418
        ],
        [
          0.1624,
          0.9409
        ],
        [
          0.1724,
          0.9399
        ],
        [
          0.1824,
          0.9399
        ],
        [
          0.1924,
          0.9399
        ],
        [
          0.2025,
          0.939
        ],
        [
          0.2129,
          0.939
        ],
        [
          0.2229,
          0.9377
        ],
        [
          0.2329,
          0.9377
        ],
        [
          0.2429,
          0.9349
        ],
        [
          0.253,
          0.9336
        ],
        [
          0.2634,
          0.9318
        ],
        [
          0.2734,
          0.9304
        ],
        [
          0.2834,
          0.9286
        ],
        [
          0.2934,
          0.9249
        ],
        [
          0.3035,
          0.9227
        ],
        [
          0.3139,
          0.9208
        ],
        [
          0.3239,
          0.9186
        ],
        [
          0.3339,
          0.9158
        ],
        [
          0.3439,
          0.9126
        ],
        [
          0.354,
          0.9076
        ],
        [
          0.3644,
          0.904
        ],
        [
          0.3744,
          0.8995
        ],
        [
          0.3844,
          0.8981
        ],
        [
          0.3944,
          0.8931
        ],
        [
          0.4049,
          0.8876
        ],
        [
          0.4149,
          0.8849
        ],
        [
          0.4249,
          0.8831
        ],
        [
          0.4349,
          0.8808
        ],
        [
          0.4449,
          0.8758
        ],
        [
          0.455,
          0.8717
        ],
        [
          0.4654,
          0.8672
        ],
        [
          0.4759,
          0.8617
        ],
        [
          0.4864,
          0.8581
        ],
        [
          0.4968,
          0.8544
        ],
        [
          0.5068,
          0.8494
        ],
        [
          0.5177,
          0.8458
        ],
        [
          0.5278,
          0.8421
        ],
        [
          0.5382,
          0.8362
        ],
        [
          0.5487,
          0.8321
        ],
        [
          0.5591,
          0.8271
        ],
        [
          0.5692,
          0.8235
        ],
        [
          0.5796,
          0.8176
        ],
        [
          0.5896,
          0.8144
        ],
        [
          0.5996,
          0.8085
        ],
        [
          0.6096,
          0.803
        ],
        [
          0.6197,
          0.7966
        ],
        [
          0.6301,
          0.7889
        ],
        [
          0.6401,
          0.7834
        ],
        [
          0.6501,
          0.7775
        ],
        [
          0.6601,
          0.7725
        ],
        [
          0.6706,
          0.7648
        ],
        [
          0.6806,
          0.7589
        ],
        [
          0.6906,
          0.7511
        ],
        [
          0.7006,
          0.7461
        ],
        [
          0.7106,
          0.7407
        ],
        [
          0.7207,
          0.7366
        ],
        [
          0.7311,
          0.7311
        ],
        [
          0.7416,
          0.7243
        ],
        [
          0.752,
          0.7184
        ],
        [
          0.7621,
          0.7134
        ],
        [
          0.7725,
          0.7052
        ],
        [
          0.7825,
          0.6984
        ],
        [
          0.7925,
          0.6911
        ],
        [
          0.803,
          0.6838
        ],
        [
          0.813,
          0.6783
        ],
        [
          0.8235,
          0.6711
        ],
        [
          0.8348,
          0.6642
        ],
        [
          0.8453,
          0.6579
        ],
        [
          0.8562,
          0.651
        ],
        [
          0.8662,
          0.6456
        ],
        [
          0.8763,
          0.6397
        ],
        [
          0.8867,
          0.6351
        ],
        [
          0.8967,
          0.6283
        ],
        [
          0.9067,
          0.6219
        ],
        [
          0.9172,
          0.616
        ],
        [
          0.9272,
          0.6083
        ],
        [
          0.9372,
          0.601
        ],
        [
          0.9472,
          0.5937
        ],
        [
          0.9572,
          0.5883
        ],
        [
          0.9672,
          0.581
        ],
        [
          0.9777,
          0.5728
        ],
        [
          0.9877,
          0.5669
        ],
        [
          0.9977,
          0.5591
        ],
        [
          1.0,
          0.5573
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.13573314180318205,
        "difference": -0.0010276602978252947,
        "worse": 0.5233333333333333,
        "kept_right": 0.13467820317819743,
        "kept_wrong": 0.0010549386249846594,
        "handed_off": 0.864266858196818
      },
      "usd_per_1000": 0.46383095083475046
    },
    {
      "decision": 334,
      "frontier": 192,
      "frontier_accuracy": 0.9331210191082803,
      "in_sample": {
        "threshold": 0.9027,
        "answered": 0.12192902638762511,
        "accuracy": 0.9331210191082803
      },
      "curve": [
        [
          0.0,
          0.9331
        ],
        [
          0.01,
          0.9331
        ],
        [
          0.0205,
          0.9331
        ],
        [
          0.0314,
          0.9331
        ],
        [
          0.0414,
          0.9331
        ],
        [
          0.0514,
          0.9331
        ],
        [
          0.0614,
          0.9331
        ],
        [
          0.0714,
          0.9331
        ],
        [
          0.0814,
          0.9331
        ],
        [
          0.0914,
          0.9331
        ],
        [
          0.1019,
          0.9331
        ],
        [
          0.1119,
          0.9331
        ],
        [
          0.1219,
          0.9331
        ],
        [
          0.1319,
          0.9322
        ],
        [
          0.1419,
          0.9322
        ],
        [
          0.152,
          0.9322
        ],
        [
          0.1624,
          0.9313
        ],
        [
          0.1724,
          0.9304
        ],
        [
          0.1824,
          0.9304
        ],
        [
          0.1924,
          0.9304
        ],
        [
          0.2025,
          0.929
        ],
        [
          0.2129,
          0.929
        ],
        [
          0.2229,
          0.9281
        ],
        [
          0.2329,
          0.9272
        ],
        [
          0.2429,
          0.9245
        ],
        [
          0.253,
          0.9227
        ],
        [
          0.2634,
          0.9204
        ],
        [
          0.2734,
          0.9186
        ],
        [
          0.2834,
          0.9158
        ],
        [
          0.2934,
          0.9131
        ],
        [
          0.3035,
          0.9108
        ],
        [
          0.3139,
          0.9095
        ],
        [
          0.3239,
          0.9072
        ],
        [
          0.3339,
          0.9045
        ],
        [
          0.3439,
          0.9013
        ],
        [
          0.354,
          0.8967
        ],
        [
          0.3644,
          0.8926
        ],
        [
          0.3744,
          0.8881
        ],
        [
          0.3844,
          0.8867
        ],
        [
          0.3944,
          0.8831
        ],
        [
          0.4049,
          0.8781
        ],
        [
          0.4149,
          0.8763
        ],
        [
          0.4249,
          0.8744
        ],
        [
          0.4349,
          0.8722
        ],
        [
          0.4449,
          0.8676
        ],
        [
          0.455,
          0.8631
        ],
        [
          0.4654,
          0.8585
        ],
        [
          0.4759,
          0.8535
        ],
        [
          0.4864,
          0.8499
        ],
        [
          0.4968,
          0.8467
        ],
        [
          0.5068,
          0.8421
        ],
        [
          0.5177,
          0.8385
        ],
        [
          0.5278,
          0.8353
        ],
        [
          0.5382,
          0.8289
        ],
        [
          0.5487,
          0.8239
        ],
        [
          0.5591,
          0.8176
        ],
        [
          0.5692,
          0.8144
        ],
        [
          0.5796,
          0.8085
        ],
        [
          0.5896,
          0.8053
        ],
        [
          0.5996,
          0.7998
        ],
        [
          0.6096,
          0.7944
        ],
        [
          0.6197,
          0.7884
        ],
        [
          0.6301,
          0.7816
        ],
        [
          0.6401,
          0.7771
        ],
        [
          0.6501,
          0.7712
        ],
        [
          0.6601,
          0.7671
        ],
        [
          0.6706,
          0.7589
        ],
        [
          0.6806,
          0.752
        ],
        [
          0.6906,
          0.7448
        ],
        [
          0.7006,
          0.7398
        ],
        [
          0.7106,
          0.7348
        ],
        [
          0.7207,
          0.7311
        ],
        [
          0.7311,
          0.7275
        ],
        [
          0.7416,
          0.7211
        ],
        [
          0.752,
          0.7152
        ],
        [
          0.7621,
          0.7088
        ],
        [
          0.7725,
          0.7015
        ],
        [
          0.7825,
          0.6947
        ],
        [
          0.7925,
          0.6874
        ],
        [
          0.803,
          0.6806
        ],
        [
          0.813,
          0.6742
        ],
        [
          0.8235,
          0.667
        ],
        [
          0.8348,
          0.6615
        ],
        [
          0.8453,
          0.6538
        ],
        [
          0.8562,
          0.6474
        ],
        [
          0.8662,
          0.6419
        ],
        [
          0.8763,
          0.6365
        ],
        [
          0.8867,
          0.6324
        ],
        [
          0.8967,
          0.6251
        ],
        [
          0.9067,
          0.6192
        ],
        [
          0.9172,
          0.6137
        ],
        [
          0.9272,
          0.6069
        ],
        [
          0.9372,
          0.6001
        ],
        [
          0.9472,
          0.5937
        ],
        [
          0.9572,
          0.5887
        ],
        [
          0.9672,
          0.581
        ],
        [
          0.9777,
          0.5732
        ],
        [
          0.9877,
          0.5669
        ],
        [
          0.9977,
          0.5591
        ],
        [
          1.0,
          0.5573
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.13462311232520893,
        "difference": -0.0009457645996054533,
        "worse": 0.5233333333333333,
        "kept_right": 0.13367432292221573,
        "kept_wrong": 0.0009487894029932331,
        "handed_off": 0.8653768876747912
      },
      "usd_per_1000": 10.065803763684812
    },
    {
      "decision": 334,
      "frontier": 123,
      "frontier_accuracy": 0.9326660600545951,
      "in_sample": {
        "threshold": 0.9027,
        "answered": 0.12192902638762511,
        "accuracy": 0.9326660600545951
      },
      "curve": [
        [
          0.0,
          0.9327
        ],
        [
          0.01,
          0.9327
        ],
        [
          0.0205,
          0.9327
        ],
        [
          0.0314,
          0.9327
        ],
        [
          0.0414,
          0.9327
        ],
        [
          0.0514,
          0.9327
        ],
        [
          0.0614,
          0.9327
        ],
        [
          0.0714,
          0.9327
        ],
        [
          0.0814,
          0.9327
        ],
        [
          0.0914,
          0.9327
        ],
        [
          0.1019,
          0.9327
        ],
        [
          0.1119,
          0.9327
        ],
        [
          0.1219,
          0.9327
        ],
        [
          0.1319,
          0.9322
        ],
        [
          0.1419,
          0.9322
        ],
        [
          0.152,
          0.9322
        ],
        [
          0.1624,
          0.9313
        ],
        [
          0.1724,
          0.9304
        ],
        [
          0.1824,
          0.9308
        ],
        [
          0.1924,
          0.9308
        ],
        [
          0.2025,
          0.9295
        ],
        [
          0.2129,
          0.929
        ],
        [
          0.2229,
          0.9277
        ],
        [
          0.2329,
          0.9268
        ],
        [
          0.2429,
          0.9245
        ],
        [
          0.253,
          0.9231
        ],
        [
          0.2634,
          0.9213
        ],
        [
          0.2734,
          0.9199
        ],
        [
          0.2834,
          0.9172
        ],
        [
          0.2934,
          0.914
        ],
        [
          0.3035,
          0.9122
        ],
        [
          0.3139,
          0.9104
        ],
        [
          0.3239,
          0.9086
        ],
        [
          0.3339,
          0.9063
        ],
        [
          0.3439,
          0.9031
        ],
        [
          0.354,
          0.8976
        ],
        [
          0.3644,
          0.894
        ],
        [
          0.3744,
          0.8894
        ],
        [
          0.3844,
          0.8876
        ],
        [
          0.3944,
          0.8835
        ],
        [
          0.4049,
          0.8781
        ],
        [
          0.4149,
          0.8763
        ],
        [
          0.4249,
          0.874
        ],
        [
          0.4349,
          0.8712
        ],
        [
          0.4449,
          0.8653
        ],
        [
          0.455,
          0.8612
        ],
        [
          0.4654,
          0.8567
        ],
        [
          0.4759,
          0.8517
        ],
        [
          0.4864,
          0.848
        ],
        [
          0.4968,
          0.8449
        ],
        [
          0.5068,
          0.8399
        ],
        [
          0.5177,
          0.8362
        ],
        [
          0.5278,
          0.8321
        ],
        [
          0.5382,
          0.8271
        ],
        [
          0.5487,
          0.8217
        ],
        [
          0.5591,
          0.8162
        ],
        [
          0.5692,
          0.8126
        ],
        [
          0.5796,
          0.8076
        ],
        [
          0.5896,
          0.8039
        ],
        [
          0.5996,
          0.798
        ],
        [
          0.6096,
          0.7916
        ],
        [
          0.6197,
          0.7857
        ],
        [
          0.6301,
          0.7784
        ],
        [
          0.6401,
          0.7739
        ],
        [
          0.6501,
          0.7684
        ],
        [
          0.6601,
          0.763
        ],
        [
          0.6706,
          0.7557
        ],
        [
          0.6806,
          0.7493
        ],
        [
          0.6906,
          0.7416
        ],
        [
          0.7006,
          0.7361
        ],
        [
          0.7106,
          0.732
        ],
        [
          0.7207,
          0.7288
        ],
        [
          0.7311,
          0.7238
        ],
        [
          0.7416,
          0.7161
        ],
        [
          0.752,
          0.7102
        ],
        [
          0.7621,
          0.7047
        ],
        [
          0.7725,
          0.6965
        ],
        [
          0.7825,
          0.6902
        ],
        [
          0.7925,
          0.6856
        ],
        [
          0.803,
          0.6788
        ],
        [
          0.813,
          0.6729
        ],
        [
          0.8235,
          0.6665
        ],
        [
          0.8348,
          0.662
        ],
        [
          0.8453,
          0.6556
        ],
        [
          0.8562,
          0.6497
        ],
        [
          0.8662,
          0.6442
        ],
        [
          0.8763,
          0.6388
        ],
        [
          0.8867,
          0.6333
        ],
        [
          0.8967,
          0.6265
        ],
        [
          0.9067,
          0.6201
        ],
        [
          0.9172,
          0.6137
        ],
        [
          0.9272,
          0.6056
        ],
        [
          0.9372,
          0.5987
        ],
        [
          0.9472,
          0.5924
        ],
        [
          0.9572,
          0.5874
        ],
        [
          0.9672,
          0.5805
        ],
        [
          0.9777,
          0.5732
        ],
        [
          0.9877,
          0.5664
        ],
        [
          0.9977,
          0.5582
        ],
        [
          1.0,
          0.5573
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.14648215945953083,
        "difference": -0.0011035367249227319,
        "worse": 0.54,
        "kept_right": 0.14509667733449189,
        "kept_wrong": 0.0013854821250388948,
        "handed_off": 0.8535178405404696
      },
      "usd_per_1000": 0.24839543374361456
    },
    {
      "decision": 334,
      "frontier": 274,
      "frontier_accuracy": 0.9085532302092811,
      "in_sample": {
        "threshold": 0.8961,
        "answered": 0.1264786169244768,
        "accuracy": 0.9085532302092811
      },
      "curve": [
        [
          0.0,
          0.9086
        ],
        [
          0.01,
          0.9086
        ],
        [
          0.0205,
          0.9086
        ],
        [
          0.0314,
          0.9086
        ],
        [
          0.0414,
          0.9086
        ],
        [
          0.0514,
          0.9086
        ],
        [
          0.0614,
          0.9086
        ],
        [
          0.0714,
          0.9086
        ],
        [
          0.0814,
          0.9086
        ],
        [
          0.0914,
          0.9086
        ],
        [
          0.1019,
          0.909
        ],
        [
          0.1119,
          0.909
        ],
        [
          0.1219,
          0.909
        ],
        [
          0.1319,
          0.9081
        ],
        [
          0.1419,
          0.9081
        ],
        [
          0.152,
          0.9081
        ],
        [
          0.1624,
          0.9072
        ],
        [
          0.1724,
          0.9067
        ],
        [
          0.1824,
          0.9067
        ],
        [
          0.1924,
          0.9067
        ],
        [
          0.2025,
          0.9058
        ],
        [
          0.2129,
          0.9058
        ],
        [
          0.2229,
          0.9045
        ],
        [
          0.2329,
          0.904
        ],
        [
          0.2429,
          0.9013
        ],
        [
          0.253,
          0.9004
        ],
        [
          0.2634,
          0.899
        ],
        [
          0.2734,
          0.8976
        ],
        [
          0.2834,
          0.8958
        ],
        [
          0.2934,
          0.8926
        ],
        [
          0.3035,
          0.8904
        ],
        [
          0.3139,
          0.8894
        ],
        [
          0.3239,
          0.8876
        ],
        [
          0.3339,
          0.8849
        ],
        [
          0.3439,
          0.8817
        ],
        [
          0.354,
          0.8776
        ],
        [
          0.3644,
          0.8744
        ],
        [
          0.3744,
          0.8699
        ],
        [
          0.3844,
          0.869
        ],
        [
          0.3944,
          0.8653
        ],
        [
          0.4049,
          0.8603
        ],
        [
          0.4149,
          0.8585
        ],
        [
          0.4249,
          0.8567
        ],
        [
          0.4349,
          0.8544
        ],
        [
          0.4449,
          0.8494
        ],
        [
          0.455,
          0.8458
        ],
        [
          0.4654,
          0.8417
        ],
        [
          0.4759,
          0.8362
        ],
        [
          0.4864,
          0.8326
        ],
        [
          0.4968,
          0.8289
        ],
        [
          0.5068,
          0.8239
        ],
        [
          0.5177,
          0.8203
        ],
        [
          0.5278,
          0.8167
        ],
        [
          0.5382,
          0.8107
        ],
        [
          0.5487,
          0.8062
        ],
        [
          0.5591,
          0.8007
        ],
        [
          0.5692,
          0.7975
        ],
        [
          0.5796,
          0.7916
        ],
        [
          0.5896,
          0.7884
        ],
        [
          0.5996,
          0.783
        ],
        [
          0.6096,
          0.778
        ],
        [
          0.6197,
          0.7725
        ],
        [
          0.6301,
          0.7657
        ],
        [
          0.6401,
          0.7616
        ],
        [
          0.6501,
          0.7566
        ],
        [
          0.6601,
          0.7516
        ],
        [
          0.6706,
          0.7448
        ],
        [
          0.6806,
          0.7379
        ],
        [
          0.6906,
          0.7311
        ],
        [
          0.7006,
          0.7261
        ],
        [
          0.7106,
          0.722
        ],
        [
          0.7207,
          0.7188
        ],
        [
          0.7311,
          0.7134
        ],
        [
          0.7416,
          0.7066
        ],
        [
          0.752,
          0.7015
        ],
        [
          0.7621,
          0.6965
        ],
        [
          0.7725,
          0.6884
        ],
        [
          0.7825,
          0.6824
        ],
        [
          0.7925,
          0.6761
        ],
        [
          0.803,
          0.6692
        ],
        [
          0.813,
          0.6638
        ],
        [
          0.8235,
          0.657
        ],
        [
          0.8348,
          0.651
        ],
        [
          0.8453,
          0.6447
        ],
        [
          0.8562,
          0.6388
        ],
        [
          0.8662,
          0.6342
        ],
        [
          0.8763,
          0.6301
        ],
        [
          0.8867,
          0.6256
        ],
        [
          0.8967,
          0.6197
        ],
        [
          0.9067,
          0.6142
        ],
        [
          0.9172,
          0.6096
        ],
        [
          0.9272,
          0.601
        ],
        [
          0.9372,
          0.5951
        ],
        [
          0.9472,
          0.5883
        ],
        [
          0.9572,
          0.5842
        ],
        [
          0.9672,
          0.5773
        ],
        [
          0.9777,
          0.5701
        ],
        [
          0.9877,
          0.5651
        ],
        [
          0.9977,
          0.5587
        ],
        [
          1.0,
          0.5573
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.15029434016997884,
        "difference": -0.0010822858503411637,
        "worse": 0.5066666666666667,
        "kept_right": 0.14855123782944302,
        "kept_wrong": 0.0017431023405358522,
        "handed_off": 0.8497056598300214
      },
      "usd_per_1000": 0.38133761429792035
    },
    {
      "decision": 334,
      "frontier": 47,
      "frontier_accuracy": 0.9044585987261147,
      "in_sample": {
        "threshold": 0.8551,
        "answered": 0.1565059144676979,
        "accuracy": 0.9044585987261147
      },
      "curve": [
        [
          0.0,
          0.9045
        ],
        [
          0.01,
          0.9045
        ],
        [
          0.0205,
          0.9045
        ],
        [
          0.0314,
          0.9049
        ],
        [
          0.0414,
          0.9049
        ],
        [
          0.0514,
          0.9049
        ],
        [
          0.0614,
          0.9049
        ],
        [
          0.0714,
          0.9049
        ],
        [
          0.0814,
          0.9049
        ],
        [
          0.0914,
          0.9049
        ],
        [
          0.1019,
          0.9049
        ],
        [
          0.1119,
          0.9049
        ],
        [
          0.1219,
          0.9049
        ],
        [
          0.1319,
          0.9045
        ],
        [
          0.1419,
          0.9045
        ],
        [
          0.152,
          0.9045
        ],
        [
          0.1624,
          0.9035
        ],
        [
          0.1724,
          0.9026
        ],
        [
          0.1824,
          0.9031
        ],
        [
          0.1924,
          0.9031
        ],
        [
          0.2025,
          0.9017
        ],
        [
          0.2129,
          0.9008
        ],
        [
          0.2229,
          0.8995
        ],
        [
          0.2329,
          0.899
        ],
        [
          0.2429,
          0.8963
        ],
        [
          0.253,
          0.8954
        ],
        [
          0.2634,
          0.8931
        ],
        [
          0.2734,
          0.8917
        ],
        [
          0.2834,
          0.8899
        ],
        [
          0.2934,
          0.8872
        ],
        [
          0.3035,
          0.8849
        ],
        [
          0.3139,
          0.884
        ],
        [
          0.3239,
          0.8822
        ],
        [
          0.3339,
          0.8794
        ],
        [
          0.3439,
          0.8767
        ],
        [
          0.354,
          0.8722
        ],
        [
          0.3644,
          0.8685
        ],
        [
          0.3744,
          0.864
        ],
        [
          0.3844,
          0.8626
        ],
        [
          0.3944,
          0.859
        ],
        [
          0.4049,
          0.854
        ],
        [
          0.4149,
          0.853
        ],
        [
          0.4249,
          0.8508
        ],
        [
          0.4349,
          0.848
        ],
        [
          0.4449,
          0.843
        ],
        [
          0.455,
          0.8394
        ],
        [
          0.4654,
          0.8358
        ],
        [
          0.4759,
          0.8303
        ],
        [
          0.4864,
          0.8267
        ],
        [
          0.4968,
          0.8239
        ],
        [
          0.5068,
          0.8194
        ],
        [
          0.5177,
          0.8157
        ],
        [
          0.5278,
          0.8112
        ],
        [
          0.5382,
          0.8071
        ],
        [
          0.5487,
          0.8016
        ],
        [
          0.5591,
          0.7962
        ],
        [
          0.5692,
          0.7925
        ],
        [
          0.5796,
          0.7866
        ],
        [
          0.5896,
          0.7839
        ],
        [
          0.5996,
          0.778
        ],
        [
          0.6096,
          0.7721
        ],
        [
          0.6197,
          0.7671
        ],
        [
          0.6301,
          0.7598
        ],
        [
          0.6401,
          0.7548
        ],
        [
          0.6501,
          0.7489
        ],
        [
          0.6601,
          0.7434
        ],
        [
          0.6706,
          0.737
        ],
        [
          0.6806,
          0.7316
        ],
        [
          0.6906,
          0.7238
        ],
        [
          0.7006,
          0.7193
        ],
        [
          0.7106,
          0.7147
        ],
        [
          0.7207,
          0.712
        ],
        [
          0.7311,
          0.7088
        ],
        [
          0.7416,
          0.7034
        ],
        [
          0.752,
          0.6984
        ],
        [
          0.7621,
          0.6929
        ],
        [
          0.7725,
          0.6852
        ],
        [
          0.7825,
          0.6793
        ],
        [
          0.7925,
          0.672
        ],
        [
          0.803,
          0.6661
        ],
        [
          0.813,
          0.6606
        ],
        [
          0.8235,
          0.6551
        ],
        [
          0.8348,
          0.6506
        ],
        [
          0.8453,
          0.6442
        ],
        [
          0.8562,
          0.6388
        ],
        [
          0.8662,
          0.6333
        ],
        [
          0.8763,
          0.6278
        ],
        [
          0.8867,
          0.6233
        ],
        [
          0.8967,
          0.6169
        ],
        [
          0.9067,
          0.6124
        ],
        [
          0.9172,
          0.6074
        ],
        [
          0.9272,
          0.6005
        ],
        [
          0.9372,
          0.5942
        ],
        [
          0.9472,
          0.5883
        ],
        [
          0.9572,
          0.5828
        ],
        [
          0.9672,
          0.5755
        ],
        [
          0.9777,
          0.5696
        ],
        [
          0.9877,
          0.5641
        ],
        [
          0.9977,
          0.5591
        ],
        [
          1.0,
          0.5573
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.16029236415640735,
        "difference": -0.001082291325212449,
        "worse": 0.42333333333333334,
        "kept_right": 0.15835518442971747,
        "kept_wrong": 0.0019371797266897236,
        "handed_off": 0.8397076358435929
      },
      "usd_per_1000": 4.159404939903585
    },
    {
      "decision": 334,
      "frontier": 193,
      "frontier_accuracy": 0.89171974522293,
      "in_sample": {
        "threshold": 0.9027,
        "answered": 0.12192902638762511,
        "accuracy": 0.89171974522293
      },
      "curve": [
        [
          0.0,
          0.8917
        ],
        [
          0.01,
          0.8917
        ],
        [
          0.0205,
          0.8917
        ],
        [
          0.0314,
          0.8917
        ],
        [
          0.0414,
          0.8917
        ],
        [
          0.0514,
          0.8917
        ],
        [
          0.0614,
          0.8917
        ],
        [
          0.0714,
          0.8917
        ],
        [
          0.0814,
          0.8917
        ],
        [
          0.0914,
          0.8917
        ],
        [
          0.1019,
          0.8917
        ],
        [
          0.1119,
          0.8917
        ],
        [
          0.1219,
          0.8917
        ],
        [
          0.1319,
          0.8908
        ],
        [
          0.1419,
          0.8908
        ],
        [
          0.152,
          0.8913
        ],
        [
          0.1624,
          0.8908
        ],
        [
          0.1724,
          0.8899
        ],
        [
          0.1824,
          0.8904
        ],
        [
          0.1924,
          0.8908
        ],
        [
          0.2025,
          0.8899
        ],
        [
          0.2129,
          0.8894
        ],
        [
          0.2229,
          0.8885
        ],
        [
          0.2329,
          0.8885
        ],
        [
          0.2429,
          0.8867
        ],
        [
          0.253,
          0.8863
        ],
        [
          0.2634,
          0.8844
        ],
        [
          0.2734,
          0.884
        ],
        [
          0.2834,
          0.8817
        ],
        [
          0.2934,
          0.8785
        ],
        [
          0.3035,
          0.8763
        ],
        [
          0.3139,
          0.8749
        ],
        [
          0.3239,
          0.8726
        ],
        [
          0.3339,
          0.8703
        ],
        [
          0.3439,
          0.8676
        ],
        [
          0.354,
          0.8635
        ],
        [
          0.3644,
          0.8603
        ],
        [
          0.3744,
          0.8562
        ],
        [
          0.3844,
          0.8544
        ],
        [
          0.3944,
          0.8503
        ],
        [
          0.4049,
          0.8458
        ],
        [
          0.4149,
          0.8439
        ],
        [
          0.4249,
          0.8421
        ],
        [
          0.4349,
          0.8403
        ],
        [
          0.4449,
          0.8353
        ],
        [
          0.455,
          0.8317
        ],
        [
          0.4654,
          0.8285
        ],
        [
          0.4759,
          0.8235
        ],
        [
          0.4864,
          0.8203
        ],
        [
          0.4968,
          0.8171
        ],
        [
          0.5068,
          0.813
        ],
        [
          0.5177,
          0.8094
        ],
        [
          0.5278,
          0.8062
        ],
        [
          0.5382,
          0.8007
        ],
        [
          0.5487,
          0.7953
        ],
        [
          0.5591,
          0.7907
        ],
        [
          0.5692,
          0.788
        ],
        [
          0.5796,
          0.7825
        ],
        [
          0.5896,
          0.7793
        ],
        [
          0.5996,
          0.7739
        ],
        [
          0.6096,
          0.7702
        ],
        [
          0.6197,
          0.7648
        ],
        [
          0.6301,
          0.7589
        ],
        [
          0.6401,
          0.7548
        ],
        [
          0.6501,
          0.7498
        ],
        [
          0.6601,
          0.7466
        ],
        [
          0.6706,
          0.7389
        ],
        [
          0.6806,
          0.7338
        ],
        [
          0.6906,
          0.7261
        ],
        [
          0.7006,
          0.7211
        ],
        [
          0.7106,
          0.7175
        ],
        [
          0.7207,
          0.7138
        ],
        [
          0.7311,
          0.7088
        ],
        [
          0.7416,
          0.7034
        ],
        [
          0.752,
          0.6979
        ],
        [
          0.7621,
          0.6924
        ],
        [
          0.7725,
          0.6852
        ],
        [
          0.7825,
          0.6793
        ],
        [
          0.7925,
          0.6729
        ],
        [
          0.803,
          0.6679
        ],
        [
          0.813,
          0.6624
        ],
        [
          0.8235,
          0.6561
        ],
        [
          0.8348,
          0.6501
        ],
        [
          0.8453,
          0.6442
        ],
        [
          0.8562,
          0.6388
        ],
        [
          0.8662,
          0.6328
        ],
        [
          0.8763,
          0.6283
        ],
        [
          0.8867,
          0.6247
        ],
        [
          0.8967,
          0.6183
        ],
        [
          0.9067,
          0.6142
        ],
        [
          0.9172,
          0.6083
        ],
        [
          0.9272,
          0.6015
        ],
        [
          0.9372,
          0.5951
        ],
        [
          0.9472,
          0.5892
        ],
        [
          0.9572,
          0.5837
        ],
        [
          0.9672,
          0.5773
        ],
        [
          0.9777,
          0.571
        ],
        [
          0.9877,
          0.5655
        ],
        [
          0.9977,
          0.5582
        ],
        [
          1.0,
          0.5573
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1744956218601239,
        "difference": -0.002867960993511937,
        "worse": 0.6966666666666667,
        "kept_right": 0.1700268202858669,
        "kept_wrong": 0.004468801574257098,
        "handed_off": 0.8255043781398763
      },
      "usd_per_1000": 6.41482370683887
    },
    {
      "decision": 334,
      "frontier": 55,
      "frontier_accuracy": 0.8503184713375797,
      "in_sample": {
        "threshold": 0.7237,
        "answered": 0.2370336669699727,
        "accuracy": 0.8503184713375797
      },
      "curve": [
        [
          0.0,
          0.8503
        ],
        [
          0.01,
          0.8503
        ],
        [
          0.0205,
          0.8503
        ],
        [
          0.0314,
          0.8503
        ],
        [
          0.0414,
          0.8508
        ],
        [
          0.0514,
          0.8512
        ],
        [
          0.0614,
          0.8512
        ],
        [
          0.0714,
          0.8512
        ],
        [
          0.0814,
          0.8512
        ],
        [
          0.0914,
          0.8512
        ],
        [
          0.1019,
          0.8512
        ],
        [
          0.1119,
          0.8517
        ],
        [
          0.1219,
          0.8521
        ],
        [
          0.1319,
          0.8517
        ],
        [
          0.1419,
          0.8517
        ],
        [
          0.152,
          0.8521
        ],
        [
          0.1624,
          0.8517
        ],
        [
          0.1724,
          0.8512
        ],
        [
          0.1824,
          0.8521
        ],
        [
          0.1924,
          0.853
        ],
        [
          0.2025,
          0.8521
        ],
        [
          0.2129,
          0.8517
        ],
        [
          0.2229,
          0.8512
        ],
        [
          0.2329,
          0.8512
        ],
        [
          0.2429,
          0.8494
        ],
        [
          0.253,
          0.849
        ],
        [
          0.2634,
          0.8467
        ],
        [
          0.2734,
          0.8467
        ],
        [
          0.2834,
          0.8444
        ],
        [
          0.2934,
          0.8421
        ],
        [
          0.3035,
          0.8399
        ],
        [
          0.3139,
          0.8399
        ],
        [
          0.3239,
          0.838
        ],
        [
          0.3339,
          0.8367
        ],
        [
          0.3439,
          0.8348
        ],
        [
          0.354,
          0.8317
        ],
        [
          0.3644,
          0.8294
        ],
        [
          0.3744,
          0.8276
        ],
        [
          0.3844,
          0.8267
        ],
        [
          0.3944,
          0.8239
        ],
        [
          0.4049,
          0.8194
        ],
        [
          0.4149,
          0.8185
        ],
        [
          0.4249,
          0.8171
        ],
        [
          0.4349,
          0.8153
        ],
        [
          0.4449,
          0.8126
        ],
        [
          0.455,
          0.8089
        ],
        [
          0.4654,
          0.8044
        ],
        [
          0.4759,
          0.7998
        ],
        [
          0.4864,
          0.7971
        ],
        [
          0.4968,
          0.7939
        ],
        [
          0.5068,
          0.7907
        ],
        [
          0.5177,
          0.7875
        ],
        [
          0.5278,
          0.7843
        ],
        [
          0.5382,
          0.7789
        ],
        [
          0.5487,
          0.7739
        ],
        [
          0.5591,
          0.7702
        ],
        [
          0.5692,
          0.7684
        ],
        [
          0.5796,
          0.7648
        ],
        [
          0.5896,
          0.7625
        ],
        [
          0.5996,
          0.7575
        ],
        [
          0.6096,
          0.7543
        ],
        [
          0.6197,
          0.7489
        ],
        [
          0.6301,
          0.7425
        ],
        [
          0.6401,
          0.7398
        ],
        [
          0.6501,
          0.7352
        ],
        [
          0.6601,
          0.7307
        ],
        [
          0.6706,
          0.7238
        ],
        [
          0.6806,
          0.7184
        ],
        [
          0.6906,
          0.7134
        ],
        [
          0.7006,
          0.7084
        ],
        [
          0.7106,
          0.7066
        ],
        [
          0.7207,
          0.7025
        ],
        [
          0.7311,
          0.6988
        ],
        [
          0.7416,
          0.6938
        ],
        [
          0.752,
          0.6888
        ],
        [
          0.7621,
          0.6838
        ],
        [
          0.7725,
          0.6774
        ],
        [
          0.7825,
          0.672
        ],
        [
          0.7925,
          0.6661
        ],
        [
          0.803,
          0.6597
        ],
        [
          0.813,
          0.6551
        ],
        [
          0.8235,
          0.6492
        ],
        [
          0.8348,
          0.6442
        ],
        [
          0.8453,
          0.6383
        ],
        [
          0.8562,
          0.6356
        ],
        [
          0.8662,
          0.6301
        ],
        [
          0.8763,
          0.6256
        ],
        [
          0.8867,
          0.6215
        ],
        [
          0.8967,
          0.6151
        ],
        [
          0.9067,
          0.6119
        ],
        [
          0.9172,
          0.6069
        ],
        [
          0.9272,
          0.6001
        ],
        [
          0.9372,
          0.5942
        ],
        [
          0.9472,
          0.5887
        ],
        [
          0.9572,
          0.5828
        ],
        [
          0.9672,
          0.576
        ],
        [
          0.9777,
          0.5705
        ],
        [
          0.9877,
          0.5646
        ],
        [
          0.9977,
          0.5587
        ],
        [
          1.0,
          0.5573
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.24364389580162465,
        "difference": -0.00208021072686255,
        "worse": 0.5633333333333334,
        "kept_right": 0.23107104267858886,
        "kept_wrong": 0.012572853123035667,
        "handed_off": 0.756356104198375
      },
      "usd_per_1000": 2.783131789513459
    },
    {
      "decision": 45,
      "frontier": 140,
      "frontier_accuracy": 0.9808917197452229,
      "in_sample": {
        "threshold": 0.4702689339045559,
        "answered": 0.10737033666969972,
        "accuracy": 0.9808917197452229
      },
      "curve": [
        [
          0.0,
          0.9809
        ],
        [
          0.01,
          0.9809
        ],
        [
          0.02,
          0.9809
        ],
        [
          0.03,
          0.9809
        ],
        [
          0.04,
          0.9809
        ],
        [
          0.05,
          0.9809
        ],
        [
          0.0601,
          0.9809
        ],
        [
          0.0705,
          0.9809
        ],
        [
          0.0805,
          0.9809
        ],
        [
          0.0905,
          0.9809
        ],
        [
          0.1005,
          0.9809
        ],
        [
          0.1106,
          0.9804
        ],
        [
          0.121,
          0.9804
        ],
        [
          0.131,
          0.9804
        ],
        [
          0.141,
          0.9804
        ],
        [
          0.151,
          0.9804
        ],
        [
          0.1611,
          0.9795
        ],
        [
          0.1715,
          0.9791
        ],
        [
          0.1815,
          0.9782
        ],
        [
          0.1915,
          0.9777
        ],
        [
          0.2015,
          0.9773
        ],
        [
          0.2116,
          0.9763
        ],
        [
          0.222,
          0.975
        ],
        [
          0.232,
          0.9745
        ],
        [
          0.242,
          0.9732
        ],
        [
          0.252,
          0.9713
        ],
        [
          0.2621,
          0.9713
        ],
        [
          0.2725,
          0.9672
        ],
        [
          0.2825,
          0.965
        ],
        [
          0.2925,
          0.9636
        ],
        [
          0.3025,
          0.9613
        ],
        [
          0.3126,
          0.9595
        ],
        [
          0.323,
          0.9572
        ],
        [
          0.333,
          0.9545
        ],
        [
          0.343,
          0.9518
        ],
        [
          0.353,
          0.9486
        ],
        [
          0.3631,
          0.9454
        ],
        [
          0.3735,
          0.9413
        ],
        [
          0.3835,
          0.9386
        ],
        [
          0.3935,
          0.9354
        ],
        [
          0.4035,
          0.9318
        ],
        [
          0.4136,
          0.9286
        ],
        [
          0.424,
          0.9236
        ],
        [
          0.434,
          0.9186
        ],
        [
          0.444,
          0.9154
        ],
        [
          0.454,
          0.9117
        ],
        [
          0.4641,
          0.9086
        ],
        [
          0.4745,
          0.9035
        ],
        [
          0.4845,
          0.8981
        ],
        [
          0.4945,
          0.8913
        ],
        [
          0.5045,
          0.8867
        ],
        [
          0.5146,
          0.8803
        ],
        [
          0.525,
          0.8749
        ],
        [
          0.535,
          0.8726
        ],
        [
          0.545,
          0.8685
        ],
        [
          0.5551,
          0.8617
        ],
        [
          0.5655,
          0.8549
        ],
        [
          0.5755,
          0.8499
        ],
        [
          0.5855,
          0.8439
        ],
        [
          0.5955,
          0.8394
        ],
        [
          0.6056,
          0.8335
        ],
        [
          0.616,
          0.8258
        ],
        [
          0.626,
          0.8198
        ],
        [
          0.636,
          0.8144
        ],
        [
          0.646,
          0.8094
        ],
        [
          0.6561,
          0.8039
        ],
        [
          0.6665,
          0.7962
        ],
        [
          0.6765,
          0.7898
        ],
        [
          0.6865,
          0.7825
        ],
        [
          0.6965,
          0.7771
        ],
        [
          0.7066,
          0.7716
        ],
        [
          0.717,
          0.7652
        ],
        [
          0.727,
          0.7584
        ],
        [
          0.737,
          0.7489
        ],
        [
          0.747,
          0.7416
        ],
        [
          0.7571,
          0.7357
        ],
        [
          0.7675,
          0.7279
        ],
        [
          0.7775,
          0.7197
        ],
        [
          0.7875,
          0.7143
        ],
        [
          0.7975,
          0.7079
        ],
        [
          0.8076,
          0.7006
        ],
        [
          0.818,
          0.6929
        ],
        [
          0.828,
          0.6847
        ],
        [
          0.838,
          0.6774
        ],
        [
          0.848,
          0.6715
        ],
        [
          0.8581,
          0.6642
        ],
        [
          0.8685,
          0.657
        ],
        [
          0.8785,
          0.6492
        ],
        [
          0.8885,
          0.6419
        ],
        [
          0.8985,
          0.6342
        ],
        [
          0.9086,
          0.6269
        ],
        [
          0.919,
          0.6183
        ],
        [
          0.929,
          0.6115
        ],
        [
          0.939,
          0.6028
        ],
        [
          0.949,
          0.5946
        ],
        [
          0.9591,
          0.5883
        ],
        [
          0.9695,
          0.581
        ],
        [
          0.9795,
          0.5742
        ],
        [
          0.9895,
          0.566
        ],
        [
          0.9995,
          0.5573
        ],
        [
          1.0,
          0.5569
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1316057238690968,
        "difference": -0.0008428442997179558,
        "worse": 0.48333333333333334,
        "kept_right": 0.13053547694655243,
        "kept_wrong": 0.001070246922544378,
        "handed_off": 0.8683942761309033
      },
      "usd_per_1000": 2.1007353392209422
    },
    {
      "decision": 45,
      "frontier": 277,
      "frontier_accuracy": 0.9795268425841674,
      "in_sample": {
        "threshold": 0.4702689339045559,
        "answered": 0.10737033666969972,
        "accuracy": 0.9795268425841674
      },
      "curve": [
        [
          0.0,
          0.9795
        ],
        [
          0.01,
          0.9795
        ],
        [
          0.02,
          0.9795
        ],
        [
          0.03,
          0.9795
        ],
        [
          0.04,
          0.9795
        ],
        [
          0.05,
          0.9795
        ],
        [
          0.0601,
          0.9795
        ],
        [
          0.0705,
          0.9795
        ],
        [
          0.0805,
          0.9795
        ],
        [
          0.0905,
          0.9795
        ],
        [
          0.1005,
          0.9795
        ],
        [
          0.1106,
          0.9791
        ],
        [
          0.121,
          0.9791
        ],
        [
          0.131,
          0.9786
        ],
        [
          0.141,
          0.9786
        ],
        [
          0.151,
          0.9786
        ],
        [
          0.1611,
          0.9777
        ],
        [
          0.1715,
          0.9773
        ],
        [
          0.1815,
          0.9763
        ],
        [
          0.1915,
          0.9759
        ],
        [
          0.2015,
          0.9754
        ],
        [
          0.2116,
          0.9741
        ],
        [
          0.222,
          0.9727
        ],
        [
          0.232,
          0.9722
        ],
        [
          0.242,
          0.9709
        ],
        [
          0.252,
          0.9691
        ],
        [
          0.2621,
          0.9691
        ],
        [
          0.2725,
          0.9654
        ],
        [
          0.2825,
          0.9631
        ],
        [
          0.2925,
          0.9618
        ],
        [
          0.3025,
          0.9595
        ],
        [
          0.3126,
          0.9577
        ],
        [
          0.323,
          0.9554
        ],
        [
          0.333,
          0.9527
        ],
        [
          0.343,
          0.9495
        ],
        [
          0.353,
          0.9463
        ],
        [
          0.3631,
          0.9431
        ],
        [
          0.3735,
          0.939
        ],
        [
          0.3835,
          0.9363
        ],
        [
          0.3935,
          0.9331
        ],
        [
          0.4035,
          0.9295
        ],
        [
          0.4136,
          0.9263
        ],
        [
          0.424,
          0.9213
        ],
        [
          0.434,
          0.9163
        ],
        [
          0.444,
          0.9131
        ],
        [
          0.454,
          0.9095
        ],
        [
          0.4641,
          0.9063
        ],
        [
          0.4745,
          0.9022
        ],
        [
          0.4845,
          0.8967
        ],
        [
          0.4945,
          0.8899
        ],
        [
          0.5045,
          0.8854
        ],
        [
          0.5146,
          0.879
        ],
        [
          0.525,
          0.8735
        ],
        [
          0.535,
          0.8712
        ],
        [
          0.545,
          0.8672
        ],
        [
          0.5551,
          0.8603
        ],
        [
          0.5655,
          0.8535
        ],
        [
          0.5755,
          0.849
        ],
        [
          0.5855,
          0.8426
        ],
        [
          0.5955,
          0.8376
        ],
        [
          0.6056,
          0.8317
        ],
        [
          0.616,
          0.8239
        ],
        [
          0.626,
          0.818
        ],
        [
          0.636,
          0.813
        ],
        [
          0.646,
          0.808
        ],
        [
          0.6561,
          0.8025
        ],
        [
          0.6665,
          0.7948
        ],
        [
          0.6765,
          0.7884
        ],
        [
          0.6865,
          0.7812
        ],
        [
          0.6965,
          0.7762
        ],
        [
          0.7066,
          0.7702
        ],
        [
          0.717,
          0.7639
        ],
        [
          0.727,
          0.7571
        ],
        [
          0.737,
          0.7475
        ],
        [
          0.747,
          0.7402
        ],
        [
          0.7571,
          0.7343
        ],
        [
          0.7675,
          0.7266
        ],
        [
          0.7775,
          0.7184
        ],
        [
          0.7875,
          0.7129
        ],
        [
          0.7975,
          0.7066
        ],
        [
          0.8076,
          0.6993
        ],
        [
          0.818,
          0.6915
        ],
        [
          0.828,
          0.6833
        ],
        [
          0.838,
          0.6761
        ],
        [
          0.848,
          0.6702
        ],
        [
          0.8581,
          0.6633
        ],
        [
          0.8685,
          0.6561
        ],
        [
          0.8785,
          0.6483
        ],
        [
          0.8885,
          0.6406
        ],
        [
          0.8985,
          0.6328
        ],
        [
          0.9086,
          0.6256
        ],
        [
          0.919,
          0.6169
        ],
        [
          0.929,
          0.6106
        ],
        [
          0.939,
          0.6019
        ],
        [
          0.949,
          0.5942
        ],
        [
          0.9591,
          0.5878
        ],
        [
          0.9695,
          0.5805
        ],
        [
          0.9795,
          0.5737
        ],
        [
          0.9895,
          0.5655
        ],
        [
          0.9995,
          0.5573
        ],
        [
          1.0,
          0.5569
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.12431172129978686,
        "difference": -0.0008519545804716065,
        "worse": 0.48333333333333334,
        "kept_right": 0.12343551321052872,
        "kept_wrong": 0.0008762080892580385,
        "handed_off": 0.8756882787002133
      },
      "usd_per_1000": 5.311535461135725
    },
    {
      "decision": 45,
      "frontier": 276,
      "frontier_accuracy": 0.9777070063694268,
      "in_sample": {
        "threshold": 0.4702689339045559,
        "answered": 0.10737033666969972,
        "accuracy": 0.9777070063694268
      },
      "curve": [
        [
          0.0,
          0.9777
        ],
        [
          0.01,
          0.9777
        ],
        [
          0.02,
          0.9777
        ],
        [
          0.03,
          0.9777
        ],
        [
          0.04,
          0.9777
        ],
        [
          0.05,
          0.9777
        ],
        [
          0.0601,
          0.9777
        ],
        [
          0.0705,
          0.9777
        ],
        [
          0.0805,
          0.9777
        ],
        [
          0.0905,
          0.9777
        ],
        [
          0.1005,
          0.9777
        ],
        [
          0.1106,
          0.9773
        ],
        [
          0.121,
          0.9773
        ],
        [
          0.131,
          0.9768
        ],
        [
          0.141,
          0.9768
        ],
        [
          0.151,
          0.9768
        ],
        [
          0.1611,
          0.9759
        ],
        [
          0.1715,
          0.975
        ],
        [
          0.1815,
          0.9741
        ],
        [
          0.1915,
          0.9736
        ],
        [
          0.2015,
          0.9732
        ],
        [
          0.2116,
          0.9722
        ],
        [
          0.222,
          0.9709
        ],
        [
          0.232,
          0.9704
        ],
        [
          0.242,
          0.9691
        ],
        [
          0.252,
          0.9672
        ],
        [
          0.2621,
          0.9672
        ],
        [
          0.2725,
          0.9636
        ],
        [
          0.2825,
          0.9613
        ],
        [
          0.2925,
          0.96
        ],
        [
          0.3025,
          0.9577
        ],
        [
          0.3126,
          0.9559
        ],
        [
          0.323,
          0.9536
        ],
        [
          0.333,
          0.9509
        ],
        [
          0.343,
          0.9477
        ],
        [
          0.353,
          0.9449
        ],
        [
          0.3631,
          0.9418
        ],
        [
          0.3735,
          0.9377
        ],
        [
          0.3835,
          0.9354
        ],
        [
          0.3935,
          0.9322
        ],
        [
          0.4035,
          0.9286
        ],
        [
          0.4136,
          0.9254
        ],
        [
          0.424,
          0.9204
        ],
        [
          0.434,
          0.9154
        ],
        [
          0.444,
          0.9122
        ],
        [
          0.454,
          0.9086
        ],
        [
          0.4641,
          0.9054
        ],
        [
          0.4745,
          0.9013
        ],
        [
          0.4845,
          0.8958
        ],
        [
          0.4945,
          0.889
        ],
        [
          0.5045,
          0.8844
        ],
        [
          0.5146,
          0.8781
        ],
        [
          0.525,
          0.8726
        ],
        [
          0.535,
          0.8703
        ],
        [
          0.545,
          0.8662
        ],
        [
          0.5551,
          0.8594
        ],
        [
          0.5655,
          0.8526
        ],
        [
          0.5755,
          0.8485
        ],
        [
          0.5855,
          0.8421
        ],
        [
          0.5955,
          0.8371
        ],
        [
          0.6056,
          0.8312
        ],
        [
          0.616,
          0.8239
        ],
        [
          0.626,
          0.818
        ],
        [
          0.636,
          0.813
        ],
        [
          0.646,
          0.808
        ],
        [
          0.6561,
          0.8025
        ],
        [
          0.6665,
          0.7948
        ],
        [
          0.6765,
          0.7884
        ],
        [
          0.6865,
          0.7812
        ],
        [
          0.6965,
          0.7766
        ],
        [
          0.7066,
          0.7712
        ],
        [
          0.717,
          0.7648
        ],
        [
          0.727,
          0.758
        ],
        [
          0.737,
          0.7484
        ],
        [
          0.747,
          0.7411
        ],
        [
          0.7571,
          0.7352
        ],
        [
          0.7675,
          0.7275
        ],
        [
          0.7775,
          0.7193
        ],
        [
          0.7875,
          0.7138
        ],
        [
          0.7975,
          0.7075
        ],
        [
          0.8076,
          0.7002
        ],
        [
          0.818,
          0.6924
        ],
        [
          0.828,
          0.6843
        ],
        [
          0.838,
          0.677
        ],
        [
          0.848,
          0.6711
        ],
        [
          0.8581,
          0.6642
        ],
        [
          0.8685,
          0.6565
        ],
        [
          0.8785,
          0.6488
        ],
        [
          0.8885,
          0.641
        ],
        [
          0.8985,
          0.6333
        ],
        [
          0.9086,
          0.626
        ],
        [
          0.919,
          0.6178
        ],
        [
          0.929,
          0.611
        ],
        [
          0.939,
          0.6024
        ],
        [
          0.949,
          0.5946
        ],
        [
          0.9591,
          0.5883
        ],
        [
          0.9695,
          0.581
        ],
        [
          0.9795,
          0.5742
        ],
        [
          0.9895,
          0.566
        ],
        [
          0.9995,
          0.5573
        ],
        [
          1.0,
          0.5569
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.12416009029531049,
        "difference": -0.0008580069597381246,
        "worse": 0.48333333333333334,
        "kept_right": 0.12329904751286433,
        "kept_wrong": 0.0008610427824460783,
        "handed_off": 0.8758399097046896
      },
      "usd_per_1000": 1.4207788945429427
    },
    {
      "decision": 45,
      "frontier": 48,
      "frontier_accuracy": 0.9758871701546861,
      "in_sample": {
        "threshold": 0.4702689339045559,
        "answered": 0.10737033666969972,
        "accuracy": 0.9758871701546861
      },
      "curve": [
        [
          0.0,
          0.9759
        ],
        [
          0.01,
          0.9759
        ],
        [
          0.02,
          0.9759
        ],
        [
          0.03,
          0.9759
        ],
        [
          0.04,
          0.9759
        ],
        [
          0.05,
          0.9759
        ],
        [
          0.0601,
          0.9759
        ],
        [
          0.0705,
          0.9759
        ],
        [
          0.0805,
          0.9759
        ],
        [
          0.0905,
          0.9759
        ],
        [
          0.1005,
          0.9759
        ],
        [
          0.1106,
          0.9754
        ],
        [
          0.121,
          0.9754
        ],
        [
          0.131,
          0.975
        ],
        [
          0.141,
          0.975
        ],
        [
          0.151,
          0.975
        ],
        [
          0.1611,
          0.9741
        ],
        [
          0.1715,
          0.9732
        ],
        [
          0.1815,
          0.9722
        ],
        [
          0.1915,
          0.9718
        ],
        [
          0.2015,
          0.9713
        ],
        [
          0.2116,
          0.97
        ],
        [
          0.222,
          0.9691
        ],
        [
          0.232,
          0.9686
        ],
        [
          0.242,
          0.9672
        ],
        [
          0.252,
          0.9654
        ],
        [
          0.2621,
          0.9654
        ],
        [
          0.2725,
          0.9613
        ],
        [
          0.2825,
          0.9591
        ],
        [
          0.2925,
          0.9577
        ],
        [
          0.3025,
          0.9554
        ],
        [
          0.3126,
          0.9536
        ],
        [
          0.323,
          0.9513
        ],
        [
          0.333,
          0.9486
        ],
        [
          0.343,
          0.9454
        ],
        [
          0.353,
          0.9422
        ],
        [
          0.3631,
          0.939
        ],
        [
          0.3735,
          0.9349
        ],
        [
          0.3835,
          0.9322
        ],
        [
          0.3935,
          0.929
        ],
        [
          0.4035,
          0.9258
        ],
        [
          0.4136,
          0.9227
        ],
        [
          0.424,
          0.9177
        ],
        [
          0.434,
          0.9126
        ],
        [
          0.444,
          0.9095
        ],
        [
          0.454,
          0.9058
        ],
        [
          0.4641,
          0.9026
        ],
        [
          0.4745,
          0.8981
        ],
        [
          0.4845,
          0.8926
        ],
        [
          0.4945,
          0.8858
        ],
        [
          0.5045,
          0.8813
        ],
        [
          0.5146,
          0.8749
        ],
        [
          0.525,
          0.869
        ],
        [
          0.535,
          0.8667
        ],
        [
          0.545,
          0.8626
        ],
        [
          0.5551,
          0.8558
        ],
        [
          0.5655,
          0.8494
        ],
        [
          0.5755,
          0.8449
        ],
        [
          0.5855,
          0.8389
        ],
        [
          0.5955,
          0.8339
        ],
        [
          0.6056,
          0.828
        ],
        [
          0.616,
          0.8203
        ],
        [
          0.626,
          0.8144
        ],
        [
          0.636,
          0.8098
        ],
        [
          0.646,
          0.8048
        ],
        [
          0.6561,
          0.7994
        ],
        [
          0.6665,
          0.7916
        ],
        [
          0.6765,
          0.7853
        ],
        [
          0.6865,
          0.778
        ],
        [
          0.6965,
          0.7734
        ],
        [
          0.7066,
          0.768
        ],
        [
          0.717,
          0.7616
        ],
        [
          0.727,
          0.7548
        ],
        [
          0.737,
          0.7452
        ],
        [
          0.747,
          0.7379
        ],
        [
          0.7571,
          0.732
        ],
        [
          0.7675,
          0.7243
        ],
        [
          0.7775,
          0.7161
        ],
        [
          0.7875,
          0.7106
        ],
        [
          0.7975,
          0.7043
        ],
        [
          0.8076,
          0.697
        ],
        [
          0.818,
          0.6893
        ],
        [
          0.828,
          0.6811
        ],
        [
          0.838,
          0.6738
        ],
        [
          0.848,
          0.6679
        ],
        [
          0.8581,
          0.6615
        ],
        [
          0.8685,
          0.6547
        ],
        [
          0.8785,
          0.647
        ],
        [
          0.8885,
          0.6401
        ],
        [
          0.8985,
          0.6328
        ],
        [
          0.9086,
          0.6256
        ],
        [
          0.919,
          0.6169
        ],
        [
          0.929,
          0.6096
        ],
        [
          0.939,
          0.6015
        ],
        [
          0.949,
          0.5937
        ],
        [
          0.9591,
          0.5878
        ],
        [
          0.9695,
          0.5805
        ],
        [
          0.9795,
          0.5737
        ],
        [
          0.9895,
          0.5655
        ],
        [
          0.9995,
          0.5573
        ],
        [
          1.0,
          0.5569
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.12416009029531049,
        "difference": -0.0008610427824460783,
        "worse": 0.48333333333333334,
        "kept_right": 0.12329904751286433,
        "kept_wrong": 0.0008610427824460783,
        "handed_off": 0.8758399097046896
      },
      "usd_per_1000": 4.710465188217546
    },
    {
      "decision": 45,
      "frontier": 191,
      "frontier_accuracy": 0.9740673339399454,
      "in_sample": {
        "threshold": 0.4702689339045559,
        "answered": 0.10737033666969972,
        "accuracy": 0.9740673339399454
      },
      "curve": [
        [
          0.0,
          0.9741
        ],
        [
          0.01,
          0.9741
        ],
        [
          0.02,
          0.9741
        ],
        [
          0.03,
          0.9741
        ],
        [
          0.04,
          0.9741
        ],
        [
          0.05,
          0.9741
        ],
        [
          0.0601,
          0.9741
        ],
        [
          0.0705,
          0.9741
        ],
        [
          0.0805,
          0.9741
        ],
        [
          0.0905,
          0.9741
        ],
        [
          0.1005,
          0.9741
        ],
        [
          0.1106,
          0.9736
        ],
        [
          0.121,
          0.9736
        ],
        [
          0.131,
          0.9732
        ],
        [
          0.141,
          0.9732
        ],
        [
          0.151,
          0.9732
        ],
        [
          0.1611,
          0.9722
        ],
        [
          0.1715,
          0.9713
        ],
        [
          0.1815,
          0.9704
        ],
        [
          0.1915,
          0.97
        ],
        [
          0.2015,
          0.9695
        ],
        [
          0.2116,
          0.9682
        ],
        [
          0.222,
          0.9668
        ],
        [
          0.232,
          0.9663
        ],
        [
          0.242,
          0.965
        ],
        [
          0.252,
          0.9631
        ],
        [
          0.2621,
          0.9631
        ],
        [
          0.2725,
          0.9591
        ],
        [
          0.2825,
          0.9568
        ],
        [
          0.2925,
          0.9554
        ],
        [
          0.3025,
          0.9531
        ],
        [
          0.3126,
          0.9513
        ],
        [
          0.323,
          0.949
        ],
        [
          0.333,
          0.9463
        ],
        [
          0.343,
          0.9431
        ],
        [
          0.353,
          0.9404
        ],
        [
          0.3631,
          0.9372
        ],
        [
          0.3735,
          0.9331
        ],
        [
          0.3835,
          0.9299
        ],
        [
          0.3935,
          0.9268
        ],
        [
          0.4035,
          0.9236
        ],
        [
          0.4136,
          0.9204
        ],
        [
          0.424,
          0.9154
        ],
        [
          0.434,
          0.9108
        ],
        [
          0.444,
          0.9076
        ],
        [
          0.454,
          0.904
        ],
        [
          0.4641,
          0.9013
        ],
        [
          0.4745,
          0.8972
        ],
        [
          0.4845,
          0.8917
        ],
        [
          0.4945,
          0.8849
        ],
        [
          0.5045,
          0.8808
        ],
        [
          0.5146,
          0.8744
        ],
        [
          0.525,
          0.8685
        ],
        [
          0.535,
          0.8662
        ],
        [
          0.545,
          0.8617
        ],
        [
          0.5551,
          0.8549
        ],
        [
          0.5655,
          0.848
        ],
        [
          0.5755,
          0.8439
        ],
        [
          0.5855,
          0.8376
        ],
        [
          0.5955,
          0.8326
        ],
        [
          0.6056,
          0.8267
        ],
        [
          0.616,
          0.8189
        ],
        [
          0.626,
          0.813
        ],
        [
          0.636,
          0.8085
        ],
        [
          0.646,
          0.8039
        ],
        [
          0.6561,
          0.7985
        ],
        [
          0.6665,
          0.7907
        ],
        [
          0.6765,
          0.7843
        ],
        [
          0.6865,
          0.7771
        ],
        [
          0.6965,
          0.7721
        ],
        [
          0.7066,
          0.7666
        ],
        [
          0.717,
          0.7602
        ],
        [
          0.727,
          0.7534
        ],
        [
          0.737,
          0.7439
        ],
        [
          0.747,
          0.7366
        ],
        [
          0.7571,
          0.7307
        ],
        [
          0.7675,
          0.7234
        ],
        [
          0.7775,
          0.7157
        ],
        [
          0.7875,
          0.7102
        ],
        [
          0.7975,
          0.7038
        ],
        [
          0.8076,
          0.6965
        ],
        [
          0.818,
          0.6888
        ],
        [
          0.828,
          0.6806
        ],
        [
          0.838,
          0.6733
        ],
        [
          0.848,
          0.6674
        ],
        [
          0.8581,
          0.6606
        ],
        [
          0.8685,
          0.6538
        ],
        [
          0.8785,
          0.6465
        ],
        [
          0.8885,
          0.6392
        ],
        [
          0.8985,
          0.6319
        ],
        [
          0.9086,
          0.6251
        ],
        [
          0.919,
          0.6169
        ],
        [
          0.929,
          0.6101
        ],
        [
          0.939,
          0.6015
        ],
        [
          0.949,
          0.5937
        ],
        [
          0.9591,
          0.5878
        ],
        [
          0.9695,
          0.581
        ],
        [
          0.9795,
          0.5737
        ],
        [
          0.9895,
          0.5655
        ],
        [
          0.9995,
          0.5569
        ],
        [
          1.0,
          0.5569
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.12416009029531049,
        "difference": -0.0008610427824460783,
        "worse": 0.48333333333333334,
        "kept_right": 0.12329904751286433,
        "kept_wrong": 0.0008610427824460783,
        "handed_off": 0.8758399097046896
      },
      "usd_per_1000": 23.417941326857925
    },
    {
      "decision": 45,
      "frontier": 188,
      "frontier_accuracy": 0.9713375796178344,
      "in_sample": {
        "threshold": 0.4186966813413095,
        "answered": 0.15332120109190173,
        "accuracy": 0.9713375796178344
      },
      "curve": [
        [
          0.0,
          0.9713
        ],
        [
          0.01,
          0.9713
        ],
        [
          0.02,
          0.9713
        ],
        [
          0.03,
          0.9713
        ],
        [
          0.04,
          0.9713
        ],
        [
          0.05,
          0.9713
        ],
        [
          0.0601,
          0.9713
        ],
        [
          0.0705,
          0.9713
        ],
        [
          0.0805,
          0.9713
        ],
        [
          0.0905,
          0.9713
        ],
        [
          0.1005,
          0.9713
        ],
        [
          0.1106,
          0.9709
        ],
        [
          0.121,
          0.9709
        ],
        [
          0.131,
          0.9709
        ],
        [
          0.141,
          0.9713
        ],
        [
          0.151,
          0.9713
        ],
        [
          0.1611,
          0.9704
        ],
        [
          0.1715,
          0.97
        ],
        [
          0.1815,
          0.9695
        ],
        [
          0.1915,
          0.9691
        ],
        [
          0.2015,
          0.9686
        ],
        [
          0.2116,
          0.9677
        ],
        [
          0.222,
          0.9663
        ],
        [
          0.232,
          0.9659
        ],
        [
          0.242,
          0.9645
        ],
        [
          0.252,
          0.9627
        ],
        [
          0.2621,
          0.9622
        ],
        [
          0.2725,
          0.9586
        ],
        [
          0.2825,
          0.9563
        ],
        [
          0.2925,
          0.9554
        ],
        [
          0.3025,
          0.9531
        ],
        [
          0.3126,
          0.9513
        ],
        [
          0.323,
          0.949
        ],
        [
          0.333,
          0.9463
        ],
        [
          0.343,
          0.9436
        ],
        [
          0.353,
          0.9409
        ],
        [
          0.3631,
          0.9381
        ],
        [
          0.3735,
          0.9345
        ],
        [
          0.3835,
          0.9322
        ],
        [
          0.3935,
          0.929
        ],
        [
          0.4035,
          0.9254
        ],
        [
          0.4136,
          0.9217
        ],
        [
          0.424,
          0.9172
        ],
        [
          0.434,
          0.9122
        ],
        [
          0.444,
          0.909
        ],
        [
          0.454,
          0.9054
        ],
        [
          0.4641,
          0.9022
        ],
        [
          0.4745,
          0.8972
        ],
        [
          0.4845,
          0.8917
        ],
        [
          0.4945,
          0.8849
        ],
        [
          0.5045,
          0.8808
        ],
        [
          0.5146,
          0.8744
        ],
        [
          0.525,
          0.869
        ],
        [
          0.535,
          0.8662
        ],
        [
          0.545,
          0.8621
        ],
        [
          0.5551,
          0.8553
        ],
        [
          0.5655,
          0.8485
        ],
        [
          0.5755,
          0.8435
        ],
        [
          0.5855,
          0.8376
        ],
        [
          0.5955,
          0.833
        ],
        [
          0.6056,
          0.8271
        ],
        [
          0.616,
          0.8198
        ],
        [
          0.626,
          0.8139
        ],
        [
          0.636,
          0.8089
        ],
        [
          0.646,
          0.8039
        ],
        [
          0.6561,
          0.7985
        ],
        [
          0.6665,
          0.7907
        ],
        [
          0.6765,
          0.7843
        ],
        [
          0.6865,
          0.7771
        ],
        [
          0.6965,
          0.7716
        ],
        [
          0.7066,
          0.7662
        ],
        [
          0.717,
          0.7602
        ],
        [
          0.727,
          0.7534
        ],
        [
          0.737,
          0.7448
        ],
        [
          0.747,
          0.7375
        ],
        [
          0.7571,
          0.7316
        ],
        [
          0.7675,
          0.7238
        ],
        [
          0.7775,
          0.7157
        ],
        [
          0.7875,
          0.7102
        ],
        [
          0.7975,
          0.7038
        ],
        [
          0.8076,
          0.697
        ],
        [
          0.818,
          0.6893
        ],
        [
          0.828,
          0.6811
        ],
        [
          0.838,
          0.6738
        ],
        [
          0.848,
          0.6679
        ],
        [
          0.8581,
          0.6611
        ],
        [
          0.8685,
          0.6542
        ],
        [
          0.8785,
          0.6465
        ],
        [
          0.8885,
          0.6397
        ],
        [
          0.8985,
          0.6319
        ],
        [
          0.9086,
          0.6247
        ],
        [
          0.919,
          0.6165
        ],
        [
          0.929,
          0.6092
        ],
        [
          0.939,
          0.601
        ],
        [
          0.949,
          0.5933
        ],
        [
          0.9591,
          0.5869
        ],
        [
          0.9695,
          0.5796
        ],
        [
          0.9795,
          0.5728
        ],
        [
          0.9895,
          0.5646
        ],
        [
          0.9995,
          0.5569
        ],
        [
          1.0,
          0.5569
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.15217647928664133,
        "difference": -0.0013459524091801623,
        "worse": 0.4666666666666667,
        "kept_right": 0.14992104609650775,
        "kept_wrong": 0.002255433190133704,
        "handed_off": 0.8478235207133592
      },
      "usd_per_1000": 0.8168269500778331
    },
    {
      "decision": 45,
      "frontier": 190,
      "frontier_accuracy": 0.9672429481346679,
      "in_sample": {
        "threshold": 0.4702689339045559,
        "answered": 0.10737033666969972,
        "accuracy": 0.9672429481346679
      },
      "curve": [
        [
          0.0,
          0.9672
        ],
        [
          0.01,
          0.9672
        ],
        [
          0.02,
          0.9672
        ],
        [
          0.03,
          0.9672
        ],
        [
          0.04,
          0.9672
        ],
        [
          0.05,
          0.9672
        ],
        [
          0.0601,
          0.9672
        ],
        [
          0.0705,
          0.9672
        ],
        [
          0.0805,
          0.9672
        ],
        [
          0.0905,
          0.9672
        ],
        [
          0.1005,
          0.9672
        ],
        [
          0.1106,
          0.9668
        ],
        [
          0.121,
          0.9668
        ],
        [
          0.131,
          0.9663
        ],
        [
          0.141,
          0.9663
        ],
        [
          0.151,
          0.9668
        ],
        [
          0.1611,
          0.9659
        ],
        [
          0.1715,
          0.965
        ],
        [
          0.1815,
          0.9641
        ],
        [
          0.1915,
          0.9636
        ],
        [
          0.2015,
          0.9631
        ],
        [
          0.2116,
          0.9618
        ],
        [
          0.222,
          0.9604
        ],
        [
          0.232,
          0.9604
        ],
        [
          0.242,
          0.9591
        ],
        [
          0.252,
          0.9572
        ],
        [
          0.2621,
          0.9572
        ],
        [
          0.2725,
          0.9536
        ],
        [
          0.2825,
          0.9513
        ],
        [
          0.2925,
          0.9504
        ],
        [
          0.3025,
          0.9481
        ],
        [
          0.3126,
          0.9463
        ],
        [
          0.323,
          0.944
        ],
        [
          0.333,
          0.9413
        ],
        [
          0.343,
          0.9381
        ],
        [
          0.353,
          0.9354
        ],
        [
          0.3631,
          0.9322
        ],
        [
          0.3735,
          0.9281
        ],
        [
          0.3835,
          0.9254
        ],
        [
          0.3935,
          0.9222
        ],
        [
          0.4035,
          0.919
        ],
        [
          0.4136,
          0.9158
        ],
        [
          0.424,
          0.9108
        ],
        [
          0.434,
          0.9063
        ],
        [
          0.444,
          0.9031
        ],
        [
          0.454,
          0.8999
        ],
        [
          0.4641,
          0.8967
        ],
        [
          0.4745,
          0.8926
        ],
        [
          0.4845,
          0.8872
        ],
        [
          0.4945,
          0.8803
        ],
        [
          0.5045,
          0.8763
        ],
        [
          0.5146,
          0.8699
        ],
        [
          0.525,
          0.8644
        ],
        [
          0.535,
          0.8626
        ],
        [
          0.545,
          0.8585
        ],
        [
          0.5551,
          0.8517
        ],
        [
          0.5655,
          0.8449
        ],
        [
          0.5755,
          0.8403
        ],
        [
          0.5855,
          0.8344
        ],
        [
          0.5955,
          0.8294
        ],
        [
          0.6056,
          0.8235
        ],
        [
          0.616,
          0.8162
        ],
        [
          0.626,
          0.8103
        ],
        [
          0.636,
          0.8057
        ],
        [
          0.646,
          0.8007
        ],
        [
          0.6561,
          0.7957
        ],
        [
          0.6665,
          0.788
        ],
        [
          0.6765,
          0.7816
        ],
        [
          0.6865,
          0.7743
        ],
        [
          0.6965,
          0.7698
        ],
        [
          0.7066,
          0.7639
        ],
        [
          0.717,
          0.7575
        ],
        [
          0.727,
          0.7507
        ],
        [
          0.737,
          0.7416
        ],
        [
          0.747,
          0.7343
        ],
        [
          0.7571,
          0.7284
        ],
        [
          0.7675,
          0.7207
        ],
        [
          0.7775,
          0.7129
        ],
        [
          0.7875,
          0.7079
        ],
        [
          0.7975,
          0.7015
        ],
        [
          0.8076,
          0.6943
        ],
        [
          0.818,
          0.6865
        ],
        [
          0.828,
          0.6783
        ],
        [
          0.838,
          0.6706
        ],
        [
          0.848,
          0.6652
        ],
        [
          0.8581,
          0.6583
        ],
        [
          0.8685,
          0.6515
        ],
        [
          0.8785,
          0.6442
        ],
        [
          0.8885,
          0.6365
        ],
        [
          0.8985,
          0.6292
        ],
        [
          0.9086,
          0.6224
        ],
        [
          0.919,
          0.6142
        ],
        [
          0.929,
          0.6078
        ],
        [
          0.939,
          0.6001
        ],
        [
          0.949,
          0.5928
        ],
        [
          0.9591,
          0.5869
        ],
        [
          0.9695,
          0.581
        ],
        [
          0.9795,
          0.5737
        ],
        [
          0.9895,
          0.5655
        ],
        [
          0.9995,
          0.5573
        ],
        [
          1.0,
          0.5569
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.13510094252937363,
        "difference": -0.0011551067025519694,
        "worse": 0.62,
        "kept_right": 0.1338488301993619,
        "kept_wrong": 0.0012521123300117832,
        "handed_off": 0.8648990574706268
      },
      "usd_per_1000": 4.816206849560034
    },
    {
      "decision": 45,
      "frontier": 124,
      "frontier_accuracy": 0.9667879890809827,
      "in_sample": {
        "threshold": 0.4702689339045559,
        "answered": 0.10737033666969972,
        "accuracy": 0.9667879890809827
      },
      "curve": [
        [
          0.0,
          0.9668
        ],
        [
          0.01,
          0.9668
        ],
        [
          0.02,
          0.9668
        ],
        [
          0.03,
          0.9668
        ],
        [
          0.04,
          0.9668
        ],
        [
          0.05,
          0.9668
        ],
        [
          0.0601,
          0.9668
        ],
        [
          0.0705,
          0.9668
        ],
        [
          0.0805,
          0.9668
        ],
        [
          0.0905,
          0.9668
        ],
        [
          0.1005,
          0.9668
        ],
        [
          0.1106,
          0.9663
        ],
        [
          0.121,
          0.9663
        ],
        [
          0.131,
          0.9659
        ],
        [
          0.141,
          0.9659
        ],
        [
          0.151,
          0.9659
        ],
        [
          0.1611,
          0.965
        ],
        [
          0.1715,
          0.9645
        ],
        [
          0.1815,
          0.9636
        ],
        [
          0.1915,
          0.9631
        ],
        [
          0.2015,
          0.9627
        ],
        [
          0.2116,
          0.9613
        ],
        [
          0.222,
          0.9604
        ],
        [
          0.232,
          0.96
        ],
        [
          0.242,
          0.9586
        ],
        [
          0.252,
          0.9568
        ],
        [
          0.2621,
          0.9568
        ],
        [
          0.2725,
          0.9531
        ],
        [
          0.2825,
          0.9509
        ],
        [
          0.2925,
          0.9495
        ],
        [
          0.3025,
          0.9472
        ],
        [
          0.3126,
          0.9454
        ],
        [
          0.323,
          0.9431
        ],
        [
          0.333,
          0.9404
        ],
        [
          0.343,
          0.9372
        ],
        [
          0.353,
          0.934
        ],
        [
          0.3631,
          0.9308
        ],
        [
          0.3735,
          0.9268
        ],
        [
          0.3835,
          0.924
        ],
        [
          0.3935,
          0.9208
        ],
        [
          0.4035,
          0.9177
        ],
        [
          0.4136,
          0.9145
        ],
        [
          0.424,
          0.9099
        ],
        [
          0.434,
          0.9054
        ],
        [
          0.444,
          0.9022
        ],
        [
          0.454,
          0.8985
        ],
        [
          0.4641,
          0.8954
        ],
        [
          0.4745,
          0.8913
        ],
        [
          0.4845,
          0.8858
        ],
        [
          0.4945,
          0.879
        ],
        [
          0.5045,
          0.8744
        ],
        [
          0.5146,
          0.8681
        ],
        [
          0.525,
          0.8631
        ],
        [
          0.535,
          0.8621
        ],
        [
          0.545,
          0.8581
        ],
        [
          0.5551,
          0.8512
        ],
        [
          0.5655,
          0.8444
        ],
        [
          0.5755,
          0.8403
        ],
        [
          0.5855,
          0.8348
        ],
        [
          0.5955,
          0.8298
        ],
        [
          0.6056,
          0.8239
        ],
        [
          0.616,
          0.8171
        ],
        [
          0.626,
          0.8112
        ],
        [
          0.636,
          0.8066
        ],
        [
          0.646,
          0.8021
        ],
        [
          0.6561,
          0.7966
        ],
        [
          0.6665,
          0.7889
        ],
        [
          0.6765,
          0.7825
        ],
        [
          0.6865,
          0.7757
        ],
        [
          0.6965,
          0.7712
        ],
        [
          0.7066,
          0.7657
        ],
        [
          0.717,
          0.7593
        ],
        [
          0.727,
          0.7525
        ],
        [
          0.737,
          0.7429
        ],
        [
          0.747,
          0.7357
        ],
        [
          0.7571,
          0.7298
        ],
        [
          0.7675,
          0.722
        ],
        [
          0.7775,
          0.7143
        ],
        [
          0.7875,
          0.7097
        ],
        [
          0.7975,
          0.7038
        ],
        [
          0.8076,
          0.697
        ],
        [
          0.818,
          0.6893
        ],
        [
          0.828,
          0.6811
        ],
        [
          0.838,
          0.6738
        ],
        [
          0.848,
          0.6679
        ],
        [
          0.8581,
          0.6611
        ],
        [
          0.8685,
          0.6542
        ],
        [
          0.8785,
          0.6465
        ],
        [
          0.8885,
          0.6388
        ],
        [
          0.8985,
          0.6319
        ],
        [
          0.9086,
          0.6247
        ],
        [
          0.919,
          0.616
        ],
        [
          0.929,
          0.6092
        ],
        [
          0.939,
          0.601
        ],
        [
          0.949,
          0.5933
        ],
        [
          0.9591,
          0.5874
        ],
        [
          0.9695,
          0.5801
        ],
        [
          0.9795,
          0.5732
        ],
        [
          0.9895,
          0.5651
        ],
        [
          0.9995,
          0.5569
        ],
        [
          1.0,
          0.5569
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.12431172129978686,
        "difference": -0.0008519545804716065,
        "worse": 0.48333333333333334,
        "kept_right": 0.12343551321052872,
        "kept_wrong": 0.0008762080892580385,
        "handed_off": 0.8756882787002133
      },
      "usd_per_1000": 0.9898544448259334
    },
    {
      "decision": 45,
      "frontier": 278,
      "frontier_accuracy": 0.9499545040946314,
      "in_sample": {
        "threshold": 0.4702689339045559,
        "answered": 0.10737033666969972,
        "accuracy": 0.9499545040946314
      },
      "curve": [
        [
          0.0,
          0.95
        ],
        [
          0.01,
          0.95
        ],
        [
          0.02,
          0.95
        ],
        [
          0.03,
          0.95
        ],
        [
          0.04,
          0.95
        ],
        [
          0.05,
          0.95
        ],
        [
          0.0601,
          0.95
        ],
        [
          0.0705,
          0.95
        ],
        [
          0.0805,
          0.95
        ],
        [
          0.0905,
          0.95
        ],
        [
          0.1005,
          0.95
        ],
        [
          0.1106,
          0.9495
        ],
        [
          0.121,
          0.9495
        ],
        [
          0.131,
          0.949
        ],
        [
          0.141,
          0.949
        ],
        [
          0.151,
          0.949
        ],
        [
          0.1611,
          0.9481
        ],
        [
          0.1715,
          0.9477
        ],
        [
          0.1815,
          0.9472
        ],
        [
          0.1915,
          0.9468
        ],
        [
          0.2015,
          0.9463
        ],
        [
          0.2116,
          0.9449
        ],
        [
          0.222,
          0.944
        ],
        [
          0.232,
          0.9436
        ],
        [
          0.242,
          0.9427
        ],
        [
          0.252,
          0.9409
        ],
        [
          0.2621,
          0.9409
        ],
        [
          0.2725,
          0.9372
        ],
        [
          0.2825,
          0.9349
        ],
        [
          0.2925,
          0.9336
        ],
        [
          0.3025,
          0.9313
        ],
        [
          0.3126,
          0.9299
        ],
        [
          0.323,
          0.9281
        ],
        [
          0.333,
          0.9254
        ],
        [
          0.343,
          0.9231
        ],
        [
          0.353,
          0.9204
        ],
        [
          0.3631,
          0.9186
        ],
        [
          0.3735,
          0.9149
        ],
        [
          0.3835,
          0.9122
        ],
        [
          0.3935,
          0.909
        ],
        [
          0.4035,
          0.9063
        ],
        [
          0.4136,
          0.904
        ],
        [
          0.424,
          0.8995
        ],
        [
          0.434,
          0.8949
        ],
        [
          0.444,
          0.8917
        ],
        [
          0.454,
          0.8885
        ],
        [
          0.4641,
          0.8854
        ],
        [
          0.4745,
          0.8808
        ],
        [
          0.4845,
          0.8753
        ],
        [
          0.4945,
          0.8685
        ],
        [
          0.5045,
          0.8644
        ],
        [
          0.5146,
          0.8599
        ],
        [
          0.525,
          0.8544
        ],
        [
          0.535,
          0.8526
        ],
        [
          0.545,
          0.8485
        ],
        [
          0.5551,
          0.8421
        ],
        [
          0.5655,
          0.8353
        ],
        [
          0.5755,
          0.8308
        ],
        [
          0.5855,
          0.8244
        ],
        [
          0.5955,
          0.8198
        ],
        [
          0.6056,
          0.8144
        ],
        [
          0.616,
          0.808
        ],
        [
          0.626,
          0.8025
        ],
        [
          0.636,
          0.7975
        ],
        [
          0.646,
          0.7925
        ],
        [
          0.6561,
          0.7871
        ],
        [
          0.6665,
          0.7793
        ],
        [
          0.6765,
          0.7734
        ],
        [
          0.6865,
          0.7666
        ],
        [
          0.6965,
          0.7611
        ],
        [
          0.7066,
          0.7557
        ],
        [
          0.717,
          0.7498
        ],
        [
          0.727,
          0.7443
        ],
        [
          0.737,
          0.7348
        ],
        [
          0.747,
          0.7279
        ],
        [
          0.7571,
          0.722
        ],
        [
          0.7675,
          0.7147
        ],
        [
          0.7775,
          0.707
        ],
        [
          0.7875,
          0.702
        ],
        [
          0.7975,
          0.6961
        ],
        [
          0.8076,
          0.6888
        ],
        [
          0.818,
          0.6811
        ],
        [
          0.828,
          0.6733
        ],
        [
          0.838,
          0.6665
        ],
        [
          0.848,
          0.6615
        ],
        [
          0.8581,
          0.6547
        ],
        [
          0.8685,
          0.6483
        ],
        [
          0.8785,
          0.6406
        ],
        [
          0.8885,
          0.6338
        ],
        [
          0.8985,
          0.6269
        ],
        [
          0.9086,
          0.6201
        ],
        [
          0.919,
          0.6124
        ],
        [
          0.929,
          0.6069
        ],
        [
          0.939,
          0.5992
        ],
        [
          0.949,
          0.5914
        ],
        [
          0.9591,
          0.5869
        ],
        [
          0.9695,
          0.5796
        ],
        [
          0.9795,
          0.5732
        ],
        [
          0.9895,
          0.5655
        ],
        [
          0.9995,
          0.5569
        ],
        [
          1.0,
          0.5569
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1274643731420146,
        "difference": -0.0010853680867970183,
        "worse": 0.5,
        "kept_right": 0.12630017570613078,
        "kept_wrong": 0.0011641974358837378,
        "handed_off": 0.8725356268579856
      },
      "usd_per_1000": 5.679182832842677
    },
    {
      "decision": 45,
      "frontier": 275,
      "frontier_accuracy": 0.9426751592356688,
      "in_sample": {
        "threshold": 0.4702689339045559,
        "answered": 0.10737033666969972,
        "accuracy": 0.9426751592356688
      },
      "curve": [
        [
          0.0,
          0.9427
        ],
        [
          0.01,
          0.9427
        ],
        [
          0.02,
          0.9427
        ],
        [
          0.03,
          0.9427
        ],
        [
          0.04,
          0.9427
        ],
        [
          0.05,
          0.9427
        ],
        [
          0.0601,
          0.9427
        ],
        [
          0.0705,
          0.9427
        ],
        [
          0.0805,
          0.9427
        ],
        [
          0.0905,
          0.9427
        ],
        [
          0.1005,
          0.9427
        ],
        [
          0.1106,
          0.9422
        ],
        [
          0.121,
          0.9422
        ],
        [
          0.131,
          0.9418
        ],
        [
          0.141,
          0.9418
        ],
        [
          0.151,
          0.9418
        ],
        [
          0.1611,
          0.9409
        ],
        [
          0.1715,
          0.9404
        ],
        [
          0.1815,
          0.9395
        ],
        [
          0.1915,
          0.939
        ],
        [
          0.2015,
          0.9386
        ],
        [
          0.2116,
          0.9372
        ],
        [
          0.222,
          0.9359
        ],
        [
          0.232,
          0.9354
        ],
        [
          0.242,
          0.9345
        ],
        [
          0.252,
          0.9327
        ],
        [
          0.2621,
          0.9327
        ],
        [
          0.2725,
          0.929
        ],
        [
          0.2825,
          0.9268
        ],
        [
          0.2925,
          0.9254
        ],
        [
          0.3025,
          0.9231
        ],
        [
          0.3126,
          0.9222
        ],
        [
          0.323,
          0.9199
        ],
        [
          0.333,
          0.9172
        ],
        [
          0.343,
          0.9145
        ],
        [
          0.353,
          0.9122
        ],
        [
          0.3631,
          0.9095
        ],
        [
          0.3735,
          0.9067
        ],
        [
          0.3835,
          0.904
        ],
        [
          0.3935,
          0.9008
        ],
        [
          0.4035,
          0.8976
        ],
        [
          0.4136,
          0.8944
        ],
        [
          0.424,
          0.8899
        ],
        [
          0.434,
          0.8854
        ],
        [
          0.444,
          0.8822
        ],
        [
          0.454,
          0.879
        ],
        [
          0.4641,
          0.8763
        ],
        [
          0.4745,
          0.8731
        ],
        [
          0.4845,
          0.8676
        ],
        [
          0.4945,
          0.8608
        ],
        [
          0.5045,
          0.8567
        ],
        [
          0.5146,
          0.8508
        ],
        [
          0.525,
          0.8458
        ],
        [
          0.535,
          0.8444
        ],
        [
          0.545,
          0.8417
        ],
        [
          0.5551,
          0.8348
        ],
        [
          0.5655,
          0.8285
        ],
        [
          0.5755,
          0.8244
        ],
        [
          0.5855,
          0.8189
        ],
        [
          0.5955,
          0.8153
        ],
        [
          0.6056,
          0.8098
        ],
        [
          0.616,
          0.8025
        ],
        [
          0.626,
          0.7966
        ],
        [
          0.636,
          0.7921
        ],
        [
          0.646,
          0.7875
        ],
        [
          0.6561,
          0.7825
        ],
        [
          0.6665,
          0.7748
        ],
        [
          0.6765,
          0.7684
        ],
        [
          0.6865,
          0.7616
        ],
        [
          0.6965,
          0.7571
        ],
        [
          0.7066,
          0.7525
        ],
        [
          0.717,
          0.7466
        ],
        [
          0.727,
          0.7407
        ],
        [
          0.737,
          0.7311
        ],
        [
          0.747,
          0.7243
        ],
        [
          0.7571,
          0.7193
        ],
        [
          0.7675,
          0.7125
        ],
        [
          0.7775,
          0.7047
        ],
        [
          0.7875,
          0.7002
        ],
        [
          0.7975,
          0.6938
        ],
        [
          0.8076,
          0.6865
        ],
        [
          0.818,
          0.6793
        ],
        [
          0.828,
          0.6711
        ],
        [
          0.838,
          0.6633
        ],
        [
          0.848,
          0.6574
        ],
        [
          0.8581,
          0.651
        ],
        [
          0.8685,
          0.6451
        ],
        [
          0.8785,
          0.6374
        ],
        [
          0.8885,
          0.6315
        ],
        [
          0.8985,
          0.6242
        ],
        [
          0.9086,
          0.6178
        ],
        [
          0.919,
          0.6101
        ],
        [
          0.929,
          0.6042
        ],
        [
          0.939,
          0.5969
        ],
        [
          0.949,
          0.5892
        ],
        [
          0.9591,
          0.5842
        ],
        [
          0.9695,
          0.5783
        ],
        [
          0.9795,
          0.5728
        ],
        [
          0.9895,
          0.5651
        ],
        [
          0.9995,
          0.5569
        ],
        [
          1.0,
          0.5569
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.12431172129978686,
        "difference": -0.0008519545804716065,
        "worse": 0.48333333333333334,
        "kept_right": 0.12343551321052872,
        "kept_wrong": 0.0008762080892580385,
        "handed_off": 0.8756882787002133
      },
      "usd_per_1000": 0.4505531730582671
    },
    {
      "decision": 45,
      "frontier": 192,
      "frontier_accuracy": 0.9331210191082803,
      "in_sample": {
        "threshold": 0.4702689339045559,
        "answered": 0.10737033666969972,
        "accuracy": 0.9331210191082803
      },
      "curve": [
        [
          0.0,
          0.9331
        ],
        [
          0.01,
          0.9331
        ],
        [
          0.02,
          0.9331
        ],
        [
          0.03,
          0.9331
        ],
        [
          0.04,
          0.9331
        ],
        [
          0.05,
          0.9331
        ],
        [
          0.0601,
          0.9331
        ],
        [
          0.0705,
          0.9331
        ],
        [
          0.0805,
          0.9331
        ],
        [
          0.0905,
          0.9331
        ],
        [
          0.1005,
          0.9331
        ],
        [
          0.1106,
          0.9327
        ],
        [
          0.121,
          0.9327
        ],
        [
          0.131,
          0.9322
        ],
        [
          0.141,
          0.9322
        ],
        [
          0.151,
          0.9322
        ],
        [
          0.1611,
          0.9318
        ],
        [
          0.1715,
          0.9308
        ],
        [
          0.1815,
          0.9299
        ],
        [
          0.1915,
          0.9295
        ],
        [
          0.2015,
          0.929
        ],
        [
          0.2116,
          0.9286
        ],
        [
          0.222,
          0.9281
        ],
        [
          0.232,
          0.9281
        ],
        [
          0.242,
          0.9272
        ],
        [
          0.252,
          0.9254
        ],
        [
          0.2621,
          0.9249
        ],
        [
          0.2725,
          0.9213
        ],
        [
          0.2825,
          0.919
        ],
        [
          0.2925,
          0.9181
        ],
        [
          0.3025,
          0.9163
        ],
        [
          0.3126,
          0.9149
        ],
        [
          0.323,
          0.9136
        ],
        [
          0.333,
          0.9108
        ],
        [
          0.343,
          0.9076
        ],
        [
          0.353,
          0.9049
        ],
        [
          0.3631,
          0.9031
        ],
        [
          0.3735,
          0.899
        ],
        [
          0.3835,
          0.8963
        ],
        [
          0.3935,
          0.8931
        ],
        [
          0.4035,
          0.8899
        ],
        [
          0.4136,
          0.8872
        ],
        [
          0.424,
          0.8826
        ],
        [
          0.434,
          0.879
        ],
        [
          0.444,
          0.8758
        ],
        [
          0.454,
          0.8726
        ],
        [
          0.4641,
          0.8703
        ],
        [
          0.4745,
          0.8662
        ],
        [
          0.4845,
          0.8612
        ],
        [
          0.4945,
          0.8549
        ],
        [
          0.5045,
          0.8517
        ],
        [
          0.5146,
          0.8467
        ],
        [
          0.525,
          0.8412
        ],
        [
          0.535,
          0.8408
        ],
        [
          0.545,
          0.8371
        ],
        [
          0.5551,
          0.8308
        ],
        [
          0.5655,
          0.8239
        ],
        [
          0.5755,
          0.8198
        ],
        [
          0.5855,
          0.8139
        ],
        [
          0.5955,
          0.8089
        ],
        [
          0.6056,
          0.8035
        ],
        [
          0.616,
          0.7966
        ],
        [
          0.626,
          0.7912
        ],
        [
          0.636,
          0.7866
        ],
        [
          0.646,
          0.7825
        ],
        [
          0.6561,
          0.7775
        ],
        [
          0.6665,
          0.7698
        ],
        [
          0.6765,
          0.7634
        ],
        [
          0.6865,
          0.7561
        ],
        [
          0.6965,
          0.7507
        ],
        [
          0.7066,
          0.7457
        ],
        [
          0.717,
          0.7393
        ],
        [
          0.727,
          0.7338
        ],
        [
          0.737,
          0.7247
        ],
        [
          0.747,
          0.7179
        ],
        [
          0.7571,
          0.7129
        ],
        [
          0.7675,
          0.7056
        ],
        [
          0.7775,
          0.6979
        ],
        [
          0.7875,
          0.6938
        ],
        [
          0.7975,
          0.6879
        ],
        [
          0.8076,
          0.682
        ],
        [
          0.818,
          0.6742
        ],
        [
          0.828,
          0.667
        ],
        [
          0.838,
          0.6606
        ],
        [
          0.848,
          0.6561
        ],
        [
          0.8581,
          0.6492
        ],
        [
          0.8685,
          0.6424
        ],
        [
          0.8785,
          0.6351
        ],
        [
          0.8885,
          0.6278
        ],
        [
          0.8985,
          0.6224
        ],
        [
          0.9086,
          0.6165
        ],
        [
          0.919,
          0.6101
        ],
        [
          0.929,
          0.6042
        ],
        [
          0.939,
          0.5969
        ],
        [
          0.949,
          0.5896
        ],
        [
          0.9591,
          0.5851
        ],
        [
          0.9695,
          0.5787
        ],
        [
          0.9795,
          0.5723
        ],
        [
          0.9895,
          0.5651
        ],
        [
          0.9995,
          0.5569
        ],
        [
          1.0,
          0.5569
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.14046933373655277,
        "difference": -0.0015764568453474725,
        "worse": 0.6333333333333333,
        "kept_right": 0.1387200585011603,
        "kept_wrong": 0.0017492752353924564,
        "handed_off": 0.8595306662634473
      },
      "usd_per_1000": 9.978777424946216
    },
    {
      "decision": 45,
      "frontier": 123,
      "frontier_accuracy": 0.9326660600545951,
      "in_sample": {
        "threshold": 0.4186966813413095,
        "answered": 0.15332120109190173,
        "accuracy": 0.9326660600545951
      },
      "curve": [
        [
          0.0,
          0.9327
        ],
        [
          0.01,
          0.9327
        ],
        [
          0.02,
          0.9327
        ],
        [
          0.03,
          0.9327
        ],
        [
          0.04,
          0.9327
        ],
        [
          0.05,
          0.9327
        ],
        [
          0.0601,
          0.9327
        ],
        [
          0.0705,
          0.9327
        ],
        [
          0.0805,
          0.9327
        ],
        [
          0.0905,
          0.9327
        ],
        [
          0.1005,
          0.9327
        ],
        [
          0.1106,
          0.9322
        ],
        [
          0.121,
          0.9322
        ],
        [
          0.131,
          0.9322
        ],
        [
          0.141,
          0.9327
        ],
        [
          0.151,
          0.9327
        ],
        [
          0.1611,
          0.9318
        ],
        [
          0.1715,
          0.9313
        ],
        [
          0.1815,
          0.9304
        ],
        [
          0.1915,
          0.9299
        ],
        [
          0.2015,
          0.9295
        ],
        [
          0.2116,
          0.9286
        ],
        [
          0.222,
          0.9277
        ],
        [
          0.232,
          0.9272
        ],
        [
          0.242,
          0.9258
        ],
        [
          0.252,
          0.924
        ],
        [
          0.2621,
          0.924
        ],
        [
          0.2725,
          0.9204
        ],
        [
          0.2825,
          0.9181
        ],
        [
          0.2925,
          0.9167
        ],
        [
          0.3025,
          0.9145
        ],
        [
          0.3126,
          0.9131
        ],
        [
          0.323,
          0.9113
        ],
        [
          0.333,
          0.9086
        ],
        [
          0.343,
          0.9058
        ],
        [
          0.353,
          0.904
        ],
        [
          0.3631,
          0.9013
        ],
        [
          0.3735,
          0.8972
        ],
        [
          0.3835,
          0.8949
        ],
        [
          0.3935,
          0.8917
        ],
        [
          0.4035,
          0.8885
        ],
        [
          0.4136,
          0.8854
        ],
        [
          0.424,
          0.8817
        ],
        [
          0.434,
          0.8776
        ],
        [
          0.444,
          0.8744
        ],
        [
          0.454,
          0.8708
        ],
        [
          0.4641,
          0.8685
        ],
        [
          0.4745,
          0.864
        ],
        [
          0.4845,
          0.8585
        ],
        [
          0.4945,
          0.8517
        ],
        [
          0.5045,
          0.848
        ],
        [
          0.5146,
          0.8435
        ],
        [
          0.525,
          0.838
        ],
        [
          0.535,
          0.8358
        ],
        [
          0.545,
          0.8321
        ],
        [
          0.5551,
          0.8253
        ],
        [
          0.5655,
          0.8198
        ],
        [
          0.5755,
          0.8148
        ],
        [
          0.5855,
          0.8089
        ],
        [
          0.5955,
          0.8048
        ],
        [
          0.6056,
          0.7994
        ],
        [
          0.616,
          0.7921
        ],
        [
          0.626,
          0.7871
        ],
        [
          0.636,
          0.7825
        ],
        [
          0.646,
          0.778
        ],
        [
          0.6561,
          0.773
        ],
        [
          0.6665,
          0.7657
        ],
        [
          0.6765,
          0.7593
        ],
        [
          0.6865,
          0.753
        ],
        [
          0.6965,
          0.7475
        ],
        [
          0.7066,
          0.7429
        ],
        [
          0.717,
          0.7379
        ],
        [
          0.727,
          0.7316
        ],
        [
          0.737,
          0.7229
        ],
        [
          0.747,
          0.7179
        ],
        [
          0.7571,
          0.7138
        ],
        [
          0.7675,
          0.7079
        ],
        [
          0.7775,
          0.7002
        ],
        [
          0.7875,
          0.6952
        ],
        [
          0.7975,
          0.6888
        ],
        [
          0.8076,
          0.6829
        ],
        [
          0.818,
          0.6752
        ],
        [
          0.828,
          0.6688
        ],
        [
          0.838,
          0.6629
        ],
        [
          0.848,
          0.6579
        ],
        [
          0.8581,
          0.652
        ],
        [
          0.8685,
          0.6456
        ],
        [
          0.8785,
          0.6383
        ],
        [
          0.8885,
          0.6324
        ],
        [
          0.8985,
          0.6265
        ],
        [
          0.9086,
          0.6201
        ],
        [
          0.919,
          0.6115
        ],
        [
          0.929,
          0.6042
        ],
        [
          0.939,
          0.5965
        ],
        [
          0.949,
          0.5887
        ],
        [
          0.9591,
          0.5846
        ],
        [
          0.9695,
          0.5792
        ],
        [
          0.9795,
          0.5723
        ],
        [
          0.9895,
          0.5651
        ],
        [
          0.9995,
          0.5569
        ],
        [
          1.0,
          0.5569
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.14740856795703655,
        "difference": -0.0011520209665515399,
        "worse": 0.4033333333333333,
        "kept_right": 0.14558655413867674,
        "kept_wrong": 0.0018220138183597842,
        "handed_off": 0.8525914320429637
      },
      "usd_per_1000": 0.22899236598179412
    },
    {
      "decision": 45,
      "frontier": 274,
      "frontier_accuracy": 0.9085532302092811,
      "in_sample": {
        "threshold": 0.4702689339045559,
        "answered": 0.10737033666969972,
        "accuracy": 0.9085532302092811
      },
      "curve": [
        [
          0.0,
          0.9086
        ],
        [
          0.01,
          0.9086
        ],
        [
          0.02,
          0.9086
        ],
        [
          0.03,
          0.9086
        ],
        [
          0.04,
          0.9086
        ],
        [
          0.05,
          0.9086
        ],
        [
          0.0601,
          0.9086
        ],
        [
          0.0705,
          0.9086
        ],
        [
          0.0805,
          0.9086
        ],
        [
          0.0905,
          0.9086
        ],
        [
          0.1005,
          0.9086
        ],
        [
          0.1106,
          0.9081
        ],
        [
          0.121,
          0.9081
        ],
        [
          0.131,
          0.9076
        ],
        [
          0.141,
          0.9076
        ],
        [
          0.151,
          0.9081
        ],
        [
          0.1611,
          0.9072
        ],
        [
          0.1715,
          0.9067
        ],
        [
          0.1815,
          0.9063
        ],
        [
          0.1915,
          0.9058
        ],
        [
          0.2015,
          0.9058
        ],
        [
          0.2116,
          0.9049
        ],
        [
          0.222,
          0.9035
        ],
        [
          0.232,
          0.9031
        ],
        [
          0.242,
          0.9022
        ],
        [
          0.252,
          0.9004
        ],
        [
          0.2621,
          0.9004
        ],
        [
          0.2725,
          0.8967
        ],
        [
          0.2825,
          0.8944
        ],
        [
          0.2925,
          0.8931
        ],
        [
          0.3025,
          0.8908
        ],
        [
          0.3126,
          0.8894
        ],
        [
          0.323,
          0.8876
        ],
        [
          0.333,
          0.8854
        ],
        [
          0.343,
          0.8831
        ],
        [
          0.353,
          0.8813
        ],
        [
          0.3631,
          0.8781
        ],
        [
          0.3735,
          0.8749
        ],
        [
          0.3835,
          0.8726
        ],
        [
          0.3935,
          0.8694
        ],
        [
          0.4035,
          0.8662
        ],
        [
          0.4136,
          0.8635
        ],
        [
          0.424,
          0.8594
        ],
        [
          0.434,
          0.8553
        ],
        [
          0.444,
          0.8521
        ],
        [
          0.454,
          0.849
        ],
        [
          0.4641,
          0.8467
        ],
        [
          0.4745,
          0.8435
        ],
        [
          0.4845,
          0.8385
        ],
        [
          0.4945,
          0.8321
        ],
        [
          0.5045,
          0.8285
        ],
        [
          0.5146,
          0.823
        ],
        [
          0.525,
          0.818
        ],
        [
          0.535,
          0.8162
        ],
        [
          0.545,
          0.8135
        ],
        [
          0.5551,
          0.8076
        ],
        [
          0.5655,
          0.8016
        ],
        [
          0.5755,
          0.798
        ],
        [
          0.5855,
          0.793
        ],
        [
          0.5955,
          0.7894
        ],
        [
          0.6056,
          0.7834
        ],
        [
          0.616,
          0.7775
        ],
        [
          0.626,
          0.773
        ],
        [
          0.636,
          0.7698
        ],
        [
          0.646,
          0.7662
        ],
        [
          0.6561,
          0.7611
        ],
        [
          0.6665,
          0.7534
        ],
        [
          0.6765,
          0.747
        ],
        [
          0.6865,
          0.7411
        ],
        [
          0.6965,
          0.7366
        ],
        [
          0.7066,
          0.732
        ],
        [
          0.717,
          0.7266
        ],
        [
          0.727,
          0.7207
        ],
        [
          0.737,
          0.7125
        ],
        [
          0.747,
          0.7066
        ],
        [
          0.7571,
          0.702
        ],
        [
          0.7675,
          0.6947
        ],
        [
          0.7775,
          0.687
        ],
        [
          0.7875,
          0.682
        ],
        [
          0.7975,
          0.6765
        ],
        [
          0.8076,
          0.6711
        ],
        [
          0.818,
          0.6642
        ],
        [
          0.828,
          0.6574
        ],
        [
          0.838,
          0.651
        ],
        [
          0.848,
          0.6456
        ],
        [
          0.8581,
          0.6397
        ],
        [
          0.8685,
          0.6338
        ],
        [
          0.8785,
          0.6265
        ],
        [
          0.8885,
          0.621
        ],
        [
          0.8985,
          0.6156
        ],
        [
          0.9086,
          0.6101
        ],
        [
          0.919,
          0.6033
        ],
        [
          0.929,
          0.5974
        ],
        [
          0.939,
          0.5905
        ],
        [
          0.949,
          0.5828
        ],
        [
          0.9591,
          0.5792
        ],
        [
          0.9695,
          0.5737
        ],
        [
          0.9795,
          0.5687
        ],
        [
          0.9895,
          0.5628
        ],
        [
          0.9995,
          0.5569
        ],
        [
          1.0,
          0.5569
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.14616175580663787,
        "difference": -0.0017370884741377316,
        "worse": 0.6666666666666666,
        "kept_right": 0.14390927516726684,
        "kept_wrong": 0.00225248063937113,
        "handed_off": 0.8538382441933622
      },
      "usd_per_1000": 0.3639448612037446
    },
    {
      "decision": 45,
      "frontier": 47,
      "frontier_accuracy": 0.9044585987261147,
      "in_sample": {
        "threshold": 0.4702689339045559,
        "answered": 0.10737033666969972,
        "accuracy": 0.9044585987261147
      },
      "curve": [
        [
          0.0,
          0.9045
        ],
        [
          0.01,
          0.9045
        ],
        [
          0.02,
          0.9045
        ],
        [
          0.03,
          0.9045
        ],
        [
          0.04,
          0.9045
        ],
        [
          0.05,
          0.9045
        ],
        [
          0.0601,
          0.9045
        ],
        [
          0.0705,
          0.9045
        ],
        [
          0.0805,
          0.9045
        ],
        [
          0.0905,
          0.9045
        ],
        [
          0.1005,
          0.9045
        ],
        [
          0.1106,
          0.904
        ],
        [
          0.121,
          0.904
        ],
        [
          0.131,
          0.9035
        ],
        [
          0.141,
          0.9035
        ],
        [
          0.151,
          0.9035
        ],
        [
          0.1611,
          0.9026
        ],
        [
          0.1715,
          0.9022
        ],
        [
          0.1815,
          0.9022
        ],
        [
          0.1915,
          0.9017
        ],
        [
          0.2015,
          0.9013
        ],
        [
          0.2116,
          0.9008
        ],
        [
          0.222,
          0.9008
        ],
        [
          0.232,
          0.9004
        ],
        [
          0.242,
          0.8995
        ],
        [
          0.252,
          0.8976
        ],
        [
          0.2621,
          0.8981
        ],
        [
          0.2725,
          0.8949
        ],
        [
          0.2825,
          0.8931
        ],
        [
          0.2925,
          0.8926
        ],
        [
          0.3025,
          0.8908
        ],
        [
          0.3126,
          0.8894
        ],
        [
          0.323,
          0.8881
        ],
        [
          0.333,
          0.8854
        ],
        [
          0.343,
          0.884
        ],
        [
          0.353,
          0.8817
        ],
        [
          0.3631,
          0.8794
        ],
        [
          0.3735,
          0.8753
        ],
        [
          0.3835,
          0.8726
        ],
        [
          0.3935,
          0.8694
        ],
        [
          0.4035,
          0.8667
        ],
        [
          0.4136,
          0.8644
        ],
        [
          0.424,
          0.8599
        ],
        [
          0.434,
          0.8558
        ],
        [
          0.444,
          0.8526
        ],
        [
          0.454,
          0.849
        ],
        [
          0.4641,
          0.8462
        ],
        [
          0.4745,
          0.8417
        ],
        [
          0.4845,
          0.8362
        ],
        [
          0.4945,
          0.8298
        ],
        [
          0.5045,
          0.8262
        ],
        [
          0.5146,
          0.8226
        ],
        [
          0.525,
          0.8176
        ],
        [
          0.535,
          0.8171
        ],
        [
          0.545,
          0.8139
        ],
        [
          0.5551,
          0.808
        ],
        [
          0.5655,
          0.8021
        ],
        [
          0.5755,
          0.798
        ],
        [
          0.5855,
          0.7921
        ],
        [
          0.5955,
          0.7884
        ],
        [
          0.6056,
          0.7834
        ],
        [
          0.616,
          0.7775
        ],
        [
          0.626,
          0.7734
        ],
        [
          0.636,
          0.7693
        ],
        [
          0.646,
          0.7652
        ],
        [
          0.6561,
          0.7602
        ],
        [
          0.6665,
          0.7525
        ],
        [
          0.6765,
          0.7466
        ],
        [
          0.6865,
          0.7402
        ],
        [
          0.6965,
          0.7357
        ],
        [
          0.7066,
          0.7325
        ],
        [
          0.717,
          0.7266
        ],
        [
          0.727,
          0.7211
        ],
        [
          0.737,
          0.712
        ],
        [
          0.747,
          0.707
        ],
        [
          0.7571,
          0.702
        ],
        [
          0.7675,
          0.6965
        ],
        [
          0.7775,
          0.6893
        ],
        [
          0.7875,
          0.6856
        ],
        [
          0.7975,
          0.6793
        ],
        [
          0.8076,
          0.6733
        ],
        [
          0.818,
          0.6661
        ],
        [
          0.828,
          0.6601
        ],
        [
          0.838,
          0.6538
        ],
        [
          0.848,
          0.6492
        ],
        [
          0.8581,
          0.6433
        ],
        [
          0.8685,
          0.6369
        ],
        [
          0.8785,
          0.6306
        ],
        [
          0.8885,
          0.6256
        ],
        [
          0.8985,
          0.6192
        ],
        [
          0.9086,
          0.6137
        ],
        [
          0.919,
          0.6078
        ],
        [
          0.929,
          0.6019
        ],
        [
          0.939,
          0.5951
        ],
        [
          0.949,
          0.5878
        ],
        [
          0.9591,
          0.5837
        ],
        [
          0.9695,
          0.5778
        ],
        [
          0.9795,
          0.5714
        ],
        [
          0.9895,
          0.5646
        ],
        [
          0.9995,
          0.5573
        ],
        [
          1.0,
          0.5569
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1346456645924226,
        "difference": -0.001506754381016898,
        "worse": 0.5266666666666666,
        "kept_right": 0.1327569317033779,
        "kept_wrong": 0.0018887328890446742,
        "handed_off": 0.8653543354075779
      },
      "usd_per_1000": 4.266703947706122
    },
    {
      "decision": 45,
      "frontier": 193,
      "frontier_accuracy": 0.89171974522293,
      "in_sample": {
        "threshold": 0.3980035171507091,
        "answered": 0.17424931756141948,
        "accuracy": 0.89171974522293
      },
      "curve": [
        [
          0.0,
          0.8917
        ],
        [
          0.01,
          0.8917
        ],
        [
          0.02,
          0.8917
        ],
        [
          0.03,
          0.8917
        ],
        [
          0.04,
          0.8917
        ],
        [
          0.05,
          0.8917
        ],
        [
          0.0601,
          0.8917
        ],
        [
          0.0705,
          0.8917
        ],
        [
          0.0805,
          0.8917
        ],
        [
          0.0905,
          0.8917
        ],
        [
          0.1005,
          0.8917
        ],
        [
          0.1106,
          0.8913
        ],
        [
          0.121,
          0.8913
        ],
        [
          0.131,
          0.8913
        ],
        [
          0.141,
          0.8913
        ],
        [
          0.151,
          0.8917
        ],
        [
          0.1611,
          0.8917
        ],
        [
          0.1715,
          0.8913
        ],
        [
          0.1815,
          0.8908
        ],
        [
          0.1915,
          0.8904
        ],
        [
          0.2015,
          0.8899
        ],
        [
          0.2116,
          0.889
        ],
        [
          0.222,
          0.8885
        ],
        [
          0.232,
          0.8881
        ],
        [
          0.242,
          0.8872
        ],
        [
          0.252,
          0.8858
        ],
        [
          0.2621,
          0.8863
        ],
        [
          0.2725,
          0.8826
        ],
        [
          0.2825,
          0.8808
        ],
        [
          0.2925,
          0.8803
        ],
        [
          0.3025,
          0.8781
        ],
        [
          0.3126,
          0.8767
        ],
        [
          0.323,
          0.8749
        ],
        [
          0.333,
          0.8717
        ],
        [
          0.343,
          0.8703
        ],
        [
          0.353,
          0.8681
        ],
        [
          0.3631,
          0.8653
        ],
        [
          0.3735,
          0.8612
        ],
        [
          0.3835,
          0.8581
        ],
        [
          0.3935,
          0.8549
        ],
        [
          0.4035,
          0.8517
        ],
        [
          0.4136,
          0.8503
        ],
        [
          0.424,
          0.8471
        ],
        [
          0.434,
          0.8435
        ],
        [
          0.444,
          0.8408
        ],
        [
          0.454,
          0.8376
        ],
        [
          0.4641,
          0.8344
        ],
        [
          0.4745,
          0.8312
        ],
        [
          0.4845,
          0.8267
        ],
        [
          0.4945,
          0.8203
        ],
        [
          0.5045,
          0.8167
        ],
        [
          0.5146,
          0.8121
        ],
        [
          0.525,
          0.8071
        ],
        [
          0.535,
          0.8062
        ],
        [
          0.545,
          0.803
        ],
        [
          0.5551,
          0.7971
        ],
        [
          0.5655,
          0.7907
        ],
        [
          0.5755,
          0.7875
        ],
        [
          0.5855,
          0.7821
        ],
        [
          0.5955,
          0.778
        ],
        [
          0.6056,
          0.7734
        ],
        [
          0.616,
          0.7666
        ],
        [
          0.626,
          0.7625
        ],
        [
          0.636,
          0.7584
        ],
        [
          0.646,
          0.7548
        ],
        [
          0.6561,
          0.7507
        ],
        [
          0.6665,
          0.7434
        ],
        [
          0.6765,
          0.7389
        ],
        [
          0.6865,
          0.732
        ],
        [
          0.6965,
          0.7275
        ],
        [
          0.7066,
          0.7234
        ],
        [
          0.717,
          0.7188
        ],
        [
          0.727,
          0.7138
        ],
        [
          0.737,
          0.7056
        ],
        [
          0.747,
          0.7006
        ],
        [
          0.7571,
          0.6956
        ],
        [
          0.7675,
          0.6893
        ],
        [
          0.7775,
          0.6824
        ],
        [
          0.7875,
          0.6779
        ],
        [
          0.7975,
          0.6733
        ],
        [
          0.8076,
          0.6674
        ],
        [
          0.818,
          0.6597
        ],
        [
          0.828,
          0.6524
        ],
        [
          0.838,
          0.6465
        ],
        [
          0.848,
          0.6424
        ],
        [
          0.8581,
          0.6369
        ],
        [
          0.8685,
          0.6315
        ],
        [
          0.8785,
          0.6251
        ],
        [
          0.8885,
          0.6187
        ],
        [
          0.8985,
          0.6133
        ],
        [
          0.9086,
          0.6074
        ],
        [
          0.919,
          0.6019
        ],
        [
          0.929,
          0.5974
        ],
        [
          0.939,
          0.5905
        ],
        [
          0.949,
          0.5842
        ],
        [
          0.9591,
          0.581
        ],
        [
          0.9695,
          0.5755
        ],
        [
          0.9795,
          0.571
        ],
        [
          0.9895,
          0.5646
        ],
        [
          0.9995,
          0.5573
        ],
        [
          1.0,
          0.5569
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1806366937442878,
        "difference": -0.0020613116814404597,
        "worse": 0.5933333333333334,
        "kept_right": 0.17611682178195945,
        "kept_wrong": 0.004519871962328211,
        "handed_off": 0.8193633062557127
      },
      "usd_per_1000": 6.348090950458158
    },
    {
      "decision": 45,
      "frontier": 55,
      "frontier_accuracy": 0.8503184713375797,
      "in_sample": {
        "threshold": 0.3470583499882852,
        "answered": 0.27070063694267515,
        "accuracy": 0.8503184713375797
      },
      "curve": [
        [
          0.0,
          0.8503
        ],
        [
          0.01,
          0.8503
        ],
        [
          0.02,
          0.8508
        ],
        [
          0.03,
          0.8508
        ],
        [
          0.04,
          0.8508
        ],
        [
          0.05,
          0.8512
        ],
        [
          0.0601,
          0.8512
        ],
        [
          0.0705,
          0.8512
        ],
        [
          0.0805,
          0.8512
        ],
        [
          0.0905,
          0.8517
        ],
        [
          0.1005,
          0.8517
        ],
        [
          0.1106,
          0.8517
        ],
        [
          0.121,
          0.8517
        ],
        [
          0.131,
          0.8512
        ],
        [
          0.141,
          0.8521
        ],
        [
          0.151,
          0.853
        ],
        [
          0.1611,
          0.854
        ],
        [
          0.1715,
          0.8544
        ],
        [
          0.1815,
          0.8544
        ],
        [
          0.1915,
          0.854
        ],
        [
          0.2015,
          0.8535
        ],
        [
          0.2116,
          0.8535
        ],
        [
          0.222,
          0.853
        ],
        [
          0.232,
          0.8526
        ],
        [
          0.242,
          0.8521
        ],
        [
          0.252,
          0.8517
        ],
        [
          0.2621,
          0.8517
        ],
        [
          0.2725,
          0.849
        ],
        [
          0.2825,
          0.8476
        ],
        [
          0.2925,
          0.8476
        ],
        [
          0.3025,
          0.8458
        ],
        [
          0.3126,
          0.8467
        ],
        [
          0.323,
          0.8449
        ],
        [
          0.333,
          0.843
        ],
        [
          0.343,
          0.8408
        ],
        [
          0.353,
          0.8394
        ],
        [
          0.3631,
          0.8371
        ],
        [
          0.3735,
          0.8335
        ],
        [
          0.3835,
          0.8317
        ],
        [
          0.3935,
          0.8294
        ],
        [
          0.4035,
          0.8285
        ],
        [
          0.4136,
          0.8271
        ],
        [
          0.424,
          0.8235
        ],
        [
          0.434,
          0.8198
        ],
        [
          0.444,
          0.8176
        ],
        [
          0.454,
          0.8139
        ],
        [
          0.4641,
          0.8116
        ],
        [
          0.4745,
          0.8089
        ],
        [
          0.4845,
          0.8057
        ],
        [
          0.4945,
          0.8003
        ],
        [
          0.5045,
          0.7975
        ],
        [
          0.5146,
          0.793
        ],
        [
          0.525,
          0.7884
        ],
        [
          0.535,
          0.788
        ],
        [
          0.545,
          0.7866
        ],
        [
          0.5551,
          0.7812
        ],
        [
          0.5655,
          0.7748
        ],
        [
          0.5755,
          0.7725
        ],
        [
          0.5855,
          0.7675
        ],
        [
          0.5955,
          0.7634
        ],
        [
          0.6056,
          0.7593
        ],
        [
          0.616,
          0.7534
        ],
        [
          0.626,
          0.7493
        ],
        [
          0.636,
          0.7448
        ],
        [
          0.646,
          0.7416
        ],
        [
          0.6561,
          0.7366
        ],
        [
          0.6665,
          0.7293
        ],
        [
          0.6765,
          0.7252
        ],
        [
          0.6865,
          0.7188
        ],
        [
          0.6965,
          0.7147
        ],
        [
          0.7066,
          0.712
        ],
        [
          0.717,
          0.7075
        ],
        [
          0.727,
          0.7038
        ],
        [
          0.737,
          0.6965
        ],
        [
          0.747,
          0.6924
        ],
        [
          0.7571,
          0.6888
        ],
        [
          0.7675,
          0.6833
        ],
        [
          0.7775,
          0.6779
        ],
        [
          0.7875,
          0.6738
        ],
        [
          0.7975,
          0.6688
        ],
        [
          0.8076,
          0.6638
        ],
        [
          0.818,
          0.6579
        ],
        [
          0.828,
          0.6515
        ],
        [
          0.838,
          0.6451
        ],
        [
          0.848,
          0.6419
        ],
        [
          0.8581,
          0.636
        ],
        [
          0.8685,
          0.6301
        ],
        [
          0.8785,
          0.6233
        ],
        [
          0.8885,
          0.6183
        ],
        [
          0.8985,
          0.6128
        ],
        [
          0.9086,
          0.6078
        ],
        [
          0.919,
          0.6015
        ],
        [
          0.929,
          0.596
        ],
        [
          0.939,
          0.5896
        ],
        [
          0.949,
          0.5837
        ],
        [
          0.9591,
          0.5805
        ],
        [
          0.9695,
          0.5751
        ],
        [
          0.9795,
          0.5696
        ],
        [
          0.9895,
          0.5632
        ],
        [
          0.9995,
          0.5573
        ],
        [
          1.0,
          0.5569
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.2727371768704905,
        "difference": -0.0014885358368746115,
        "worse": 0.4533333333333333,
        "kept_right": 0.2559024626620125,
        "kept_wrong": 0.016834714208477923,
        "handed_off": 0.727262823129509
      },
      "usd_per_1000": 2.6576609850724644
    },
    {
      "decision": 127,
      "frontier": 140,
      "frontier_accuracy": 0.9808917197452229,
      "in_sample": {
        "threshold": 0.9693852514507183,
        "answered": 0.09008189262966333,
        "accuracy": 0.9808917197452229
      },
      "curve": [
        [
          0.0,
          0.9809
        ],
        [
          0.01,
          0.9809
        ],
        [
          0.02,
          0.9809
        ],
        [
          0.03,
          0.9809
        ],
        [
          0.04,
          0.9809
        ],
        [
          0.05,
          0.9809
        ],
        [
          0.0601,
          0.9809
        ],
        [
          0.0705,
          0.9809
        ],
        [
          0.0805,
          0.9809
        ],
        [
          0.0905,
          0.9804
        ],
        [
          0.1005,
          0.9804
        ],
        [
          0.1106,
          0.9804
        ],
        [
          0.121,
          0.9804
        ],
        [
          0.131,
          0.9804
        ],
        [
          0.141,
          0.9804
        ],
        [
          0.151,
          0.9791
        ],
        [
          0.1611,
          0.9786
        ],
        [
          0.1715,
          0.9782
        ],
        [
          0.1815,
          0.9777
        ],
        [
          0.1915,
          0.9768
        ],
        [
          0.2015,
          0.9754
        ],
        [
          0.2116,
          0.9736
        ],
        [
          0.222,
          0.9713
        ],
        [
          0.232,
          0.9695
        ],
        [
          0.242,
          0.9682
        ],
        [
          0.252,
          0.9663
        ],
        [
          0.2621,
          0.965
        ],
        [
          0.2725,
          0.9636
        ],
        [
          0.2825,
          0.9627
        ],
        [
          0.2925,
          0.96
        ],
        [
          0.3025,
          0.9581
        ],
        [
          0.3126,
          0.9559
        ],
        [
          0.323,
          0.9522
        ],
        [
          0.333,
          0.9495
        ],
        [
          0.343,
          0.9481
        ],
        [
          0.353,
          0.9449
        ],
        [
          0.3631,
          0.9418
        ],
        [
          0.3735,
          0.9363
        ],
        [
          0.3835,
          0.9318
        ],
        [
          0.3935,
          0.929
        ],
        [
          0.4035,
          0.9245
        ],
        [
          0.4136,
          0.9204
        ],
        [
          0.424,
          0.9163
        ],
        [
          0.434,
          0.9117
        ],
        [
          0.444,
          0.9072
        ],
        [
          0.454,
          0.9035
        ],
        [
          0.4641,
          0.8999
        ],
        [
          0.4745,
          0.8944
        ],
        [
          0.4845,
          0.8908
        ],
        [
          0.4945,
          0.8858
        ],
        [
          0.5045,
          0.8803
        ],
        [
          0.5146,
          0.8753
        ],
        [
          0.525,
          0.8699
        ],
        [
          0.535,
          0.8635
        ],
        [
          0.545,
          0.8576
        ],
        [
          0.5551,
          0.8526
        ],
        [
          0.5655,
          0.849
        ],
        [
          0.5755,
          0.8408
        ],
        [
          0.5855,
          0.8367
        ],
        [
          0.5955,
          0.8317
        ],
        [
          0.6056,
          0.8258
        ],
        [
          0.616,
          0.8194
        ],
        [
          0.626,
          0.8148
        ],
        [
          0.636,
          0.8089
        ],
        [
          0.646,
          0.8021
        ],
        [
          0.6561,
          0.7962
        ],
        [
          0.6665,
          0.7903
        ],
        [
          0.6765,
          0.783
        ],
        [
          0.6865,
          0.7757
        ],
        [
          0.6965,
          0.7707
        ],
        [
          0.7066,
          0.7634
        ],
        [
          0.717,
          0.7552
        ],
        [
          0.727,
          0.748
        ],
        [
          0.737,
          0.742
        ],
        [
          0.747,
          0.7352
        ],
        [
          0.7571,
          0.7307
        ],
        [
          0.7675,
          0.7238
        ],
        [
          0.7775,
          0.7166
        ],
        [
          0.7875,
          0.7106
        ],
        [
          0.7975,
          0.7043
        ],
        [
          0.8076,
          0.6961
        ],
        [
          0.818,
          0.6893
        ],
        [
          0.828,
          0.682
        ],
        [
          0.838,
          0.6733
        ],
        [
          0.848,
          0.6661
        ],
        [
          0.8581,
          0.6592
        ],
        [
          0.8685,
          0.6542
        ],
        [
          0.8785,
          0.6465
        ],
        [
          0.8885,
          0.6392
        ],
        [
          0.8985,
          0.6315
        ],
        [
          0.9086,
          0.6247
        ],
        [
          0.919,
          0.6165
        ],
        [
          0.929,
          0.6087
        ],
        [
          0.939,
          0.6033
        ],
        [
          0.949,
          0.5937
        ],
        [
          0.9591,
          0.5855
        ],
        [
          0.9695,
          0.5787
        ],
        [
          0.9795,
          0.5723
        ],
        [
          0.9895,
          0.5646
        ],
        [
          0.9995,
          0.5573
        ],
        [
          1.0,
          0.5569
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1168033676744158,
        "difference": -0.0008489241270820754,
        "worse": 0.48,
        "kept_right": 0.11595444354733364,
        "kept_wrong": 0.0008489241270820754,
        "handed_off": 0.8831966323255847
      },
      "usd_per_1000": 2.1365437658960333
    },
    {
      "decision": 127,
      "frontier": 277,
      "frontier_accuracy": 0.9795268425841674,
      "in_sample": {
        "threshold": 0.9693852514507183,
        "answered": 0.09008189262966333,
        "accuracy": 0.9795268425841674
      },
      "curve": [
        [
          0.0,
          0.9795
        ],
        [
          0.01,
          0.9795
        ],
        [
          0.02,
          0.9795
        ],
        [
          0.03,
          0.9795
        ],
        [
          0.04,
          0.9795
        ],
        [
          0.05,
          0.9795
        ],
        [
          0.0601,
          0.9795
        ],
        [
          0.0705,
          0.9795
        ],
        [
          0.0805,
          0.9795
        ],
        [
          0.0905,
          0.9791
        ],
        [
          0.1005,
          0.9791
        ],
        [
          0.1106,
          0.9791
        ],
        [
          0.121,
          0.9791
        ],
        [
          0.131,
          0.9791
        ],
        [
          0.141,
          0.9791
        ],
        [
          0.151,
          0.9777
        ],
        [
          0.1611,
          0.9773
        ],
        [
          0.1715,
          0.9768
        ],
        [
          0.1815,
          0.9763
        ],
        [
          0.1915,
          0.9754
        ],
        [
          0.2015,
          0.9741
        ],
        [
          0.2116,
          0.9722
        ],
        [
          0.222,
          0.97
        ],
        [
          0.232,
          0.9677
        ],
        [
          0.242,
          0.9663
        ],
        [
          0.252,
          0.965
        ],
        [
          0.2621,
          0.9636
        ],
        [
          0.2725,
          0.9622
        ],
        [
          0.2825,
          0.9609
        ],
        [
          0.2925,
          0.9581
        ],
        [
          0.3025,
          0.9563
        ],
        [
          0.3126,
          0.954
        ],
        [
          0.323,
          0.9504
        ],
        [
          0.333,
          0.9477
        ],
        [
          0.343,
          0.9463
        ],
        [
          0.353,
          0.9431
        ],
        [
          0.3631,
          0.9409
        ],
        [
          0.3735,
          0.9349
        ],
        [
          0.3835,
          0.9304
        ],
        [
          0.3935,
          0.9272
        ],
        [
          0.4035,
          0.9227
        ],
        [
          0.4136,
          0.9186
        ],
        [
          0.424,
          0.9145
        ],
        [
          0.434,
          0.9099
        ],
        [
          0.444,
          0.9054
        ],
        [
          0.454,
          0.9017
        ],
        [
          0.4641,
          0.8981
        ],
        [
          0.4745,
          0.8926
        ],
        [
          0.4845,
          0.889
        ],
        [
          0.4945,
          0.8835
        ],
        [
          0.5045,
          0.8785
        ],
        [
          0.5146,
          0.8735
        ],
        [
          0.525,
          0.8685
        ],
        [
          0.535,
          0.8617
        ],
        [
          0.545,
          0.8562
        ],
        [
          0.5551,
          0.8512
        ],
        [
          0.5655,
          0.8476
        ],
        [
          0.5755,
          0.8394
        ],
        [
          0.5855,
          0.8353
        ],
        [
          0.5955,
          0.8298
        ],
        [
          0.6056,
          0.8239
        ],
        [
          0.616,
          0.8176
        ],
        [
          0.626,
          0.813
        ],
        [
          0.636,
          0.8071
        ],
        [
          0.646,
          0.8003
        ],
        [
          0.6561,
          0.7939
        ],
        [
          0.6665,
          0.788
        ],
        [
          0.6765,
          0.7807
        ],
        [
          0.6865,
          0.7734
        ],
        [
          0.6965,
          0.7684
        ],
        [
          0.7066,
          0.7611
        ],
        [
          0.717,
          0.753
        ],
        [
          0.727,
          0.7452
        ],
        [
          0.737,
          0.7393
        ],
        [
          0.747,
          0.7325
        ],
        [
          0.7571,
          0.7279
        ],
        [
          0.7675,
          0.7211
        ],
        [
          0.7775,
          0.7138
        ],
        [
          0.7875,
          0.7079
        ],
        [
          0.7975,
          0.7015
        ],
        [
          0.8076,
          0.6934
        ],
        [
          0.818,
          0.6865
        ],
        [
          0.828,
          0.6793
        ],
        [
          0.838,
          0.6711
        ],
        [
          0.848,
          0.6638
        ],
        [
          0.8581,
          0.657
        ],
        [
          0.8685,
          0.6524
        ],
        [
          0.8785,
          0.6447
        ],
        [
          0.8885,
          0.6379
        ],
        [
          0.8985,
          0.6306
        ],
        [
          0.9086,
          0.6242
        ],
        [
          0.919,
          0.616
        ],
        [
          0.929,
          0.6087
        ],
        [
          0.939,
          0.6033
        ],
        [
          0.949,
          0.5937
        ],
        [
          0.9591,
          0.5855
        ],
        [
          0.9695,
          0.5783
        ],
        [
          0.9795,
          0.5723
        ],
        [
          0.9895,
          0.5646
        ],
        [
          0.9995,
          0.5573
        ],
        [
          1.0,
          0.5569
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1168033676744158,
        "difference": -0.0008489241270820754,
        "worse": 0.48,
        "kept_right": 0.11595444354733364,
        "kept_wrong": 0.0008489241270820754,
        "handed_off": 0.8831966323255847
      },
      "usd_per_1000": 5.357077793385622
    },
    {
      "decision": 127,
      "frontier": 276,
      "frontier_accuracy": 0.9777070063694268,
      "in_sample": {
        "threshold": 0.9693852514507183,
        "answered": 0.09008189262966333,
        "accuracy": 0.9777070063694268
      },
      "curve": [
        [
          0.0,
          0.9777
        ],
        [
          0.01,
          0.9777
        ],
        [
          0.02,
          0.9777
        ],
        [
          0.03,
          0.9777
        ],
        [
          0.04,
          0.9777
        ],
        [
          0.05,
          0.9777
        ],
        [
          0.0601,
          0.9777
        ],
        [
          0.0705,
          0.9777
        ],
        [
          0.0805,
          0.9777
        ],
        [
          0.0905,
          0.9773
        ],
        [
          0.1005,
          0.9773
        ],
        [
          0.1106,
          0.9773
        ],
        [
          0.121,
          0.9773
        ],
        [
          0.131,
          0.9773
        ],
        [
          0.141,
          0.9773
        ],
        [
          0.151,
          0.9759
        ],
        [
          0.1611,
          0.9754
        ],
        [
          0.1715,
          0.975
        ],
        [
          0.1815,
          0.9745
        ],
        [
          0.1915,
          0.9736
        ],
        [
          0.2015,
          0.9722
        ],
        [
          0.2116,
          0.97
        ],
        [
          0.222,
          0.9677
        ],
        [
          0.232,
          0.9659
        ],
        [
          0.242,
          0.9645
        ],
        [
          0.252,
          0.9631
        ],
        [
          0.2621,
          0.9613
        ],
        [
          0.2725,
          0.96
        ],
        [
          0.2825,
          0.9591
        ],
        [
          0.2925,
          0.9563
        ],
        [
          0.3025,
          0.9545
        ],
        [
          0.3126,
          0.9522
        ],
        [
          0.323,
          0.9486
        ],
        [
          0.333,
          0.9459
        ],
        [
          0.343,
          0.9449
        ],
        [
          0.353,
          0.9418
        ],
        [
          0.3631,
          0.9395
        ],
        [
          0.3735,
          0.934
        ],
        [
          0.3835,
          0.9295
        ],
        [
          0.3935,
          0.9263
        ],
        [
          0.4035,
          0.9222
        ],
        [
          0.4136,
          0.9181
        ],
        [
          0.424,
          0.914
        ],
        [
          0.434,
          0.9095
        ],
        [
          0.444,
          0.9049
        ],
        [
          0.454,
          0.9013
        ],
        [
          0.4641,
          0.8976
        ],
        [
          0.4745,
          0.8922
        ],
        [
          0.4845,
          0.8885
        ],
        [
          0.4945,
          0.8831
        ],
        [
          0.5045,
          0.8781
        ],
        [
          0.5146,
          0.8731
        ],
        [
          0.525,
          0.8681
        ],
        [
          0.535,
          0.8612
        ],
        [
          0.545,
          0.8558
        ],
        [
          0.5551,
          0.8508
        ],
        [
          0.5655,
          0.8471
        ],
        [
          0.5755,
          0.8389
        ],
        [
          0.5855,
          0.8348
        ],
        [
          0.5955,
          0.8294
        ],
        [
          0.6056,
          0.8235
        ],
        [
          0.616,
          0.8171
        ],
        [
          0.626,
          0.8126
        ],
        [
          0.636,
          0.8066
        ],
        [
          0.646,
          0.7998
        ],
        [
          0.6561,
          0.7934
        ],
        [
          0.6665,
          0.7875
        ],
        [
          0.6765,
          0.7807
        ],
        [
          0.6865,
          0.7734
        ],
        [
          0.6965,
          0.7689
        ],
        [
          0.7066,
          0.7616
        ],
        [
          0.717,
          0.7534
        ],
        [
          0.727,
          0.7457
        ],
        [
          0.737,
          0.7398
        ],
        [
          0.747,
          0.7329
        ],
        [
          0.7571,
          0.7284
        ],
        [
          0.7675,
          0.7216
        ],
        [
          0.7775,
          0.7143
        ],
        [
          0.7875,
          0.7084
        ],
        [
          0.7975,
          0.702
        ],
        [
          0.8076,
          0.6938
        ],
        [
          0.818,
          0.687
        ],
        [
          0.828,
          0.6797
        ],
        [
          0.838,
          0.6715
        ],
        [
          0.848,
          0.6642
        ],
        [
          0.8581,
          0.6574
        ],
        [
          0.8685,
          0.6524
        ],
        [
          0.8785,
          0.6456
        ],
        [
          0.8885,
          0.6388
        ],
        [
          0.8985,
          0.6315
        ],
        [
          0.9086,
          0.6251
        ],
        [
          0.919,
          0.6169
        ],
        [
          0.929,
          0.6092
        ],
        [
          0.939,
          0.6033
        ],
        [
          0.949,
          0.5937
        ],
        [
          0.9591,
          0.5855
        ],
        [
          0.9695,
          0.5783
        ],
        [
          0.9795,
          0.5723
        ],
        [
          0.9895,
          0.5646
        ],
        [
          0.9995,
          0.5573
        ],
        [
          1.0,
          0.5569
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1168033676744158,
        "difference": -0.0008489241270820754,
        "worse": 0.48,
        "kept_right": 0.11595444354733364,
        "kept_wrong": 0.0008489241270820754,
        "handed_off": 0.8831966323255847
      },
      "usd_per_1000": 1.4327128976831947
    },
    {
      "decision": 127,
      "frontier": 48,
      "frontier_accuracy": 0.9758871701546861,
      "in_sample": {
        "threshold": 0.9693852514507183,
        "answered": 0.09008189262966333,
        "accuracy": 0.9758871701546861
      },
      "curve": [
        [
          0.0,
          0.9759
        ],
        [
          0.01,
          0.9759
        ],
        [
          0.02,
          0.9759
        ],
        [
          0.03,
          0.9759
        ],
        [
          0.04,
          0.9759
        ],
        [
          0.05,
          0.9759
        ],
        [
          0.0601,
          0.9759
        ],
        [
          0.0705,
          0.9759
        ],
        [
          0.0805,
          0.9759
        ],
        [
          0.0905,
          0.9754
        ],
        [
          0.1005,
          0.9754
        ],
        [
          0.1106,
          0.9754
        ],
        [
          0.121,
          0.9754
        ],
        [
          0.131,
          0.9754
        ],
        [
          0.141,
          0.9754
        ],
        [
          0.151,
          0.9741
        ],
        [
          0.1611,
          0.9736
        ],
        [
          0.1715,
          0.9732
        ],
        [
          0.1815,
          0.9727
        ],
        [
          0.1915,
          0.9718
        ],
        [
          0.2015,
          0.9704
        ],
        [
          0.2116,
          0.9686
        ],
        [
          0.222,
          0.9663
        ],
        [
          0.232,
          0.9641
        ],
        [
          0.242,
          0.9627
        ],
        [
          0.252,
          0.9613
        ],
        [
          0.2621,
          0.9595
        ],
        [
          0.2725,
          0.9581
        ],
        [
          0.2825,
          0.9568
        ],
        [
          0.2925,
          0.954
        ],
        [
          0.3025,
          0.9522
        ],
        [
          0.3126,
          0.95
        ],
        [
          0.323,
          0.9463
        ],
        [
          0.333,
          0.944
        ],
        [
          0.343,
          0.9427
        ],
        [
          0.353,
          0.9395
        ],
        [
          0.3631,
          0.9368
        ],
        [
          0.3735,
          0.9313
        ],
        [
          0.3835,
          0.9268
        ],
        [
          0.3935,
          0.9236
        ],
        [
          0.4035,
          0.919
        ],
        [
          0.4136,
          0.9149
        ],
        [
          0.424,
          0.9108
        ],
        [
          0.434,
          0.9063
        ],
        [
          0.444,
          0.9017
        ],
        [
          0.454,
          0.8981
        ],
        [
          0.4641,
          0.8944
        ],
        [
          0.4745,
          0.889
        ],
        [
          0.4845,
          0.8854
        ],
        [
          0.4945,
          0.8799
        ],
        [
          0.5045,
          0.8753
        ],
        [
          0.5146,
          0.8703
        ],
        [
          0.525,
          0.8649
        ],
        [
          0.535,
          0.8581
        ],
        [
          0.545,
          0.8526
        ],
        [
          0.5551,
          0.8476
        ],
        [
          0.5655,
          0.8439
        ],
        [
          0.5755,
          0.8358
        ],
        [
          0.5855,
          0.8317
        ],
        [
          0.5955,
          0.8262
        ],
        [
          0.6056,
          0.8203
        ],
        [
          0.616,
          0.8139
        ],
        [
          0.626,
          0.8094
        ],
        [
          0.636,
          0.8035
        ],
        [
          0.646,
          0.7966
        ],
        [
          0.6561,
          0.7907
        ],
        [
          0.6665,
          0.7848
        ],
        [
          0.6765,
          0.7771
        ],
        [
          0.6865,
          0.7698
        ],
        [
          0.6965,
          0.7652
        ],
        [
          0.7066,
          0.758
        ],
        [
          0.717,
          0.7502
        ],
        [
          0.727,
          0.7425
        ],
        [
          0.737,
          0.7366
        ],
        [
          0.747,
          0.7298
        ],
        [
          0.7571,
          0.7257
        ],
        [
          0.7675,
          0.7188
        ],
        [
          0.7775,
          0.7116
        ],
        [
          0.7875,
          0.7056
        ],
        [
          0.7975,
          0.6993
        ],
        [
          0.8076,
          0.6911
        ],
        [
          0.818,
          0.6843
        ],
        [
          0.828,
          0.677
        ],
        [
          0.838,
          0.6688
        ],
        [
          0.848,
          0.6615
        ],
        [
          0.8581,
          0.6547
        ],
        [
          0.8685,
          0.6501
        ],
        [
          0.8785,
          0.6429
        ],
        [
          0.8885,
          0.636
        ],
        [
          0.8985,
          0.6288
        ],
        [
          0.9086,
          0.6224
        ],
        [
          0.919,
          0.6142
        ],
        [
          0.929,
          0.6065
        ],
        [
          0.939,
          0.601
        ],
        [
          0.949,
          0.5928
        ],
        [
          0.9591,
          0.5846
        ],
        [
          0.9695,
          0.5778
        ],
        [
          0.9795,
          0.5719
        ],
        [
          0.9895,
          0.5646
        ],
        [
          0.9995,
          0.5573
        ],
        [
          1.0,
          0.5569
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1168033676744158,
        "difference": -0.0008489241270820754,
        "worse": 0.48,
        "kept_right": 0.11595444354733364,
        "kept_wrong": 0.0008489241270820754,
        "handed_off": 0.8831966323255847
      },
      "usd_per_1000": 4.7500313068896
    },
    {
      "decision": 127,
      "frontier": 191,
      "frontier_accuracy": 0.9740673339399454,
      "in_sample": {
        "threshold": 0.9693852514507183,
        "answered": 0.09008189262966333,
        "accuracy": 0.9740673339399454
      },
      "curve": [
        [
          0.0,
          0.9741
        ],
        [
          0.01,
          0.9741
        ],
        [
          0.02,
          0.9741
        ],
        [
          0.03,
          0.9741
        ],
        [
          0.04,
          0.9741
        ],
        [
          0.05,
          0.9741
        ],
        [
          0.0601,
          0.9741
        ],
        [
          0.0705,
          0.9741
        ],
        [
          0.0805,
          0.9741
        ],
        [
          0.0905,
          0.9736
        ],
        [
          0.1005,
          0.9736
        ],
        [
          0.1106,
          0.9736
        ],
        [
          0.121,
          0.9736
        ],
        [
          0.131,
          0.9736
        ],
        [
          0.141,
          0.9736
        ],
        [
          0.151,
          0.9727
        ],
        [
          0.1611,
          0.9722
        ],
        [
          0.1715,
          0.9718
        ],
        [
          0.1815,
          0.9713
        ],
        [
          0.1915,
          0.9704
        ],
        [
          0.2015,
          0.9691
        ],
        [
          0.2116,
          0.9672
        ],
        [
          0.222,
          0.965
        ],
        [
          0.232,
          0.9627
        ],
        [
          0.242,
          0.9613
        ],
        [
          0.252,
          0.96
        ],
        [
          0.2621,
          0.9581
        ],
        [
          0.2725,
          0.9568
        ],
        [
          0.2825,
          0.9554
        ],
        [
          0.2925,
          0.9531
        ],
        [
          0.3025,
          0.9518
        ],
        [
          0.3126,
          0.9495
        ],
        [
          0.323,
          0.9459
        ],
        [
          0.333,
          0.9431
        ],
        [
          0.343,
          0.9418
        ],
        [
          0.353,
          0.9386
        ],
        [
          0.3631,
          0.9363
        ],
        [
          0.3735,
          0.9308
        ],
        [
          0.3835,
          0.9263
        ],
        [
          0.3935,
          0.9231
        ],
        [
          0.4035,
          0.9186
        ],
        [
          0.4136,
          0.9145
        ],
        [
          0.424,
          0.9108
        ],
        [
          0.434,
          0.9058
        ],
        [
          0.444,
          0.9013
        ],
        [
          0.454,
          0.8976
        ],
        [
          0.4641,
          0.894
        ],
        [
          0.4745,
          0.8885
        ],
        [
          0.4845,
          0.8854
        ],
        [
          0.4945,
          0.8799
        ],
        [
          0.5045,
          0.8744
        ],
        [
          0.5146,
          0.8694
        ],
        [
          0.525,
          0.8644
        ],
        [
          0.535,
          0.8581
        ],
        [
          0.545,
          0.8526
        ],
        [
          0.5551,
          0.8476
        ],
        [
          0.5655,
          0.8439
        ],
        [
          0.5755,
          0.8358
        ],
        [
          0.5855,
          0.8317
        ],
        [
          0.5955,
          0.8262
        ],
        [
          0.6056,
          0.8203
        ],
        [
          0.616,
          0.8139
        ],
        [
          0.626,
          0.8094
        ],
        [
          0.636,
          0.8035
        ],
        [
          0.646,
          0.7966
        ],
        [
          0.6561,
          0.7903
        ],
        [
          0.6665,
          0.7843
        ],
        [
          0.6765,
          0.7775
        ],
        [
          0.6865,
          0.7702
        ],
        [
          0.6965,
          0.7652
        ],
        [
          0.7066,
          0.758
        ],
        [
          0.717,
          0.7498
        ],
        [
          0.727,
          0.742
        ],
        [
          0.737,
          0.7361
        ],
        [
          0.747,
          0.7298
        ],
        [
          0.7571,
          0.7257
        ],
        [
          0.7675,
          0.7188
        ],
        [
          0.7775,
          0.712
        ],
        [
          0.7875,
          0.7061
        ],
        [
          0.7975,
          0.6993
        ],
        [
          0.8076,
          0.6911
        ],
        [
          0.818,
          0.6843
        ],
        [
          0.828,
          0.677
        ],
        [
          0.838,
          0.6688
        ],
        [
          0.848,
          0.6615
        ],
        [
          0.8581,
          0.6547
        ],
        [
          0.8685,
          0.6497
        ],
        [
          0.8785,
          0.6429
        ],
        [
          0.8885,
          0.636
        ],
        [
          0.8985,
          0.6288
        ],
        [
          0.9086,
          0.6224
        ],
        [
          0.919,
          0.6142
        ],
        [
          0.929,
          0.6065
        ],
        [
          0.939,
          0.6015
        ],
        [
          0.949,
          0.5924
        ],
        [
          0.9591,
          0.5842
        ],
        [
          0.9695,
          0.5773
        ],
        [
          0.9795,
          0.5719
        ],
        [
          0.9895,
          0.5646
        ],
        [
          0.9995,
          0.5573
        ],
        [
          1.0,
          0.5569
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.11835538019236308,
        "difference": -0.000818576924057526,
        "worse": 0.48,
        "kept_right": 0.11731852200557595,
        "kept_wrong": 0.0010368581867870514,
        "handed_off": 0.8816446198076374
      },
      "usd_per_1000": 23.573145901465715
    },
    {
      "decision": 127,
      "frontier": 188,
      "frontier_accuracy": 0.9713375796178344,
      "in_sample": {
        "threshold": 0.9693852514507183,
        "answered": 0.09008189262966333,
        "accuracy": 0.9713375796178344
      },
      "curve": [
        [
          0.0,
          0.9713
        ],
        [
          0.01,
          0.9713
        ],
        [
          0.02,
          0.9713
        ],
        [
          0.03,
          0.9713
        ],
        [
          0.04,
          0.9713
        ],
        [
          0.05,
          0.9713
        ],
        [
          0.0601,
          0.9713
        ],
        [
          0.0705,
          0.9713
        ],
        [
          0.0805,
          0.9713
        ],
        [
          0.0905,
          0.9709
        ],
        [
          0.1005,
          0.9709
        ],
        [
          0.1106,
          0.9709
        ],
        [
          0.121,
          0.9709
        ],
        [
          0.131,
          0.9709
        ],
        [
          0.141,
          0.9709
        ],
        [
          0.151,
          0.9695
        ],
        [
          0.1611,
          0.9691
        ],
        [
          0.1715,
          0.9686
        ],
        [
          0.1815,
          0.9682
        ],
        [
          0.1915,
          0.9677
        ],
        [
          0.2015,
          0.9663
        ],
        [
          0.2116,
          0.9645
        ],
        [
          0.222,
          0.9622
        ],
        [
          0.232,
          0.9604
        ],
        [
          0.242,
          0.9591
        ],
        [
          0.252,
          0.9572
        ],
        [
          0.2621,
          0.9559
        ],
        [
          0.2725,
          0.9545
        ],
        [
          0.2825,
          0.9536
        ],
        [
          0.2925,
          0.9509
        ],
        [
          0.3025,
          0.949
        ],
        [
          0.3126,
          0.9468
        ],
        [
          0.323,
          0.9431
        ],
        [
          0.333,
          0.9404
        ],
        [
          0.343,
          0.939
        ],
        [
          0.353,
          0.9363
        ],
        [
          0.3631,
          0.9336
        ],
        [
          0.3735,
          0.9277
        ],
        [
          0.3835,
          0.9236
        ],
        [
          0.3935,
          0.9204
        ],
        [
          0.4035,
          0.9158
        ],
        [
          0.4136,
          0.9117
        ],
        [
          0.424,
          0.9076
        ],
        [
          0.434,
          0.9035
        ],
        [
          0.444,
          0.899
        ],
        [
          0.454,
          0.8954
        ],
        [
          0.4641,
          0.8917
        ],
        [
          0.4745,
          0.8863
        ],
        [
          0.4845,
          0.8831
        ],
        [
          0.4945,
          0.8785
        ],
        [
          0.5045,
          0.8731
        ],
        [
          0.5146,
          0.8681
        ],
        [
          0.525,
          0.8626
        ],
        [
          0.535,
          0.8562
        ],
        [
          0.545,
          0.8508
        ],
        [
          0.5551,
          0.8458
        ],
        [
          0.5655,
          0.8426
        ],
        [
          0.5755,
          0.8344
        ],
        [
          0.5855,
          0.8308
        ],
        [
          0.5955,
          0.8258
        ],
        [
          0.6056,
          0.8198
        ],
        [
          0.616,
          0.8135
        ],
        [
          0.626,
          0.8094
        ],
        [
          0.636,
          0.8035
        ],
        [
          0.646,
          0.7971
        ],
        [
          0.6561,
          0.7912
        ],
        [
          0.6665,
          0.7853
        ],
        [
          0.6765,
          0.7775
        ],
        [
          0.6865,
          0.7702
        ],
        [
          0.6965,
          0.7652
        ],
        [
          0.7066,
          0.758
        ],
        [
          0.717,
          0.7498
        ],
        [
          0.727,
          0.7425
        ],
        [
          0.737,
          0.737
        ],
        [
          0.747,
          0.7302
        ],
        [
          0.7571,
          0.7257
        ],
        [
          0.7675,
          0.7193
        ],
        [
          0.7775,
          0.712
        ],
        [
          0.7875,
          0.7061
        ],
        [
          0.7975,
          0.6997
        ],
        [
          0.8076,
          0.6915
        ],
        [
          0.818,
          0.6852
        ],
        [
          0.828,
          0.6779
        ],
        [
          0.838,
          0.6697
        ],
        [
          0.848,
          0.6624
        ],
        [
          0.8581,
          0.6556
        ],
        [
          0.8685,
          0.6506
        ],
        [
          0.8785,
          0.6442
        ],
        [
          0.8885,
          0.6365
        ],
        [
          0.8985,
          0.6292
        ],
        [
          0.9086,
          0.6224
        ],
        [
          0.919,
          0.6142
        ],
        [
          0.929,
          0.6065
        ],
        [
          0.939,
          0.6015
        ],
        [
          0.949,
          0.5924
        ],
        [
          0.9591,
          0.5846
        ],
        [
          0.9695,
          0.5778
        ],
        [
          0.9795,
          0.5714
        ],
        [
          0.9895,
          0.5646
        ],
        [
          0.9995,
          0.5573
        ],
        [
          1.0,
          0.5569
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.11747011504545547,
        "difference": -0.00091561292758318,
        "worse": 0.48333333333333334,
        "kept_right": 0.1165484304724563,
        "kept_wrong": 0.0009216845729990876,
        "handed_off": 0.8825298849545451
      },
      "usd_per_1000": 0.8502644438000707
    },
    {
      "decision": 127,
      "frontier": 190,
      "frontier_accuracy": 0.9672429481346679,
      "in_sample": {
        "threshold": 0.9528763014370394,
        "answered": 0.1451319381255687,
        "accuracy": 0.9672429481346679
      },
      "curve": [
        [
          0.0,
          0.9672
        ],
        [
          0.01,
          0.9672
        ],
        [
          0.02,
          0.9672
        ],
        [
          0.03,
          0.9672
        ],
        [
          0.04,
          0.9672
        ],
        [
          0.05,
          0.9672
        ],
        [
          0.0601,
          0.9672
        ],
        [
          0.0705,
          0.9672
        ],
        [
          0.0805,
          0.9672
        ],
        [
          0.0905,
          0.9668
        ],
        [
          0.1005,
          0.9668
        ],
        [
          0.1106,
          0.9668
        ],
        [
          0.121,
          0.9668
        ],
        [
          0.131,
          0.9668
        ],
        [
          0.141,
          0.9672
        ],
        [
          0.151,
          0.9663
        ],
        [
          0.1611,
          0.9659
        ],
        [
          0.1715,
          0.9654
        ],
        [
          0.1815,
          0.965
        ],
        [
          0.1915,
          0.9641
        ],
        [
          0.2015,
          0.9627
        ],
        [
          0.2116,
          0.9609
        ],
        [
          0.222,
          0.9591
        ],
        [
          0.232,
          0.9572
        ],
        [
          0.242,
          0.9559
        ],
        [
          0.252,
          0.9545
        ],
        [
          0.2621,
          0.9527
        ],
        [
          0.2725,
          0.9513
        ],
        [
          0.2825,
          0.95
        ],
        [
          0.2925,
          0.9477
        ],
        [
          0.3025,
          0.9459
        ],
        [
          0.3126,
          0.9436
        ],
        [
          0.323,
          0.9399
        ],
        [
          0.333,
          0.9377
        ],
        [
          0.343,
          0.9368
        ],
        [
          0.353,
          0.9336
        ],
        [
          0.3631,
          0.9313
        ],
        [
          0.3735,
          0.9258
        ],
        [
          0.3835,
          0.9213
        ],
        [
          0.3935,
          0.9181
        ],
        [
          0.4035,
          0.914
        ],
        [
          0.4136,
          0.9099
        ],
        [
          0.424,
          0.9063
        ],
        [
          0.434,
          0.9017
        ],
        [
          0.444,
          0.8972
        ],
        [
          0.454,
          0.8935
        ],
        [
          0.4641,
          0.8899
        ],
        [
          0.4745,
          0.8849
        ],
        [
          0.4845,
          0.8813
        ],
        [
          0.4945,
          0.8758
        ],
        [
          0.5045,
          0.8708
        ],
        [
          0.5146,
          0.8658
        ],
        [
          0.525,
          0.8608
        ],
        [
          0.535,
          0.854
        ],
        [
          0.545,
          0.8485
        ],
        [
          0.5551,
          0.8435
        ],
        [
          0.5655,
          0.8399
        ],
        [
          0.5755,
          0.8317
        ],
        [
          0.5855,
          0.828
        ],
        [
          0.5955,
          0.8226
        ],
        [
          0.6056,
          0.8167
        ],
        [
          0.616,
          0.8103
        ],
        [
          0.626,
          0.8057
        ],
        [
          0.636,
          0.8003
        ],
        [
          0.646,
          0.7934
        ],
        [
          0.6561,
          0.7871
        ],
        [
          0.6665,
          0.7816
        ],
        [
          0.6765,
          0.7743
        ],
        [
          0.6865,
          0.7671
        ],
        [
          0.6965,
          0.7625
        ],
        [
          0.7066,
          0.7552
        ],
        [
          0.717,
          0.747
        ],
        [
          0.727,
          0.7393
        ],
        [
          0.737,
          0.7343
        ],
        [
          0.747,
          0.7284
        ],
        [
          0.7571,
          0.7243
        ],
        [
          0.7675,
          0.7175
        ],
        [
          0.7775,
          0.7106
        ],
        [
          0.7875,
          0.7047
        ],
        [
          0.7975,
          0.6979
        ],
        [
          0.8076,
          0.6897
        ],
        [
          0.818,
          0.6833
        ],
        [
          0.828,
          0.6761
        ],
        [
          0.838,
          0.6683
        ],
        [
          0.848,
          0.6611
        ],
        [
          0.8581,
          0.6542
        ],
        [
          0.8685,
          0.6497
        ],
        [
          0.8785,
          0.6424
        ],
        [
          0.8885,
          0.6351
        ],
        [
          0.8985,
          0.6278
        ],
        [
          0.9086,
          0.6215
        ],
        [
          0.919,
          0.6133
        ],
        [
          0.929,
          0.606
        ],
        [
          0.939,
          0.601
        ],
        [
          0.949,
          0.5928
        ],
        [
          0.9591,
          0.5846
        ],
        [
          0.9695,
          0.5773
        ],
        [
          0.9795,
          0.5723
        ],
        [
          0.9895,
          0.5646
        ],
        [
          0.9995,
          0.5573
        ],
        [
          1.0,
          0.5569
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1374342756318103,
        "difference": -0.001045927026992256,
        "worse": 0.48,
        "kept_right": 0.1358092926178333,
        "kept_wrong": 0.0016249830139768156,
        "handed_off": 0.8625657243681903
      },
      "usd_per_1000": 4.803213639805447
    },
    {
      "decision": 127,
      "frontier": 124,
      "frontier_accuracy": 0.9667879890809827,
      "in_sample": {
        "threshold": 0.9693852514507183,
        "answered": 0.09008189262966333,
        "accuracy": 0.9667879890809827
      },
      "curve": [
        [
          0.0,
          0.9668
        ],
        [
          0.01,
          0.9668
        ],
        [
          0.02,
          0.9668
        ],
        [
          0.03,
          0.9668
        ],
        [
          0.04,
          0.9668
        ],
        [
          0.05,
          0.9668
        ],
        [
          0.0601,
          0.9668
        ],
        [
          0.0705,
          0.9668
        ],
        [
          0.0805,
          0.9668
        ],
        [
          0.0905,
          0.9663
        ],
        [
          0.1005,
          0.9663
        ],
        [
          0.1106,
          0.9663
        ],
        [
          0.121,
          0.9663
        ],
        [
          0.131,
          0.9663
        ],
        [
          0.141,
          0.9663
        ],
        [
          0.151,
          0.9654
        ],
        [
          0.1611,
          0.965
        ],
        [
          0.1715,
          0.9645
        ],
        [
          0.1815,
          0.9641
        ],
        [
          0.1915,
          0.9631
        ],
        [
          0.2015,
          0.9618
        ],
        [
          0.2116,
          0.96
        ],
        [
          0.222,
          0.9581
        ],
        [
          0.232,
          0.9563
        ],
        [
          0.242,
          0.955
        ],
        [
          0.252,
          0.9536
        ],
        [
          0.2621,
          0.9522
        ],
        [
          0.2725,
          0.9509
        ],
        [
          0.2825,
          0.9495
        ],
        [
          0.2925,
          0.9468
        ],
        [
          0.3025,
          0.9449
        ],
        [
          0.3126,
          0.9427
        ],
        [
          0.323,
          0.939
        ],
        [
          0.333,
          0.9368
        ],
        [
          0.343,
          0.9359
        ],
        [
          0.353,
          0.9327
        ],
        [
          0.3631,
          0.9308
        ],
        [
          0.3735,
          0.9254
        ],
        [
          0.3835,
          0.9208
        ],
        [
          0.3935,
          0.9181
        ],
        [
          0.4035,
          0.9136
        ],
        [
          0.4136,
          0.9095
        ],
        [
          0.424,
          0.9054
        ],
        [
          0.434,
          0.9008
        ],
        [
          0.444,
          0.8963
        ],
        [
          0.454,
          0.8926
        ],
        [
          0.4641,
          0.889
        ],
        [
          0.4745,
          0.8844
        ],
        [
          0.4845,
          0.8808
        ],
        [
          0.4945,
          0.8753
        ],
        [
          0.5045,
          0.8703
        ],
        [
          0.5146,
          0.8653
        ],
        [
          0.525,
          0.8603
        ],
        [
          0.535,
          0.854
        ],
        [
          0.545,
          0.8485
        ],
        [
          0.5551,
          0.8435
        ],
        [
          0.5655,
          0.8399
        ],
        [
          0.5755,
          0.8317
        ],
        [
          0.5855,
          0.8276
        ],
        [
          0.5955,
          0.8226
        ],
        [
          0.6056,
          0.8167
        ],
        [
          0.616,
          0.8103
        ],
        [
          0.626,
          0.8057
        ],
        [
          0.636,
          0.7998
        ],
        [
          0.646,
          0.793
        ],
        [
          0.6561,
          0.7866
        ],
        [
          0.6665,
          0.7807
        ],
        [
          0.6765,
          0.7739
        ],
        [
          0.6865,
          0.7666
        ],
        [
          0.6965,
          0.7621
        ],
        [
          0.7066,
          0.7548
        ],
        [
          0.717,
          0.747
        ],
        [
          0.727,
          0.7393
        ],
        [
          0.737,
          0.7338
        ],
        [
          0.747,
          0.7275
        ],
        [
          0.7571,
          0.7234
        ],
        [
          0.7675,
          0.7175
        ],
        [
          0.7775,
          0.7106
        ],
        [
          0.7875,
          0.7047
        ],
        [
          0.7975,
          0.6984
        ],
        [
          0.8076,
          0.6902
        ],
        [
          0.818,
          0.6833
        ],
        [
          0.828,
          0.6761
        ],
        [
          0.838,
          0.6688
        ],
        [
          0.848,
          0.6615
        ],
        [
          0.8581,
          0.6547
        ],
        [
          0.8685,
          0.6501
        ],
        [
          0.8785,
          0.6429
        ],
        [
          0.8885,
          0.6356
        ],
        [
          0.8985,
          0.6288
        ],
        [
          0.9086,
          0.6224
        ],
        [
          0.919,
          0.6142
        ],
        [
          0.929,
          0.6069
        ],
        [
          0.939,
          0.6019
        ],
        [
          0.949,
          0.5933
        ],
        [
          0.9591,
          0.5851
        ],
        [
          0.9695,
          0.5778
        ],
        [
          0.9795,
          0.5719
        ],
        [
          0.9895,
          0.5646
        ],
        [
          0.9995,
          0.5573
        ],
        [
          1.0,
          0.5569
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.11835538019236308,
        "difference": -0.000818576924057526,
        "worse": 0.48,
        "kept_right": 0.11731852200557595,
        "kept_wrong": 0.0010368581867870514,
        "handed_off": 0.8816446198076374
      },
      "usd_per_1000": 0.9965873323881994
    },
    {
      "decision": 127,
      "frontier": 278,
      "frontier_accuracy": 0.9499545040946314,
      "in_sample": {
        "threshold": 0.9693852514507183,
        "answered": 0.09008189262966333,
        "accuracy": 0.9499545040946314
      },
      "curve": [
        [
          0.0,
          0.95
        ],
        [
          0.01,
          0.95
        ],
        [
          0.02,
          0.95
        ],
        [
          0.03,
          0.95
        ],
        [
          0.04,
          0.95
        ],
        [
          0.05,
          0.95
        ],
        [
          0.0601,
          0.95
        ],
        [
          0.0705,
          0.95
        ],
        [
          0.0805,
          0.95
        ],
        [
          0.0905,
          0.9495
        ],
        [
          0.1005,
          0.9495
        ],
        [
          0.1106,
          0.9495
        ],
        [
          0.121,
          0.9495
        ],
        [
          0.131,
          0.9495
        ],
        [
          0.141,
          0.9495
        ],
        [
          0.151,
          0.9486
        ],
        [
          0.1611,
          0.9481
        ],
        [
          0.1715,
          0.9477
        ],
        [
          0.1815,
          0.9477
        ],
        [
          0.1915,
          0.9468
        ],
        [
          0.2015,
          0.9454
        ],
        [
          0.2116,
          0.944
        ],
        [
          0.222,
          0.9418
        ],
        [
          0.232,
          0.9395
        ],
        [
          0.242,
          0.9386
        ],
        [
          0.252,
          0.9372
        ],
        [
          0.2621,
          0.9359
        ],
        [
          0.2725,
          0.9354
        ],
        [
          0.2825,
          0.934
        ],
        [
          0.2925,
          0.9313
        ],
        [
          0.3025,
          0.9299
        ],
        [
          0.3126,
          0.9281
        ],
        [
          0.323,
          0.9245
        ],
        [
          0.333,
          0.9217
        ],
        [
          0.343,
          0.9204
        ],
        [
          0.353,
          0.9172
        ],
        [
          0.3631,
          0.9154
        ],
        [
          0.3735,
          0.9095
        ],
        [
          0.3835,
          0.9058
        ],
        [
          0.3935,
          0.9031
        ],
        [
          0.4035,
          0.899
        ],
        [
          0.4136,
          0.8954
        ],
        [
          0.424,
          0.8913
        ],
        [
          0.434,
          0.8876
        ],
        [
          0.444,
          0.8831
        ],
        [
          0.454,
          0.8799
        ],
        [
          0.4641,
          0.8763
        ],
        [
          0.4745,
          0.8717
        ],
        [
          0.4845,
          0.8685
        ],
        [
          0.4945,
          0.864
        ],
        [
          0.5045,
          0.8585
        ],
        [
          0.5146,
          0.8535
        ],
        [
          0.525,
          0.8485
        ],
        [
          0.535,
          0.843
        ],
        [
          0.545,
          0.8385
        ],
        [
          0.5551,
          0.8335
        ],
        [
          0.5655,
          0.8303
        ],
        [
          0.5755,
          0.8226
        ],
        [
          0.5855,
          0.8189
        ],
        [
          0.5955,
          0.8139
        ],
        [
          0.6056,
          0.8089
        ],
        [
          0.616,
          0.8025
        ],
        [
          0.626,
          0.798
        ],
        [
          0.636,
          0.7925
        ],
        [
          0.646,
          0.7857
        ],
        [
          0.6561,
          0.7798
        ],
        [
          0.6665,
          0.7739
        ],
        [
          0.6765,
          0.7671
        ],
        [
          0.6865,
          0.7598
        ],
        [
          0.6965,
          0.7552
        ],
        [
          0.7066,
          0.7484
        ],
        [
          0.717,
          0.7407
        ],
        [
          0.727,
          0.7329
        ],
        [
          0.737,
          0.727
        ],
        [
          0.747,
          0.7202
        ],
        [
          0.7571,
          0.7161
        ],
        [
          0.7675,
          0.7106
        ],
        [
          0.7775,
          0.7043
        ],
        [
          0.7875,
          0.6984
        ],
        [
          0.7975,
          0.6924
        ],
        [
          0.8076,
          0.6847
        ],
        [
          0.818,
          0.6779
        ],
        [
          0.828,
          0.6711
        ],
        [
          0.838,
          0.6629
        ],
        [
          0.848,
          0.6556
        ],
        [
          0.8581,
          0.6488
        ],
        [
          0.8685,
          0.6451
        ],
        [
          0.8785,
          0.6383
        ],
        [
          0.8885,
          0.6319
        ],
        [
          0.8985,
          0.6256
        ],
        [
          0.9086,
          0.6192
        ],
        [
          0.919,
          0.6115
        ],
        [
          0.929,
          0.6046
        ],
        [
          0.939,
          0.6001
        ],
        [
          0.949,
          0.5914
        ],
        [
          0.9591,
          0.5842
        ],
        [
          0.9695,
          0.5787
        ],
        [
          0.9795,
          0.5723
        ],
        [
          0.9895,
          0.5651
        ],
        [
          0.9995,
          0.5573
        ],
        [
          1.0,
          0.5569
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1185160331275383,
        "difference": -0.000824618188590318,
        "worse": 0.48,
        "kept_right": 0.11746098760791417,
        "kept_wrong": 0.001055045519624066,
        "handed_off": 0.8814839668724621
      },
      "usd_per_1000": 5.737426023640117
    },
    {
      "decision": 127,
      "frontier": 275,
      "frontier_accuracy": 0.9426751592356688,
      "in_sample": {
        "threshold": 0.9693852514507183,
        "answered": 0.09008189262966333,
        "accuracy": 0.9426751592356688
      },
      "curve": [
        [
          0.0,
          0.9427
        ],
        [
          0.01,
          0.9427
        ],
        [
          0.02,
          0.9427
        ],
        [
          0.03,
          0.9427
        ],
        [
          0.04,
          0.9427
        ],
        [
          0.05,
          0.9427
        ],
        [
          0.0601,
          0.9427
        ],
        [
          0.0705,
          0.9427
        ],
        [
          0.0805,
          0.9427
        ],
        [
          0.0905,
          0.9422
        ],
        [
          0.1005,
          0.9422
        ],
        [
          0.1106,
          0.9422
        ],
        [
          0.121,
          0.9422
        ],
        [
          0.131,
          0.9422
        ],
        [
          0.141,
          0.9422
        ],
        [
          0.151,
          0.9413
        ],
        [
          0.1611,
          0.9409
        ],
        [
          0.1715,
          0.9404
        ],
        [
          0.1815,
          0.9404
        ],
        [
          0.1915,
          0.9395
        ],
        [
          0.2015,
          0.9381
        ],
        [
          0.2116,
          0.9368
        ],
        [
          0.222,
          0.9345
        ],
        [
          0.232,
          0.9322
        ],
        [
          0.242,
          0.9308
        ],
        [
          0.252,
          0.9295
        ],
        [
          0.2621,
          0.9281
        ],
        [
          0.2725,
          0.9272
        ],
        [
          0.2825,
          0.9258
        ],
        [
          0.2925,
          0.9231
        ],
        [
          0.3025,
          0.9217
        ],
        [
          0.3126,
          0.9204
        ],
        [
          0.323,
          0.9167
        ],
        [
          0.333,
          0.914
        ],
        [
          0.343,
          0.9131
        ],
        [
          0.353,
          0.9099
        ],
        [
          0.3631,
          0.9086
        ],
        [
          0.3735,
          0.9031
        ],
        [
          0.3835,
          0.8995
        ],
        [
          0.3935,
          0.8963
        ],
        [
          0.4035,
          0.8913
        ],
        [
          0.4136,
          0.8876
        ],
        [
          0.424,
          0.884
        ],
        [
          0.434,
          0.8794
        ],
        [
          0.444,
          0.8749
        ],
        [
          0.454,
          0.8717
        ],
        [
          0.4641,
          0.8681
        ],
        [
          0.4745,
          0.8631
        ],
        [
          0.4845,
          0.8599
        ],
        [
          0.4945,
          0.8544
        ],
        [
          0.5045,
          0.8499
        ],
        [
          0.5146,
          0.8458
        ],
        [
          0.525,
          0.8408
        ],
        [
          0.535,
          0.8353
        ],
        [
          0.545,
          0.8308
        ],
        [
          0.5551,
          0.8258
        ],
        [
          0.5655,
          0.8221
        ],
        [
          0.5755,
          0.8139
        ],
        [
          0.5855,
          0.8107
        ],
        [
          0.5955,
          0.8057
        ],
        [
          0.6056,
          0.7998
        ],
        [
          0.616,
          0.7934
        ],
        [
          0.626,
          0.7889
        ],
        [
          0.636,
          0.783
        ],
        [
          0.646,
          0.7766
        ],
        [
          0.6561,
          0.7716
        ],
        [
          0.6665,
          0.7657
        ],
        [
          0.6765,
          0.7589
        ],
        [
          0.6865,
          0.7516
        ],
        [
          0.6965,
          0.747
        ],
        [
          0.7066,
          0.7407
        ],
        [
          0.717,
          0.7325
        ],
        [
          0.727,
          0.7252
        ],
        [
          0.737,
          0.7197
        ],
        [
          0.747,
          0.7134
        ],
        [
          0.7571,
          0.7102
        ],
        [
          0.7675,
          0.7047
        ],
        [
          0.7775,
          0.6988
        ],
        [
          0.7875,
          0.6938
        ],
        [
          0.7975,
          0.6888
        ],
        [
          0.8076,
          0.6811
        ],
        [
          0.818,
          0.6747
        ],
        [
          0.828,
          0.6679
        ],
        [
          0.838,
          0.6606
        ],
        [
          0.848,
          0.6533
        ],
        [
          0.8581,
          0.6474
        ],
        [
          0.8685,
          0.6429
        ],
        [
          0.8785,
          0.6365
        ],
        [
          0.8885,
          0.6292
        ],
        [
          0.8985,
          0.6224
        ],
        [
          0.9086,
          0.616
        ],
        [
          0.919,
          0.6083
        ],
        [
          0.929,
          0.6019
        ],
        [
          0.939,
          0.5978
        ],
        [
          0.949,
          0.5901
        ],
        [
          0.9591,
          0.5823
        ],
        [
          0.9695,
          0.576
        ],
        [
          0.9795,
          0.5705
        ],
        [
          0.9895,
          0.5646
        ],
        [
          0.9995,
          0.5573
        ],
        [
          1.0,
          0.5569
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1185160331275383,
        "difference": -0.000824618188590318,
        "worse": 0.48,
        "kept_right": 0.11746098760791417,
        "kept_wrong": 0.001055045519624066,
        "handed_off": 0.8814839668724621
      },
      "usd_per_1000": 0.45353513109011234
    },
    {
      "decision": 127,
      "frontier": 192,
      "frontier_accuracy": 0.9331210191082803,
      "in_sample": {
        "threshold": 0.9693852514507183,
        "answered": 0.09008189262966333,
        "accuracy": 0.9331210191082803
      },
      "curve": [
        [
          0.0,
          0.9331
        ],
        [
          0.01,
          0.9331
        ],
        [
          0.02,
          0.9331
        ],
        [
          0.03,
          0.9331
        ],
        [
          0.04,
          0.9331
        ],
        [
          0.05,
          0.9331
        ],
        [
          0.0601,
          0.9331
        ],
        [
          0.0705,
          0.9331
        ],
        [
          0.0805,
          0.9331
        ],
        [
          0.0905,
          0.9327
        ],
        [
          0.1005,
          0.9327
        ],
        [
          0.1106,
          0.9327
        ],
        [
          0.121,
          0.9327
        ],
        [
          0.131,
          0.9327
        ],
        [
          0.141,
          0.9327
        ],
        [
          0.151,
          0.9318
        ],
        [
          0.1611,
          0.9318
        ],
        [
          0.1715,
          0.9313
        ],
        [
          0.1815,
          0.9313
        ],
        [
          0.1915,
          0.9304
        ],
        [
          0.2015,
          0.929
        ],
        [
          0.2116,
          0.9272
        ],
        [
          0.222,
          0.9254
        ],
        [
          0.232,
          0.9245
        ],
        [
          0.242,
          0.9236
        ],
        [
          0.252,
          0.9222
        ],
        [
          0.2621,
          0.9208
        ],
        [
          0.2725,
          0.9204
        ],
        [
          0.2825,
          0.919
        ],
        [
          0.2925,
          0.9163
        ],
        [
          0.3025,
          0.9149
        ],
        [
          0.3126,
          0.9131
        ],
        [
          0.323,
          0.9099
        ],
        [
          0.333,
          0.9067
        ],
        [
          0.343,
          0.9054
        ],
        [
          0.353,
          0.9022
        ],
        [
          0.3631,
          0.9008
        ],
        [
          0.3735,
          0.8958
        ],
        [
          0.3835,
          0.8922
        ],
        [
          0.3935,
          0.8899
        ],
        [
          0.4035,
          0.8854
        ],
        [
          0.4136,
          0.8817
        ],
        [
          0.424,
          0.8781
        ],
        [
          0.434,
          0.874
        ],
        [
          0.444,
          0.8694
        ],
        [
          0.454,
          0.8672
        ],
        [
          0.4641,
          0.8635
        ],
        [
          0.4745,
          0.859
        ],
        [
          0.4845,
          0.8562
        ],
        [
          0.4945,
          0.8512
        ],
        [
          0.5045,
          0.8471
        ],
        [
          0.5146,
          0.8421
        ],
        [
          0.525,
          0.838
        ],
        [
          0.535,
          0.8335
        ],
        [
          0.545,
          0.8289
        ],
        [
          0.5551,
          0.8239
        ],
        [
          0.5655,
          0.8212
        ],
        [
          0.5755,
          0.813
        ],
        [
          0.5855,
          0.8094
        ],
        [
          0.5955,
          0.8048
        ],
        [
          0.6056,
          0.7998
        ],
        [
          0.616,
          0.7939
        ],
        [
          0.626,
          0.7894
        ],
        [
          0.636,
          0.7839
        ],
        [
          0.646,
          0.778
        ],
        [
          0.6561,
          0.7721
        ],
        [
          0.6665,
          0.7671
        ],
        [
          0.6765,
          0.7602
        ],
        [
          0.6865,
          0.753
        ],
        [
          0.6965,
          0.7484
        ],
        [
          0.7066,
          0.7416
        ],
        [
          0.717,
          0.7343
        ],
        [
          0.727,
          0.727
        ],
        [
          0.737,
          0.7216
        ],
        [
          0.747,
          0.717
        ],
        [
          0.7571,
          0.7125
        ],
        [
          0.7675,
          0.7066
        ],
        [
          0.7775,
          0.7002
        ],
        [
          0.7875,
          0.6943
        ],
        [
          0.7975,
          0.6897
        ],
        [
          0.8076,
          0.6815
        ],
        [
          0.818,
          0.6752
        ],
        [
          0.828,
          0.6683
        ],
        [
          0.838,
          0.6611
        ],
        [
          0.848,
          0.6547
        ],
        [
          0.8581,
          0.6483
        ],
        [
          0.8685,
          0.6447
        ],
        [
          0.8785,
          0.6383
        ],
        [
          0.8885,
          0.6319
        ],
        [
          0.8985,
          0.6256
        ],
        [
          0.9086,
          0.6192
        ],
        [
          0.919,
          0.6115
        ],
        [
          0.929,
          0.6042
        ],
        [
          0.939,
          0.6001
        ],
        [
          0.949,
          0.5919
        ],
        [
          0.9591,
          0.5842
        ],
        [
          0.9695,
          0.5773
        ],
        [
          0.9795,
          0.5714
        ],
        [
          0.9895,
          0.5641
        ],
        [
          0.9995,
          0.5573
        ],
        [
          1.0,
          0.5569
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.11915836138446338,
        "difference": -0.0008215465004376748,
        "worse": 0.48,
        "kept_right": 0.11804878139452524,
        "kept_wrong": 0.0011095799899380533,
        "handed_off": 0.8808416386155371
      },
      "usd_per_1000": 10.226188550758804
    },
    {
      "decision": 127,
      "frontier": 123,
      "frontier_accuracy": 0.9326660600545951,
      "in_sample": {
        "threshold": 0.9693852514507183,
        "answered": 0.09008189262966333,
        "accuracy": 0.9326660600545951
      },
      "curve": [
        [
          0.0,
          0.9327
        ],
        [
          0.01,
          0.9327
        ],
        [
          0.02,
          0.9327
        ],
        [
          0.03,
          0.9327
        ],
        [
          0.04,
          0.9327
        ],
        [
          0.05,
          0.9327
        ],
        [
          0.0601,
          0.9327
        ],
        [
          0.0705,
          0.9327
        ],
        [
          0.0805,
          0.9327
        ],
        [
          0.0905,
          0.9322
        ],
        [
          0.1005,
          0.9322
        ],
        [
          0.1106,
          0.9322
        ],
        [
          0.121,
          0.9322
        ],
        [
          0.131,
          0.9322
        ],
        [
          0.141,
          0.9322
        ],
        [
          0.151,
          0.9313
        ],
        [
          0.1611,
          0.9308
        ],
        [
          0.1715,
          0.9304
        ],
        [
          0.1815,
          0.9299
        ],
        [
          0.1915,
          0.929
        ],
        [
          0.2015,
          0.9281
        ],
        [
          0.2116,
          0.9263
        ],
        [
          0.222,
          0.924
        ],
        [
          0.232,
          0.9227
        ],
        [
          0.242,
          0.9213
        ],
        [
          0.252,
          0.9199
        ],
        [
          0.2621,
          0.919
        ],
        [
          0.2725,
          0.9177
        ],
        [
          0.2825,
          0.9172
        ],
        [
          0.2925,
          0.9145
        ],
        [
          0.3025,
          0.9131
        ],
        [
          0.3126,
          0.9108
        ],
        [
          0.323,
          0.9076
        ],
        [
          0.333,
          0.9054
        ],
        [
          0.343,
          0.904
        ],
        [
          0.353,
          0.9017
        ],
        [
          0.3631,
          0.8995
        ],
        [
          0.3735,
          0.894
        ],
        [
          0.3835,
          0.8894
        ],
        [
          0.3935,
          0.8867
        ],
        [
          0.4035,
          0.8822
        ],
        [
          0.4136,
          0.8781
        ],
        [
          0.424,
          0.8744
        ],
        [
          0.434,
          0.8712
        ],
        [
          0.444,
          0.8667
        ],
        [
          0.454,
          0.8631
        ],
        [
          0.4641,
          0.8594
        ],
        [
          0.4745,
          0.854
        ],
        [
          0.4845,
          0.8503
        ],
        [
          0.4945,
          0.8462
        ],
        [
          0.5045,
          0.8412
        ],
        [
          0.5146,
          0.8371
        ],
        [
          0.525,
          0.833
        ],
        [
          0.535,
          0.8289
        ],
        [
          0.545,
          0.8244
        ],
        [
          0.5551,
          0.8189
        ],
        [
          0.5655,
          0.8157
        ],
        [
          0.5755,
          0.8076
        ],
        [
          0.5855,
          0.8048
        ],
        [
          0.5955,
          0.8007
        ],
        [
          0.6056,
          0.7957
        ],
        [
          0.616,
          0.7894
        ],
        [
          0.626,
          0.7853
        ],
        [
          0.636,
          0.7803
        ],
        [
          0.646,
          0.7734
        ],
        [
          0.6561,
          0.7684
        ],
        [
          0.6665,
          0.7625
        ],
        [
          0.6765,
          0.7548
        ],
        [
          0.6865,
          0.7484
        ],
        [
          0.6965,
          0.7434
        ],
        [
          0.7066,
          0.737
        ],
        [
          0.717,
          0.7302
        ],
        [
          0.727,
          0.7243
        ],
        [
          0.737,
          0.7197
        ],
        [
          0.747,
          0.7134
        ],
        [
          0.7571,
          0.7093
        ],
        [
          0.7675,
          0.7034
        ],
        [
          0.7775,
          0.6965
        ],
        [
          0.7875,
          0.6915
        ],
        [
          0.7975,
          0.6856
        ],
        [
          0.8076,
          0.677
        ],
        [
          0.818,
          0.6706
        ],
        [
          0.828,
          0.6633
        ],
        [
          0.838,
          0.6556
        ],
        [
          0.848,
          0.6492
        ],
        [
          0.8581,
          0.6429
        ],
        [
          0.8685,
          0.6392
        ],
        [
          0.8785,
          0.6333
        ],
        [
          0.8885,
          0.6265
        ],
        [
          0.8985,
          0.6201
        ],
        [
          0.9086,
          0.6142
        ],
        [
          0.919,
          0.6078
        ],
        [
          0.929,
          0.601
        ],
        [
          0.939,
          0.5965
        ],
        [
          0.949,
          0.5883
        ],
        [
          0.9591,
          0.5801
        ],
        [
          0.9695,
          0.5742
        ],
        [
          0.9795,
          0.5687
        ],
        [
          0.9895,
          0.5641
        ],
        [
          0.9995,
          0.5573
        ],
        [
          1.0,
          0.5569
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.11920487245081217,
        "difference": -0.000882287805908394,
        "worse": 0.48333333333333334,
        "kept_right": 0.11808912978831203,
        "kept_wrong": 0.0011157426625000375,
        "handed_off": 0.8807951275491882
      },
      "usd_per_1000": 0.23656742564187638
    },
    {
      "decision": 127,
      "frontier": 274,
      "frontier_accuracy": 0.9085532302092811,
      "in_sample": {
        "threshold": 0.9528763014370394,
        "answered": 0.1451319381255687,
        "accuracy": 0.9085532302092811
      },
      "curve": [
        [
          0.0,
          0.9086
        ],
        [
          0.01,
          0.9086
        ],
        [
          0.02,
          0.9086
        ],
        [
          0.03,
          0.9086
        ],
        [
          0.04,
          0.9086
        ],
        [
          0.05,
          0.9086
        ],
        [
          0.0601,
          0.9086
        ],
        [
          0.0705,
          0.9086
        ],
        [
          0.0805,
          0.9086
        ],
        [
          0.0905,
          0.9081
        ],
        [
          0.1005,
          0.9081
        ],
        [
          0.1106,
          0.9086
        ],
        [
          0.121,
          0.9086
        ],
        [
          0.131,
          0.9086
        ],
        [
          0.141,
          0.9086
        ],
        [
          0.151,
          0.9076
        ],
        [
          0.1611,
          0.9076
        ],
        [
          0.1715,
          0.9072
        ],
        [
          0.1815,
          0.9072
        ],
        [
          0.1915,
          0.9063
        ],
        [
          0.2015,
          0.9054
        ],
        [
          0.2116,
          0.9049
        ],
        [
          0.222,
          0.9026
        ],
        [
          0.232,
          0.9004
        ],
        [
          0.242,
          0.8995
        ],
        [
          0.252,
          0.8981
        ],
        [
          0.2621,
          0.8967
        ],
        [
          0.2725,
          0.8963
        ],
        [
          0.2825,
          0.8958
        ],
        [
          0.2925,
          0.8935
        ],
        [
          0.3025,
          0.8926
        ],
        [
          0.3126,
          0.8917
        ],
        [
          0.323,
          0.889
        ],
        [
          0.333,
          0.8863
        ],
        [
          0.343,
          0.8858
        ],
        [
          0.353,
          0.8831
        ],
        [
          0.3631,
          0.8817
        ],
        [
          0.3735,
          0.8763
        ],
        [
          0.3835,
          0.8726
        ],
        [
          0.3935,
          0.8694
        ],
        [
          0.4035,
          0.8649
        ],
        [
          0.4136,
          0.8612
        ],
        [
          0.424,
          0.8581
        ],
        [
          0.434,
          0.8535
        ],
        [
          0.444,
          0.849
        ],
        [
          0.454,
          0.8453
        ],
        [
          0.4641,
          0.8421
        ],
        [
          0.4745,
          0.8371
        ],
        [
          0.4845,
          0.8344
        ],
        [
          0.4945,
          0.8303
        ],
        [
          0.5045,
          0.8262
        ],
        [
          0.5146,
          0.8221
        ],
        [
          0.525,
          0.8176
        ],
        [
          0.535,
          0.8126
        ],
        [
          0.545,
          0.8085
        ],
        [
          0.5551,
          0.803
        ],
        [
          0.5655,
          0.8003
        ],
        [
          0.5755,
          0.7934
        ],
        [
          0.5855,
          0.7903
        ],
        [
          0.5955,
          0.7862
        ],
        [
          0.6056,
          0.7803
        ],
        [
          0.616,
          0.7748
        ],
        [
          0.626,
          0.7702
        ],
        [
          0.636,
          0.7652
        ],
        [
          0.646,
          0.7593
        ],
        [
          0.6561,
          0.7543
        ],
        [
          0.6665,
          0.7484
        ],
        [
          0.6765,
          0.742
        ],
        [
          0.6865,
          0.7361
        ],
        [
          0.6965,
          0.732
        ],
        [
          0.7066,
          0.7257
        ],
        [
          0.717,
          0.7175
        ],
        [
          0.727,
          0.7111
        ],
        [
          0.737,
          0.707
        ],
        [
          0.747,
          0.7011
        ],
        [
          0.7571,
          0.6979
        ],
        [
          0.7675,
          0.6929
        ],
        [
          0.7775,
          0.6865
        ],
        [
          0.7875,
          0.682
        ],
        [
          0.7975,
          0.6774
        ],
        [
          0.8076,
          0.6711
        ],
        [
          0.818,
          0.6647
        ],
        [
          0.828,
          0.6592
        ],
        [
          0.838,
          0.6524
        ],
        [
          0.848,
          0.646
        ],
        [
          0.8581,
          0.641
        ],
        [
          0.8685,
          0.6374
        ],
        [
          0.8785,
          0.6306
        ],
        [
          0.8885,
          0.6242
        ],
        [
          0.8985,
          0.6187
        ],
        [
          0.9086,
          0.6124
        ],
        [
          0.919,
          0.6056
        ],
        [
          0.929,
          0.5987
        ],
        [
          0.939,
          0.5951
        ],
        [
          0.949,
          0.5887
        ],
        [
          0.9591,
          0.581
        ],
        [
          0.9695,
          0.5751
        ],
        [
          0.9795,
          0.5705
        ],
        [
          0.9895,
          0.5646
        ],
        [
          0.9995,
          0.5573
        ],
        [
          1.0,
          0.5569
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.14013794050365058,
        "difference": -0.0011155248779727497,
        "worse": 0.5066666666666667,
        "kept_right": 0.1381826209854485,
        "kept_wrong": 0.001955319518202027,
        "handed_off": 0.8598620594963499
      },
      "usd_per_1000": 0.3665124864410445
    },
    {
      "decision": 127,
      "frontier": 47,
      "frontier_accuracy": 0.9044585987261147,
      "in_sample": {
        "threshold": 0.9693852514507183,
        "answered": 0.09008189262966333,
        "accuracy": 0.9044585987261147
      },
      "curve": [
        [
          0.0,
          0.9045
        ],
        [
          0.01,
          0.9045
        ],
        [
          0.02,
          0.9045
        ],
        [
          0.03,
          0.9045
        ],
        [
          0.04,
          0.9045
        ],
        [
          0.05,
          0.9045
        ],
        [
          0.0601,
          0.9045
        ],
        [
          0.0705,
          0.9045
        ],
        [
          0.0805,
          0.9045
        ],
        [
          0.0905,
          0.904
        ],
        [
          0.1005,
          0.904
        ],
        [
          0.1106,
          0.904
        ],
        [
          0.121,
          0.904
        ],
        [
          0.131,
          0.904
        ],
        [
          0.141,
          0.904
        ],
        [
          0.151,
          0.9031
        ],
        [
          0.1611,
          0.9035
        ],
        [
          0.1715,
          0.9031
        ],
        [
          0.1815,
          0.9031
        ],
        [
          0.1915,
          0.9022
        ],
        [
          0.2015,
          0.9008
        ],
        [
          0.2116,
          0.8985
        ],
        [
          0.222,
          0.8972
        ],
        [
          0.232,
          0.8954
        ],
        [
          0.242,
          0.8944
        ],
        [
          0.252,
          0.8931
        ],
        [
          0.2621,
          0.8926
        ],
        [
          0.2725,
          0.8922
        ],
        [
          0.2825,
          0.8908
        ],
        [
          0.2925,
          0.8881
        ],
        [
          0.3025,
          0.8872
        ],
        [
          0.3126,
          0.8858
        ],
        [
          0.323,
          0.8835
        ],
        [
          0.333,
          0.8808
        ],
        [
          0.343,
          0.8794
        ],
        [
          0.353,
          0.8763
        ],
        [
          0.3631,
          0.8753
        ],
        [
          0.3735,
          0.8703
        ],
        [
          0.3835,
          0.8667
        ],
        [
          0.3935,
          0.8635
        ],
        [
          0.4035,
          0.859
        ],
        [
          0.4136,
          0.8562
        ],
        [
          0.424,
          0.853
        ],
        [
          0.434,
          0.849
        ],
        [
          0.444,
          0.8449
        ],
        [
          0.454,
          0.8421
        ],
        [
          0.4641,
          0.8389
        ],
        [
          0.4745,
          0.8339
        ],
        [
          0.4845,
          0.8308
        ],
        [
          0.4945,
          0.8258
        ],
        [
          0.5045,
          0.8212
        ],
        [
          0.5146,
          0.8162
        ],
        [
          0.525,
          0.8116
        ],
        [
          0.535,
          0.8066
        ],
        [
          0.545,
          0.8021
        ],
        [
          0.5551,
          0.798
        ],
        [
          0.5655,
          0.7953
        ],
        [
          0.5755,
          0.788
        ],
        [
          0.5855,
          0.7853
        ],
        [
          0.5955,
          0.7812
        ],
        [
          0.6056,
          0.7766
        ],
        [
          0.616,
          0.7707
        ],
        [
          0.626,
          0.7662
        ],
        [
          0.636,
          0.7611
        ],
        [
          0.646,
          0.7552
        ],
        [
          0.6561,
          0.7507
        ],
        [
          0.6665,
          0.7461
        ],
        [
          0.6765,
          0.7398
        ],
        [
          0.6865,
          0.7338
        ],
        [
          0.6965,
          0.7298
        ],
        [
          0.7066,
          0.7238
        ],
        [
          0.717,
          0.7166
        ],
        [
          0.727,
          0.7102
        ],
        [
          0.737,
          0.7075
        ],
        [
          0.747,
          0.7015
        ],
        [
          0.7571,
          0.6979
        ],
        [
          0.7675,
          0.692
        ],
        [
          0.7775,
          0.6861
        ],
        [
          0.7875,
          0.6824
        ],
        [
          0.7975,
          0.677
        ],
        [
          0.8076,
          0.6692
        ],
        [
          0.818,
          0.6629
        ],
        [
          0.828,
          0.6565
        ],
        [
          0.838,
          0.6492
        ],
        [
          0.848,
          0.6424
        ],
        [
          0.8581,
          0.6369
        ],
        [
          0.8685,
          0.6351
        ],
        [
          0.8785,
          0.6301
        ],
        [
          0.8885,
          0.6242
        ],
        [
          0.8985,
          0.6183
        ],
        [
          0.9086,
          0.6119
        ],
        [
          0.919,
          0.6046
        ],
        [
          0.929,
          0.5983
        ],
        [
          0.939,
          0.5946
        ],
        [
          0.949,
          0.5869
        ],
        [
          0.9591,
          0.581
        ],
        [
          0.9695,
          0.576
        ],
        [
          0.9795,
          0.5705
        ],
        [
          0.9895,
          0.5641
        ],
        [
          0.9995,
          0.5573
        ],
        [
          1.0,
          0.5569
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.12654532369133745,
        "difference": -0.0010428360323053573,
        "worse": 0.5266666666666666,
        "kept_right": 0.12502348423683918,
        "kept_wrong": 0.001521839454498152,
        "handed_off": 0.8734546763086629
      },
      "usd_per_1000": 4.3066433749283215
    },
    {
      "decision": 127,
      "frontier": 193,
      "frontier_accuracy": 0.89171974522293,
      "in_sample": {
        "threshold": 0.9528763014370394,
        "answered": 0.1451319381255687,
        "accuracy": 0.89171974522293
      },
      "curve": [
        [
          0.0,
          0.8917
        ],
        [
          0.01,
          0.8917
        ],
        [
          0.02,
          0.8917
        ],
        [
          0.03,
          0.8917
        ],
        [
          0.04,
          0.8917
        ],
        [
          0.05,
          0.8917
        ],
        [
          0.0601,
          0.8917
        ],
        [
          0.0705,
          0.8917
        ],
        [
          0.0805,
          0.8917
        ],
        [
          0.0905,
          0.8913
        ],
        [
          0.1005,
          0.8913
        ],
        [
          0.1106,
          0.8917
        ],
        [
          0.121,
          0.8917
        ],
        [
          0.131,
          0.8917
        ],
        [
          0.141,
          0.8917
        ],
        [
          0.151,
          0.8908
        ],
        [
          0.1611,
          0.8904
        ],
        [
          0.1715,
          0.8899
        ],
        [
          0.1815,
          0.8899
        ],
        [
          0.1915,
          0.889
        ],
        [
          0.2015,
          0.8872
        ],
        [
          0.2116,
          0.8867
        ],
        [
          0.222,
          0.8849
        ],
        [
          0.232,
          0.884
        ],
        [
          0.242,
          0.8835
        ],
        [
          0.252,
          0.8822
        ],
        [
          0.2621,
          0.8817
        ],
        [
          0.2725,
          0.8813
        ],
        [
          0.2825,
          0.8799
        ],
        [
          0.2925,
          0.8781
        ],
        [
          0.3025,
          0.8772
        ],
        [
          0.3126,
          0.8767
        ],
        [
          0.323,
          0.8735
        ],
        [
          0.333,
          0.8708
        ],
        [
          0.343,
          0.8703
        ],
        [
          0.353,
          0.8672
        ],
        [
          0.3631,
          0.8662
        ],
        [
          0.3735,
          0.8612
        ],
        [
          0.3835,
          0.8581
        ],
        [
          0.3935,
          0.8562
        ],
        [
          0.4035,
          0.8526
        ],
        [
          0.4136,
          0.8503
        ],
        [
          0.424,
          0.8476
        ],
        [
          0.434,
          0.8435
        ],
        [
          0.444,
          0.8394
        ],
        [
          0.454,
          0.8371
        ],
        [
          0.4641,
          0.8339
        ],
        [
          0.4745,
          0.8298
        ],
        [
          0.4845,
          0.8271
        ],
        [
          0.4945,
          0.823
        ],
        [
          0.5045,
          0.8189
        ],
        [
          0.5146,
          0.8148
        ],
        [
          0.525,
          0.8098
        ],
        [
          0.535,
          0.8057
        ],
        [
          0.545,
          0.8021
        ],
        [
          0.5551,
          0.7994
        ],
        [
          0.5655,
          0.7971
        ],
        [
          0.5755,
          0.7898
        ],
        [
          0.5855,
          0.7862
        ],
        [
          0.5955,
          0.7825
        ],
        [
          0.6056,
          0.7789
        ],
        [
          0.616,
          0.7734
        ],
        [
          0.626,
          0.7689
        ],
        [
          0.636,
          0.7639
        ],
        [
          0.646,
          0.758
        ],
        [
          0.6561,
          0.7534
        ],
        [
          0.6665,
          0.748
        ],
        [
          0.6765,
          0.7411
        ],
        [
          0.6865,
          0.7343
        ],
        [
          0.6965,
          0.7311
        ],
        [
          0.7066,
          0.7247
        ],
        [
          0.717,
          0.7175
        ],
        [
          0.727,
          0.7116
        ],
        [
          0.737,
          0.7066
        ],
        [
          0.747,
          0.7002
        ],
        [
          0.7571,
          0.6965
        ],
        [
          0.7675,
          0.6906
        ],
        [
          0.7775,
          0.6843
        ],
        [
          0.7875,
          0.6793
        ],
        [
          0.7975,
          0.6747
        ],
        [
          0.8076,
          0.667
        ],
        [
          0.818,
          0.6624
        ],
        [
          0.828,
          0.6561
        ],
        [
          0.838,
          0.6483
        ],
        [
          0.848,
          0.6424
        ],
        [
          0.8581,
          0.6374
        ],
        [
          0.8685,
          0.6328
        ],
        [
          0.8785,
          0.6265
        ],
        [
          0.8885,
          0.6201
        ],
        [
          0.8985,
          0.6151
        ],
        [
          0.9086,
          0.6087
        ],
        [
          0.919,
          0.6015
        ],
        [
          0.929,
          0.5946
        ],
        [
          0.939,
          0.5924
        ],
        [
          0.949,
          0.5846
        ],
        [
          0.9591,
          0.5787
        ],
        [
          0.9695,
          0.5732
        ],
        [
          0.9795,
          0.5692
        ],
        [
          0.9895,
          0.5637
        ],
        [
          0.9995,
          0.5573
        ],
        [
          1.0,
          0.5569
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1377408203181731,
        "difference": -0.0011095719184569229,
        "worse": 0.5066666666666667,
        "kept_right": 0.13604611256055693,
        "kept_wrong": 0.0016947077576162002,
        "handed_off": 0.8622591796818274
      },
      "usd_per_1000": 6.6804305900652725
    },
    {
      "decision": 127,
      "frontier": 55,
      "frontier_accuracy": 0.8503184713375797,
      "in_sample": {
        "threshold": 0.9124882818124339,
        "answered": 0.22156505914467697,
        "accuracy": 0.8503184713375797
      },
      "curve": [
        [
          0.0,
          0.8503
        ],
        [
          0.01,
          0.8503
        ],
        [
          0.02,
          0.8503
        ],
        [
          0.03,
          0.8503
        ],
        [
          0.04,
          0.8503
        ],
        [
          0.05,
          0.8503
        ],
        [
          0.0601,
          0.8503
        ],
        [
          0.0705,
          0.8503
        ],
        [
          0.0805,
          0.8508
        ],
        [
          0.0905,
          0.8503
        ],
        [
          0.1005,
          0.8503
        ],
        [
          0.1106,
          0.8508
        ],
        [
          0.121,
          0.8512
        ],
        [
          0.131,
          0.8512
        ],
        [
          0.141,
          0.8521
        ],
        [
          0.151,
          0.8517
        ],
        [
          0.1611,
          0.8517
        ],
        [
          0.1715,
          0.8517
        ],
        [
          0.1815,
          0.8521
        ],
        [
          0.1915,
          0.8521
        ],
        [
          0.2015,
          0.8517
        ],
        [
          0.2116,
          0.8517
        ],
        [
          0.222,
          0.8499
        ],
        [
          0.232,
          0.8494
        ],
        [
          0.242,
          0.849
        ],
        [
          0.252,
          0.8485
        ],
        [
          0.2621,
          0.8485
        ],
        [
          0.2725,
          0.8485
        ],
        [
          0.2825,
          0.8485
        ],
        [
          0.2925,
          0.8471
        ],
        [
          0.3025,
          0.8458
        ],
        [
          0.3126,
          0.8449
        ],
        [
          0.323,
          0.8421
        ],
        [
          0.333,
          0.8408
        ],
        [
          0.343,
          0.8408
        ],
        [
          0.353,
          0.838
        ],
        [
          0.3631,
          0.8362
        ],
        [
          0.3735,
          0.8321
        ],
        [
          0.3835,
          0.8294
        ],
        [
          0.3935,
          0.8271
        ],
        [
          0.4035,
          0.8244
        ],
        [
          0.4136,
          0.8217
        ],
        [
          0.424,
          0.818
        ],
        [
          0.434,
          0.8148
        ],
        [
          0.444,
          0.8107
        ],
        [
          0.454,
          0.8089
        ],
        [
          0.4641,
          0.8062
        ],
        [
          0.4745,
          0.8025
        ],
        [
          0.4845,
          0.8007
        ],
        [
          0.4945,
          0.7985
        ],
        [
          0.5045,
          0.7939
        ],
        [
          0.5146,
          0.7912
        ],
        [
          0.525,
          0.7875
        ],
        [
          0.535,
          0.7839
        ],
        [
          0.545,
          0.7812
        ],
        [
          0.5551,
          0.778
        ],
        [
          0.5655,
          0.7762
        ],
        [
          0.5755,
          0.7693
        ],
        [
          0.5855,
          0.7671
        ],
        [
          0.5955,
          0.7643
        ],
        [
          0.6056,
          0.7611
        ],
        [
          0.616,
          0.7557
        ],
        [
          0.626,
          0.7516
        ],
        [
          0.636,
          0.7466
        ],
        [
          0.646,
          0.7407
        ],
        [
          0.6561,
          0.7366
        ],
        [
          0.6665,
          0.7316
        ],
        [
          0.6765,
          0.7261
        ],
        [
          0.6865,
          0.7197
        ],
        [
          0.6965,
          0.717
        ],
        [
          0.7066,
          0.7111
        ],
        [
          0.717,
          0.7038
        ],
        [
          0.727,
          0.6975
        ],
        [
          0.737,
          0.6938
        ],
        [
          0.747,
          0.6884
        ],
        [
          0.7571,
          0.6856
        ],
        [
          0.7675,
          0.6806
        ],
        [
          0.7775,
          0.6738
        ],
        [
          0.7875,
          0.6692
        ],
        [
          0.7975,
          0.6647
        ],
        [
          0.8076,
          0.6579
        ],
        [
          0.818,
          0.652
        ],
        [
          0.828,
          0.646
        ],
        [
          0.838,
          0.6401
        ],
        [
          0.848,
          0.6333
        ],
        [
          0.8581,
          0.6283
        ],
        [
          0.8685,
          0.6256
        ],
        [
          0.8785,
          0.6192
        ],
        [
          0.8885,
          0.6133
        ],
        [
          0.8985,
          0.6083
        ],
        [
          0.9086,
          0.6033
        ],
        [
          0.919,
          0.5955
        ],
        [
          0.929,
          0.5892
        ],
        [
          0.939,
          0.5869
        ],
        [
          0.949,
          0.5801
        ],
        [
          0.9591,
          0.5755
        ],
        [
          0.9695,
          0.5714
        ],
        [
          0.9795,
          0.5678
        ],
        [
          0.9895,
          0.5637
        ],
        [
          0.9995,
          0.5573
        ],
        [
          1.0,
          0.5569
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.2416815443483455,
        "difference": -0.001573759325693657,
        "worse": 0.5333333333333333,
        "kept_right": 0.22689897140477952,
        "kept_wrong": 0.014782572943566073,
        "handed_off": 0.7583184556516548
      },
      "usd_per_1000": 2.7711486270856938
    },
    {
      "decision": 128,
      "frontier": 140,
      "frontier_accuracy": 0.9808917197452229,
      "in_sample": {
        "threshold": 0.9911303026027772,
        "answered": 0.06187443130118289,
        "accuracy": 0.9808917197452229
      },
      "curve": [
        [
          0.0,
          0.9809
        ],
        [
          0.01,
          0.9809
        ],
        [
          0.02,
          0.9809
        ],
        [
          0.03,
          0.9809
        ],
        [
          0.04,
          0.9809
        ],
        [
          0.05,
          0.9809
        ],
        [
          0.0601,
          0.9809
        ],
        [
          0.0705,
          0.9804
        ],
        [
          0.0805,
          0.9804
        ],
        [
          0.0905,
          0.9804
        ],
        [
          0.1005,
          0.9804
        ],
        [
          0.1106,
          0.9804
        ],
        [
          0.121,
          0.9804
        ],
        [
          0.131,
          0.9804
        ],
        [
          0.141,
          0.9804
        ],
        [
          0.151,
          0.98
        ],
        [
          0.1611,
          0.9795
        ],
        [
          0.1715,
          0.9795
        ],
        [
          0.1815,
          0.9786
        ],
        [
          0.1915,
          0.9782
        ],
        [
          0.2015,
          0.9773
        ],
        [
          0.2116,
          0.9768
        ],
        [
          0.222,
          0.9745
        ],
        [
          0.232,
          0.9722
        ],
        [
          0.242,
          0.97
        ],
        [
          0.252,
          0.9672
        ],
        [
          0.2621,
          0.965
        ],
        [
          0.2725,
          0.9618
        ],
        [
          0.2825,
          0.9591
        ],
        [
          0.2925,
          0.9581
        ],
        [
          0.3025,
          0.9563
        ],
        [
          0.3126,
          0.9522
        ],
        [
          0.323,
          0.9481
        ],
        [
          0.333,
          0.9445
        ],
        [
          0.343,
          0.9431
        ],
        [
          0.353,
          0.9377
        ],
        [
          0.3631,
          0.9331
        ],
        [
          0.3735,
          0.9295
        ],
        [
          0.3835,
          0.9227
        ],
        [
          0.3935,
          0.9195
        ],
        [
          0.4035,
          0.914
        ],
        [
          0.4136,
          0.9117
        ],
        [
          0.424,
          0.9072
        ],
        [
          0.434,
          0.9022
        ],
        [
          0.444,
          0.899
        ],
        [
          0.454,
          0.8958
        ],
        [
          0.4641,
          0.8913
        ],
        [
          0.4745,
          0.8854
        ],
        [
          0.4845,
          0.8817
        ],
        [
          0.4945,
          0.8758
        ],
        [
          0.5045,
          0.8717
        ],
        [
          0.5146,
          0.8662
        ],
        [
          0.525,
          0.8603
        ],
        [
          0.535,
          0.8549
        ],
        [
          0.545,
          0.849
        ],
        [
          0.5551,
          0.8449
        ],
        [
          0.5655,
          0.838
        ],
        [
          0.5755,
          0.8339
        ],
        [
          0.5855,
          0.8289
        ],
        [
          0.5955,
          0.8226
        ],
        [
          0.6056,
          0.8162
        ],
        [
          0.616,
          0.8112
        ],
        [
          0.626,
          0.8044
        ],
        [
          0.636,
          0.7998
        ],
        [
          0.646,
          0.7939
        ],
        [
          0.6561,
          0.7889
        ],
        [
          0.6665,
          0.783
        ],
        [
          0.6765,
          0.7762
        ],
        [
          0.6865,
          0.7707
        ],
        [
          0.6965,
          0.7666
        ],
        [
          0.7066,
          0.7598
        ],
        [
          0.717,
          0.752
        ],
        [
          0.727,
          0.7457
        ],
        [
          0.737,
          0.7393
        ],
        [
          0.747,
          0.7307
        ],
        [
          0.7571,
          0.7225
        ],
        [
          0.7675,
          0.7157
        ],
        [
          0.7775,
          0.7084
        ],
        [
          0.7875,
          0.7015
        ],
        [
          0.7975,
          0.6956
        ],
        [
          0.8076,
          0.6879
        ],
        [
          0.818,
          0.6802
        ],
        [
          0.828,
          0.6765
        ],
        [
          0.838,
          0.6715
        ],
        [
          0.848,
          0.6633
        ],
        [
          0.8581,
          0.6574
        ],
        [
          0.8685,
          0.651
        ],
        [
          0.8785,
          0.6456
        ],
        [
          0.8885,
          0.6369
        ],
        [
          0.8985,
          0.6301
        ],
        [
          0.9086,
          0.6228
        ],
        [
          0.919,
          0.6151
        ],
        [
          0.929,
          0.6083
        ],
        [
          0.939,
          0.6024
        ],
        [
          0.949,
          0.5937
        ],
        [
          0.9591,
          0.586
        ],
        [
          0.9695,
          0.5778
        ],
        [
          0.9795,
          0.5705
        ],
        [
          0.9895,
          0.5628
        ],
        [
          0.9995,
          0.5551
        ],
        [
          1.0,
          0.5546
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.10634845801091258,
        "difference": -0.0008215853335784767,
        "worse": 0.4666666666666667,
        "kept_right": 0.10495086902740888,
        "kept_wrong": 0.0013975889835037318,
        "handed_off": 0.893651541989087
      },
      "usd_per_1000": 2.161835270921077
    },
    {
      "decision": 128,
      "frontier": 277,
      "frontier_accuracy": 0.9795268425841674,
      "in_sample": {
        "threshold": 0.9911303026027772,
        "answered": 0.06187443130118289,
        "accuracy": 0.9795268425841674
      },
      "curve": [
        [
          0.0,
          0.9795
        ],
        [
          0.01,
          0.9795
        ],
        [
          0.02,
          0.9795
        ],
        [
          0.03,
          0.9795
        ],
        [
          0.04,
          0.9795
        ],
        [
          0.05,
          0.9795
        ],
        [
          0.0601,
          0.9795
        ],
        [
          0.0705,
          0.9791
        ],
        [
          0.0805,
          0.9791
        ],
        [
          0.0905,
          0.9791
        ],
        [
          0.1005,
          0.9791
        ],
        [
          0.1106,
          0.9791
        ],
        [
          0.121,
          0.9791
        ],
        [
          0.131,
          0.9791
        ],
        [
          0.141,
          0.9791
        ],
        [
          0.151,
          0.9786
        ],
        [
          0.1611,
          0.9782
        ],
        [
          0.1715,
          0.9782
        ],
        [
          0.1815,
          0.9768
        ],
        [
          0.1915,
          0.9763
        ],
        [
          0.2015,
          0.9754
        ],
        [
          0.2116,
          0.975
        ],
        [
          0.222,
          0.9727
        ],
        [
          0.232,
          0.9704
        ],
        [
          0.242,
          0.9686
        ],
        [
          0.252,
          0.9659
        ],
        [
          0.2621,
          0.9636
        ],
        [
          0.2725,
          0.9604
        ],
        [
          0.2825,
          0.9577
        ],
        [
          0.2925,
          0.9568
        ],
        [
          0.3025,
          0.9545
        ],
        [
          0.3126,
          0.95
        ],
        [
          0.323,
          0.9463
        ],
        [
          0.333,
          0.9427
        ],
        [
          0.343,
          0.9409
        ],
        [
          0.353,
          0.9354
        ],
        [
          0.3631,
          0.9308
        ],
        [
          0.3735,
          0.9272
        ],
        [
          0.3835,
          0.9204
        ],
        [
          0.3935,
          0.9172
        ],
        [
          0.4035,
          0.9113
        ],
        [
          0.4136,
          0.909
        ],
        [
          0.424,
          0.9045
        ],
        [
          0.434,
          0.8995
        ],
        [
          0.444,
          0.8963
        ],
        [
          0.454,
          0.8935
        ],
        [
          0.4641,
          0.8894
        ],
        [
          0.4745,
          0.8835
        ],
        [
          0.4845,
          0.8803
        ],
        [
          0.4945,
          0.8744
        ],
        [
          0.5045,
          0.8703
        ],
        [
          0.5146,
          0.8649
        ],
        [
          0.525,
          0.8585
        ],
        [
          0.535,
          0.853
        ],
        [
          0.545,
          0.8471
        ],
        [
          0.5551,
          0.8435
        ],
        [
          0.5655,
          0.8367
        ],
        [
          0.5755,
          0.8321
        ],
        [
          0.5855,
          0.8271
        ],
        [
          0.5955,
          0.8203
        ],
        [
          0.6056,
          0.8144
        ],
        [
          0.616,
          0.8094
        ],
        [
          0.626,
          0.803
        ],
        [
          0.636,
          0.7985
        ],
        [
          0.646,
          0.7925
        ],
        [
          0.6561,
          0.7875
        ],
        [
          0.6665,
          0.7816
        ],
        [
          0.6765,
          0.7748
        ],
        [
          0.6865,
          0.7698
        ],
        [
          0.6965,
          0.7652
        ],
        [
          0.7066,
          0.7584
        ],
        [
          0.717,
          0.7507
        ],
        [
          0.727,
          0.7439
        ],
        [
          0.737,
          0.7375
        ],
        [
          0.747,
          0.7288
        ],
        [
          0.7571,
          0.7207
        ],
        [
          0.7675,
          0.7138
        ],
        [
          0.7775,
          0.7066
        ],
        [
          0.7875,
          0.6997
        ],
        [
          0.7975,
          0.6938
        ],
        [
          0.8076,
          0.6865
        ],
        [
          0.818,
          0.6788
        ],
        [
          0.828,
          0.6752
        ],
        [
          0.838,
          0.6711
        ],
        [
          0.848,
          0.6629
        ],
        [
          0.8581,
          0.657
        ],
        [
          0.8685,
          0.6501
        ],
        [
          0.8785,
          0.6447
        ],
        [
          0.8885,
          0.636
        ],
        [
          0.8985,
          0.6292
        ],
        [
          0.9086,
          0.6224
        ],
        [
          0.919,
          0.6146
        ],
        [
          0.929,
          0.6078
        ],
        [
          0.939,
          0.6019
        ],
        [
          0.949,
          0.5933
        ],
        [
          0.9591,
          0.5855
        ],
        [
          0.9695,
          0.5778
        ],
        [
          0.9795,
          0.5705
        ],
        [
          0.9895,
          0.5628
        ],
        [
          0.9995,
          0.5551
        ],
        [
          1.0,
          0.5546
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.10618774419871489,
        "difference": -0.0008337065557035924,
        "worse": 0.4666666666666667,
        "kept_right": 0.1048053039880614,
        "kept_wrong": 0.0013824402106535065,
        "handed_off": 0.8938122558012848
      },
      "usd_per_1000": 5.421467441968041
    },
    {
      "decision": 128,
      "frontier": 276,
      "frontier_accuracy": 0.9777070063694268,
      "in_sample": {
        "threshold": 0.9911303026027772,
        "answered": 0.06187443130118289,
        "accuracy": 0.9777070063694268
      },
      "curve": [
        [
          0.0,
          0.9777
        ],
        [
          0.01,
          0.9777
        ],
        [
          0.02,
          0.9777
        ],
        [
          0.03,
          0.9777
        ],
        [
          0.04,
          0.9777
        ],
        [
          0.05,
          0.9777
        ],
        [
          0.0601,
          0.9777
        ],
        [
          0.0705,
          0.9773
        ],
        [
          0.0805,
          0.9773
        ],
        [
          0.0905,
          0.9773
        ],
        [
          0.1005,
          0.9768
        ],
        [
          0.1106,
          0.9768
        ],
        [
          0.121,
          0.9768
        ],
        [
          0.131,
          0.9768
        ],
        [
          0.141,
          0.9768
        ],
        [
          0.151,
          0.9763
        ],
        [
          0.1611,
          0.9759
        ],
        [
          0.1715,
          0.9759
        ],
        [
          0.1815,
          0.9745
        ],
        [
          0.1915,
          0.9741
        ],
        [
          0.2015,
          0.9736
        ],
        [
          0.2116,
          0.9732
        ],
        [
          0.222,
          0.9709
        ],
        [
          0.232,
          0.9686
        ],
        [
          0.242,
          0.9663
        ],
        [
          0.252,
          0.9636
        ],
        [
          0.2621,
          0.9613
        ],
        [
          0.2725,
          0.9581
        ],
        [
          0.2825,
          0.9554
        ],
        [
          0.2925,
          0.9545
        ],
        [
          0.3025,
          0.9522
        ],
        [
          0.3126,
          0.9477
        ],
        [
          0.323,
          0.944
        ],
        [
          0.333,
          0.9404
        ],
        [
          0.343,
          0.9386
        ],
        [
          0.353,
          0.9336
        ],
        [
          0.3631,
          0.929
        ],
        [
          0.3735,
          0.9254
        ],
        [
          0.3835,
          0.9186
        ],
        [
          0.3935,
          0.9154
        ],
        [
          0.4035,
          0.9099
        ],
        [
          0.4136,
          0.9081
        ],
        [
          0.424,
          0.904
        ],
        [
          0.434,
          0.899
        ],
        [
          0.444,
          0.8958
        ],
        [
          0.454,
          0.8931
        ],
        [
          0.4641,
          0.889
        ],
        [
          0.4745,
          0.8831
        ],
        [
          0.4845,
          0.8799
        ],
        [
          0.4945,
          0.874
        ],
        [
          0.5045,
          0.8699
        ],
        [
          0.5146,
          0.8649
        ],
        [
          0.525,
          0.8585
        ],
        [
          0.535,
          0.8535
        ],
        [
          0.545,
          0.848
        ],
        [
          0.5551,
          0.8444
        ],
        [
          0.5655,
          0.8376
        ],
        [
          0.5755,
          0.833
        ],
        [
          0.5855,
          0.828
        ],
        [
          0.5955,
          0.8212
        ],
        [
          0.6056,
          0.8153
        ],
        [
          0.616,
          0.8103
        ],
        [
          0.626,
          0.8039
        ],
        [
          0.636,
          0.7994
        ],
        [
          0.646,
          0.7934
        ],
        [
          0.6561,
          0.7884
        ],
        [
          0.6665,
          0.7825
        ],
        [
          0.6765,
          0.7757
        ],
        [
          0.6865,
          0.7707
        ],
        [
          0.6965,
          0.7662
        ],
        [
          0.7066,
          0.7593
        ],
        [
          0.717,
          0.7516
        ],
        [
          0.727,
          0.7448
        ],
        [
          0.737,
          0.7384
        ],
        [
          0.747,
          0.7298
        ],
        [
          0.7571,
          0.7216
        ],
        [
          0.7675,
          0.7147
        ],
        [
          0.7775,
          0.7075
        ],
        [
          0.7875,
          0.7006
        ],
        [
          0.7975,
          0.6947
        ],
        [
          0.8076,
          0.687
        ],
        [
          0.818,
          0.6793
        ],
        [
          0.828,
          0.6756
        ],
        [
          0.838,
          0.6715
        ],
        [
          0.848,
          0.6633
        ],
        [
          0.8581,
          0.6574
        ],
        [
          0.8685,
          0.6506
        ],
        [
          0.8785,
          0.6451
        ],
        [
          0.8885,
          0.6365
        ],
        [
          0.8985,
          0.6297
        ],
        [
          0.9086,
          0.6224
        ],
        [
          0.919,
          0.6146
        ],
        [
          0.929,
          0.6083
        ],
        [
          0.939,
          0.6024
        ],
        [
          0.949,
          0.5937
        ],
        [
          0.9591,
          0.586
        ],
        [
          0.9695,
          0.5778
        ],
        [
          0.9795,
          0.5705
        ],
        [
          0.9895,
          0.5628
        ],
        [
          0.9995,
          0.5551
        ],
        [
          1.0,
          0.5546
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.0920657465585808,
        "difference": -0.0008610234606109113,
        "worse": 0.4666666666666667,
        "kept_right": 0.09103800390176463,
        "kept_wrong": 0.0010277426568161116,
        "handed_off": 0.9079342534414188
      },
      "usd_per_1000": 1.472842023557841
    },
    {
      "decision": 128,
      "frontier": 48,
      "frontier_accuracy": 0.9758871701546861,
      "in_sample": {
        "threshold": 0.9911303026027772,
        "answered": 0.06187443130118289,
        "accuracy": 0.9758871701546861
      },
      "curve": [
        [
          0.0,
          0.9759
        ],
        [
          0.01,
          0.9759
        ],
        [
          0.02,
          0.9759
        ],
        [
          0.03,
          0.9759
        ],
        [
          0.04,
          0.9759
        ],
        [
          0.05,
          0.9759
        ],
        [
          0.0601,
          0.9759
        ],
        [
          0.0705,
          0.9754
        ],
        [
          0.0805,
          0.9754
        ],
        [
          0.0905,
          0.9754
        ],
        [
          0.1005,
          0.975
        ],
        [
          0.1106,
          0.975
        ],
        [
          0.121,
          0.975
        ],
        [
          0.131,
          0.975
        ],
        [
          0.141,
          0.975
        ],
        [
          0.151,
          0.9745
        ],
        [
          0.1611,
          0.9741
        ],
        [
          0.1715,
          0.9741
        ],
        [
          0.1815,
          0.9727
        ],
        [
          0.1915,
          0.9722
        ],
        [
          0.2015,
          0.9713
        ],
        [
          0.2116,
          0.9709
        ],
        [
          0.222,
          0.9686
        ],
        [
          0.232,
          0.9663
        ],
        [
          0.242,
          0.9645
        ],
        [
          0.252,
          0.9618
        ],
        [
          0.2621,
          0.9595
        ],
        [
          0.2725,
          0.9563
        ],
        [
          0.2825,
          0.954
        ],
        [
          0.2925,
          0.9531
        ],
        [
          0.3025,
          0.9509
        ],
        [
          0.3126,
          0.9463
        ],
        [
          0.323,
          0.9427
        ],
        [
          0.333,
          0.9386
        ],
        [
          0.343,
          0.9368
        ],
        [
          0.353,
          0.9318
        ],
        [
          0.3631,
          0.9272
        ],
        [
          0.3735,
          0.9236
        ],
        [
          0.3835,
          0.9167
        ],
        [
          0.3935,
          0.9136
        ],
        [
          0.4035,
          0.9086
        ],
        [
          0.4136,
          0.9063
        ],
        [
          0.424,
          0.9022
        ],
        [
          0.434,
          0.8972
        ],
        [
          0.444,
          0.894
        ],
        [
          0.454,
          0.8913
        ],
        [
          0.4641,
          0.8867
        ],
        [
          0.4745,
          0.8808
        ],
        [
          0.4845,
          0.8781
        ],
        [
          0.4945,
          0.8722
        ],
        [
          0.5045,
          0.8681
        ],
        [
          0.5146,
          0.8626
        ],
        [
          0.525,
          0.8562
        ],
        [
          0.535,
          0.8512
        ],
        [
          0.545,
          0.8453
        ],
        [
          0.5551,
          0.8412
        ],
        [
          0.5655,
          0.8344
        ],
        [
          0.5755,
          0.8298
        ],
        [
          0.5855,
          0.8248
        ],
        [
          0.5955,
          0.818
        ],
        [
          0.6056,
          0.8121
        ],
        [
          0.616,
          0.8071
        ],
        [
          0.626,
          0.8007
        ],
        [
          0.636,
          0.7966
        ],
        [
          0.646,
          0.7907
        ],
        [
          0.6561,
          0.7857
        ],
        [
          0.6665,
          0.7798
        ],
        [
          0.6765,
          0.773
        ],
        [
          0.6865,
          0.768
        ],
        [
          0.6965,
          0.7639
        ],
        [
          0.7066,
          0.7575
        ],
        [
          0.717,
          0.7498
        ],
        [
          0.727,
          0.7434
        ],
        [
          0.737,
          0.737
        ],
        [
          0.747,
          0.7284
        ],
        [
          0.7571,
          0.7202
        ],
        [
          0.7675,
          0.7134
        ],
        [
          0.7775,
          0.7061
        ],
        [
          0.7875,
          0.6993
        ],
        [
          0.7975,
          0.6938
        ],
        [
          0.8076,
          0.6865
        ],
        [
          0.818,
          0.6788
        ],
        [
          0.828,
          0.6752
        ],
        [
          0.838,
          0.6706
        ],
        [
          0.848,
          0.6624
        ],
        [
          0.8581,
          0.6565
        ],
        [
          0.8685,
          0.6497
        ],
        [
          0.8785,
          0.6442
        ],
        [
          0.8885,
          0.6356
        ],
        [
          0.8985,
          0.6288
        ],
        [
          0.9086,
          0.6219
        ],
        [
          0.919,
          0.6142
        ],
        [
          0.929,
          0.6074
        ],
        [
          0.939,
          0.6019
        ],
        [
          0.949,
          0.5933
        ],
        [
          0.9591,
          0.5855
        ],
        [
          0.9695,
          0.5773
        ],
        [
          0.9795,
          0.5701
        ],
        [
          0.9895,
          0.5623
        ],
        [
          0.9995,
          0.5551
        ],
        [
          1.0,
          0.5546
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.0920657465585808,
        "difference": -0.0008610234606109113,
        "worse": 0.4666666666666667,
        "kept_right": 0.09103800390176463,
        "kept_wrong": 0.0010277426568161116,
        "handed_off": 0.9079342534414188
      },
      "usd_per_1000": 4.883075829997419
    },
    {
      "decision": 128,
      "frontier": 191,
      "frontier_accuracy": 0.9740673339399454,
      "in_sample": {
        "threshold": 0.9911303026027772,
        "answered": 0.06187443130118289,
        "accuracy": 0.9740673339399454
      },
      "curve": [
        [
          0.0,
          0.9741
        ],
        [
          0.01,
          0.9741
        ],
        [
          0.02,
          0.9741
        ],
        [
          0.03,
          0.9741
        ],
        [
          0.04,
          0.9741
        ],
        [
          0.05,
          0.9741
        ],
        [
          0.0601,
          0.9741
        ],
        [
          0.0705,
          0.9736
        ],
        [
          0.0805,
          0.9736
        ],
        [
          0.0905,
          0.9736
        ],
        [
          0.1005,
          0.9732
        ],
        [
          0.1106,
          0.9732
        ],
        [
          0.121,
          0.9732
        ],
        [
          0.131,
          0.9732
        ],
        [
          0.141,
          0.9732
        ],
        [
          0.151,
          0.9727
        ],
        [
          0.1611,
          0.9722
        ],
        [
          0.1715,
          0.9722
        ],
        [
          0.1815,
          0.9709
        ],
        [
          0.1915,
          0.9704
        ],
        [
          0.2015,
          0.9695
        ],
        [
          0.2116,
          0.9691
        ],
        [
          0.222,
          0.9672
        ],
        [
          0.232,
          0.965
        ],
        [
          0.242,
          0.9631
        ],
        [
          0.252,
          0.9604
        ],
        [
          0.2621,
          0.9581
        ],
        [
          0.2725,
          0.955
        ],
        [
          0.2825,
          0.9522
        ],
        [
          0.2925,
          0.9513
        ],
        [
          0.3025,
          0.9495
        ],
        [
          0.3126,
          0.9449
        ],
        [
          0.323,
          0.9413
        ],
        [
          0.333,
          0.9372
        ],
        [
          0.343,
          0.9354
        ],
        [
          0.353,
          0.9304
        ],
        [
          0.3631,
          0.9263
        ],
        [
          0.3735,
          0.9227
        ],
        [
          0.3835,
          0.9158
        ],
        [
          0.3935,
          0.9131
        ],
        [
          0.4035,
          0.9081
        ],
        [
          0.4136,
          0.9058
        ],
        [
          0.424,
          0.9013
        ],
        [
          0.434,
          0.8963
        ],
        [
          0.444,
          0.8931
        ],
        [
          0.454,
          0.8904
        ],
        [
          0.4641,
          0.8858
        ],
        [
          0.4745,
          0.8803
        ],
        [
          0.4845,
          0.8772
        ],
        [
          0.4945,
          0.8712
        ],
        [
          0.5045,
          0.8672
        ],
        [
          0.5146,
          0.8621
        ],
        [
          0.525,
          0.8558
        ],
        [
          0.535,
          0.8508
        ],
        [
          0.545,
          0.8453
        ],
        [
          0.5551,
          0.8412
        ],
        [
          0.5655,
          0.8344
        ],
        [
          0.5755,
          0.8298
        ],
        [
          0.5855,
          0.8244
        ],
        [
          0.5955,
          0.8176
        ],
        [
          0.6056,
          0.8116
        ],
        [
          0.616,
          0.8066
        ],
        [
          0.626,
          0.7998
        ],
        [
          0.636,
          0.7957
        ],
        [
          0.646,
          0.7903
        ],
        [
          0.6561,
          0.7853
        ],
        [
          0.6665,
          0.7793
        ],
        [
          0.6765,
          0.7725
        ],
        [
          0.6865,
          0.7675
        ],
        [
          0.6965,
          0.7625
        ],
        [
          0.7066,
          0.7557
        ],
        [
          0.717,
          0.748
        ],
        [
          0.727,
          0.7416
        ],
        [
          0.737,
          0.7352
        ],
        [
          0.747,
          0.727
        ],
        [
          0.7571,
          0.7188
        ],
        [
          0.7675,
          0.712
        ],
        [
          0.7775,
          0.7047
        ],
        [
          0.7875,
          0.6984
        ],
        [
          0.7975,
          0.6924
        ],
        [
          0.8076,
          0.6847
        ],
        [
          0.818,
          0.677
        ],
        [
          0.828,
          0.6733
        ],
        [
          0.838,
          0.6697
        ],
        [
          0.848,
          0.6615
        ],
        [
          0.8581,
          0.6561
        ],
        [
          0.8685,
          0.6492
        ],
        [
          0.8785,
          0.6438
        ],
        [
          0.8885,
          0.6351
        ],
        [
          0.8985,
          0.6283
        ],
        [
          0.9086,
          0.6215
        ],
        [
          0.919,
          0.6137
        ],
        [
          0.929,
          0.6074
        ],
        [
          0.939,
          0.6015
        ],
        [
          0.949,
          0.5928
        ],
        [
          0.9591,
          0.5851
        ],
        [
          0.9695,
          0.5769
        ],
        [
          0.9795,
          0.5701
        ],
        [
          0.9895,
          0.5623
        ],
        [
          0.9995,
          0.5551
        ],
        [
          1.0,
          0.5546
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.0920657465585808,
        "difference": -0.0008610234606109113,
        "worse": 0.4666666666666667,
        "kept_right": 0.09103800390176463,
        "kept_wrong": 0.0010277426568161116,
        "handed_off": 0.9079342534414188
      },
      "usd_per_1000": 24.27607013581362
    },
    {
      "decision": 128,
      "frontier": 188,
      "frontier_accuracy": 0.9713375796178344,
      "in_sample": {
        "threshold": 0.9808269846664879,
        "answered": 0.14740673339399454,
        "accuracy": 0.9713375796178344
      },
      "curve": [
        [
          0.0,
          0.9713
        ],
        [
          0.01,
          0.9713
        ],
        [
          0.02,
          0.9713
        ],
        [
          0.03,
          0.9713
        ],
        [
          0.04,
          0.9713
        ],
        [
          0.05,
          0.9713
        ],
        [
          0.0601,
          0.9713
        ],
        [
          0.0705,
          0.9709
        ],
        [
          0.0805,
          0.9709
        ],
        [
          0.0905,
          0.9709
        ],
        [
          0.1005,
          0.9709
        ],
        [
          0.1106,
          0.9709
        ],
        [
          0.121,
          0.9709
        ],
        [
          0.131,
          0.9709
        ],
        [
          0.141,
          0.9709
        ],
        [
          0.151,
          0.9709
        ],
        [
          0.1611,
          0.9704
        ],
        [
          0.1715,
          0.9704
        ],
        [
          0.1815,
          0.9695
        ],
        [
          0.1915,
          0.9691
        ],
        [
          0.2015,
          0.9682
        ],
        [
          0.2116,
          0.9682
        ],
        [
          0.222,
          0.9659
        ],
        [
          0.232,
          0.9636
        ],
        [
          0.242,
          0.9613
        ],
        [
          0.252,
          0.9586
        ],
        [
          0.2621,
          0.9559
        ],
        [
          0.2725,
          0.9527
        ],
        [
          0.2825,
          0.95
        ],
        [
          0.2925,
          0.949
        ],
        [
          0.3025,
          0.9472
        ],
        [
          0.3126,
          0.9431
        ],
        [
          0.323,
          0.939
        ],
        [
          0.333,
          0.9354
        ],
        [
          0.343,
          0.934
        ],
        [
          0.353,
          0.9286
        ],
        [
          0.3631,
          0.924
        ],
        [
          0.3735,
          0.9204
        ],
        [
          0.3835,
          0.9136
        ],
        [
          0.3935,
          0.9104
        ],
        [
          0.4035,
          0.9054
        ],
        [
          0.4136,
          0.9031
        ],
        [
          0.424,
          0.8985
        ],
        [
          0.434,
          0.8935
        ],
        [
          0.444,
          0.8904
        ],
        [
          0.454,
          0.8876
        ],
        [
          0.4641,
          0.8835
        ],
        [
          0.4745,
          0.8776
        ],
        [
          0.4845,
          0.874
        ],
        [
          0.4945,
          0.8681
        ],
        [
          0.5045,
          0.864
        ],
        [
          0.5146,
          0.8585
        ],
        [
          0.525,
          0.8526
        ],
        [
          0.535,
          0.8476
        ],
        [
          0.545,
          0.8421
        ],
        [
          0.5551,
          0.8376
        ],
        [
          0.5655,
          0.8308
        ],
        [
          0.5755,
          0.8262
        ],
        [
          0.5855,
          0.8217
        ],
        [
          0.5955,
          0.8153
        ],
        [
          0.6056,
          0.8089
        ],
        [
          0.616,
          0.8039
        ],
        [
          0.626,
          0.798
        ],
        [
          0.636,
          0.7934
        ],
        [
          0.646,
          0.788
        ],
        [
          0.6561,
          0.7839
        ],
        [
          0.6665,
          0.7784
        ],
        [
          0.6765,
          0.7716
        ],
        [
          0.6865,
          0.7666
        ],
        [
          0.6965,
          0.7621
        ],
        [
          0.7066,
          0.7552
        ],
        [
          0.717,
          0.7475
        ],
        [
          0.727,
          0.7411
        ],
        [
          0.737,
          0.7352
        ],
        [
          0.747,
          0.7266
        ],
        [
          0.7571,
          0.7184
        ],
        [
          0.7675,
          0.7116
        ],
        [
          0.7775,
          0.7047
        ],
        [
          0.7875,
          0.6979
        ],
        [
          0.7975,
          0.692
        ],
        [
          0.8076,
          0.6843
        ],
        [
          0.818,
          0.6765
        ],
        [
          0.828,
          0.6733
        ],
        [
          0.838,
          0.6688
        ],
        [
          0.848,
          0.6611
        ],
        [
          0.8581,
          0.6551
        ],
        [
          0.8685,
          0.6488
        ],
        [
          0.8785,
          0.6433
        ],
        [
          0.8885,
          0.6347
        ],
        [
          0.8985,
          0.6278
        ],
        [
          0.9086,
          0.621
        ],
        [
          0.919,
          0.6133
        ],
        [
          0.929,
          0.6069
        ],
        [
          0.939,
          0.6019
        ],
        [
          0.949,
          0.5937
        ],
        [
          0.9591,
          0.5864
        ],
        [
          0.9695,
          0.5778
        ],
        [
          0.9795,
          0.5705
        ],
        [
          0.9895,
          0.5628
        ],
        [
          0.9995,
          0.5551
        ],
        [
          1.0,
          0.5546
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.13405050387096498,
        "difference": -0.0010793294890224195,
        "worse": 0.49,
        "kept_right": 0.13175852230316396,
        "kept_wrong": 0.002291981567801134,
        "handed_off": 0.865949496129035
      },
      "usd_per_1000": 0.834290236781079
    },
    {
      "decision": 128,
      "frontier": 190,
      "frontier_accuracy": 0.9672429481346679,
      "in_sample": {
        "threshold": 0.9911303026027772,
        "answered": 0.06187443130118289,
        "accuracy": 0.9672429481346679
      },
      "curve": [
        [
          0.0,
          0.9672
        ],
        [
          0.01,
          0.9672
        ],
        [
          0.02,
          0.9672
        ],
        [
          0.03,
          0.9672
        ],
        [
          0.04,
          0.9672
        ],
        [
          0.05,
          0.9672
        ],
        [
          0.0601,
          0.9672
        ],
        [
          0.0705,
          0.9668
        ],
        [
          0.0805,
          0.9668
        ],
        [
          0.0905,
          0.9668
        ],
        [
          0.1005,
          0.9663
        ],
        [
          0.1106,
          0.9663
        ],
        [
          0.121,
          0.9663
        ],
        [
          0.131,
          0.9663
        ],
        [
          0.141,
          0.9663
        ],
        [
          0.151,
          0.9659
        ],
        [
          0.1611,
          0.9654
        ],
        [
          0.1715,
          0.9654
        ],
        [
          0.1815,
          0.9645
        ],
        [
          0.1915,
          0.9641
        ],
        [
          0.2015,
          0.9631
        ],
        [
          0.2116,
          0.9627
        ],
        [
          0.222,
          0.9618
        ],
        [
          0.232,
          0.9595
        ],
        [
          0.242,
          0.9577
        ],
        [
          0.252,
          0.955
        ],
        [
          0.2621,
          0.9527
        ],
        [
          0.2725,
          0.9495
        ],
        [
          0.2825,
          0.9468
        ],
        [
          0.2925,
          0.9459
        ],
        [
          0.3025,
          0.944
        ],
        [
          0.3126,
          0.9395
        ],
        [
          0.323,
          0.9359
        ],
        [
          0.333,
          0.9322
        ],
        [
          0.343,
          0.9304
        ],
        [
          0.353,
          0.9258
        ],
        [
          0.3631,
          0.9213
        ],
        [
          0.3735,
          0.9177
        ],
        [
          0.3835,
          0.9108
        ],
        [
          0.3935,
          0.9081
        ],
        [
          0.4035,
          0.9026
        ],
        [
          0.4136,
          0.9008
        ],
        [
          0.424,
          0.8967
        ],
        [
          0.434,
          0.8917
        ],
        [
          0.444,
          0.8885
        ],
        [
          0.454,
          0.8863
        ],
        [
          0.4641,
          0.8822
        ],
        [
          0.4745,
          0.8767
        ],
        [
          0.4845,
          0.8735
        ],
        [
          0.4945,
          0.8676
        ],
        [
          0.5045,
          0.8635
        ],
        [
          0.5146,
          0.8585
        ],
        [
          0.525,
          0.8526
        ],
        [
          0.535,
          0.8471
        ],
        [
          0.545,
          0.8417
        ],
        [
          0.5551,
          0.838
        ],
        [
          0.5655,
          0.8312
        ],
        [
          0.5755,
          0.8267
        ],
        [
          0.5855,
          0.8212
        ],
        [
          0.5955,
          0.8144
        ],
        [
          0.6056,
          0.8094
        ],
        [
          0.616,
          0.8044
        ],
        [
          0.626,
          0.798
        ],
        [
          0.636,
          0.7934
        ],
        [
          0.646,
          0.7875
        ],
        [
          0.6561,
          0.7825
        ],
        [
          0.6665,
          0.7766
        ],
        [
          0.6765,
          0.7698
        ],
        [
          0.6865,
          0.7648
        ],
        [
          0.6965,
          0.7602
        ],
        [
          0.7066,
          0.7543
        ],
        [
          0.717,
          0.7466
        ],
        [
          0.727,
          0.7402
        ],
        [
          0.737,
          0.7338
        ],
        [
          0.747,
          0.7252
        ],
        [
          0.7571,
          0.717
        ],
        [
          0.7675,
          0.7102
        ],
        [
          0.7775,
          0.7029
        ],
        [
          0.7875,
          0.6961
        ],
        [
          0.7975,
          0.6902
        ],
        [
          0.8076,
          0.6829
        ],
        [
          0.818,
          0.6752
        ],
        [
          0.828,
          0.6715
        ],
        [
          0.838,
          0.6674
        ],
        [
          0.848,
          0.6592
        ],
        [
          0.8581,
          0.6538
        ],
        [
          0.8685,
          0.6474
        ],
        [
          0.8785,
          0.6419
        ],
        [
          0.8885,
          0.6333
        ],
        [
          0.8985,
          0.6274
        ],
        [
          0.9086,
          0.6206
        ],
        [
          0.919,
          0.6128
        ],
        [
          0.929,
          0.6065
        ],
        [
          0.939,
          0.601
        ],
        [
          0.949,
          0.5924
        ],
        [
          0.9591,
          0.5846
        ],
        [
          0.9695,
          0.5769
        ],
        [
          0.9795,
          0.5701
        ],
        [
          0.9895,
          0.5623
        ],
        [
          0.9995,
          0.5551
        ],
        [
          1.0,
          0.5546
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.09327227645141704,
        "difference": -0.0009337618077548226,
        "worse": 0.47,
        "kept_right": 0.09215058881586981,
        "kept_wrong": 0.0011216876355471595,
        "handed_off": 0.9067277235485826
      },
      "usd_per_1000": 5.049130572082944
    },
    {
      "decision": 128,
      "frontier": 124,
      "frontier_accuracy": 0.9667879890809827,
      "in_sample": {
        "threshold": 0.9911303026027772,
        "answered": 0.06187443130118289,
        "accuracy": 0.9667879890809827
      },
      "curve": [
        [
          0.0,
          0.9668
        ],
        [
          0.01,
          0.9668
        ],
        [
          0.02,
          0.9668
        ],
        [
          0.03,
          0.9668
        ],
        [
          0.04,
          0.9668
        ],
        [
          0.05,
          0.9668
        ],
        [
          0.0601,
          0.9668
        ],
        [
          0.0705,
          0.9663
        ],
        [
          0.0805,
          0.9663
        ],
        [
          0.0905,
          0.9663
        ],
        [
          0.1005,
          0.9663
        ],
        [
          0.1106,
          0.9663
        ],
        [
          0.121,
          0.9663
        ],
        [
          0.131,
          0.9663
        ],
        [
          0.141,
          0.9663
        ],
        [
          0.151,
          0.9659
        ],
        [
          0.1611,
          0.9654
        ],
        [
          0.1715,
          0.9654
        ],
        [
          0.1815,
          0.9641
        ],
        [
          0.1915,
          0.9636
        ],
        [
          0.2015,
          0.9631
        ],
        [
          0.2116,
          0.9627
        ],
        [
          0.222,
          0.9609
        ],
        [
          0.232,
          0.9586
        ],
        [
          0.242,
          0.9568
        ],
        [
          0.252,
          0.954
        ],
        [
          0.2621,
          0.9518
        ],
        [
          0.2725,
          0.9486
        ],
        [
          0.2825,
          0.9463
        ],
        [
          0.2925,
          0.9454
        ],
        [
          0.3025,
          0.9431
        ],
        [
          0.3126,
          0.9386
        ],
        [
          0.323,
          0.9349
        ],
        [
          0.333,
          0.9313
        ],
        [
          0.343,
          0.9295
        ],
        [
          0.353,
          0.9249
        ],
        [
          0.3631,
          0.9204
        ],
        [
          0.3735,
          0.9167
        ],
        [
          0.3835,
          0.9099
        ],
        [
          0.3935,
          0.9067
        ],
        [
          0.4035,
          0.9017
        ],
        [
          0.4136,
          0.8999
        ],
        [
          0.424,
          0.8958
        ],
        [
          0.434,
          0.8908
        ],
        [
          0.444,
          0.8876
        ],
        [
          0.454,
          0.8849
        ],
        [
          0.4641,
          0.8813
        ],
        [
          0.4745,
          0.8758
        ],
        [
          0.4845,
          0.8726
        ],
        [
          0.4945,
          0.8667
        ],
        [
          0.5045,
          0.8626
        ],
        [
          0.5146,
          0.8576
        ],
        [
          0.525,
          0.8512
        ],
        [
          0.535,
          0.8458
        ],
        [
          0.545,
          0.8399
        ],
        [
          0.5551,
          0.8358
        ],
        [
          0.5655,
          0.8289
        ],
        [
          0.5755,
          0.8248
        ],
        [
          0.5855,
          0.8198
        ],
        [
          0.5955,
          0.813
        ],
        [
          0.6056,
          0.8076
        ],
        [
          0.616,
          0.8025
        ],
        [
          0.626,
          0.7957
        ],
        [
          0.636,
          0.7916
        ],
        [
          0.646,
          0.7862
        ],
        [
          0.6561,
          0.7812
        ],
        [
          0.6665,
          0.7753
        ],
        [
          0.6765,
          0.7684
        ],
        [
          0.6865,
          0.7634
        ],
        [
          0.6965,
          0.7589
        ],
        [
          0.7066,
          0.752
        ],
        [
          0.717,
          0.7443
        ],
        [
          0.727,
          0.7379
        ],
        [
          0.737,
          0.7316
        ],
        [
          0.747,
          0.7238
        ],
        [
          0.7571,
          0.7157
        ],
        [
          0.7675,
          0.7088
        ],
        [
          0.7775,
          0.7015
        ],
        [
          0.7875,
          0.6947
        ],
        [
          0.7975,
          0.6897
        ],
        [
          0.8076,
          0.6824
        ],
        [
          0.818,
          0.6747
        ],
        [
          0.828,
          0.6724
        ],
        [
          0.838,
          0.6683
        ],
        [
          0.848,
          0.6601
        ],
        [
          0.8581,
          0.6547
        ],
        [
          0.8685,
          0.6479
        ],
        [
          0.8785,
          0.6424
        ],
        [
          0.8885,
          0.6338
        ],
        [
          0.8985,
          0.6274
        ],
        [
          0.9086,
          0.6201
        ],
        [
          0.919,
          0.6128
        ],
        [
          0.929,
          0.6065
        ],
        [
          0.939,
          0.601
        ],
        [
          0.949,
          0.5924
        ],
        [
          0.9591,
          0.5851
        ],
        [
          0.9695,
          0.5773
        ],
        [
          0.9795,
          0.5701
        ],
        [
          0.9895,
          0.5623
        ],
        [
          0.9995,
          0.5551
        ],
        [
          1.0,
          0.5546
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.10670242781966555,
        "difference": -0.0008609545121068626,
        "worse": 0.47,
        "kept_right": 0.10528365700047435,
        "kept_wrong": 0.0014187708191911994,
        "handed_off": 0.893297572180334
      },
      "usd_per_1000": 1.0097595158946187
    },
    {
      "decision": 128,
      "frontier": 278,
      "frontier_accuracy": 0.9499545040946314,
      "in_sample": {
        "threshold": 0.9911303026027772,
        "answered": 0.06187443130118289,
        "accuracy": 0.9499545040946314
      },
      "curve": [
        [
          0.0,
          0.95
        ],
        [
          0.01,
          0.95
        ],
        [
          0.02,
          0.95
        ],
        [
          0.03,
          0.95
        ],
        [
          0.04,
          0.95
        ],
        [
          0.05,
          0.95
        ],
        [
          0.0601,
          0.95
        ],
        [
          0.0705,
          0.9495
        ],
        [
          0.0805,
          0.9495
        ],
        [
          0.0905,
          0.9495
        ],
        [
          0.1005,
          0.9495
        ],
        [
          0.1106,
          0.9495
        ],
        [
          0.121,
          0.9495
        ],
        [
          0.131,
          0.9495
        ],
        [
          0.141,
          0.9495
        ],
        [
          0.151,
          0.949
        ],
        [
          0.1611,
          0.9486
        ],
        [
          0.1715,
          0.9486
        ],
        [
          0.1815,
          0.9472
        ],
        [
          0.1915,
          0.9468
        ],
        [
          0.2015,
          0.9459
        ],
        [
          0.2116,
          0.9454
        ],
        [
          0.222,
          0.9436
        ],
        [
          0.232,
          0.9418
        ],
        [
          0.242,
          0.9399
        ],
        [
          0.252,
          0.9372
        ],
        [
          0.2621,
          0.9349
        ],
        [
          0.2725,
          0.9322
        ],
        [
          0.2825,
          0.9304
        ],
        [
          0.2925,
          0.9295
        ],
        [
          0.3025,
          0.9272
        ],
        [
          0.3126,
          0.9231
        ],
        [
          0.323,
          0.9199
        ],
        [
          0.333,
          0.9167
        ],
        [
          0.343,
          0.9154
        ],
        [
          0.353,
          0.9099
        ],
        [
          0.3631,
          0.9058
        ],
        [
          0.3735,
          0.9022
        ],
        [
          0.3835,
          0.8958
        ],
        [
          0.3935,
          0.8926
        ],
        [
          0.4035,
          0.8885
        ],
        [
          0.4136,
          0.8863
        ],
        [
          0.424,
          0.8822
        ],
        [
          0.434,
          0.8772
        ],
        [
          0.444,
          0.8749
        ],
        [
          0.454,
          0.8722
        ],
        [
          0.4641,
          0.8694
        ],
        [
          0.4745,
          0.8644
        ],
        [
          0.4845,
          0.8612
        ],
        [
          0.4945,
          0.8558
        ],
        [
          0.5045,
          0.853
        ],
        [
          0.5146,
          0.848
        ],
        [
          0.525,
          0.8417
        ],
        [
          0.535,
          0.838
        ],
        [
          0.545,
          0.833
        ],
        [
          0.5551,
          0.8289
        ],
        [
          0.5655,
          0.8226
        ],
        [
          0.5755,
          0.8185
        ],
        [
          0.5855,
          0.8135
        ],
        [
          0.5955,
          0.8066
        ],
        [
          0.6056,
          0.8007
        ],
        [
          0.616,
          0.7962
        ],
        [
          0.626,
          0.7903
        ],
        [
          0.636,
          0.7866
        ],
        [
          0.646,
          0.7812
        ],
        [
          0.6561,
          0.7762
        ],
        [
          0.6665,
          0.7707
        ],
        [
          0.6765,
          0.7639
        ],
        [
          0.6865,
          0.7584
        ],
        [
          0.6965,
          0.7552
        ],
        [
          0.7066,
          0.7489
        ],
        [
          0.717,
          0.7411
        ],
        [
          0.727,
          0.7352
        ],
        [
          0.737,
          0.7288
        ],
        [
          0.747,
          0.7207
        ],
        [
          0.7571,
          0.7129
        ],
        [
          0.7675,
          0.7061
        ],
        [
          0.7775,
          0.6988
        ],
        [
          0.7875,
          0.6929
        ],
        [
          0.7975,
          0.6879
        ],
        [
          0.8076,
          0.6802
        ],
        [
          0.818,
          0.6724
        ],
        [
          0.828,
          0.6697
        ],
        [
          0.838,
          0.6652
        ],
        [
          0.848,
          0.6574
        ],
        [
          0.8581,
          0.6515
        ],
        [
          0.8685,
          0.6456
        ],
        [
          0.8785,
          0.6401
        ],
        [
          0.8885,
          0.6324
        ],
        [
          0.8985,
          0.6265
        ],
        [
          0.9086,
          0.6197
        ],
        [
          0.919,
          0.6124
        ],
        [
          0.929,
          0.6056
        ],
        [
          0.939,
          0.6001
        ],
        [
          0.949,
          0.5919
        ],
        [
          0.9591,
          0.5842
        ],
        [
          0.9695,
          0.5769
        ],
        [
          0.9795,
          0.5696
        ],
        [
          0.9895,
          0.5619
        ],
        [
          0.9995,
          0.5551
        ],
        [
          1.0,
          0.5546
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.10618774419871489,
        "difference": -0.0008337065557035924,
        "worse": 0.4666666666666667,
        "kept_right": 0.1048053039880614,
        "kept_wrong": 0.0013824402106535065,
        "handed_off": 0.8938122558012848
      },
      "usd_per_1000": 5.817668714812536
    },
    {
      "decision": 128,
      "frontier": 275,
      "frontier_accuracy": 0.9426751592356688,
      "in_sample": {
        "threshold": 0.9911303026027772,
        "answered": 0.06187443130118289,
        "accuracy": 0.9426751592356688
      },
      "curve": [
        [
          0.0,
          0.9427
        ],
        [
          0.01,
          0.9427
        ],
        [
          0.02,
          0.9427
        ],
        [
          0.03,
          0.9427
        ],
        [
          0.04,
          0.9427
        ],
        [
          0.05,
          0.9427
        ],
        [
          0.0601,
          0.9427
        ],
        [
          0.0705,
          0.9422
        ],
        [
          0.0805,
          0.9422
        ],
        [
          0.0905,
          0.9422
        ],
        [
          0.1005,
          0.9422
        ],
        [
          0.1106,
          0.9422
        ],
        [
          0.121,
          0.9422
        ],
        [
          0.131,
          0.9422
        ],
        [
          0.141,
          0.9422
        ],
        [
          0.151,
          0.9418
        ],
        [
          0.1611,
          0.9413
        ],
        [
          0.1715,
          0.9413
        ],
        [
          0.1815,
          0.9399
        ],
        [
          0.1915,
          0.9395
        ],
        [
          0.2015,
          0.9386
        ],
        [
          0.2116,
          0.9381
        ],
        [
          0.222,
          0.9363
        ],
        [
          0.232,
          0.934
        ],
        [
          0.242,
          0.9322
        ],
        [
          0.252,
          0.9295
        ],
        [
          0.2621,
          0.9272
        ],
        [
          0.2725,
          0.924
        ],
        [
          0.2825,
          0.9217
        ],
        [
          0.2925,
          0.9208
        ],
        [
          0.3025,
          0.9186
        ],
        [
          0.3126,
          0.9145
        ],
        [
          0.323,
          0.9113
        ],
        [
          0.333,
          0.9081
        ],
        [
          0.343,
          0.9063
        ],
        [
          0.353,
          0.9013
        ],
        [
          0.3631,
          0.8967
        ],
        [
          0.3735,
          0.8931
        ],
        [
          0.3835,
          0.8863
        ],
        [
          0.3935,
          0.884
        ],
        [
          0.4035,
          0.879
        ],
        [
          0.4136,
          0.8767
        ],
        [
          0.424,
          0.8731
        ],
        [
          0.434,
          0.8681
        ],
        [
          0.444,
          0.8662
        ],
        [
          0.454,
          0.8631
        ],
        [
          0.4641,
          0.8599
        ],
        [
          0.4745,
          0.8549
        ],
        [
          0.4845,
          0.8526
        ],
        [
          0.4945,
          0.8467
        ],
        [
          0.5045,
          0.8435
        ],
        [
          0.5146,
          0.8385
        ],
        [
          0.525,
          0.8326
        ],
        [
          0.535,
          0.8285
        ],
        [
          0.545,
          0.8226
        ],
        [
          0.5551,
          0.8194
        ],
        [
          0.5655,
          0.8126
        ],
        [
          0.5755,
          0.808
        ],
        [
          0.5855,
          0.803
        ],
        [
          0.5955,
          0.7962
        ],
        [
          0.6056,
          0.7912
        ],
        [
          0.616,
          0.7866
        ],
        [
          0.626,
          0.7807
        ],
        [
          0.636,
          0.7762
        ],
        [
          0.646,
          0.7712
        ],
        [
          0.6561,
          0.7662
        ],
        [
          0.6665,
          0.7611
        ],
        [
          0.6765,
          0.7543
        ],
        [
          0.6865,
          0.7493
        ],
        [
          0.6965,
          0.7452
        ],
        [
          0.7066,
          0.7384
        ],
        [
          0.717,
          0.7307
        ],
        [
          0.727,
          0.7252
        ],
        [
          0.737,
          0.7197
        ],
        [
          0.747,
          0.7125
        ],
        [
          0.7571,
          0.7047
        ],
        [
          0.7675,
          0.6979
        ],
        [
          0.7775,
          0.6915
        ],
        [
          0.7875,
          0.6852
        ],
        [
          0.7975,
          0.6806
        ],
        [
          0.8076,
          0.6729
        ],
        [
          0.818,
          0.6652
        ],
        [
          0.828,
          0.662
        ],
        [
          0.838,
          0.6583
        ],
        [
          0.848,
          0.6501
        ],
        [
          0.8581,
          0.6447
        ],
        [
          0.8685,
          0.6397
        ],
        [
          0.8785,
          0.6342
        ],
        [
          0.8885,
          0.626
        ],
        [
          0.8985,
          0.6201
        ],
        [
          0.9086,
          0.6137
        ],
        [
          0.919,
          0.6069
        ],
        [
          0.929,
          0.601
        ],
        [
          0.939,
          0.5965
        ],
        [
          0.949,
          0.5887
        ],
        [
          0.9591,
          0.581
        ],
        [
          0.9695,
          0.5737
        ],
        [
          0.9795,
          0.5678
        ],
        [
          0.9895,
          0.5605
        ],
        [
          0.9995,
          0.5551
        ],
        [
          1.0,
          0.5546
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.10618774419871489,
        "difference": -0.0008337065557035924,
        "worse": 0.4666666666666667,
        "kept_right": 0.1048053039880614,
        "kept_wrong": 0.0013824402106535065,
        "handed_off": 0.8938122558012848
      },
      "usd_per_1000": 0.45987819840112526
    },
    {
      "decision": 128,
      "frontier": 192,
      "frontier_accuracy": 0.9331210191082803,
      "in_sample": {
        "threshold": 0.9911303026027772,
        "answered": 0.06187443130118289,
        "accuracy": 0.9331210191082803
      },
      "curve": [
        [
          0.0,
          0.9331
        ],
        [
          0.01,
          0.9331
        ],
        [
          0.02,
          0.9331
        ],
        [
          0.03,
          0.9331
        ],
        [
          0.04,
          0.9331
        ],
        [
          0.05,
          0.9331
        ],
        [
          0.0601,
          0.9331
        ],
        [
          0.0705,
          0.9327
        ],
        [
          0.0805,
          0.9327
        ],
        [
          0.0905,
          0.9327
        ],
        [
          0.1005,
          0.9322
        ],
        [
          0.1106,
          0.9322
        ],
        [
          0.121,
          0.9322
        ],
        [
          0.131,
          0.9322
        ],
        [
          0.141,
          0.9322
        ],
        [
          0.151,
          0.9318
        ],
        [
          0.1611,
          0.9313
        ],
        [
          0.1715,
          0.9313
        ],
        [
          0.1815,
          0.9304
        ],
        [
          0.1915,
          0.9299
        ],
        [
          0.2015,
          0.929
        ],
        [
          0.2116,
          0.9286
        ],
        [
          0.222,
          0.9281
        ],
        [
          0.232,
          0.9268
        ],
        [
          0.242,
          0.9249
        ],
        [
          0.252,
          0.9222
        ],
        [
          0.2621,
          0.9195
        ],
        [
          0.2725,
          0.9172
        ],
        [
          0.2825,
          0.9149
        ],
        [
          0.2925,
          0.914
        ],
        [
          0.3025,
          0.9117
        ],
        [
          0.3126,
          0.9076
        ],
        [
          0.323,
          0.9045
        ],
        [
          0.333,
          0.9022
        ],
        [
          0.343,
          0.9008
        ],
        [
          0.353,
          0.8967
        ],
        [
          0.3631,
          0.894
        ],
        [
          0.3735,
          0.8904
        ],
        [
          0.3835,
          0.884
        ],
        [
          0.3935,
          0.8817
        ],
        [
          0.4035,
          0.8767
        ],
        [
          0.4136,
          0.8749
        ],
        [
          0.424,
          0.8708
        ],
        [
          0.434,
          0.8658
        ],
        [
          0.444,
          0.8631
        ],
        [
          0.454,
          0.8608
        ],
        [
          0.4641,
          0.8581
        ],
        [
          0.4745,
          0.853
        ],
        [
          0.4845,
          0.8503
        ],
        [
          0.4945,
          0.8444
        ],
        [
          0.5045,
          0.8408
        ],
        [
          0.5146,
          0.8358
        ],
        [
          0.525,
          0.8303
        ],
        [
          0.535,
          0.8258
        ],
        [
          0.545,
          0.8207
        ],
        [
          0.5551,
          0.8176
        ],
        [
          0.5655,
          0.8112
        ],
        [
          0.5755,
          0.8071
        ],
        [
          0.5855,
          0.8016
        ],
        [
          0.5955,
          0.7953
        ],
        [
          0.6056,
          0.7894
        ],
        [
          0.616,
          0.7843
        ],
        [
          0.626,
          0.7793
        ],
        [
          0.636,
          0.7771
        ],
        [
          0.646,
          0.7721
        ],
        [
          0.6561,
          0.7675
        ],
        [
          0.6665,
          0.7621
        ],
        [
          0.6765,
          0.7561
        ],
        [
          0.6865,
          0.7511
        ],
        [
          0.6965,
          0.7461
        ],
        [
          0.7066,
          0.7407
        ],
        [
          0.717,
          0.7329
        ],
        [
          0.727,
          0.7266
        ],
        [
          0.737,
          0.7207
        ],
        [
          0.747,
          0.7129
        ],
        [
          0.7571,
          0.7047
        ],
        [
          0.7675,
          0.6979
        ],
        [
          0.7775,
          0.6906
        ],
        [
          0.7875,
          0.6847
        ],
        [
          0.7975,
          0.6797
        ],
        [
          0.8076,
          0.6724
        ],
        [
          0.818,
          0.6656
        ],
        [
          0.828,
          0.6624
        ],
        [
          0.838,
          0.6588
        ],
        [
          0.848,
          0.651
        ],
        [
          0.8581,
          0.6456
        ],
        [
          0.8685,
          0.6406
        ],
        [
          0.8785,
          0.6356
        ],
        [
          0.8885,
          0.6278
        ],
        [
          0.8985,
          0.6228
        ],
        [
          0.9086,
          0.6165
        ],
        [
          0.919,
          0.6092
        ],
        [
          0.929,
          0.6033
        ],
        [
          0.939,
          0.5983
        ],
        [
          0.949,
          0.5905
        ],
        [
          0.9591,
          0.5837
        ],
        [
          0.9695,
          0.5764
        ],
        [
          0.9795,
          0.5692
        ],
        [
          0.9895,
          0.5619
        ],
        [
          0.9995,
          0.5551
        ],
        [
          1.0,
          0.5546
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.09253267035895453,
        "difference": -0.0008943926944088663,
        "worse": 0.4666666666666667,
        "kept_right": 0.09142608179404639,
        "kept_wrong": 0.0011065885649080645,
        "handed_off": 0.907467329641045
      },
      "usd_per_1000": 10.535301250232292
    },
    {
      "decision": 128,
      "frontier": 123,
      "frontier_accuracy": 0.9326660600545951,
      "in_sample": {
        "threshold": 0.9911303026027772,
        "answered": 0.06187443130118289,
        "accuracy": 0.9326660600545951
      },
      "curve": [
        [
          0.0,
          0.9327
        ],
        [
          0.01,
          0.9327
        ],
        [
          0.02,
          0.9327
        ],
        [
          0.03,
          0.9327
        ],
        [
          0.04,
          0.9327
        ],
        [
          0.05,
          0.9327
        ],
        [
          0.0601,
          0.9327
        ],
        [
          0.0705,
          0.9322
        ],
        [
          0.0805,
          0.9322
        ],
        [
          0.0905,
          0.9322
        ],
        [
          0.1005,
          0.9322
        ],
        [
          0.1106,
          0.9322
        ],
        [
          0.121,
          0.9322
        ],
        [
          0.131,
          0.9322
        ],
        [
          0.141,
          0.9322
        ],
        [
          0.151,
          0.9318
        ],
        [
          0.1611,
          0.9313
        ],
        [
          0.1715,
          0.9313
        ],
        [
          0.1815,
          0.9304
        ],
        [
          0.1915,
          0.9295
        ],
        [
          0.2015,
          0.929
        ],
        [
          0.2116,
          0.9286
        ],
        [
          0.222,
          0.9268
        ],
        [
          0.232,
          0.9249
        ],
        [
          0.242,
          0.9227
        ],
        [
          0.252,
          0.9204
        ],
        [
          0.2621,
          0.9181
        ],
        [
          0.2725,
          0.9149
        ],
        [
          0.2825,
          0.9131
        ],
        [
          0.2925,
          0.9122
        ],
        [
          0.3025,
          0.9108
        ],
        [
          0.3126,
          0.9067
        ],
        [
          0.323,
          0.9035
        ],
        [
          0.333,
          0.8995
        ],
        [
          0.343,
          0.8981
        ],
        [
          0.353,
          0.8926
        ],
        [
          0.3631,
          0.8881
        ],
        [
          0.3735,
          0.8844
        ],
        [
          0.3835,
          0.8781
        ],
        [
          0.3935,
          0.8758
        ],
        [
          0.4035,
          0.8712
        ],
        [
          0.4136,
          0.8694
        ],
        [
          0.424,
          0.8653
        ],
        [
          0.434,
          0.8603
        ],
        [
          0.444,
          0.8571
        ],
        [
          0.454,
          0.854
        ],
        [
          0.4641,
          0.8503
        ],
        [
          0.4745,
          0.8449
        ],
        [
          0.4845,
          0.8426
        ],
        [
          0.4945,
          0.8371
        ],
        [
          0.5045,
          0.8335
        ],
        [
          0.5146,
          0.828
        ],
        [
          0.525,
          0.823
        ],
        [
          0.535,
          0.8189
        ],
        [
          0.545,
          0.8139
        ],
        [
          0.5551,
          0.8098
        ],
        [
          0.5655,
          0.8035
        ],
        [
          0.5755,
          0.8003
        ],
        [
          0.5855,
          0.7948
        ],
        [
          0.5955,
          0.7884
        ],
        [
          0.6056,
          0.7834
        ],
        [
          0.616,
          0.7789
        ],
        [
          0.626,
          0.773
        ],
        [
          0.636,
          0.7689
        ],
        [
          0.646,
          0.7634
        ],
        [
          0.6561,
          0.7589
        ],
        [
          0.6665,
          0.7539
        ],
        [
          0.6765,
          0.7475
        ],
        [
          0.6865,
          0.7425
        ],
        [
          0.6965,
          0.7389
        ],
        [
          0.7066,
          0.7325
        ],
        [
          0.717,
          0.7252
        ],
        [
          0.727,
          0.7188
        ],
        [
          0.737,
          0.7138
        ],
        [
          0.747,
          0.7061
        ],
        [
          0.7571,
          0.6993
        ],
        [
          0.7675,
          0.6929
        ],
        [
          0.7775,
          0.6861
        ],
        [
          0.7875,
          0.6802
        ],
        [
          0.7975,
          0.6747
        ],
        [
          0.8076,
          0.6674
        ],
        [
          0.818,
          0.6606
        ],
        [
          0.828,
          0.6579
        ],
        [
          0.838,
          0.6556
        ],
        [
          0.848,
          0.6483
        ],
        [
          0.8581,
          0.6429
        ],
        [
          0.8685,
          0.6388
        ],
        [
          0.8785,
          0.6347
        ],
        [
          0.8885,
          0.626
        ],
        [
          0.8985,
          0.6201
        ],
        [
          0.9086,
          0.6133
        ],
        [
          0.919,
          0.606
        ],
        [
          0.929,
          0.6005
        ],
        [
          0.939,
          0.5955
        ],
        [
          0.949,
          0.5878
        ],
        [
          0.9591,
          0.5805
        ],
        [
          0.9695,
          0.5755
        ],
        [
          0.9795,
          0.5692
        ],
        [
          0.9895,
          0.5619
        ],
        [
          0.9995,
          0.5551
        ],
        [
          1.0,
          0.5546
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1063241935280494,
        "difference": -0.0008215853335784767,
        "worse": 0.4666666666666667,
        "kept_right": 0.10492660454454567,
        "kept_wrong": 0.0013975889835037318,
        "handed_off": 0.8936758064719504
      },
      "usd_per_1000": 0.24002696913612373
    },
    {
      "decision": 128,
      "frontier": 274,
      "frontier_accuracy": 0.9085532302092811,
      "in_sample": {
        "threshold": 0.9911303026027772,
        "answered": 0.06187443130118289,
        "accuracy": 0.9085532302092811
      },
      "curve": [
        [
          0.0,
          0.9086
        ],
        [
          0.01,
          0.9086
        ],
        [
          0.02,
          0.9086
        ],
        [
          0.03,
          0.9086
        ],
        [
          0.04,
          0.9086
        ],
        [
          0.05,
          0.9086
        ],
        [
          0.0601,
          0.9086
        ],
        [
          0.0705,
          0.9081
        ],
        [
          0.0805,
          0.9081
        ],
        [
          0.0905,
          0.9081
        ],
        [
          0.1005,
          0.9081
        ],
        [
          0.1106,
          0.9081
        ],
        [
          0.121,
          0.9081
        ],
        [
          0.131,
          0.9081
        ],
        [
          0.141,
          0.9081
        ],
        [
          0.151,
          0.9076
        ],
        [
          0.1611,
          0.9072
        ],
        [
          0.1715,
          0.9076
        ],
        [
          0.1815,
          0.9063
        ],
        [
          0.1915,
          0.9058
        ],
        [
          0.2015,
          0.9049
        ],
        [
          0.2116,
          0.9049
        ],
        [
          0.222,
          0.9031
        ],
        [
          0.232,
          0.9017
        ],
        [
          0.242,
          0.9004
        ],
        [
          0.252,
          0.8981
        ],
        [
          0.2621,
          0.8963
        ],
        [
          0.2725,
          0.8935
        ],
        [
          0.2825,
          0.8913
        ],
        [
          0.2925,
          0.8908
        ],
        [
          0.3025,
          0.8885
        ],
        [
          0.3126,
          0.8849
        ],
        [
          0.323,
          0.8822
        ],
        [
          0.333,
          0.8799
        ],
        [
          0.343,
          0.8785
        ],
        [
          0.353,
          0.8735
        ],
        [
          0.3631,
          0.8694
        ],
        [
          0.3735,
          0.8662
        ],
        [
          0.3835,
          0.8599
        ],
        [
          0.3935,
          0.8581
        ],
        [
          0.4035,
          0.854
        ],
        [
          0.4136,
          0.8521
        ],
        [
          0.424,
          0.848
        ],
        [
          0.434,
          0.843
        ],
        [
          0.444,
          0.8408
        ],
        [
          0.454,
          0.8376
        ],
        [
          0.4641,
          0.8353
        ],
        [
          0.4745,
          0.8303
        ],
        [
          0.4845,
          0.8289
        ],
        [
          0.4945,
          0.8235
        ],
        [
          0.5045,
          0.8203
        ],
        [
          0.5146,
          0.8157
        ],
        [
          0.525,
          0.8098
        ],
        [
          0.535,
          0.8057
        ],
        [
          0.545,
          0.8012
        ],
        [
          0.5551,
          0.7975
        ],
        [
          0.5655,
          0.7912
        ],
        [
          0.5755,
          0.7871
        ],
        [
          0.5855,
          0.7821
        ],
        [
          0.5955,
          0.7757
        ],
        [
          0.6056,
          0.7707
        ],
        [
          0.616,
          0.7662
        ],
        [
          0.626,
          0.7607
        ],
        [
          0.636,
          0.7571
        ],
        [
          0.646,
          0.7525
        ],
        [
          0.6561,
          0.7484
        ],
        [
          0.6665,
          0.7429
        ],
        [
          0.6765,
          0.737
        ],
        [
          0.6865,
          0.732
        ],
        [
          0.6965,
          0.7279
        ],
        [
          0.7066,
          0.722
        ],
        [
          0.717,
          0.7147
        ],
        [
          0.727,
          0.7097
        ],
        [
          0.737,
          0.7052
        ],
        [
          0.747,
          0.6979
        ],
        [
          0.7571,
          0.6902
        ],
        [
          0.7675,
          0.6843
        ],
        [
          0.7775,
          0.6779
        ],
        [
          0.7875,
          0.6729
        ],
        [
          0.7975,
          0.6688
        ],
        [
          0.8076,
          0.6615
        ],
        [
          0.818,
          0.6547
        ],
        [
          0.828,
          0.6538
        ],
        [
          0.838,
          0.6506
        ],
        [
          0.848,
          0.6433
        ],
        [
          0.8581,
          0.6374
        ],
        [
          0.8685,
          0.6324
        ],
        [
          0.8785,
          0.6274
        ],
        [
          0.8885,
          0.6201
        ],
        [
          0.8985,
          0.6156
        ],
        [
          0.9086,
          0.6092
        ],
        [
          0.919,
          0.6028
        ],
        [
          0.929,
          0.5974
        ],
        [
          0.939,
          0.5924
        ],
        [
          0.949,
          0.5846
        ],
        [
          0.9591,
          0.5783
        ],
        [
          0.9695,
          0.5728
        ],
        [
          0.9795,
          0.5669
        ],
        [
          0.9895,
          0.5605
        ],
        [
          0.9995,
          0.5551
        ],
        [
          1.0,
          0.5546
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.11810525538375967,
        "difference": -0.0011913486895201533,
        "worse": 0.5366666666666666,
        "kept_right": 0.11616816180678373,
        "kept_wrong": 0.0019370935769759748,
        "handed_off": 0.8818947446162396
      },
      "usd_per_1000": 0.37590382324568705
    },
    {
      "decision": 128,
      "frontier": 47,
      "frontier_accuracy": 0.9044585987261147,
      "in_sample": {
        "threshold": 0.9792831161462286,
        "answered": 0.1583257506824386,
        "accuracy": 0.9044585987261147
      },
      "curve": [
        [
          0.0,
          0.9045
        ],
        [
          0.01,
          0.9045
        ],
        [
          0.02,
          0.9045
        ],
        [
          0.03,
          0.9045
        ],
        [
          0.04,
          0.9045
        ],
        [
          0.05,
          0.9045
        ],
        [
          0.0601,
          0.9045
        ],
        [
          0.0705,
          0.904
        ],
        [
          0.0805,
          0.904
        ],
        [
          0.0905,
          0.904
        ],
        [
          0.1005,
          0.904
        ],
        [
          0.1106,
          0.904
        ],
        [
          0.121,
          0.9045
        ],
        [
          0.131,
          0.9045
        ],
        [
          0.141,
          0.9049
        ],
        [
          0.151,
          0.9045
        ],
        [
          0.1611,
          0.904
        ],
        [
          0.1715,
          0.904
        ],
        [
          0.1815,
          0.9031
        ],
        [
          0.1915,
          0.9022
        ],
        [
          0.2015,
          0.9017
        ],
        [
          0.2116,
          0.9013
        ],
        [
          0.222,
          0.8995
        ],
        [
          0.232,
          0.8981
        ],
        [
          0.242,
          0.8967
        ],
        [
          0.252,
          0.894
        ],
        [
          0.2621,
          0.8917
        ],
        [
          0.2725,
          0.8899
        ],
        [
          0.2825,
          0.8876
        ],
        [
          0.2925,
          0.8872
        ],
        [
          0.3025,
          0.8854
        ],
        [
          0.3126,
          0.8808
        ],
        [
          0.323,
          0.8781
        ],
        [
          0.333,
          0.8758
        ],
        [
          0.343,
          0.8744
        ],
        [
          0.353,
          0.8699
        ],
        [
          0.3631,
          0.8662
        ],
        [
          0.3735,
          0.8626
        ],
        [
          0.3835,
          0.8553
        ],
        [
          0.3935,
          0.8535
        ],
        [
          0.4035,
          0.849
        ],
        [
          0.4136,
          0.8467
        ],
        [
          0.424,
          0.843
        ],
        [
          0.434,
          0.8385
        ],
        [
          0.444,
          0.8362
        ],
        [
          0.454,
          0.8348
        ],
        [
          0.4641,
          0.8321
        ],
        [
          0.4745,
          0.8276
        ],
        [
          0.4845,
          0.8258
        ],
        [
          0.4945,
          0.8207
        ],
        [
          0.5045,
          0.8176
        ],
        [
          0.5146,
          0.8126
        ],
        [
          0.525,
          0.8066
        ],
        [
          0.535,
          0.8021
        ],
        [
          0.545,
          0.7966
        ],
        [
          0.5551,
          0.793
        ],
        [
          0.5655,
          0.7871
        ],
        [
          0.5755,
          0.7825
        ],
        [
          0.5855,
          0.7775
        ],
        [
          0.5955,
          0.7712
        ],
        [
          0.6056,
          0.7657
        ],
        [
          0.616,
          0.7611
        ],
        [
          0.626,
          0.7557
        ],
        [
          0.636,
          0.7511
        ],
        [
          0.646,
          0.747
        ],
        [
          0.6561,
          0.7434
        ],
        [
          0.6665,
          0.7384
        ],
        [
          0.6765,
          0.732
        ],
        [
          0.6865,
          0.7279
        ],
        [
          0.6965,
          0.7247
        ],
        [
          0.7066,
          0.7197
        ],
        [
          0.717,
          0.7129
        ],
        [
          0.727,
          0.7075
        ],
        [
          0.737,
          0.702
        ],
        [
          0.747,
          0.6952
        ],
        [
          0.7571,
          0.6884
        ],
        [
          0.7675,
          0.6824
        ],
        [
          0.7775,
          0.6756
        ],
        [
          0.7875,
          0.6697
        ],
        [
          0.7975,
          0.6656
        ],
        [
          0.8076,
          0.6588
        ],
        [
          0.818,
          0.652
        ],
        [
          0.828,
          0.651
        ],
        [
          0.838,
          0.6483
        ],
        [
          0.848,
          0.6424
        ],
        [
          0.8581,
          0.6379
        ],
        [
          0.8685,
          0.6319
        ],
        [
          0.8785,
          0.6265
        ],
        [
          0.8885,
          0.6201
        ],
        [
          0.8985,
          0.6146
        ],
        [
          0.9086,
          0.6096
        ],
        [
          0.919,
          0.6024
        ],
        [
          0.929,
          0.5965
        ],
        [
          0.939,
          0.5933
        ],
        [
          0.949,
          0.5855
        ],
        [
          0.9591,
          0.5792
        ],
        [
          0.9695,
          0.5732
        ],
        [
          0.9795,
          0.5669
        ],
        [
          0.9895,
          0.5605
        ],
        [
          0.9995,
          0.5551
        ],
        [
          1.0,
          0.5546
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.15553161070786198,
        "difference": -0.0009760727430123939,
        "worse": 0.47,
        "kept_right": 0.15245455096980617,
        "kept_wrong": 0.003077059738055834,
        "handed_off": 0.8444683892921387
      },
      "usd_per_1000": 4.163723994759621
    },
    {
      "decision": 128,
      "frontier": 193,
      "frontier_accuracy": 0.89171974522293,
      "in_sample": {
        "threshold": 0.9911303026027772,
        "answered": 0.06187443130118289,
        "accuracy": 0.89171974522293
      },
      "curve": [
        [
          0.0,
          0.8917
        ],
        [
          0.01,
          0.8917
        ],
        [
          0.02,
          0.8917
        ],
        [
          0.03,
          0.8917
        ],
        [
          0.04,
          0.8917
        ],
        [
          0.05,
          0.8917
        ],
        [
          0.0601,
          0.8917
        ],
        [
          0.0705,
          0.8913
        ],
        [
          0.0805,
          0.8913
        ],
        [
          0.0905,
          0.8913
        ],
        [
          0.1005,
          0.8908
        ],
        [
          0.1106,
          0.8908
        ],
        [
          0.121,
          0.8908
        ],
        [
          0.131,
          0.8908
        ],
        [
          0.141,
          0.8908
        ],
        [
          0.151,
          0.8904
        ],
        [
          0.1611,
          0.8899
        ],
        [
          0.1715,
          0.8904
        ],
        [
          0.1815,
          0.8899
        ],
        [
          0.1915,
          0.889
        ],
        [
          0.2015,
          0.8885
        ],
        [
          0.2116,
          0.8881
        ],
        [
          0.222,
          0.8876
        ],
        [
          0.232,
          0.8867
        ],
        [
          0.242,
          0.8858
        ],
        [
          0.252,
          0.8831
        ],
        [
          0.2621,
          0.8813
        ],
        [
          0.2725,
          0.8785
        ],
        [
          0.2825,
          0.8767
        ],
        [
          0.2925,
          0.8758
        ],
        [
          0.3025,
          0.874
        ],
        [
          0.3126,
          0.8708
        ],
        [
          0.323,
          0.8681
        ],
        [
          0.333,
          0.8662
        ],
        [
          0.343,
          0.8653
        ],
        [
          0.353,
          0.8608
        ],
        [
          0.3631,
          0.8571
        ],
        [
          0.3735,
          0.8535
        ],
        [
          0.3835,
          0.8467
        ],
        [
          0.3935,
          0.8449
        ],
        [
          0.4035,
          0.8412
        ],
        [
          0.4136,
          0.8394
        ],
        [
          0.424,
          0.8371
        ],
        [
          0.434,
          0.833
        ],
        [
          0.444,
          0.8317
        ],
        [
          0.454,
          0.8289
        ],
        [
          0.4641,
          0.8276
        ],
        [
          0.4745,
          0.823
        ],
        [
          0.4845,
          0.8212
        ],
        [
          0.4945,
          0.8157
        ],
        [
          0.5045,
          0.813
        ],
        [
          0.5146,
          0.8085
        ],
        [
          0.525,
          0.8025
        ],
        [
          0.535,
          0.798
        ],
        [
          0.545,
          0.793
        ],
        [
          0.5551,
          0.7898
        ],
        [
          0.5655,
          0.7839
        ],
        [
          0.5755,
          0.7798
        ],
        [
          0.5855,
          0.7753
        ],
        [
          0.5955,
          0.7698
        ],
        [
          0.6056,
          0.7648
        ],
        [
          0.616,
          0.7602
        ],
        [
          0.626,
          0.7552
        ],
        [
          0.636,
          0.753
        ],
        [
          0.646,
          0.7489
        ],
        [
          0.6561,
          0.7448
        ],
        [
          0.6665,
          0.7402
        ],
        [
          0.6765,
          0.7348
        ],
        [
          0.6865,
          0.7307
        ],
        [
          0.6965,
          0.727
        ],
        [
          0.7066,
          0.7216
        ],
        [
          0.717,
          0.7138
        ],
        [
          0.727,
          0.7075
        ],
        [
          0.737,
          0.7025
        ],
        [
          0.747,
          0.6956
        ],
        [
          0.7571,
          0.6888
        ],
        [
          0.7675,
          0.6829
        ],
        [
          0.7775,
          0.6765
        ],
        [
          0.7875,
          0.6711
        ],
        [
          0.7975,
          0.6661
        ],
        [
          0.8076,
          0.6592
        ],
        [
          0.818,
          0.652
        ],
        [
          0.828,
          0.6506
        ],
        [
          0.838,
          0.6474
        ],
        [
          0.848,
          0.6415
        ],
        [
          0.8581,
          0.6356
        ],
        [
          0.8685,
          0.6306
        ],
        [
          0.8785,
          0.6269
        ],
        [
          0.8885,
          0.6192
        ],
        [
          0.8985,
          0.6137
        ],
        [
          0.9086,
          0.6074
        ],
        [
          0.919,
          0.601
        ],
        [
          0.929,
          0.5969
        ],
        [
          0.939,
          0.5919
        ],
        [
          0.949,
          0.5851
        ],
        [
          0.9591,
          0.5792
        ],
        [
          0.9695,
          0.5728
        ],
        [
          0.9795,
          0.5673
        ],
        [
          0.9895,
          0.5605
        ],
        [
          0.9995,
          0.5551
        ],
        [
          1.0,
          0.5546
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1026090170442903,
        "difference": -0.0012278725373157284,
        "worse": 0.5066666666666667,
        "kept_right": 0.10092036420315582,
        "kept_wrong": 0.0016886528411344307,
        "handed_off": 0.8973909829557094
      },
      "usd_per_1000": 6.952617397472304
    },
    {
      "decision": 128,
      "frontier": 55,
      "frontier_accuracy": 0.8503184713375797,
      "in_sample": {
        "threshold": 0.9668741844737082,
        "answered": 0.2356687898089172,
        "accuracy": 0.8503184713375797
      },
      "curve": [
        [
          0.0,
          0.8503
        ],
        [
          0.01,
          0.8503
        ],
        [
          0.02,
          0.8503
        ],
        [
          0.03,
          0.8503
        ],
        [
          0.04,
          0.8503
        ],
        [
          0.05,
          0.8503
        ],
        [
          0.0601,
          0.8503
        ],
        [
          0.0705,
          0.8503
        ],
        [
          0.0805,
          0.8508
        ],
        [
          0.0905,
          0.8508
        ],
        [
          0.1005,
          0.8508
        ],
        [
          0.1106,
          0.8508
        ],
        [
          0.121,
          0.8517
        ],
        [
          0.131,
          0.8517
        ],
        [
          0.141,
          0.853
        ],
        [
          0.151,
          0.8526
        ],
        [
          0.1611,
          0.853
        ],
        [
          0.1715,
          0.854
        ],
        [
          0.1815,
          0.853
        ],
        [
          0.1915,
          0.8526
        ],
        [
          0.2015,
          0.8526
        ],
        [
          0.2116,
          0.8526
        ],
        [
          0.222,
          0.8508
        ],
        [
          0.232,
          0.8503
        ],
        [
          0.242,
          0.849
        ],
        [
          0.252,
          0.8471
        ],
        [
          0.2621,
          0.8462
        ],
        [
          0.2725,
          0.8444
        ],
        [
          0.2825,
          0.8435
        ],
        [
          0.2925,
          0.8435
        ],
        [
          0.3025,
          0.843
        ],
        [
          0.3126,
          0.8403
        ],
        [
          0.323,
          0.8394
        ],
        [
          0.333,
          0.838
        ],
        [
          0.343,
          0.8371
        ],
        [
          0.353,
          0.8326
        ],
        [
          0.3631,
          0.8294
        ],
        [
          0.3735,
          0.8267
        ],
        [
          0.3835,
          0.8212
        ],
        [
          0.3935,
          0.8194
        ],
        [
          0.4035,
          0.8153
        ],
        [
          0.4136,
          0.8148
        ],
        [
          0.424,
          0.8135
        ],
        [
          0.434,
          0.8098
        ],
        [
          0.444,
          0.808
        ],
        [
          0.454,
          0.8053
        ],
        [
          0.4641,
          0.8035
        ],
        [
          0.4745,
          0.7994
        ],
        [
          0.4845,
          0.7975
        ],
        [
          0.4945,
          0.793
        ],
        [
          0.5045,
          0.7907
        ],
        [
          0.5146,
          0.7862
        ],
        [
          0.525,
          0.7807
        ],
        [
          0.535,
          0.7771
        ],
        [
          0.545,
          0.7734
        ],
        [
          0.5551,
          0.7707
        ],
        [
          0.5655,
          0.7666
        ],
        [
          0.5755,
          0.763
        ],
        [
          0.5855,
          0.7598
        ],
        [
          0.5955,
          0.7548
        ],
        [
          0.6056,
          0.7502
        ],
        [
          0.616,
          0.7475
        ],
        [
          0.626,
          0.7425
        ],
        [
          0.636,
          0.7393
        ],
        [
          0.646,
          0.7352
        ],
        [
          0.6561,
          0.7316
        ],
        [
          0.6665,
          0.7284
        ],
        [
          0.6765,
          0.7234
        ],
        [
          0.6865,
          0.7202
        ],
        [
          0.6965,
          0.717
        ],
        [
          0.7066,
          0.7134
        ],
        [
          0.717,
          0.7061
        ],
        [
          0.727,
          0.7006
        ],
        [
          0.737,
          0.6947
        ],
        [
          0.747,
          0.6879
        ],
        [
          0.7571,
          0.682
        ],
        [
          0.7675,
          0.677
        ],
        [
          0.7775,
          0.672
        ],
        [
          0.7875,
          0.6674
        ],
        [
          0.7975,
          0.6624
        ],
        [
          0.8076,
          0.6565
        ],
        [
          0.818,
          0.6492
        ],
        [
          0.828,
          0.6488
        ],
        [
          0.838,
          0.6447
        ],
        [
          0.848,
          0.6392
        ],
        [
          0.8581,
          0.6342
        ],
        [
          0.8685,
          0.6297
        ],
        [
          0.8785,
          0.6247
        ],
        [
          0.8885,
          0.6178
        ],
        [
          0.8985,
          0.6133
        ],
        [
          0.9086,
          0.6069
        ],
        [
          0.919,
          0.6015
        ],
        [
          0.929,
          0.596
        ],
        [
          0.939,
          0.5919
        ],
        [
          0.949,
          0.5851
        ],
        [
          0.9591,
          0.5801
        ],
        [
          0.9695,
          0.5737
        ],
        [
          0.9795,
          0.5682
        ],
        [
          0.9895,
          0.561
        ],
        [
          0.9995,
          0.5551
        ],
        [
          1.0,
          0.5546
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.23167516391729762,
        "difference": -0.0012582901200723917,
        "worse": 0.46,
        "kept_right": 0.219493790829597,
        "kept_wrong": 0.012181373087700658,
        "handed_off": 0.7683248360827024
      },
      "usd_per_1000": 2.8077152795084768
    },
    {
      "decision": 373,
      "frontier": 140,
      "frontier_accuracy": 0.9808917197452229,
      "in_sample": {
        "threshold": 0.988141,
        "answered": 0.0805277525022748,
        "accuracy": 0.9808917197452229
      },
      "curve": [
        [
          0.0,
          0.9809
        ],
        [
          0.01,
          0.9809
        ],
        [
          0.02,
          0.9809
        ],
        [
          0.03,
          0.9809
        ],
        [
          0.04,
          0.9809
        ],
        [
          0.05,
          0.9809
        ],
        [
          0.0601,
          0.9809
        ],
        [
          0.0705,
          0.9809
        ],
        [
          0.0805,
          0.9809
        ],
        [
          0.0905,
          0.9804
        ],
        [
          0.1005,
          0.9804
        ],
        [
          0.1106,
          0.98
        ],
        [
          0.121,
          0.9791
        ],
        [
          0.131,
          0.9791
        ],
        [
          0.141,
          0.9782
        ],
        [
          0.151,
          0.9782
        ],
        [
          0.1611,
          0.9773
        ],
        [
          0.1715,
          0.9763
        ],
        [
          0.1815,
          0.9754
        ],
        [
          0.1915,
          0.9727
        ],
        [
          0.2015,
          0.9722
        ],
        [
          0.2116,
          0.9713
        ],
        [
          0.222,
          0.9695
        ],
        [
          0.232,
          0.9677
        ],
        [
          0.242,
          0.9654
        ],
        [
          0.252,
          0.9641
        ],
        [
          0.2621,
          0.9613
        ],
        [
          0.2725,
          0.9586
        ],
        [
          0.2825,
          0.9545
        ],
        [
          0.2925,
          0.9527
        ],
        [
          0.3025,
          0.9495
        ],
        [
          0.3126,
          0.9459
        ],
        [
          0.323,
          0.9418
        ],
        [
          0.333,
          0.9368
        ],
        [
          0.343,
          0.9331
        ],
        [
          0.353,
          0.9295
        ],
        [
          0.3631,
          0.9227
        ],
        [
          0.3735,
          0.9172
        ],
        [
          0.3835,
          0.9131
        ],
        [
          0.3935,
          0.9099
        ],
        [
          0.4035,
          0.9054
        ],
        [
          0.4136,
          0.9004
        ],
        [
          0.424,
          0.8958
        ],
        [
          0.434,
          0.8908
        ],
        [
          0.444,
          0.8863
        ],
        [
          0.454,
          0.8817
        ],
        [
          0.4641,
          0.8758
        ],
        [
          0.4745,
          0.8694
        ],
        [
          0.4845,
          0.8626
        ],
        [
          0.4945,
          0.8576
        ],
        [
          0.5045,
          0.8517
        ],
        [
          0.5146,
          0.8467
        ],
        [
          0.525,
          0.8412
        ],
        [
          0.535,
          0.8362
        ],
        [
          0.545,
          0.8312
        ],
        [
          0.5551,
          0.8276
        ],
        [
          0.5655,
          0.8212
        ],
        [
          0.5755,
          0.8157
        ],
        [
          0.5855,
          0.8107
        ],
        [
          0.5955,
          0.8048
        ],
        [
          0.6056,
          0.7971
        ],
        [
          0.616,
          0.7907
        ],
        [
          0.626,
          0.7839
        ],
        [
          0.636,
          0.7789
        ],
        [
          0.646,
          0.7757
        ],
        [
          0.6561,
          0.7684
        ],
        [
          0.6665,
          0.7625
        ],
        [
          0.6765,
          0.7548
        ],
        [
          0.6865,
          0.748
        ],
        [
          0.6965,
          0.7411
        ],
        [
          0.7066,
          0.7361
        ],
        [
          0.717,
          0.7298
        ],
        [
          0.727,
          0.7243
        ],
        [
          0.737,
          0.7166
        ],
        [
          0.747,
          0.7079
        ],
        [
          0.7571,
          0.702
        ],
        [
          0.7675,
          0.6943
        ],
        [
          0.7775,
          0.6879
        ],
        [
          0.7875,
          0.6811
        ],
        [
          0.7975,
          0.6747
        ],
        [
          0.8076,
          0.6688
        ],
        [
          0.818,
          0.6601
        ],
        [
          0.828,
          0.6524
        ],
        [
          0.838,
          0.6451
        ],
        [
          0.848,
          0.6383
        ],
        [
          0.8581,
          0.6301
        ],
        [
          0.8685,
          0.6215
        ],
        [
          0.8785,
          0.6137
        ],
        [
          0.8885,
          0.6078
        ],
        [
          0.8985,
          0.6005
        ],
        [
          0.9086,
          0.5919
        ],
        [
          0.919,
          0.5823
        ],
        [
          0.929,
          0.5769
        ],
        [
          0.939,
          0.5687
        ],
        [
          0.949,
          0.5614
        ],
        [
          0.9591,
          0.5546
        ],
        [
          0.9695,
          0.5469
        ],
        [
          0.9795,
          0.54
        ],
        [
          0.9895,
          0.5318
        ],
        [
          0.9995,
          0.5237
        ],
        [
          1.0,
          0.5237
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.09971627907638507,
        "difference": -0.0010429712723115327,
        "worse": 0.55,
        "kept_right": 0.09862780908613032,
        "kept_wrong": 0.0010884699902547485,
        "handed_off": 0.9002837209236145
      },
      "usd_per_1000": 2.191747508054224
    },
    {
      "decision": 373,
      "frontier": 277,
      "frontier_accuracy": 0.9795268425841674,
      "in_sample": {
        "threshold": 0.988141,
        "answered": 0.0805277525022748,
        "accuracy": 0.9795268425841674
      },
      "curve": [
        [
          0.0,
          0.9795
        ],
        [
          0.01,
          0.9795
        ],
        [
          0.02,
          0.9795
        ],
        [
          0.03,
          0.9795
        ],
        [
          0.04,
          0.9795
        ],
        [
          0.05,
          0.9795
        ],
        [
          0.0601,
          0.9795
        ],
        [
          0.0705,
          0.9795
        ],
        [
          0.0805,
          0.9795
        ],
        [
          0.0905,
          0.9791
        ],
        [
          0.1005,
          0.9791
        ],
        [
          0.1106,
          0.9786
        ],
        [
          0.121,
          0.9777
        ],
        [
          0.131,
          0.9777
        ],
        [
          0.141,
          0.9768
        ],
        [
          0.151,
          0.9768
        ],
        [
          0.1611,
          0.9759
        ],
        [
          0.1715,
          0.975
        ],
        [
          0.1815,
          0.9741
        ],
        [
          0.1915,
          0.9713
        ],
        [
          0.2015,
          0.9709
        ],
        [
          0.2116,
          0.9691
        ],
        [
          0.222,
          0.9672
        ],
        [
          0.232,
          0.9654
        ],
        [
          0.242,
          0.9631
        ],
        [
          0.252,
          0.9618
        ],
        [
          0.2621,
          0.9591
        ],
        [
          0.2725,
          0.9563
        ],
        [
          0.2825,
          0.9522
        ],
        [
          0.2925,
          0.9504
        ],
        [
          0.3025,
          0.9472
        ],
        [
          0.3126,
          0.9436
        ],
        [
          0.323,
          0.9399
        ],
        [
          0.333,
          0.9349
        ],
        [
          0.343,
          0.9313
        ],
        [
          0.353,
          0.9277
        ],
        [
          0.3631,
          0.9208
        ],
        [
          0.3735,
          0.9154
        ],
        [
          0.3835,
          0.9113
        ],
        [
          0.3935,
          0.9081
        ],
        [
          0.4035,
          0.904
        ],
        [
          0.4136,
          0.899
        ],
        [
          0.424,
          0.8949
        ],
        [
          0.434,
          0.8899
        ],
        [
          0.444,
          0.8858
        ],
        [
          0.454,
          0.8813
        ],
        [
          0.4641,
          0.8753
        ],
        [
          0.4745,
          0.8685
        ],
        [
          0.4845,
          0.8621
        ],
        [
          0.4945,
          0.8571
        ],
        [
          0.5045,
          0.8508
        ],
        [
          0.5146,
          0.8449
        ],
        [
          0.525,
          0.8394
        ],
        [
          0.535,
          0.8348
        ],
        [
          0.545,
          0.8298
        ],
        [
          0.5551,
          0.8262
        ],
        [
          0.5655,
          0.8198
        ],
        [
          0.5755,
          0.8144
        ],
        [
          0.5855,
          0.8094
        ],
        [
          0.5955,
          0.8035
        ],
        [
          0.6056,
          0.7957
        ],
        [
          0.616,
          0.7894
        ],
        [
          0.626,
          0.783
        ],
        [
          0.636,
          0.778
        ],
        [
          0.646,
          0.7748
        ],
        [
          0.6561,
          0.7675
        ],
        [
          0.6665,
          0.7616
        ],
        [
          0.6765,
          0.7539
        ],
        [
          0.6865,
          0.747
        ],
        [
          0.6965,
          0.7402
        ],
        [
          0.7066,
          0.7352
        ],
        [
          0.717,
          0.7284
        ],
        [
          0.727,
          0.7234
        ],
        [
          0.737,
          0.7157
        ],
        [
          0.747,
          0.707
        ],
        [
          0.7571,
          0.7011
        ],
        [
          0.7675,
          0.6934
        ],
        [
          0.7775,
          0.687
        ],
        [
          0.7875,
          0.6793
        ],
        [
          0.7975,
          0.6729
        ],
        [
          0.8076,
          0.667
        ],
        [
          0.818,
          0.6583
        ],
        [
          0.828,
          0.651
        ],
        [
          0.838,
          0.6438
        ],
        [
          0.848,
          0.6369
        ],
        [
          0.8581,
          0.6288
        ],
        [
          0.8685,
          0.6201
        ],
        [
          0.8785,
          0.6119
        ],
        [
          0.8885,
          0.606
        ],
        [
          0.8985,
          0.5987
        ],
        [
          0.9086,
          0.5905
        ],
        [
          0.919,
          0.581
        ],
        [
          0.929,
          0.5751
        ],
        [
          0.939,
          0.5669
        ],
        [
          0.949,
          0.5596
        ],
        [
          0.9591,
          0.5532
        ],
        [
          0.9695,
          0.5455
        ],
        [
          0.9795,
          0.5387
        ],
        [
          0.9895,
          0.5318
        ],
        [
          0.9995,
          0.5237
        ],
        [
          1.0,
          0.5237
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.09971627907638507,
        "difference": -0.0010429712723115327,
        "worse": 0.55,
        "kept_right": 0.09862780908613032,
        "kept_wrong": 0.0010884699902547485,
        "handed_off": 0.9002837209236145
      },
      "usd_per_1000": 5.474588782023893
    },
    {
      "decision": 373,
      "frontier": 276,
      "frontier_accuracy": 0.9777070063694268,
      "in_sample": {
        "threshold": 0.988141,
        "answered": 0.0805277525022748,
        "accuracy": 0.9777070063694268
      },
      "curve": [
        [
          0.0,
          0.9777
        ],
        [
          0.01,
          0.9777
        ],
        [
          0.02,
          0.9777
        ],
        [
          0.03,
          0.9777
        ],
        [
          0.04,
          0.9777
        ],
        [
          0.05,
          0.9777
        ],
        [
          0.0601,
          0.9777
        ],
        [
          0.0705,
          0.9777
        ],
        [
          0.0805,
          0.9777
        ],
        [
          0.0905,
          0.9773
        ],
        [
          0.1005,
          0.9773
        ],
        [
          0.1106,
          0.9768
        ],
        [
          0.121,
          0.9759
        ],
        [
          0.131,
          0.9759
        ],
        [
          0.141,
          0.975
        ],
        [
          0.151,
          0.9754
        ],
        [
          0.1611,
          0.9745
        ],
        [
          0.1715,
          0.9736
        ],
        [
          0.1815,
          0.9727
        ],
        [
          0.1915,
          0.97
        ],
        [
          0.2015,
          0.9695
        ],
        [
          0.2116,
          0.9677
        ],
        [
          0.222,
          0.9659
        ],
        [
          0.232,
          0.9636
        ],
        [
          0.242,
          0.9613
        ],
        [
          0.252,
          0.96
        ],
        [
          0.2621,
          0.9572
        ],
        [
          0.2725,
          0.9545
        ],
        [
          0.2825,
          0.9504
        ],
        [
          0.2925,
          0.9486
        ],
        [
          0.3025,
          0.9454
        ],
        [
          0.3126,
          0.9418
        ],
        [
          0.323,
          0.9381
        ],
        [
          0.333,
          0.9331
        ],
        [
          0.343,
          0.9295
        ],
        [
          0.353,
          0.9258
        ],
        [
          0.3631,
          0.9195
        ],
        [
          0.3735,
          0.914
        ],
        [
          0.3835,
          0.9099
        ],
        [
          0.3935,
          0.9072
        ],
        [
          0.4035,
          0.9031
        ],
        [
          0.4136,
          0.8985
        ],
        [
          0.424,
          0.8944
        ],
        [
          0.434,
          0.8894
        ],
        [
          0.444,
          0.8858
        ],
        [
          0.454,
          0.8813
        ],
        [
          0.4641,
          0.8753
        ],
        [
          0.4745,
          0.8685
        ],
        [
          0.4845,
          0.8621
        ],
        [
          0.4945,
          0.8571
        ],
        [
          0.5045,
          0.8508
        ],
        [
          0.5146,
          0.8449
        ],
        [
          0.525,
          0.8394
        ],
        [
          0.535,
          0.8344
        ],
        [
          0.545,
          0.8294
        ],
        [
          0.5551,
          0.8258
        ],
        [
          0.5655,
          0.8194
        ],
        [
          0.5755,
          0.8139
        ],
        [
          0.5855,
          0.8089
        ],
        [
          0.5955,
          0.803
        ],
        [
          0.6056,
          0.7953
        ],
        [
          0.616,
          0.7889
        ],
        [
          0.626,
          0.783
        ],
        [
          0.636,
          0.778
        ],
        [
          0.646,
          0.7748
        ],
        [
          0.6561,
          0.7675
        ],
        [
          0.6665,
          0.7616
        ],
        [
          0.6765,
          0.7539
        ],
        [
          0.6865,
          0.747
        ],
        [
          0.6965,
          0.7402
        ],
        [
          0.7066,
          0.7352
        ],
        [
          0.717,
          0.7288
        ],
        [
          0.727,
          0.7238
        ],
        [
          0.737,
          0.7161
        ],
        [
          0.747,
          0.7075
        ],
        [
          0.7571,
          0.7015
        ],
        [
          0.7675,
          0.6938
        ],
        [
          0.7775,
          0.6874
        ],
        [
          0.7875,
          0.6797
        ],
        [
          0.7975,
          0.6733
        ],
        [
          0.8076,
          0.6674
        ],
        [
          0.818,
          0.6588
        ],
        [
          0.828,
          0.651
        ],
        [
          0.838,
          0.6438
        ],
        [
          0.848,
          0.6369
        ],
        [
          0.8581,
          0.6288
        ],
        [
          0.8685,
          0.6201
        ],
        [
          0.8785,
          0.6119
        ],
        [
          0.8885,
          0.606
        ],
        [
          0.8985,
          0.5987
        ],
        [
          0.9086,
          0.5905
        ],
        [
          0.919,
          0.581
        ],
        [
          0.929,
          0.5755
        ],
        [
          0.939,
          0.5673
        ],
        [
          0.949,
          0.5601
        ],
        [
          0.9591,
          0.5532
        ],
        [
          0.9695,
          0.5455
        ],
        [
          0.9795,
          0.5387
        ],
        [
          0.9895,
          0.5314
        ],
        [
          0.9995,
          0.5237
        ],
        [
          1.0,
          0.5237
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.10212604301616059,
        "difference": -0.001218767681414942,
        "worse": 0.5566666666666666,
        "kept_right": 0.1008345093841498,
        "kept_wrong": 0.0012915336320107663,
        "handed_off": 0.8978739569838391
      },
      "usd_per_1000": 1.4703906216099616
    },
    {
      "decision": 373,
      "frontier": 48,
      "frontier_accuracy": 0.9758871701546861,
      "in_sample": {
        "threshold": 0.988141,
        "answered": 0.0805277525022748,
        "accuracy": 0.9758871701546861
      },
      "curve": [
        [
          0.0,
          0.9759
        ],
        [
          0.01,
          0.9759
        ],
        [
          0.02,
          0.9759
        ],
        [
          0.03,
          0.9759
        ],
        [
          0.04,
          0.9759
        ],
        [
          0.05,
          0.9759
        ],
        [
          0.0601,
          0.9759
        ],
        [
          0.0705,
          0.9759
        ],
        [
          0.0805,
          0.9759
        ],
        [
          0.0905,
          0.9754
        ],
        [
          0.1005,
          0.9754
        ],
        [
          0.1106,
          0.975
        ],
        [
          0.121,
          0.9741
        ],
        [
          0.131,
          0.9741
        ],
        [
          0.141,
          0.9732
        ],
        [
          0.151,
          0.9736
        ],
        [
          0.1611,
          0.9727
        ],
        [
          0.1715,
          0.9718
        ],
        [
          0.1815,
          0.9709
        ],
        [
          0.1915,
          0.9682
        ],
        [
          0.2015,
          0.9677
        ],
        [
          0.2116,
          0.9659
        ],
        [
          0.222,
          0.9641
        ],
        [
          0.232,
          0.9622
        ],
        [
          0.242,
          0.96
        ],
        [
          0.252,
          0.9586
        ],
        [
          0.2621,
          0.9559
        ],
        [
          0.2725,
          0.9531
        ],
        [
          0.2825,
          0.949
        ],
        [
          0.2925,
          0.9472
        ],
        [
          0.3025,
          0.944
        ],
        [
          0.3126,
          0.9409
        ],
        [
          0.323,
          0.9372
        ],
        [
          0.333,
          0.9322
        ],
        [
          0.343,
          0.9286
        ],
        [
          0.353,
          0.9249
        ],
        [
          0.3631,
          0.9181
        ],
        [
          0.3735,
          0.9126
        ],
        [
          0.3835,
          0.9086
        ],
        [
          0.3935,
          0.9058
        ],
        [
          0.4035,
          0.9017
        ],
        [
          0.4136,
          0.8967
        ],
        [
          0.424,
          0.8926
        ],
        [
          0.434,
          0.8881
        ],
        [
          0.444,
          0.884
        ],
        [
          0.454,
          0.879
        ],
        [
          0.4641,
          0.8731
        ],
        [
          0.4745,
          0.8662
        ],
        [
          0.4845,
          0.8599
        ],
        [
          0.4945,
          0.8549
        ],
        [
          0.5045,
          0.849
        ],
        [
          0.5146,
          0.8435
        ],
        [
          0.525,
          0.838
        ],
        [
          0.535,
          0.8339
        ],
        [
          0.545,
          0.8289
        ],
        [
          0.5551,
          0.8258
        ],
        [
          0.5655,
          0.8194
        ],
        [
          0.5755,
          0.8139
        ],
        [
          0.5855,
          0.8089
        ],
        [
          0.5955,
          0.803
        ],
        [
          0.6056,
          0.7953
        ],
        [
          0.616,
          0.7889
        ],
        [
          0.626,
          0.7825
        ],
        [
          0.636,
          0.778
        ],
        [
          0.646,
          0.7743
        ],
        [
          0.6561,
          0.7671
        ],
        [
          0.6665,
          0.7611
        ],
        [
          0.6765,
          0.7534
        ],
        [
          0.6865,
          0.7466
        ],
        [
          0.6965,
          0.7398
        ],
        [
          0.7066,
          0.7348
        ],
        [
          0.717,
          0.7284
        ],
        [
          0.727,
          0.7229
        ],
        [
          0.737,
          0.7152
        ],
        [
          0.747,
          0.7066
        ],
        [
          0.7571,
          0.7006
        ],
        [
          0.7675,
          0.6929
        ],
        [
          0.7775,
          0.6865
        ],
        [
          0.7875,
          0.6788
        ],
        [
          0.7975,
          0.6724
        ],
        [
          0.8076,
          0.6665
        ],
        [
          0.818,
          0.6579
        ],
        [
          0.828,
          0.6501
        ],
        [
          0.838,
          0.6429
        ],
        [
          0.848,
          0.636
        ],
        [
          0.8581,
          0.6278
        ],
        [
          0.8685,
          0.6192
        ],
        [
          0.8785,
          0.6115
        ],
        [
          0.8885,
          0.606
        ],
        [
          0.8985,
          0.5987
        ],
        [
          0.9086,
          0.5901
        ],
        [
          0.919,
          0.5805
        ],
        [
          0.929,
          0.5746
        ],
        [
          0.939,
          0.5664
        ],
        [
          0.949,
          0.5591
        ],
        [
          0.9591,
          0.5528
        ],
        [
          0.9695,
          0.545
        ],
        [
          0.9795,
          0.5382
        ],
        [
          0.9895,
          0.5314
        ],
        [
          0.9995,
          0.5237
        ],
        [
          1.0,
          0.5237
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.10212604301616059,
        "difference": -0.001218767681414942,
        "worse": 0.5566666666666666,
        "kept_right": 0.1008345093841498,
        "kept_wrong": 0.0012915336320107663,
        "handed_off": 0.8978739569838391
      },
      "usd_per_1000": 4.842837591665295
    },
    {
      "decision": 373,
      "frontier": 191,
      "frontier_accuracy": 0.9740673339399454,
      "in_sample": {
        "threshold": 0.988141,
        "answered": 0.0805277525022748,
        "accuracy": 0.9740673339399454
      },
      "curve": [
        [
          0.0,
          0.9741
        ],
        [
          0.01,
          0.9741
        ],
        [
          0.02,
          0.9741
        ],
        [
          0.03,
          0.9741
        ],
        [
          0.04,
          0.9741
        ],
        [
          0.05,
          0.9741
        ],
        [
          0.0601,
          0.9741
        ],
        [
          0.0705,
          0.9741
        ],
        [
          0.0805,
          0.9741
        ],
        [
          0.0905,
          0.9736
        ],
        [
          0.1005,
          0.9736
        ],
        [
          0.1106,
          0.9732
        ],
        [
          0.121,
          0.9722
        ],
        [
          0.131,
          0.9722
        ],
        [
          0.141,
          0.9713
        ],
        [
          0.151,
          0.9718
        ],
        [
          0.1611,
          0.9709
        ],
        [
          0.1715,
          0.9695
        ],
        [
          0.1815,
          0.9686
        ],
        [
          0.1915,
          0.9659
        ],
        [
          0.2015,
          0.9654
        ],
        [
          0.2116,
          0.9636
        ],
        [
          0.222,
          0.9618
        ],
        [
          0.232,
          0.9595
        ],
        [
          0.242,
          0.9572
        ],
        [
          0.252,
          0.9559
        ],
        [
          0.2621,
          0.9531
        ],
        [
          0.2725,
          0.9504
        ],
        [
          0.2825,
          0.9463
        ],
        [
          0.2925,
          0.9445
        ],
        [
          0.3025,
          0.9413
        ],
        [
          0.3126,
          0.9377
        ],
        [
          0.323,
          0.934
        ],
        [
          0.333,
          0.929
        ],
        [
          0.343,
          0.9254
        ],
        [
          0.353,
          0.9217
        ],
        [
          0.3631,
          0.9149
        ],
        [
          0.3735,
          0.9095
        ],
        [
          0.3835,
          0.9054
        ],
        [
          0.3935,
          0.9026
        ],
        [
          0.4035,
          0.8985
        ],
        [
          0.4136,
          0.894
        ],
        [
          0.424,
          0.8899
        ],
        [
          0.434,
          0.8849
        ],
        [
          0.444,
          0.8808
        ],
        [
          0.454,
          0.8763
        ],
        [
          0.4641,
          0.8703
        ],
        [
          0.4745,
          0.8635
        ],
        [
          0.4845,
          0.8571
        ],
        [
          0.4945,
          0.8526
        ],
        [
          0.5045,
          0.8467
        ],
        [
          0.5146,
          0.8412
        ],
        [
          0.525,
          0.8358
        ],
        [
          0.535,
          0.8308
        ],
        [
          0.545,
          0.8258
        ],
        [
          0.5551,
          0.8226
        ],
        [
          0.5655,
          0.8162
        ],
        [
          0.5755,
          0.8112
        ],
        [
          0.5855,
          0.8062
        ],
        [
          0.5955,
          0.8007
        ],
        [
          0.6056,
          0.793
        ],
        [
          0.616,
          0.7866
        ],
        [
          0.626,
          0.7798
        ],
        [
          0.636,
          0.7753
        ],
        [
          0.646,
          0.7716
        ],
        [
          0.6561,
          0.7643
        ],
        [
          0.6665,
          0.7584
        ],
        [
          0.6765,
          0.7511
        ],
        [
          0.6865,
          0.7443
        ],
        [
          0.6965,
          0.7375
        ],
        [
          0.7066,
          0.7329
        ],
        [
          0.717,
          0.727
        ],
        [
          0.727,
          0.722
        ],
        [
          0.737,
          0.7143
        ],
        [
          0.747,
          0.7056
        ],
        [
          0.7571,
          0.6997
        ],
        [
          0.7675,
          0.692
        ],
        [
          0.7775,
          0.6856
        ],
        [
          0.7875,
          0.6779
        ],
        [
          0.7975,
          0.6715
        ],
        [
          0.8076,
          0.6656
        ],
        [
          0.818,
          0.657
        ],
        [
          0.828,
          0.6492
        ],
        [
          0.838,
          0.6419
        ],
        [
          0.848,
          0.6351
        ],
        [
          0.8581,
          0.6269
        ],
        [
          0.8685,
          0.6183
        ],
        [
          0.8785,
          0.6101
        ],
        [
          0.8885,
          0.6042
        ],
        [
          0.8985,
          0.5969
        ],
        [
          0.9086,
          0.5887
        ],
        [
          0.919,
          0.5792
        ],
        [
          0.929,
          0.5737
        ],
        [
          0.939,
          0.566
        ],
        [
          0.949,
          0.5591
        ],
        [
          0.9591,
          0.5532
        ],
        [
          0.9695,
          0.5455
        ],
        [
          0.9795,
          0.5387
        ],
        [
          0.9895,
          0.5318
        ],
        [
          0.9995,
          0.5237
        ],
        [
          1.0,
          0.5237
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.10176532371478976,
        "difference": -0.0011854343180289918,
        "worse": 0.5566666666666666,
        "kept_right": 0.10051015374919525,
        "kept_wrong": 0.0012551699655945134,
        "handed_off": 0.8982346762852099
      },
      "usd_per_1000": 24.03059410173699
    },
    {
      "decision": 373,
      "frontier": 188,
      "frontier_accuracy": 0.9713375796178344,
      "in_sample": {
        "threshold": 0.988141,
        "answered": 0.0805277525022748,
        "accuracy": 0.9713375796178344
      },
      "curve": [
        [
          0.0,
          0.9713
        ],
        [
          0.01,
          0.9713
        ],
        [
          0.02,
          0.9713
        ],
        [
          0.03,
          0.9713
        ],
        [
          0.04,
          0.9713
        ],
        [
          0.05,
          0.9713
        ],
        [
          0.0601,
          0.9713
        ],
        [
          0.0705,
          0.9713
        ],
        [
          0.0805,
          0.9713
        ],
        [
          0.0905,
          0.9709
        ],
        [
          0.1005,
          0.9709
        ],
        [
          0.1106,
          0.9704
        ],
        [
          0.121,
          0.9695
        ],
        [
          0.131,
          0.9695
        ],
        [
          0.141,
          0.9686
        ],
        [
          0.151,
          0.9691
        ],
        [
          0.1611,
          0.9682
        ],
        [
          0.1715,
          0.9672
        ],
        [
          0.1815,
          0.9663
        ],
        [
          0.1915,
          0.9641
        ],
        [
          0.2015,
          0.9641
        ],
        [
          0.2116,
          0.9631
        ],
        [
          0.222,
          0.9613
        ],
        [
          0.232,
          0.9595
        ],
        [
          0.242,
          0.9572
        ],
        [
          0.252,
          0.9559
        ],
        [
          0.2621,
          0.9536
        ],
        [
          0.2725,
          0.9509
        ],
        [
          0.2825,
          0.9468
        ],
        [
          0.2925,
          0.9449
        ],
        [
          0.3025,
          0.9418
        ],
        [
          0.3126,
          0.9381
        ],
        [
          0.323,
          0.934
        ],
        [
          0.333,
          0.929
        ],
        [
          0.343,
          0.9254
        ],
        [
          0.353,
          0.9213
        ],
        [
          0.3631,
          0.9149
        ],
        [
          0.3735,
          0.9095
        ],
        [
          0.3835,
          0.9054
        ],
        [
          0.3935,
          0.9022
        ],
        [
          0.4035,
          0.8976
        ],
        [
          0.4136,
          0.8931
        ],
        [
          0.424,
          0.889
        ],
        [
          0.434,
          0.884
        ],
        [
          0.444,
          0.8794
        ],
        [
          0.454,
          0.8749
        ],
        [
          0.4641,
          0.8694
        ],
        [
          0.4745,
          0.8626
        ],
        [
          0.4845,
          0.8562
        ],
        [
          0.4945,
          0.8512
        ],
        [
          0.5045,
          0.8453
        ],
        [
          0.5146,
          0.8403
        ],
        [
          0.525,
          0.8348
        ],
        [
          0.535,
          0.8298
        ],
        [
          0.545,
          0.8248
        ],
        [
          0.5551,
          0.8212
        ],
        [
          0.5655,
          0.8153
        ],
        [
          0.5755,
          0.8098
        ],
        [
          0.5855,
          0.8048
        ],
        [
          0.5955,
          0.7989
        ],
        [
          0.6056,
          0.7916
        ],
        [
          0.616,
          0.7853
        ],
        [
          0.626,
          0.7789
        ],
        [
          0.636,
          0.7739
        ],
        [
          0.646,
          0.7707
        ],
        [
          0.6561,
          0.7643
        ],
        [
          0.6665,
          0.7584
        ],
        [
          0.6765,
          0.7507
        ],
        [
          0.6865,
          0.7439
        ],
        [
          0.6965,
          0.737
        ],
        [
          0.7066,
          0.732
        ],
        [
          0.717,
          0.7261
        ],
        [
          0.727,
          0.7211
        ],
        [
          0.737,
          0.7134
        ],
        [
          0.747,
          0.7047
        ],
        [
          0.7571,
          0.6988
        ],
        [
          0.7675,
          0.6906
        ],
        [
          0.7775,
          0.6843
        ],
        [
          0.7875,
          0.6774
        ],
        [
          0.7975,
          0.6711
        ],
        [
          0.8076,
          0.6652
        ],
        [
          0.818,
          0.6565
        ],
        [
          0.828,
          0.6488
        ],
        [
          0.838,
          0.6415
        ],
        [
          0.848,
          0.6347
        ],
        [
          0.8581,
          0.6269
        ],
        [
          0.8685,
          0.6187
        ],
        [
          0.8785,
          0.611
        ],
        [
          0.8885,
          0.6051
        ],
        [
          0.8985,
          0.5978
        ],
        [
          0.9086,
          0.5896
        ],
        [
          0.919,
          0.5801
        ],
        [
          0.929,
          0.5751
        ],
        [
          0.939,
          0.5673
        ],
        [
          0.949,
          0.5601
        ],
        [
          0.9591,
          0.5532
        ],
        [
          0.9695,
          0.5455
        ],
        [
          0.9795,
          0.5391
        ],
        [
          0.9895,
          0.5318
        ],
        [
          0.9995,
          0.5237
        ],
        [
          1.0,
          0.5237
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.10218969885678059,
        "difference": -0.001230902715345112,
        "worse": 0.5566666666666666,
        "kept_right": 0.10088906049386996,
        "kept_wrong": 0.0013006383629106336,
        "handed_off": 0.8978103011432191
      },
      "usd_per_1000": 0.8788545175025719
    },
    {
      "decision": 373,
      "frontier": 190,
      "frontier_accuracy": 0.9672429481346679,
      "in_sample": {
        "threshold": 0.980397,
        "answered": 0.10828025477707007,
        "accuracy": 0.9672429481346679
      },
      "curve": [
        [
          0.0,
          0.9672
        ],
        [
          0.01,
          0.9672
        ],
        [
          0.02,
          0.9672
        ],
        [
          0.03,
          0.9672
        ],
        [
          0.04,
          0.9672
        ],
        [
          0.05,
          0.9672
        ],
        [
          0.0601,
          0.9672
        ],
        [
          0.0705,
          0.9672
        ],
        [
          0.0805,
          0.9672
        ],
        [
          0.0905,
          0.9668
        ],
        [
          0.1005,
          0.9668
        ],
        [
          0.1106,
          0.9668
        ],
        [
          0.121,
          0.9659
        ],
        [
          0.131,
          0.9659
        ],
        [
          0.141,
          0.965
        ],
        [
          0.151,
          0.965
        ],
        [
          0.1611,
          0.9641
        ],
        [
          0.1715,
          0.9627
        ],
        [
          0.1815,
          0.9618
        ],
        [
          0.1915,
          0.9591
        ],
        [
          0.2015,
          0.9586
        ],
        [
          0.2116,
          0.9568
        ],
        [
          0.222,
          0.955
        ],
        [
          0.232,
          0.9527
        ],
        [
          0.242,
          0.9509
        ],
        [
          0.252,
          0.9495
        ],
        [
          0.2621,
          0.9472
        ],
        [
          0.2725,
          0.9445
        ],
        [
          0.2825,
          0.9404
        ],
        [
          0.2925,
          0.9386
        ],
        [
          0.3025,
          0.9354
        ],
        [
          0.3126,
          0.9322
        ],
        [
          0.323,
          0.9286
        ],
        [
          0.333,
          0.9236
        ],
        [
          0.343,
          0.9199
        ],
        [
          0.353,
          0.9163
        ],
        [
          0.3631,
          0.9099
        ],
        [
          0.3735,
          0.9045
        ],
        [
          0.3835,
          0.9004
        ],
        [
          0.3935,
          0.8976
        ],
        [
          0.4035,
          0.894
        ],
        [
          0.4136,
          0.8894
        ],
        [
          0.424,
          0.8854
        ],
        [
          0.434,
          0.8803
        ],
        [
          0.444,
          0.8763
        ],
        [
          0.454,
          0.8717
        ],
        [
          0.4641,
          0.8658
        ],
        [
          0.4745,
          0.859
        ],
        [
          0.4845,
          0.8526
        ],
        [
          0.4945,
          0.848
        ],
        [
          0.5045,
          0.8421
        ],
        [
          0.5146,
          0.8362
        ],
        [
          0.525,
          0.8308
        ],
        [
          0.535,
          0.8262
        ],
        [
          0.545,
          0.8212
        ],
        [
          0.5551,
          0.818
        ],
        [
          0.5655,
          0.8121
        ],
        [
          0.5755,
          0.8071
        ],
        [
          0.5855,
          0.8021
        ],
        [
          0.5955,
          0.7966
        ],
        [
          0.6056,
          0.7889
        ],
        [
          0.616,
          0.7825
        ],
        [
          0.626,
          0.7762
        ],
        [
          0.636,
          0.7716
        ],
        [
          0.646,
          0.7684
        ],
        [
          0.6561,
          0.7611
        ],
        [
          0.6665,
          0.7552
        ],
        [
          0.6765,
          0.7475
        ],
        [
          0.6865,
          0.7407
        ],
        [
          0.6965,
          0.7338
        ],
        [
          0.7066,
          0.7293
        ],
        [
          0.717,
          0.7225
        ],
        [
          0.727,
          0.7179
        ],
        [
          0.737,
          0.7102
        ],
        [
          0.747,
          0.7015
        ],
        [
          0.7571,
          0.6956
        ],
        [
          0.7675,
          0.6879
        ],
        [
          0.7775,
          0.6815
        ],
        [
          0.7875,
          0.6747
        ],
        [
          0.7975,
          0.6683
        ],
        [
          0.8076,
          0.6624
        ],
        [
          0.818,
          0.6542
        ],
        [
          0.828,
          0.647
        ],
        [
          0.838,
          0.6397
        ],
        [
          0.848,
          0.6333
        ],
        [
          0.8581,
          0.6256
        ],
        [
          0.8685,
          0.6169
        ],
        [
          0.8785,
          0.6092
        ],
        [
          0.8885,
          0.6033
        ],
        [
          0.8985,
          0.596
        ],
        [
          0.9086,
          0.5878
        ],
        [
          0.919,
          0.5783
        ],
        [
          0.929,
          0.5728
        ],
        [
          0.939,
          0.5651
        ],
        [
          0.949,
          0.5582
        ],
        [
          0.9591,
          0.5523
        ],
        [
          0.9695,
          0.5446
        ],
        [
          0.9795,
          0.5387
        ],
        [
          0.9895,
          0.5314
        ],
        [
          0.9995,
          0.5237
        ],
        [
          1.0,
          0.5237
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.11056945615566122,
        "difference": -0.001200621708738382,
        "worse": 0.54,
        "kept_right": 0.10908385605153552,
        "kept_wrong": 0.0014856001041256644,
        "handed_off": 0.8894305438443388
      },
      "usd_per_1000": 4.9666792100563155
    },
    {
      "decision": 373,
      "frontier": 124,
      "frontier_accuracy": 0.9667879890809827,
      "in_sample": {
        "threshold": 0.988141,
        "answered": 0.0805277525022748,
        "accuracy": 0.9667879890809827
      },
      "curve": [
        [
          0.0,
          0.9668
        ],
        [
          0.01,
          0.9668
        ],
        [
          0.02,
          0.9668
        ],
        [
          0.03,
          0.9668
        ],
        [
          0.04,
          0.9668
        ],
        [
          0.05,
          0.9668
        ],
        [
          0.0601,
          0.9668
        ],
        [
          0.0705,
          0.9668
        ],
        [
          0.0805,
          0.9668
        ],
        [
          0.0905,
          0.9663
        ],
        [
          0.1005,
          0.9663
        ],
        [
          0.1106,
          0.9659
        ],
        [
          0.121,
          0.965
        ],
        [
          0.131,
          0.965
        ],
        [
          0.141,
          0.9641
        ],
        [
          0.151,
          0.9645
        ],
        [
          0.1611,
          0.9636
        ],
        [
          0.1715,
          0.9627
        ],
        [
          0.1815,
          0.9618
        ],
        [
          0.1915,
          0.9591
        ],
        [
          0.2015,
          0.9586
        ],
        [
          0.2116,
          0.9568
        ],
        [
          0.222,
          0.955
        ],
        [
          0.232,
          0.9531
        ],
        [
          0.242,
          0.9509
        ],
        [
          0.252,
          0.9495
        ],
        [
          0.2621,
          0.9468
        ],
        [
          0.2725,
          0.944
        ],
        [
          0.2825,
          0.9399
        ],
        [
          0.2925,
          0.9386
        ],
        [
          0.3025,
          0.9354
        ],
        [
          0.3126,
          0.9322
        ],
        [
          0.323,
          0.9286
        ],
        [
          0.333,
          0.9236
        ],
        [
          0.343,
          0.9199
        ],
        [
          0.353,
          0.9163
        ],
        [
          0.3631,
          0.9099
        ],
        [
          0.3735,
          0.9045
        ],
        [
          0.3835,
          0.9004
        ],
        [
          0.3935,
          0.8976
        ],
        [
          0.4035,
          0.8935
        ],
        [
          0.4136,
          0.8885
        ],
        [
          0.424,
          0.8844
        ],
        [
          0.434,
          0.8799
        ],
        [
          0.444,
          0.8758
        ],
        [
          0.454,
          0.8708
        ],
        [
          0.4641,
          0.8649
        ],
        [
          0.4745,
          0.8581
        ],
        [
          0.4845,
          0.8517
        ],
        [
          0.4945,
          0.8467
        ],
        [
          0.5045,
          0.8408
        ],
        [
          0.5146,
          0.8358
        ],
        [
          0.525,
          0.8303
        ],
        [
          0.535,
          0.8262
        ],
        [
          0.545,
          0.8212
        ],
        [
          0.5551,
          0.818
        ],
        [
          0.5655,
          0.8121
        ],
        [
          0.5755,
          0.8066
        ],
        [
          0.5855,
          0.8021
        ],
        [
          0.5955,
          0.7971
        ],
        [
          0.6056,
          0.7894
        ],
        [
          0.616,
          0.783
        ],
        [
          0.626,
          0.7766
        ],
        [
          0.636,
          0.7721
        ],
        [
          0.646,
          0.7689
        ],
        [
          0.6561,
          0.7616
        ],
        [
          0.6665,
          0.7557
        ],
        [
          0.6765,
          0.7484
        ],
        [
          0.6865,
          0.7416
        ],
        [
          0.6965,
          0.7352
        ],
        [
          0.7066,
          0.7302
        ],
        [
          0.717,
          0.7243
        ],
        [
          0.727,
          0.7197
        ],
        [
          0.737,
          0.712
        ],
        [
          0.747,
          0.7034
        ],
        [
          0.7571,
          0.6975
        ],
        [
          0.7675,
          0.6897
        ],
        [
          0.7775,
          0.6833
        ],
        [
          0.7875,
          0.6761
        ],
        [
          0.7975,
          0.6697
        ],
        [
          0.8076,
          0.6642
        ],
        [
          0.818,
          0.6556
        ],
        [
          0.828,
          0.6483
        ],
        [
          0.838,
          0.641
        ],
        [
          0.848,
          0.6342
        ],
        [
          0.8581,
          0.626
        ],
        [
          0.8685,
          0.6178
        ],
        [
          0.8785,
          0.6101
        ],
        [
          0.8885,
          0.6042
        ],
        [
          0.8985,
          0.5974
        ],
        [
          0.9086,
          0.5892
        ],
        [
          0.919,
          0.5801
        ],
        [
          0.929,
          0.5742
        ],
        [
          0.939,
          0.566
        ],
        [
          0.949,
          0.5587
        ],
        [
          0.9591,
          0.5523
        ],
        [
          0.9695,
          0.5446
        ],
        [
          0.9795,
          0.5391
        ],
        [
          0.9895,
          0.5314
        ],
        [
          0.9995,
          0.5237
        ],
        [
          1.0,
          0.5237
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.10212604301616059,
        "difference": -0.001218767681414942,
        "worse": 0.5566666666666666,
        "kept_right": 0.1008345093841498,
        "kept_wrong": 0.0012915336320107663,
        "handed_off": 0.8978739569838391
      },
      "usd_per_1000": 1.0288008501200037
    },
    {
      "decision": 373,
      "frontier": 278,
      "frontier_accuracy": 0.9499545040946314,
      "in_sample": {
        "threshold": 0.980397,
        "answered": 0.10828025477707007,
        "accuracy": 0.9499545040946314
      },
      "curve": [
        [
          0.0,
          0.95
        ],
        [
          0.01,
          0.95
        ],
        [
          0.02,
          0.95
        ],
        [
          0.03,
          0.95
        ],
        [
          0.04,
          0.95
        ],
        [
          0.05,
          0.95
        ],
        [
          0.0601,
          0.95
        ],
        [
          0.0705,
          0.95
        ],
        [
          0.0805,
          0.95
        ],
        [
          0.0905,
          0.95
        ],
        [
          0.1005,
          0.95
        ],
        [
          0.1106,
          0.9495
        ],
        [
          0.121,
          0.9486
        ],
        [
          0.131,
          0.949
        ],
        [
          0.141,
          0.9486
        ],
        [
          0.151,
          0.949
        ],
        [
          0.1611,
          0.9481
        ],
        [
          0.1715,
          0.9472
        ],
        [
          0.1815,
          0.9463
        ],
        [
          0.1915,
          0.9436
        ],
        [
          0.2015,
          0.9431
        ],
        [
          0.2116,
          0.9418
        ],
        [
          0.222,
          0.9404
        ],
        [
          0.232,
          0.9386
        ],
        [
          0.242,
          0.9363
        ],
        [
          0.252,
          0.9349
        ],
        [
          0.2621,
          0.9322
        ],
        [
          0.2725,
          0.9299
        ],
        [
          0.2825,
          0.9263
        ],
        [
          0.2925,
          0.9249
        ],
        [
          0.3025,
          0.9222
        ],
        [
          0.3126,
          0.919
        ],
        [
          0.323,
          0.9154
        ],
        [
          0.333,
          0.9108
        ],
        [
          0.343,
          0.9076
        ],
        [
          0.353,
          0.904
        ],
        [
          0.3631,
          0.8972
        ],
        [
          0.3735,
          0.8922
        ],
        [
          0.3835,
          0.8885
        ],
        [
          0.3935,
          0.8854
        ],
        [
          0.4035,
          0.8813
        ],
        [
          0.4136,
          0.8772
        ],
        [
          0.424,
          0.8731
        ],
        [
          0.434,
          0.8685
        ],
        [
          0.444,
          0.8649
        ],
        [
          0.454,
          0.8608
        ],
        [
          0.4641,
          0.8558
        ],
        [
          0.4745,
          0.849
        ],
        [
          0.4845,
          0.8421
        ],
        [
          0.4945,
          0.8371
        ],
        [
          0.5045,
          0.8312
        ],
        [
          0.5146,
          0.8267
        ],
        [
          0.525,
          0.8212
        ],
        [
          0.535,
          0.8167
        ],
        [
          0.545,
          0.8116
        ],
        [
          0.5551,
          0.8089
        ],
        [
          0.5655,
          0.8039
        ],
        [
          0.5755,
          0.7989
        ],
        [
          0.5855,
          0.7939
        ],
        [
          0.5955,
          0.7884
        ],
        [
          0.6056,
          0.7807
        ],
        [
          0.616,
          0.7743
        ],
        [
          0.626,
          0.768
        ],
        [
          0.636,
          0.7634
        ],
        [
          0.646,
          0.7598
        ],
        [
          0.6561,
          0.7525
        ],
        [
          0.6665,
          0.747
        ],
        [
          0.6765,
          0.7402
        ],
        [
          0.6865,
          0.7338
        ],
        [
          0.6965,
          0.7279
        ],
        [
          0.7066,
          0.7229
        ],
        [
          0.717,
          0.7175
        ],
        [
          0.727,
          0.7134
        ],
        [
          0.737,
          0.707
        ],
        [
          0.747,
          0.6984
        ],
        [
          0.7571,
          0.6924
        ],
        [
          0.7675,
          0.6847
        ],
        [
          0.7775,
          0.6783
        ],
        [
          0.7875,
          0.6711
        ],
        [
          0.7975,
          0.6647
        ],
        [
          0.8076,
          0.6592
        ],
        [
          0.818,
          0.6506
        ],
        [
          0.828,
          0.6438
        ],
        [
          0.838,
          0.6365
        ],
        [
          0.848,
          0.6306
        ],
        [
          0.8581,
          0.6233
        ],
        [
          0.8685,
          0.6151
        ],
        [
          0.8785,
          0.6078
        ],
        [
          0.8885,
          0.6019
        ],
        [
          0.8985,
          0.5946
        ],
        [
          0.9086,
          0.5864
        ],
        [
          0.919,
          0.5769
        ],
        [
          0.929,
          0.5723
        ],
        [
          0.939,
          0.5646
        ],
        [
          0.949,
          0.5573
        ],
        [
          0.9591,
          0.551
        ],
        [
          0.9695,
          0.5441
        ],
        [
          0.9795,
          0.5378
        ],
        [
          0.9895,
          0.5314
        ],
        [
          0.9995,
          0.5237
        ],
        [
          1.0,
          0.5237
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.13019858043870763,
        "difference": -0.0015461867074406225,
        "worse": 0.6033333333333334,
        "kept_right": 0.12761861534437627,
        "kept_wrong": 0.0025799650943314485,
        "handed_off": 0.8698014195612925
      },
      "usd_per_1000": 5.675254664591427
    },
    {
      "decision": 373,
      "frontier": 275,
      "frontier_accuracy": 0.9426751592356688,
      "in_sample": {
        "threshold": 0.980397,
        "answered": 0.10828025477707007,
        "accuracy": 0.9426751592356688
      },
      "curve": [
        [
          0.0,
          0.9427
        ],
        [
          0.01,
          0.9427
        ],
        [
          0.02,
          0.9427
        ],
        [
          0.03,
          0.9427
        ],
        [
          0.04,
          0.9427
        ],
        [
          0.05,
          0.9427
        ],
        [
          0.0601,
          0.9427
        ],
        [
          0.0705,
          0.9427
        ],
        [
          0.0805,
          0.9427
        ],
        [
          0.0905,
          0.9427
        ],
        [
          0.1005,
          0.9427
        ],
        [
          0.1106,
          0.9422
        ],
        [
          0.121,
          0.9413
        ],
        [
          0.131,
          0.9413
        ],
        [
          0.141,
          0.9404
        ],
        [
          0.151,
          0.9409
        ],
        [
          0.1611,
          0.9399
        ],
        [
          0.1715,
          0.939
        ],
        [
          0.1815,
          0.9381
        ],
        [
          0.1915,
          0.9354
        ],
        [
          0.2015,
          0.9349
        ],
        [
          0.2116,
          0.9336
        ],
        [
          0.222,
          0.9322
        ],
        [
          0.232,
          0.9308
        ],
        [
          0.242,
          0.9286
        ],
        [
          0.252,
          0.9272
        ],
        [
          0.2621,
          0.9245
        ],
        [
          0.2725,
          0.9222
        ],
        [
          0.2825,
          0.9186
        ],
        [
          0.2925,
          0.9172
        ],
        [
          0.3025,
          0.9145
        ],
        [
          0.3126,
          0.9108
        ],
        [
          0.323,
          0.9072
        ],
        [
          0.333,
          0.9026
        ],
        [
          0.343,
          0.899
        ],
        [
          0.353,
          0.8954
        ],
        [
          0.3631,
          0.8885
        ],
        [
          0.3735,
          0.8835
        ],
        [
          0.3835,
          0.8803
        ],
        [
          0.3935,
          0.8776
        ],
        [
          0.4035,
          0.8735
        ],
        [
          0.4136,
          0.869
        ],
        [
          0.424,
          0.8653
        ],
        [
          0.434,
          0.8612
        ],
        [
          0.444,
          0.8581
        ],
        [
          0.454,
          0.8535
        ],
        [
          0.4641,
          0.8476
        ],
        [
          0.4745,
          0.8408
        ],
        [
          0.4845,
          0.8344
        ],
        [
          0.4945,
          0.8298
        ],
        [
          0.5045,
          0.8235
        ],
        [
          0.5146,
          0.8189
        ],
        [
          0.525,
          0.8135
        ],
        [
          0.535,
          0.8089
        ],
        [
          0.545,
          0.8039
        ],
        [
          0.5551,
          0.8016
        ],
        [
          0.5655,
          0.7962
        ],
        [
          0.5755,
          0.7912
        ],
        [
          0.5855,
          0.7875
        ],
        [
          0.5955,
          0.783
        ],
        [
          0.6056,
          0.7762
        ],
        [
          0.616,
          0.7698
        ],
        [
          0.626,
          0.7648
        ],
        [
          0.636,
          0.7607
        ],
        [
          0.646,
          0.758
        ],
        [
          0.6561,
          0.7507
        ],
        [
          0.6665,
          0.7452
        ],
        [
          0.6765,
          0.7384
        ],
        [
          0.6865,
          0.7316
        ],
        [
          0.6965,
          0.7252
        ],
        [
          0.7066,
          0.7211
        ],
        [
          0.717,
          0.7152
        ],
        [
          0.727,
          0.7106
        ],
        [
          0.737,
          0.7034
        ],
        [
          0.747,
          0.6947
        ],
        [
          0.7571,
          0.6893
        ],
        [
          0.7675,
          0.682
        ],
        [
          0.7775,
          0.6761
        ],
        [
          0.7875,
          0.6683
        ],
        [
          0.7975,
          0.6624
        ],
        [
          0.8076,
          0.657
        ],
        [
          0.818,
          0.6488
        ],
        [
          0.828,
          0.6415
        ],
        [
          0.838,
          0.6347
        ],
        [
          0.848,
          0.6278
        ],
        [
          0.8581,
          0.6201
        ],
        [
          0.8685,
          0.6128
        ],
        [
          0.8785,
          0.6056
        ],
        [
          0.8885,
          0.6001
        ],
        [
          0.8985,
          0.5928
        ],
        [
          0.9086,
          0.5851
        ],
        [
          0.919,
          0.5764
        ],
        [
          0.929,
          0.571
        ],
        [
          0.939,
          0.5628
        ],
        [
          0.949,
          0.5564
        ],
        [
          0.9591,
          0.551
        ],
        [
          0.9695,
          0.5441
        ],
        [
          0.9795,
          0.5378
        ],
        [
          0.9895,
          0.5305
        ],
        [
          0.9995,
          0.5237
        ],
        [
          1.0,
          0.5237
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.11780150869653748,
        "difference": -0.001139880458378212,
        "worse": 0.5066666666666667,
        "kept_right": 0.11606438213610365,
        "kept_wrong": 0.0017371265604338778,
        "handed_off": 0.8821984913034623
      },
      "usd_per_1000": 0.4677710754390745
    },
    {
      "decision": 373,
      "frontier": 192,
      "frontier_accuracy": 0.9331210191082803,
      "in_sample": {
        "threshold": 0.980397,
        "answered": 0.10828025477707007,
        "accuracy": 0.9331210191082803
      },
      "curve": [
        [
          0.0,
          0.9331
        ],
        [
          0.01,
          0.9331
        ],
        [
          0.02,
          0.9331
        ],
        [
          0.03,
          0.9331
        ],
        [
          0.04,
          0.9331
        ],
        [
          0.05,
          0.9331
        ],
        [
          0.0601,
          0.9331
        ],
        [
          0.0705,
          0.9331
        ],
        [
          0.0805,
          0.9331
        ],
        [
          0.0905,
          0.9331
        ],
        [
          0.1005,
          0.9331
        ],
        [
          0.1106,
          0.9327
        ],
        [
          0.121,
          0.9318
        ],
        [
          0.131,
          0.9318
        ],
        [
          0.141,
          0.9313
        ],
        [
          0.151,
          0.9313
        ],
        [
          0.1611,
          0.9304
        ],
        [
          0.1715,
          0.929
        ],
        [
          0.1815,
          0.9281
        ],
        [
          0.1915,
          0.9254
        ],
        [
          0.2015,
          0.9249
        ],
        [
          0.2116,
          0.9231
        ],
        [
          0.222,
          0.9213
        ],
        [
          0.232,
          0.9195
        ],
        [
          0.242,
          0.9177
        ],
        [
          0.252,
          0.9172
        ],
        [
          0.2621,
          0.9145
        ],
        [
          0.2725,
          0.9122
        ],
        [
          0.2825,
          0.9081
        ],
        [
          0.2925,
          0.9067
        ],
        [
          0.3025,
          0.9035
        ],
        [
          0.3126,
          0.9008
        ],
        [
          0.323,
          0.8981
        ],
        [
          0.333,
          0.8931
        ],
        [
          0.343,
          0.8894
        ],
        [
          0.353,
          0.8854
        ],
        [
          0.3631,
          0.8785
        ],
        [
          0.3735,
          0.8735
        ],
        [
          0.3835,
          0.8699
        ],
        [
          0.3935,
          0.8672
        ],
        [
          0.4035,
          0.8635
        ],
        [
          0.4136,
          0.8599
        ],
        [
          0.424,
          0.8558
        ],
        [
          0.434,
          0.8508
        ],
        [
          0.444,
          0.8467
        ],
        [
          0.454,
          0.8426
        ],
        [
          0.4641,
          0.838
        ],
        [
          0.4745,
          0.8317
        ],
        [
          0.4845,
          0.8258
        ],
        [
          0.4945,
          0.8212
        ],
        [
          0.5045,
          0.8148
        ],
        [
          0.5146,
          0.8098
        ],
        [
          0.525,
          0.8048
        ],
        [
          0.535,
          0.8012
        ],
        [
          0.545,
          0.7966
        ],
        [
          0.5551,
          0.7939
        ],
        [
          0.5655,
          0.7884
        ],
        [
          0.5755,
          0.7843
        ],
        [
          0.5855,
          0.7793
        ],
        [
          0.5955,
          0.7748
        ],
        [
          0.6056,
          0.7671
        ],
        [
          0.616,
          0.7611
        ],
        [
          0.626,
          0.7548
        ],
        [
          0.636,
          0.7498
        ],
        [
          0.646,
          0.7466
        ],
        [
          0.6561,
          0.7402
        ],
        [
          0.6665,
          0.7357
        ],
        [
          0.6765,
          0.7302
        ],
        [
          0.6865,
          0.7252
        ],
        [
          0.6965,
          0.7193
        ],
        [
          0.7066,
          0.7147
        ],
        [
          0.717,
          0.7097
        ],
        [
          0.727,
          0.7056
        ],
        [
          0.737,
          0.7002
        ],
        [
          0.747,
          0.6924
        ],
        [
          0.7571,
          0.687
        ],
        [
          0.7675,
          0.6802
        ],
        [
          0.7775,
          0.6742
        ],
        [
          0.7875,
          0.6665
        ],
        [
          0.7975,
          0.662
        ],
        [
          0.8076,
          0.6565
        ],
        [
          0.818,
          0.6488
        ],
        [
          0.828,
          0.6415
        ],
        [
          0.838,
          0.6351
        ],
        [
          0.848,
          0.6292
        ],
        [
          0.8581,
          0.6224
        ],
        [
          0.8685,
          0.6137
        ],
        [
          0.8785,
          0.6065
        ],
        [
          0.8885,
          0.6005
        ],
        [
          0.8985,
          0.5933
        ],
        [
          0.9086,
          0.5855
        ],
        [
          0.919,
          0.5769
        ],
        [
          0.929,
          0.5719
        ],
        [
          0.939,
          0.5641
        ],
        [
          0.949,
          0.5573
        ],
        [
          0.9591,
          0.551
        ],
        [
          0.9695,
          0.5441
        ],
        [
          0.9795,
          0.5382
        ],
        [
          0.9895,
          0.5314
        ],
        [
          0.9995,
          0.5237
        ],
        [
          1.0,
          0.5237
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.11472414893539785,
        "difference": -0.000845802902007393,
        "worse": 0.48333333333333334,
        "kept_right": 0.11329018807803459,
        "kept_wrong": 0.001433960857363309,
        "handed_off": 0.8852758510646019
      },
      "usd_per_1000": 10.291536141984642
    },
    {
      "decision": 373,
      "frontier": 123,
      "frontier_accuracy": 0.9326660600545951,
      "in_sample": {
        "threshold": 0.988141,
        "answered": 0.0805277525022748,
        "accuracy": 0.9326660600545951
      },
      "curve": [
        [
          0.0,
          0.9327
        ],
        [
          0.01,
          0.9327
        ],
        [
          0.02,
          0.9327
        ],
        [
          0.03,
          0.9327
        ],
        [
          0.04,
          0.9327
        ],
        [
          0.05,
          0.9327
        ],
        [
          0.0601,
          0.9327
        ],
        [
          0.0705,
          0.9327
        ],
        [
          0.0805,
          0.9327
        ],
        [
          0.0905,
          0.9322
        ],
        [
          0.1005,
          0.9322
        ],
        [
          0.1106,
          0.9318
        ],
        [
          0.121,
          0.9308
        ],
        [
          0.131,
          0.9308
        ],
        [
          0.141,
          0.9299
        ],
        [
          0.151,
          0.9308
        ],
        [
          0.1611,
          0.9299
        ],
        [
          0.1715,
          0.929
        ],
        [
          0.1815,
          0.9281
        ],
        [
          0.1915,
          0.9254
        ],
        [
          0.2015,
          0.9249
        ],
        [
          0.2116,
          0.924
        ],
        [
          0.222,
          0.9222
        ],
        [
          0.232,
          0.9208
        ],
        [
          0.242,
          0.9181
        ],
        [
          0.252,
          0.9167
        ],
        [
          0.2621,
          0.914
        ],
        [
          0.2725,
          0.9117
        ],
        [
          0.2825,
          0.9076
        ],
        [
          0.2925,
          0.9063
        ],
        [
          0.3025,
          0.9035
        ],
        [
          0.3126,
          0.9004
        ],
        [
          0.323,
          0.8967
        ],
        [
          0.333,
          0.8922
        ],
        [
          0.343,
          0.8885
        ],
        [
          0.353,
          0.8854
        ],
        [
          0.3631,
          0.8785
        ],
        [
          0.3735,
          0.8735
        ],
        [
          0.3835,
          0.8703
        ],
        [
          0.3935,
          0.8672
        ],
        [
          0.4035,
          0.8626
        ],
        [
          0.4136,
          0.8585
        ],
        [
          0.424,
          0.8544
        ],
        [
          0.434,
          0.8503
        ],
        [
          0.444,
          0.8471
        ],
        [
          0.454,
          0.8421
        ],
        [
          0.4641,
          0.8362
        ],
        [
          0.4745,
          0.8303
        ],
        [
          0.4845,
          0.8239
        ],
        [
          0.4945,
          0.8198
        ],
        [
          0.5045,
          0.8139
        ],
        [
          0.5146,
          0.8107
        ],
        [
          0.525,
          0.8053
        ],
        [
          0.535,
          0.8003
        ],
        [
          0.545,
          0.7957
        ],
        [
          0.5551,
          0.7921
        ],
        [
          0.5655,
          0.7866
        ],
        [
          0.5755,
          0.7816
        ],
        [
          0.5855,
          0.7775
        ],
        [
          0.5955,
          0.773
        ],
        [
          0.6056,
          0.7657
        ],
        [
          0.616,
          0.7607
        ],
        [
          0.626,
          0.7557
        ],
        [
          0.636,
          0.7511
        ],
        [
          0.646,
          0.7484
        ],
        [
          0.6561,
          0.7429
        ],
        [
          0.6665,
          0.737
        ],
        [
          0.6765,
          0.7293
        ],
        [
          0.6865,
          0.7225
        ],
        [
          0.6965,
          0.7161
        ],
        [
          0.7066,
          0.712
        ],
        [
          0.717,
          0.7066
        ],
        [
          0.727,
          0.7015
        ],
        [
          0.737,
          0.6943
        ],
        [
          0.747,
          0.6865
        ],
        [
          0.7571,
          0.6815
        ],
        [
          0.7675,
          0.6747
        ],
        [
          0.7775,
          0.6688
        ],
        [
          0.7875,
          0.6629
        ],
        [
          0.7975,
          0.657
        ],
        [
          0.8076,
          0.652
        ],
        [
          0.818,
          0.6438
        ],
        [
          0.828,
          0.6369
        ],
        [
          0.838,
          0.6301
        ],
        [
          0.848,
          0.6233
        ],
        [
          0.8581,
          0.6156
        ],
        [
          0.8685,
          0.6078
        ],
        [
          0.8785,
          0.6005
        ],
        [
          0.8885,
          0.596
        ],
        [
          0.8985,
          0.5892
        ],
        [
          0.9086,
          0.5819
        ],
        [
          0.919,
          0.5728
        ],
        [
          0.929,
          0.5687
        ],
        [
          0.939,
          0.5614
        ],
        [
          0.949,
          0.556
        ],
        [
          0.9591,
          0.5505
        ],
        [
          0.9695,
          0.5441
        ],
        [
          0.9795,
          0.5387
        ],
        [
          0.9895,
          0.5314
        ],
        [
          0.9995,
          0.5237
        ],
        [
          1.0,
          0.5237
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.10798393900938104,
        "difference": -0.0015644321099259678,
        "worse": 0.5733333333333334,
        "kept_right": 0.10625577937335375,
        "kept_wrong": 0.001728159636027294,
        "handed_off": 0.8920160609906188
      },
      "usd_per_1000": 0.2534495002573519
    },
    {
      "decision": 373,
      "frontier": 274,
      "frontier_accuracy": 0.9085532302092811,
      "in_sample": {
        "threshold": 0.980397,
        "answered": 0.10828025477707007,
        "accuracy": 0.9085532302092811
      },
      "curve": [
        [
          0.0,
          0.9086
        ],
        [
          0.01,
          0.9086
        ],
        [
          0.02,
          0.9086
        ],
        [
          0.03,
          0.9086
        ],
        [
          0.04,
          0.9086
        ],
        [
          0.05,
          0.9086
        ],
        [
          0.0601,
          0.9086
        ],
        [
          0.0705,
          0.9086
        ],
        [
          0.0805,
          0.9086
        ],
        [
          0.0905,
          0.9086
        ],
        [
          0.1005,
          0.9086
        ],
        [
          0.1106,
          0.9081
        ],
        [
          0.121,
          0.9072
        ],
        [
          0.131,
          0.9072
        ],
        [
          0.141,
          0.9067
        ],
        [
          0.151,
          0.9067
        ],
        [
          0.1611,
          0.9063
        ],
        [
          0.1715,
          0.9054
        ],
        [
          0.1815,
          0.9045
        ],
        [
          0.1915,
          0.9017
        ],
        [
          0.2015,
          0.9013
        ],
        [
          0.2116,
          0.8999
        ],
        [
          0.222,
          0.899
        ],
        [
          0.232,
          0.8976
        ],
        [
          0.242,
          0.8954
        ],
        [
          0.252,
          0.8944
        ],
        [
          0.2621,
          0.8922
        ],
        [
          0.2725,
          0.8899
        ],
        [
          0.2825,
          0.8858
        ],
        [
          0.2925,
          0.8849
        ],
        [
          0.3025,
          0.8826
        ],
        [
          0.3126,
          0.879
        ],
        [
          0.323,
          0.8758
        ],
        [
          0.333,
          0.8708
        ],
        [
          0.343,
          0.8672
        ],
        [
          0.353,
          0.8649
        ],
        [
          0.3631,
          0.8594
        ],
        [
          0.3735,
          0.8549
        ],
        [
          0.3835,
          0.8521
        ],
        [
          0.3935,
          0.8494
        ],
        [
          0.4035,
          0.8458
        ],
        [
          0.4136,
          0.8412
        ],
        [
          0.424,
          0.8371
        ],
        [
          0.434,
          0.8335
        ],
        [
          0.444,
          0.8317
        ],
        [
          0.454,
          0.8271
        ],
        [
          0.4641,
          0.8226
        ],
        [
          0.4745,
          0.8171
        ],
        [
          0.4845,
          0.8107
        ],
        [
          0.4945,
          0.8066
        ],
        [
          0.5045,
          0.8007
        ],
        [
          0.5146,
          0.7966
        ],
        [
          0.525,
          0.7921
        ],
        [
          0.535,
          0.788
        ],
        [
          0.545,
          0.7834
        ],
        [
          0.5551,
          0.7807
        ],
        [
          0.5655,
          0.7762
        ],
        [
          0.5755,
          0.7712
        ],
        [
          0.5855,
          0.7684
        ],
        [
          0.5955,
          0.7643
        ],
        [
          0.6056,
          0.758
        ],
        [
          0.616,
          0.7516
        ],
        [
          0.626,
          0.7457
        ],
        [
          0.636,
          0.7416
        ],
        [
          0.646,
          0.7398
        ],
        [
          0.6561,
          0.7329
        ],
        [
          0.6665,
          0.727
        ],
        [
          0.6765,
          0.7202
        ],
        [
          0.6865,
          0.7134
        ],
        [
          0.6965,
          0.7079
        ],
        [
          0.7066,
          0.7034
        ],
        [
          0.717,
          0.6988
        ],
        [
          0.727,
          0.6947
        ],
        [
          0.737,
          0.6888
        ],
        [
          0.747,
          0.6811
        ],
        [
          0.7571,
          0.6756
        ],
        [
          0.7675,
          0.6688
        ],
        [
          0.7775,
          0.6629
        ],
        [
          0.7875,
          0.6561
        ],
        [
          0.7975,
          0.6515
        ],
        [
          0.8076,
          0.6474
        ],
        [
          0.818,
          0.6392
        ],
        [
          0.828,
          0.6324
        ],
        [
          0.838,
          0.6251
        ],
        [
          0.848,
          0.6197
        ],
        [
          0.8581,
          0.6119
        ],
        [
          0.8685,
          0.6042
        ],
        [
          0.8785,
          0.5965
        ],
        [
          0.8885,
          0.5919
        ],
        [
          0.8985,
          0.5855
        ],
        [
          0.9086,
          0.5783
        ],
        [
          0.919,
          0.5714
        ],
        [
          0.929,
          0.5664
        ],
        [
          0.939,
          0.5591
        ],
        [
          0.949,
          0.5537
        ],
        [
          0.9591,
          0.5491
        ],
        [
          0.9695,
          0.5423
        ],
        [
          0.9795,
          0.5364
        ],
        [
          0.9895,
          0.53
        ],
        [
          0.9995,
          0.5237
        ],
        [
          1.0,
          0.5237
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.12045085540026816,
        "difference": -0.0011822825868936621,
        "worse": 0.5066666666666667,
        "kept_right": 0.11854401527111615,
        "kept_wrong": 0.0019068401291520432,
        "handed_off": 0.8795491445997319
      },
      "usd_per_1000": 0.3887723334785048
    },
    {
      "decision": 373,
      "frontier": 47,
      "frontier_accuracy": 0.9044585987261147,
      "in_sample": {
        "threshold": 0.959943,
        "answered": 0.15969062784349408,
        "accuracy": 0.9044585987261147
      },
      "curve": [
        [
          0.0,
          0.9045
        ],
        [
          0.01,
          0.9045
        ],
        [
          0.02,
          0.9045
        ],
        [
          0.03,
          0.9045
        ],
        [
          0.04,
          0.9045
        ],
        [
          0.05,
          0.9045
        ],
        [
          0.0601,
          0.9045
        ],
        [
          0.0705,
          0.9045
        ],
        [
          0.0805,
          0.9045
        ],
        [
          0.0905,
          0.9045
        ],
        [
          0.1005,
          0.9045
        ],
        [
          0.1106,
          0.904
        ],
        [
          0.121,
          0.9035
        ],
        [
          0.131,
          0.904
        ],
        [
          0.141,
          0.9035
        ],
        [
          0.151,
          0.9045
        ],
        [
          0.1611,
          0.904
        ],
        [
          0.1715,
          0.9031
        ],
        [
          0.1815,
          0.9022
        ],
        [
          0.1915,
          0.8995
        ],
        [
          0.2015,
          0.899
        ],
        [
          0.2116,
          0.8976
        ],
        [
          0.222,
          0.8958
        ],
        [
          0.232,
          0.8949
        ],
        [
          0.242,
          0.8922
        ],
        [
          0.252,
          0.8917
        ],
        [
          0.2621,
          0.889
        ],
        [
          0.2725,
          0.8872
        ],
        [
          0.2825,
          0.8831
        ],
        [
          0.2925,
          0.8813
        ],
        [
          0.3025,
          0.8785
        ],
        [
          0.3126,
          0.8758
        ],
        [
          0.323,
          0.8726
        ],
        [
          0.333,
          0.8685
        ],
        [
          0.343,
          0.8653
        ],
        [
          0.353,
          0.8621
        ],
        [
          0.3631,
          0.8558
        ],
        [
          0.3735,
          0.8517
        ],
        [
          0.3835,
          0.8485
        ],
        [
          0.3935,
          0.8453
        ],
        [
          0.4035,
          0.8421
        ],
        [
          0.4136,
          0.838
        ],
        [
          0.424,
          0.8344
        ],
        [
          0.434,
          0.8294
        ],
        [
          0.444,
          0.828
        ],
        [
          0.454,
          0.823
        ],
        [
          0.4641,
          0.8194
        ],
        [
          0.4745,
          0.813
        ],
        [
          0.4845,
          0.8071
        ],
        [
          0.4945,
          0.803
        ],
        [
          0.5045,
          0.7966
        ],
        [
          0.5146,
          0.7925
        ],
        [
          0.525,
          0.7889
        ],
        [
          0.535,
          0.7839
        ],
        [
          0.545,
          0.7789
        ],
        [
          0.5551,
          0.7762
        ],
        [
          0.5655,
          0.7712
        ],
        [
          0.5755,
          0.7675
        ],
        [
          0.5855,
          0.7625
        ],
        [
          0.5955,
          0.7571
        ],
        [
          0.6056,
          0.7502
        ],
        [
          0.616,
          0.7448
        ],
        [
          0.626,
          0.7398
        ],
        [
          0.636,
          0.7357
        ],
        [
          0.646,
          0.7338
        ],
        [
          0.6561,
          0.7275
        ],
        [
          0.6665,
          0.7225
        ],
        [
          0.6765,
          0.7161
        ],
        [
          0.6865,
          0.7116
        ],
        [
          0.6965,
          0.7047
        ],
        [
          0.7066,
          0.7002
        ],
        [
          0.717,
          0.6947
        ],
        [
          0.727,
          0.6911
        ],
        [
          0.737,
          0.6847
        ],
        [
          0.747,
          0.677
        ],
        [
          0.7571,
          0.6711
        ],
        [
          0.7675,
          0.6652
        ],
        [
          0.7775,
          0.6597
        ],
        [
          0.7875,
          0.6529
        ],
        [
          0.7975,
          0.6479
        ],
        [
          0.8076,
          0.6429
        ],
        [
          0.818,
          0.6351
        ],
        [
          0.828,
          0.6292
        ],
        [
          0.838,
          0.6233
        ],
        [
          0.848,
          0.6174
        ],
        [
          0.8581,
          0.6101
        ],
        [
          0.8685,
          0.6033
        ],
        [
          0.8785,
          0.5955
        ],
        [
          0.8885,
          0.5905
        ],
        [
          0.8985,
          0.5833
        ],
        [
          0.9086,
          0.576
        ],
        [
          0.919,
          0.5682
        ],
        [
          0.929,
          0.5632
        ],
        [
          0.939,
          0.5555
        ],
        [
          0.949,
          0.5514
        ],
        [
          0.9591,
          0.5455
        ],
        [
          0.9695,
          0.5409
        ],
        [
          0.9795,
          0.5359
        ],
        [
          0.9895,
          0.5296
        ],
        [
          0.9995,
          0.5237
        ],
        [
          1.0,
          0.5237
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.15085078172217523,
        "difference": -0.00163415414638064,
        "worse": 0.6233333333333333,
        "kept_right": 0.1466063479147157,
        "kept_wrong": 0.004244433807459616,
        "handed_off": 0.8491492182778251
      },
      "usd_per_1000": 4.200671536599672
    },
    {
      "decision": 373,
      "frontier": 193,
      "frontier_accuracy": 0.89171974522293,
      "in_sample": {
        "threshold": 0.979399,
        "answered": 0.11282984531392175,
        "accuracy": 0.89171974522293
      },
      "curve": [
        [
          0.0,
          0.8917
        ],
        [
          0.01,
          0.8917
        ],
        [
          0.02,
          0.8917
        ],
        [
          0.03,
          0.8917
        ],
        [
          0.04,
          0.8917
        ],
        [
          0.05,
          0.8917
        ],
        [
          0.0601,
          0.8917
        ],
        [
          0.0705,
          0.8917
        ],
        [
          0.0805,
          0.8917
        ],
        [
          0.0905,
          0.8917
        ],
        [
          0.1005,
          0.8917
        ],
        [
          0.1106,
          0.8917
        ],
        [
          0.121,
          0.8908
        ],
        [
          0.131,
          0.8913
        ],
        [
          0.141,
          0.8908
        ],
        [
          0.151,
          0.8913
        ],
        [
          0.1611,
          0.8904
        ],
        [
          0.1715,
          0.8899
        ],
        [
          0.1815,
          0.889
        ],
        [
          0.1915,
          0.8867
        ],
        [
          0.2015,
          0.8863
        ],
        [
          0.2116,
          0.8849
        ],
        [
          0.222,
          0.8831
        ],
        [
          0.232,
          0.8813
        ],
        [
          0.242,
          0.8785
        ],
        [
          0.252,
          0.8781
        ],
        [
          0.2621,
          0.8753
        ],
        [
          0.2725,
          0.8735
        ],
        [
          0.2825,
          0.8699
        ],
        [
          0.2925,
          0.8685
        ],
        [
          0.3025,
          0.8658
        ],
        [
          0.3126,
          0.8635
        ],
        [
          0.323,
          0.8599
        ],
        [
          0.333,
          0.8553
        ],
        [
          0.343,
          0.853
        ],
        [
          0.353,
          0.8508
        ],
        [
          0.3631,
          0.8449
        ],
        [
          0.3735,
          0.8403
        ],
        [
          0.3835,
          0.838
        ],
        [
          0.3935,
          0.8348
        ],
        [
          0.4035,
          0.8312
        ],
        [
          0.4136,
          0.8276
        ],
        [
          0.424,
          0.8244
        ],
        [
          0.434,
          0.8194
        ],
        [
          0.444,
          0.8157
        ],
        [
          0.454,
          0.8112
        ],
        [
          0.4641,
          0.8071
        ],
        [
          0.4745,
          0.8016
        ],
        [
          0.4845,
          0.7962
        ],
        [
          0.4945,
          0.7921
        ],
        [
          0.5045,
          0.7862
        ],
        [
          0.5146,
          0.7821
        ],
        [
          0.525,
          0.7775
        ],
        [
          0.535,
          0.7734
        ],
        [
          0.545,
          0.7693
        ],
        [
          0.5551,
          0.7675
        ],
        [
          0.5655,
          0.7625
        ],
        [
          0.5755,
          0.7584
        ],
        [
          0.5855,
          0.7543
        ],
        [
          0.5955,
          0.7498
        ],
        [
          0.6056,
          0.7429
        ],
        [
          0.616,
          0.737
        ],
        [
          0.626,
          0.7307
        ],
        [
          0.636,
          0.7266
        ],
        [
          0.646,
          0.7238
        ],
        [
          0.6561,
          0.7179
        ],
        [
          0.6665,
          0.7134
        ],
        [
          0.6765,
          0.7075
        ],
        [
          0.6865,
          0.7025
        ],
        [
          0.6965,
          0.6975
        ],
        [
          0.7066,
          0.6938
        ],
        [
          0.717,
          0.6888
        ],
        [
          0.727,
          0.6865
        ],
        [
          0.737,
          0.6824
        ],
        [
          0.747,
          0.6752
        ],
        [
          0.7571,
          0.6706
        ],
        [
          0.7675,
          0.6652
        ],
        [
          0.7775,
          0.6592
        ],
        [
          0.7875,
          0.6529
        ],
        [
          0.7975,
          0.6474
        ],
        [
          0.8076,
          0.6429
        ],
        [
          0.818,
          0.636
        ],
        [
          0.828,
          0.6288
        ],
        [
          0.838,
          0.6242
        ],
        [
          0.848,
          0.6192
        ],
        [
          0.8581,
          0.6124
        ],
        [
          0.8685,
          0.6046
        ],
        [
          0.8785,
          0.5987
        ],
        [
          0.8885,
          0.5933
        ],
        [
          0.8985,
          0.586
        ],
        [
          0.9086,
          0.5805
        ],
        [
          0.919,
          0.5723
        ],
        [
          0.929,
          0.5687
        ],
        [
          0.939,
          0.561
        ],
        [
          0.949,
          0.5541
        ],
        [
          0.9591,
          0.5487
        ],
        [
          0.9695,
          0.5423
        ],
        [
          0.9795,
          0.5359
        ],
        [
          0.9895,
          0.5318
        ],
        [
          0.9995,
          0.5237
        ],
        [
          1.0,
          0.5237
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.14573131477049092,
        "difference": -0.0019553525905589373,
        "worse": 0.6833333333333333,
        "kept_right": 0.1416932260867294,
        "kept_wrong": 0.004038088683761591,
        "handed_off": 0.8542686852295096
      },
      "usd_per_1000": 6.632391827326058
    },
    {
      "decision": 373,
      "frontier": 55,
      "frontier_accuracy": 0.8503184713375797,
      "in_sample": {
        "threshold": 0.94883,
        "answered": 0.18198362147406733,
        "accuracy": 0.8503184713375797
      },
      "curve": [
        [
          0.0,
          0.8503
        ],
        [
          0.01,
          0.8503
        ],
        [
          0.02,
          0.8503
        ],
        [
          0.03,
          0.8503
        ],
        [
          0.04,
          0.8503
        ],
        [
          0.05,
          0.8503
        ],
        [
          0.0601,
          0.8503
        ],
        [
          0.0705,
          0.8503
        ],
        [
          0.0805,
          0.8503
        ],
        [
          0.0905,
          0.8503
        ],
        [
          0.1005,
          0.8508
        ],
        [
          0.1106,
          0.8512
        ],
        [
          0.121,
          0.8512
        ],
        [
          0.131,
          0.8512
        ],
        [
          0.141,
          0.8508
        ],
        [
          0.151,
          0.8512
        ],
        [
          0.1611,
          0.8503
        ],
        [
          0.1715,
          0.8503
        ],
        [
          0.1815,
          0.8503
        ],
        [
          0.1915,
          0.848
        ],
        [
          0.2015,
          0.8476
        ],
        [
          0.2116,
          0.8467
        ],
        [
          0.222,
          0.8462
        ],
        [
          0.232,
          0.8458
        ],
        [
          0.242,
          0.8439
        ],
        [
          0.252,
          0.8426
        ],
        [
          0.2621,
          0.8417
        ],
        [
          0.2725,
          0.8403
        ],
        [
          0.2825,
          0.8371
        ],
        [
          0.2925,
          0.8367
        ],
        [
          0.3025,
          0.8353
        ],
        [
          0.3126,
          0.8344
        ],
        [
          0.323,
          0.8326
        ],
        [
          0.333,
          0.8285
        ],
        [
          0.343,
          0.8262
        ],
        [
          0.353,
          0.8239
        ],
        [
          0.3631,
          0.8189
        ],
        [
          0.3735,
          0.8144
        ],
        [
          0.3835,
          0.813
        ],
        [
          0.3935,
          0.8103
        ],
        [
          0.4035,
          0.8066
        ],
        [
          0.4136,
          0.803
        ],
        [
          0.424,
          0.8007
        ],
        [
          0.434,
          0.7975
        ],
        [
          0.444,
          0.7944
        ],
        [
          0.454,
          0.7912
        ],
        [
          0.4641,
          0.7871
        ],
        [
          0.4745,
          0.7816
        ],
        [
          0.4845,
          0.7771
        ],
        [
          0.4945,
          0.7734
        ],
        [
          0.5045,
          0.768
        ],
        [
          0.5146,
          0.7662
        ],
        [
          0.525,
          0.7611
        ],
        [
          0.535,
          0.7571
        ],
        [
          0.545,
          0.753
        ],
        [
          0.5551,
          0.7511
        ],
        [
          0.5655,
          0.7457
        ],
        [
          0.5755,
          0.7416
        ],
        [
          0.5855,
          0.7384
        ],
        [
          0.5955,
          0.7338
        ],
        [
          0.6056,
          0.7279
        ],
        [
          0.616,
          0.7216
        ],
        [
          0.626,
          0.7157
        ],
        [
          0.636,
          0.712
        ],
        [
          0.646,
          0.7097
        ],
        [
          0.6561,
          0.7038
        ],
        [
          0.6665,
          0.7006
        ],
        [
          0.6765,
          0.6943
        ],
        [
          0.6865,
          0.6906
        ],
        [
          0.6965,
          0.6852
        ],
        [
          0.7066,
          0.6811
        ],
        [
          0.717,
          0.6765
        ],
        [
          0.727,
          0.6738
        ],
        [
          0.737,
          0.6697
        ],
        [
          0.747,
          0.6633
        ],
        [
          0.7571,
          0.6588
        ],
        [
          0.7675,
          0.6538
        ],
        [
          0.7775,
          0.6497
        ],
        [
          0.7875,
          0.6438
        ],
        [
          0.7975,
          0.6401
        ],
        [
          0.8076,
          0.6351
        ],
        [
          0.818,
          0.6274
        ],
        [
          0.828,
          0.6206
        ],
        [
          0.838,
          0.6151
        ],
        [
          0.848,
          0.6119
        ],
        [
          0.8581,
          0.6056
        ],
        [
          0.8685,
          0.5983
        ],
        [
          0.8785,
          0.5928
        ],
        [
          0.8885,
          0.5874
        ],
        [
          0.8985,
          0.5819
        ],
        [
          0.9086,
          0.5773
        ],
        [
          0.919,
          0.5687
        ],
        [
          0.929,
          0.5637
        ],
        [
          0.939,
          0.5564
        ],
        [
          0.949,
          0.5514
        ],
        [
          0.9591,
          0.5464
        ],
        [
          0.9695,
          0.5409
        ],
        [
          0.9795,
          0.5359
        ],
        [
          0.9895,
          0.53
        ],
        [
          0.9995,
          0.5237
        ],
        [
          1.0,
          0.5237
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.1786183851371922,
        "difference": -0.0016946690581215885,
        "worse": 0.5066666666666667,
        "kept_right": 0.17082715006967586,
        "kept_wrong": 0.007791235067516351,
        "handed_off": 0.8213816148628081
      },
      "usd_per_1000": 3.015470762070628
    },
    {
      "decision": 223,
      "frontier": 140,
      "frontier_accuracy": 0.9808917197452229,
      "in_sample": {
        "threshold": 0.9532922221023848,
        "answered": 0.0632393084622384,
        "accuracy": 0.9808917197452229
      },
      "curve": [
        [
          0.0,
          0.9809
        ],
        [
          0.01,
          0.9809
        ],
        [
          0.02,
          0.9809
        ],
        [
          0.03,
          0.9809
        ],
        [
          0.04,
          0.9809
        ],
        [
          0.05,
          0.9809
        ],
        [
          0.0601,
          0.9809
        ],
        [
          0.0705,
          0.9804
        ],
        [
          0.0805,
          0.98
        ],
        [
          0.0905,
          0.9795
        ],
        [
          0.1005,
          0.9782
        ],
        [
          0.1106,
          0.9768
        ],
        [
          0.121,
          0.9763
        ],
        [
          0.131,
          0.975
        ],
        [
          0.141,
          0.9722
        ],
        [
          0.151,
          0.9709
        ],
        [
          0.1611,
          0.9677
        ],
        [
          0.1715,
          0.9659
        ],
        [
          0.1815,
          0.9627
        ],
        [
          0.1915,
          0.9609
        ],
        [
          0.2015,
          0.9595
        ],
        [
          0.2116,
          0.9577
        ],
        [
          0.222,
          0.9545
        ],
        [
          0.232,
          0.9513
        ],
        [
          0.242,
          0.9472
        ],
        [
          0.252,
          0.9427
        ],
        [
          0.2621,
          0.9386
        ],
        [
          0.2725,
          0.9359
        ],
        [
          0.2825,
          0.9299
        ],
        [
          0.2925,
          0.9258
        ],
        [
          0.3025,
          0.9222
        ],
        [
          0.3126,
          0.9181
        ],
        [
          0.323,
          0.9122
        ],
        [
          0.333,
          0.9081
        ],
        [
          0.343,
          0.9049
        ],
        [
          0.353,
          0.9017
        ],
        [
          0.3631,
          0.8963
        ],
        [
          0.3735,
          0.8908
        ],
        [
          0.3835,
          0.8858
        ],
        [
          0.3935,
          0.8794
        ],
        [
          0.4035,
          0.8744
        ],
        [
          0.4136,
          0.8676
        ],
        [
          0.424,
          0.8621
        ],
        [
          0.434,
          0.8544
        ],
        [
          0.444,
          0.849
        ],
        [
          0.454,
          0.8421
        ],
        [
          0.4641,
          0.8371
        ],
        [
          0.4745,
          0.8303
        ],
        [
          0.4845,
          0.8253
        ],
        [
          0.4945,
          0.8185
        ],
        [
          0.5045,
          0.8135
        ],
        [
          0.5146,
          0.8089
        ],
        [
          0.525,
          0.8035
        ],
        [
          0.535,
          0.7994
        ],
        [
          0.545,
          0.7934
        ],
        [
          0.5551,
          0.7875
        ],
        [
          0.5655,
          0.7807
        ],
        [
          0.5755,
          0.7748
        ],
        [
          0.5855,
          0.7675
        ],
        [
          0.5955,
          0.7607
        ],
        [
          0.6056,
          0.7552
        ],
        [
          0.616,
          0.747
        ],
        [
          0.626,
          0.7407
        ],
        [
          0.636,
          0.7348
        ],
        [
          0.646,
          0.7298
        ],
        [
          0.6561,
          0.7211
        ],
        [
          0.6665,
          0.7157
        ],
        [
          0.6765,
          0.7084
        ],
        [
          0.6865,
          0.7006
        ],
        [
          0.6965,
          0.6938
        ],
        [
          0.7066,
          0.6879
        ],
        [
          0.717,
          0.6806
        ],
        [
          0.727,
          0.6756
        ],
        [
          0.737,
          0.6679
        ],
        [
          0.747,
          0.6611
        ],
        [
          0.7571,
          0.657
        ],
        [
          0.7675,
          0.6483
        ],
        [
          0.7775,
          0.6429
        ],
        [
          0.7875,
          0.6351
        ],
        [
          0.7975,
          0.6269
        ],
        [
          0.8076,
          0.6206
        ],
        [
          0.818,
          0.6146
        ],
        [
          0.828,
          0.6078
        ],
        [
          0.838,
          0.601
        ],
        [
          0.848,
          0.5951
        ],
        [
          0.8581,
          0.5869
        ],
        [
          0.8685,
          0.5796
        ],
        [
          0.8785,
          0.5723
        ],
        [
          0.8885,
          0.5651
        ],
        [
          0.8985,
          0.5591
        ],
        [
          0.9086,
          0.55
        ],
        [
          0.919,
          0.5419
        ],
        [
          0.929,
          0.535
        ],
        [
          0.939,
          0.5268
        ],
        [
          0.949,
          0.5187
        ],
        [
          0.9591,
          0.5096
        ],
        [
          0.9695,
          0.5018
        ],
        [
          0.9795,
          0.4936
        ],
        [
          0.9895,
          0.485
        ],
        [
          0.9995,
          0.4777
        ],
        [
          1.0,
          0.4777
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.07302788731665506,
        "difference": -0.0008760594234234772,
        "worse": 0.48,
        "kept_right": 0.07215182789323157,
        "kept_wrong": 0.0008760594234234772,
        "handed_off": 0.9269721126833453
      },
      "usd_per_1000": 2.246323543732308
    },
    {
      "decision": 223,
      "frontier": 277,
      "frontier_accuracy": 0.9795268425841674,
      "in_sample": {
        "threshold": 0.9532922221023848,
        "answered": 0.0632393084622384,
        "accuracy": 0.9795268425841674
      },
      "curve": [
        [
          0.0,
          0.9795
        ],
        [
          0.01,
          0.9795
        ],
        [
          0.02,
          0.9795
        ],
        [
          0.03,
          0.9795
        ],
        [
          0.04,
          0.9795
        ],
        [
          0.05,
          0.9795
        ],
        [
          0.0601,
          0.9795
        ],
        [
          0.0705,
          0.9791
        ],
        [
          0.0805,
          0.9786
        ],
        [
          0.0905,
          0.9782
        ],
        [
          0.1005,
          0.9768
        ],
        [
          0.1106,
          0.9754
        ],
        [
          0.121,
          0.9754
        ],
        [
          0.131,
          0.9741
        ],
        [
          0.141,
          0.9713
        ],
        [
          0.151,
          0.97
        ],
        [
          0.1611,
          0.9663
        ],
        [
          0.1715,
          0.9641
        ],
        [
          0.1815,
          0.9609
        ],
        [
          0.1915,
          0.9591
        ],
        [
          0.2015,
          0.9577
        ],
        [
          0.2116,
          0.9554
        ],
        [
          0.222,
          0.9522
        ],
        [
          0.232,
          0.949
        ],
        [
          0.242,
          0.9454
        ],
        [
          0.252,
          0.9409
        ],
        [
          0.2621,
          0.9368
        ],
        [
          0.2725,
          0.934
        ],
        [
          0.2825,
          0.9281
        ],
        [
          0.2925,
          0.9245
        ],
        [
          0.3025,
          0.9213
        ],
        [
          0.3126,
          0.9167
        ],
        [
          0.323,
          0.9108
        ],
        [
          0.333,
          0.9067
        ],
        [
          0.343,
          0.9035
        ],
        [
          0.353,
          0.9004
        ],
        [
          0.3631,
          0.8949
        ],
        [
          0.3735,
          0.889
        ],
        [
          0.3835,
          0.884
        ],
        [
          0.3935,
          0.8776
        ],
        [
          0.4035,
          0.8726
        ],
        [
          0.4136,
          0.8662
        ],
        [
          0.424,
          0.8603
        ],
        [
          0.434,
          0.8526
        ],
        [
          0.444,
          0.8471
        ],
        [
          0.454,
          0.8408
        ],
        [
          0.4641,
          0.8358
        ],
        [
          0.4745,
          0.8289
        ],
        [
          0.4845,
          0.8239
        ],
        [
          0.4945,
          0.8171
        ],
        [
          0.5045,
          0.8121
        ],
        [
          0.5146,
          0.8076
        ],
        [
          0.525,
          0.8021
        ],
        [
          0.535,
          0.798
        ],
        [
          0.545,
          0.7921
        ],
        [
          0.5551,
          0.7862
        ],
        [
          0.5655,
          0.7798
        ],
        [
          0.5755,
          0.7739
        ],
        [
          0.5855,
          0.7666
        ],
        [
          0.5955,
          0.7593
        ],
        [
          0.6056,
          0.7539
        ],
        [
          0.616,
          0.7461
        ],
        [
          0.626,
          0.7393
        ],
        [
          0.636,
          0.7334
        ],
        [
          0.646,
          0.7284
        ],
        [
          0.6561,
          0.7197
        ],
        [
          0.6665,
          0.7143
        ],
        [
          0.6765,
          0.7066
        ],
        [
          0.6865,
          0.6988
        ],
        [
          0.6965,
          0.6924
        ],
        [
          0.7066,
          0.6865
        ],
        [
          0.717,
          0.6793
        ],
        [
          0.727,
          0.6747
        ],
        [
          0.737,
          0.6674
        ],
        [
          0.747,
          0.6601
        ],
        [
          0.7571,
          0.6556
        ],
        [
          0.7675,
          0.6483
        ],
        [
          0.7775,
          0.6429
        ],
        [
          0.7875,
          0.6351
        ],
        [
          0.7975,
          0.6269
        ],
        [
          0.8076,
          0.6206
        ],
        [
          0.818,
          0.6146
        ],
        [
          0.828,
          0.6074
        ],
        [
          0.838,
          0.6005
        ],
        [
          0.848,
          0.5946
        ],
        [
          0.8581,
          0.5864
        ],
        [
          0.8685,
          0.5792
        ],
        [
          0.8785,
          0.5719
        ],
        [
          0.8885,
          0.5646
        ],
        [
          0.8985,
          0.5587
        ],
        [
          0.9086,
          0.5496
        ],
        [
          0.919,
          0.5414
        ],
        [
          0.929,
          0.5346
        ],
        [
          0.939,
          0.5264
        ],
        [
          0.949,
          0.5182
        ],
        [
          0.9591,
          0.5091
        ],
        [
          0.9695,
          0.5018
        ],
        [
          0.9795,
          0.4936
        ],
        [
          0.9895,
          0.485
        ],
        [
          0.9995,
          0.4777
        ],
        [
          1.0,
          0.4777
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.07382515643292784,
        "difference": -0.000985200174807337,
        "worse": 0.48,
        "kept_right": 0.07283086257166686,
        "kept_wrong": 0.0009942938612609719,
        "handed_off": 0.9261748435670725
      },
      "usd_per_1000": 5.6216468745096115
    },
    {
      "decision": 223,
      "frontier": 276,
      "frontier_accuracy": 0.9777070063694268,
      "in_sample": {
        "threshold": 0.9532922221023848,
        "answered": 0.0632393084622384,
        "accuracy": 0.9777070063694268
      },
      "curve": [
        [
          0.0,
          0.9777
        ],
        [
          0.01,
          0.9777
        ],
        [
          0.02,
          0.9777
        ],
        [
          0.03,
          0.9777
        ],
        [
          0.04,
          0.9777
        ],
        [
          0.05,
          0.9777
        ],
        [
          0.0601,
          0.9777
        ],
        [
          0.0705,
          0.9773
        ],
        [
          0.0805,
          0.9768
        ],
        [
          0.0905,
          0.9763
        ],
        [
          0.1005,
          0.975
        ],
        [
          0.1106,
          0.9736
        ],
        [
          0.121,
          0.9736
        ],
        [
          0.131,
          0.9722
        ],
        [
          0.141,
          0.9695
        ],
        [
          0.151,
          0.9682
        ],
        [
          0.1611,
          0.9645
        ],
        [
          0.1715,
          0.9622
        ],
        [
          0.1815,
          0.9591
        ],
        [
          0.1915,
          0.9577
        ],
        [
          0.2015,
          0.9568
        ],
        [
          0.2116,
          0.9545
        ],
        [
          0.222,
          0.9513
        ],
        [
          0.232,
          0.9481
        ],
        [
          0.242,
          0.944
        ],
        [
          0.252,
          0.9395
        ],
        [
          0.2621,
          0.9354
        ],
        [
          0.2725,
          0.9331
        ],
        [
          0.2825,
          0.9272
        ],
        [
          0.2925,
          0.9231
        ],
        [
          0.3025,
          0.9199
        ],
        [
          0.3126,
          0.9154
        ],
        [
          0.323,
          0.9095
        ],
        [
          0.333,
          0.9054
        ],
        [
          0.343,
          0.9017
        ],
        [
          0.353,
          0.8985
        ],
        [
          0.3631,
          0.8931
        ],
        [
          0.3735,
          0.8872
        ],
        [
          0.3835,
          0.8822
        ],
        [
          0.3935,
          0.8758
        ],
        [
          0.4035,
          0.8708
        ],
        [
          0.4136,
          0.8644
        ],
        [
          0.424,
          0.8585
        ],
        [
          0.434,
          0.8508
        ],
        [
          0.444,
          0.8453
        ],
        [
          0.454,
          0.8389
        ],
        [
          0.4641,
          0.8339
        ],
        [
          0.4745,
          0.8271
        ],
        [
          0.4845,
          0.8221
        ],
        [
          0.4945,
          0.8153
        ],
        [
          0.5045,
          0.8103
        ],
        [
          0.5146,
          0.8057
        ],
        [
          0.525,
          0.8003
        ],
        [
          0.535,
          0.7962
        ],
        [
          0.545,
          0.7903
        ],
        [
          0.5551,
          0.7843
        ],
        [
          0.5655,
          0.778
        ],
        [
          0.5755,
          0.7725
        ],
        [
          0.5855,
          0.7652
        ],
        [
          0.5955,
          0.758
        ],
        [
          0.6056,
          0.7525
        ],
        [
          0.616,
          0.7448
        ],
        [
          0.626,
          0.7379
        ],
        [
          0.636,
          0.732
        ],
        [
          0.646,
          0.727
        ],
        [
          0.6561,
          0.7184
        ],
        [
          0.6665,
          0.7129
        ],
        [
          0.6765,
          0.7056
        ],
        [
          0.6865,
          0.6979
        ],
        [
          0.6965,
          0.6915
        ],
        [
          0.7066,
          0.6856
        ],
        [
          0.717,
          0.6783
        ],
        [
          0.727,
          0.6738
        ],
        [
          0.737,
          0.667
        ],
        [
          0.747,
          0.6597
        ],
        [
          0.7571,
          0.6556
        ],
        [
          0.7675,
          0.6479
        ],
        [
          0.7775,
          0.6429
        ],
        [
          0.7875,
          0.6351
        ],
        [
          0.7975,
          0.6269
        ],
        [
          0.8076,
          0.6206
        ],
        [
          0.818,
          0.6146
        ],
        [
          0.828,
          0.6074
        ],
        [
          0.838,
          0.6005
        ],
        [
          0.848,
          0.5946
        ],
        [
          0.8581,
          0.5864
        ],
        [
          0.8685,
          0.5792
        ],
        [
          0.8785,
          0.5719
        ],
        [
          0.8885,
          0.5646
        ],
        [
          0.8985,
          0.5587
        ],
        [
          0.9086,
          0.5496
        ],
        [
          0.919,
          0.5414
        ],
        [
          0.929,
          0.5346
        ],
        [
          0.939,
          0.5268
        ],
        [
          0.949,
          0.5187
        ],
        [
          0.9591,
          0.5096
        ],
        [
          0.9695,
          0.5023
        ],
        [
          0.9795,
          0.4941
        ],
        [
          0.9895,
          0.4854
        ],
        [
          0.9995,
          0.4777
        ],
        [
          1.0,
          0.4777
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.07382515643292784,
        "difference": -0.000985200174807337,
        "worse": 0.48,
        "kept_right": 0.07283086257166686,
        "kept_wrong": 0.0009942938612609719,
        "handed_off": 0.9261748435670725
      },
      "usd_per_1000": 1.5063141044935482
    },
    {
      "decision": 223,
      "frontier": 48,
      "frontier_accuracy": 0.9758871701546861,
      "in_sample": {
        "threshold": 0.9532922221023848,
        "answered": 0.0632393084622384,
        "accuracy": 0.9758871701546861
      },
      "curve": [
        [
          0.0,
          0.9759
        ],
        [
          0.01,
          0.9759
        ],
        [
          0.02,
          0.9759
        ],
        [
          0.03,
          0.9759
        ],
        [
          0.04,
          0.9759
        ],
        [
          0.05,
          0.9759
        ],
        [
          0.0601,
          0.9759
        ],
        [
          0.0705,
          0.9754
        ],
        [
          0.0805,
          0.975
        ],
        [
          0.0905,
          0.9745
        ],
        [
          0.1005,
          0.9736
        ],
        [
          0.1106,
          0.9722
        ],
        [
          0.121,
          0.9722
        ],
        [
          0.131,
          0.9709
        ],
        [
          0.141,
          0.9682
        ],
        [
          0.151,
          0.9668
        ],
        [
          0.1611,
          0.9631
        ],
        [
          0.1715,
          0.9609
        ],
        [
          0.1815,
          0.9577
        ],
        [
          0.1915,
          0.9559
        ],
        [
          0.2015,
          0.9545
        ],
        [
          0.2116,
          0.9522
        ],
        [
          0.222,
          0.949
        ],
        [
          0.232,
          0.9459
        ],
        [
          0.242,
          0.9422
        ],
        [
          0.252,
          0.9377
        ],
        [
          0.2621,
          0.9336
        ],
        [
          0.2725,
          0.9313
        ],
        [
          0.2825,
          0.9254
        ],
        [
          0.2925,
          0.9213
        ],
        [
          0.3025,
          0.9177
        ],
        [
          0.3126,
          0.9131
        ],
        [
          0.323,
          0.9072
        ],
        [
          0.333,
          0.9031
        ],
        [
          0.343,
          0.8995
        ],
        [
          0.353,
          0.8963
        ],
        [
          0.3631,
          0.8908
        ],
        [
          0.3735,
          0.8854
        ],
        [
          0.3835,
          0.8803
        ],
        [
          0.3935,
          0.874
        ],
        [
          0.4035,
          0.869
        ],
        [
          0.4136,
          0.8626
        ],
        [
          0.424,
          0.8567
        ],
        [
          0.434,
          0.849
        ],
        [
          0.444,
          0.8435
        ],
        [
          0.454,
          0.8371
        ],
        [
          0.4641,
          0.8321
        ],
        [
          0.4745,
          0.8253
        ],
        [
          0.4845,
          0.8203
        ],
        [
          0.4945,
          0.8135
        ],
        [
          0.5045,
          0.8085
        ],
        [
          0.5146,
          0.8039
        ],
        [
          0.525,
          0.7985
        ],
        [
          0.535,
          0.7944
        ],
        [
          0.545,
          0.7884
        ],
        [
          0.5551,
          0.7825
        ],
        [
          0.5655,
          0.7762
        ],
        [
          0.5755,
          0.7702
        ],
        [
          0.5855,
          0.7634
        ],
        [
          0.5955,
          0.7561
        ],
        [
          0.6056,
          0.7507
        ],
        [
          0.616,
          0.7429
        ],
        [
          0.626,
          0.7361
        ],
        [
          0.636,
          0.7302
        ],
        [
          0.646,
          0.7261
        ],
        [
          0.6561,
          0.7175
        ],
        [
          0.6665,
          0.712
        ],
        [
          0.6765,
          0.7047
        ],
        [
          0.6865,
          0.697
        ],
        [
          0.6965,
          0.6906
        ],
        [
          0.7066,
          0.6847
        ],
        [
          0.717,
          0.6774
        ],
        [
          0.727,
          0.6729
        ],
        [
          0.737,
          0.6656
        ],
        [
          0.747,
          0.6588
        ],
        [
          0.7571,
          0.6542
        ],
        [
          0.7675,
          0.646
        ],
        [
          0.7775,
          0.641
        ],
        [
          0.7875,
          0.6333
        ],
        [
          0.7975,
          0.6251
        ],
        [
          0.8076,
          0.6187
        ],
        [
          0.818,
          0.6128
        ],
        [
          0.828,
          0.6056
        ],
        [
          0.838,
          0.5987
        ],
        [
          0.848,
          0.5928
        ],
        [
          0.8581,
          0.5851
        ],
        [
          0.8685,
          0.5778
        ],
        [
          0.8785,
          0.5705
        ],
        [
          0.8885,
          0.5632
        ],
        [
          0.8985,
          0.5569
        ],
        [
          0.9086,
          0.5482
        ],
        [
          0.919,
          0.54
        ],
        [
          0.929,
          0.5332
        ],
        [
          0.939,
          0.5255
        ],
        [
          0.949,
          0.5173
        ],
        [
          0.9591,
          0.5082
        ],
        [
          0.9695,
          0.5009
        ],
        [
          0.9795,
          0.4936
        ],
        [
          0.9895,
          0.485
        ],
        [
          0.9995,
          0.4777
        ],
        [
          1.0,
          0.4777
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.07826689032487401,
        "difference": -0.0013551843383423341,
        "worse": 0.55,
        "kept_right": 0.07683894554567815,
        "kept_wrong": 0.0014279447791959356,
        "handed_off": 0.9217331096751261
      },
      "usd_per_1000": 4.96117157562173
    },
    {
      "decision": 223,
      "frontier": 191,
      "frontier_accuracy": 0.9740673339399454,
      "in_sample": {
        "threshold": 0.9532922221023848,
        "answered": 0.0632393084622384,
        "accuracy": 0.9740673339399454
      },
      "curve": [
        [
          0.0,
          0.9741
        ],
        [
          0.01,
          0.9741
        ],
        [
          0.02,
          0.9741
        ],
        [
          0.03,
          0.9741
        ],
        [
          0.04,
          0.9741
        ],
        [
          0.05,
          0.9741
        ],
        [
          0.0601,
          0.9741
        ],
        [
          0.0705,
          0.9736
        ],
        [
          0.0805,
          0.9732
        ],
        [
          0.0905,
          0.9727
        ],
        [
          0.1005,
          0.9713
        ],
        [
          0.1106,
          0.97
        ],
        [
          0.121,
          0.9695
        ],
        [
          0.131,
          0.9682
        ],
        [
          0.141,
          0.9654
        ],
        [
          0.151,
          0.9641
        ],
        [
          0.1611,
          0.9604
        ],
        [
          0.1715,
          0.9581
        ],
        [
          0.1815,
          0.9554
        ],
        [
          0.1915,
          0.9536
        ],
        [
          0.2015,
          0.9522
        ],
        [
          0.2116,
          0.95
        ],
        [
          0.222,
          0.9468
        ],
        [
          0.232,
          0.9436
        ],
        [
          0.242,
          0.9395
        ],
        [
          0.252,
          0.9349
        ],
        [
          0.2621,
          0.9308
        ],
        [
          0.2725,
          0.9286
        ],
        [
          0.2825,
          0.9227
        ],
        [
          0.2925,
          0.9186
        ],
        [
          0.3025,
          0.9149
        ],
        [
          0.3126,
          0.9108
        ],
        [
          0.323,
          0.9049
        ],
        [
          0.333,
          0.9008
        ],
        [
          0.343,
          0.8972
        ],
        [
          0.353,
          0.8935
        ],
        [
          0.3631,
          0.8881
        ],
        [
          0.3735,
          0.8822
        ],
        [
          0.3835,
          0.8772
        ],
        [
          0.3935,
          0.8708
        ],
        [
          0.4035,
          0.8662
        ],
        [
          0.4136,
          0.8594
        ],
        [
          0.424,
          0.8535
        ],
        [
          0.434,
          0.8458
        ],
        [
          0.444,
          0.8403
        ],
        [
          0.454,
          0.8339
        ],
        [
          0.4641,
          0.8289
        ],
        [
          0.4745,
          0.8221
        ],
        [
          0.4845,
          0.8171
        ],
        [
          0.4945,
          0.8103
        ],
        [
          0.5045,
          0.8053
        ],
        [
          0.5146,
          0.8007
        ],
        [
          0.525,
          0.7957
        ],
        [
          0.535,
          0.7916
        ],
        [
          0.545,
          0.7857
        ],
        [
          0.5551,
          0.7798
        ],
        [
          0.5655,
          0.7734
        ],
        [
          0.5755,
          0.7675
        ],
        [
          0.5855,
          0.7607
        ],
        [
          0.5955,
          0.7534
        ],
        [
          0.6056,
          0.7475
        ],
        [
          0.616,
          0.7398
        ],
        [
          0.626,
          0.7329
        ],
        [
          0.636,
          0.727
        ],
        [
          0.646,
          0.722
        ],
        [
          0.6561,
          0.7134
        ],
        [
          0.6665,
          0.7079
        ],
        [
          0.6765,
          0.7011
        ],
        [
          0.6865,
          0.6934
        ],
        [
          0.6965,
          0.687
        ],
        [
          0.7066,
          0.6811
        ],
        [
          0.717,
          0.6738
        ],
        [
          0.727,
          0.6688
        ],
        [
          0.737,
          0.662
        ],
        [
          0.747,
          0.6547
        ],
        [
          0.7571,
          0.6506
        ],
        [
          0.7675,
          0.6433
        ],
        [
          0.7775,
          0.6383
        ],
        [
          0.7875,
          0.6306
        ],
        [
          0.7975,
          0.6224
        ],
        [
          0.8076,
          0.616
        ],
        [
          0.818,
          0.6106
        ],
        [
          0.828,
          0.6033
        ],
        [
          0.838,
          0.5965
        ],
        [
          0.848,
          0.5905
        ],
        [
          0.8581,
          0.5828
        ],
        [
          0.8685,
          0.5755
        ],
        [
          0.8785,
          0.5682
        ],
        [
          0.8885,
          0.561
        ],
        [
          0.8985,
          0.5555
        ],
        [
          0.9086,
          0.5469
        ],
        [
          0.919,
          0.5387
        ],
        [
          0.929,
          0.5318
        ],
        [
          0.939,
          0.5237
        ],
        [
          0.949,
          0.5155
        ],
        [
          0.9591,
          0.5068
        ],
        [
          0.9695,
          0.5005
        ],
        [
          0.9795,
          0.4932
        ],
        [
          0.9895,
          0.4845
        ],
        [
          0.9995,
          0.4777
        ],
        [
          1.0,
          0.4777
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.07302788731665506,
        "difference": -0.0008760594234234772,
        "worse": 0.48,
        "kept_right": 0.07215182789323157,
        "kept_wrong": 0.0008760594234234772,
        "handed_off": 0.9269721126833453
      },
      "usd_per_1000": 24.788981005264276
    },
    {
      "decision": 223,
      "frontier": 188,
      "frontier_accuracy": 0.9713375796178344,
      "in_sample": {
        "threshold": 0.9532922221023848,
        "answered": 0.0632393084622384,
        "accuracy": 0.9713375796178344
      },
      "curve": [
        [
          0.0,
          0.9713
        ],
        [
          0.01,
          0.9713
        ],
        [
          0.02,
          0.9713
        ],
        [
          0.03,
          0.9713
        ],
        [
          0.04,
          0.9713
        ],
        [
          0.05,
          0.9713
        ],
        [
          0.0601,
          0.9713
        ],
        [
          0.0705,
          0.9709
        ],
        [
          0.0805,
          0.9704
        ],
        [
          0.0905,
          0.97
        ],
        [
          0.1005,
          0.9686
        ],
        [
          0.1106,
          0.9672
        ],
        [
          0.121,
          0.9668
        ],
        [
          0.131,
          0.9654
        ],
        [
          0.141,
          0.9627
        ],
        [
          0.151,
          0.9613
        ],
        [
          0.1611,
          0.9577
        ],
        [
          0.1715,
          0.9559
        ],
        [
          0.1815,
          0.9527
        ],
        [
          0.1915,
          0.9509
        ],
        [
          0.2015,
          0.95
        ],
        [
          0.2116,
          0.9481
        ],
        [
          0.222,
          0.9449
        ],
        [
          0.232,
          0.9418
        ],
        [
          0.242,
          0.9377
        ],
        [
          0.252,
          0.9336
        ],
        [
          0.2621,
          0.9295
        ],
        [
          0.2725,
          0.9268
        ],
        [
          0.2825,
          0.9208
        ],
        [
          0.2925,
          0.9167
        ],
        [
          0.3025,
          0.914
        ],
        [
          0.3126,
          0.9099
        ],
        [
          0.323,
          0.904
        ],
        [
          0.333,
          0.8999
        ],
        [
          0.343,
          0.8967
        ],
        [
          0.353,
          0.8931
        ],
        [
          0.3631,
          0.8876
        ],
        [
          0.3735,
          0.8822
        ],
        [
          0.3835,
          0.8772
        ],
        [
          0.3935,
          0.8708
        ],
        [
          0.4035,
          0.8658
        ],
        [
          0.4136,
          0.859
        ],
        [
          0.424,
          0.8535
        ],
        [
          0.434,
          0.8458
        ],
        [
          0.444,
          0.8403
        ],
        [
          0.454,
          0.8335
        ],
        [
          0.4641,
          0.8285
        ],
        [
          0.4745,
          0.8217
        ],
        [
          0.4845,
          0.8167
        ],
        [
          0.4945,
          0.8098
        ],
        [
          0.5045,
          0.8048
        ],
        [
          0.5146,
          0.8003
        ],
        [
          0.525,
          0.7953
        ],
        [
          0.535,
          0.7921
        ],
        [
          0.545,
          0.7862
        ],
        [
          0.5551,
          0.7803
        ],
        [
          0.5655,
          0.7739
        ],
        [
          0.5755,
          0.768
        ],
        [
          0.5855,
          0.7607
        ],
        [
          0.5955,
          0.7539
        ],
        [
          0.6056,
          0.7493
        ],
        [
          0.616,
          0.7411
        ],
        [
          0.626,
          0.7352
        ],
        [
          0.636,
          0.7293
        ],
        [
          0.646,
          0.7243
        ],
        [
          0.6561,
          0.7157
        ],
        [
          0.6665,
          0.7102
        ],
        [
          0.6765,
          0.7034
        ],
        [
          0.6865,
          0.6961
        ],
        [
          0.6965,
          0.6897
        ],
        [
          0.7066,
          0.6838
        ],
        [
          0.717,
          0.6765
        ],
        [
          0.727,
          0.672
        ],
        [
          0.737,
          0.6642
        ],
        [
          0.747,
          0.6579
        ],
        [
          0.7571,
          0.6542
        ],
        [
          0.7675,
          0.6456
        ],
        [
          0.7775,
          0.6406
        ],
        [
          0.7875,
          0.6328
        ],
        [
          0.7975,
          0.6251
        ],
        [
          0.8076,
          0.6187
        ],
        [
          0.818,
          0.6128
        ],
        [
          0.828,
          0.6056
        ],
        [
          0.838,
          0.5987
        ],
        [
          0.848,
          0.5928
        ],
        [
          0.8581,
          0.5846
        ],
        [
          0.8685,
          0.5773
        ],
        [
          0.8785,
          0.5701
        ],
        [
          0.8885,
          0.5628
        ],
        [
          0.8985,
          0.5564
        ],
        [
          0.9086,
          0.5487
        ],
        [
          0.919,
          0.54
        ],
        [
          0.929,
          0.5337
        ],
        [
          0.939,
          0.5255
        ],
        [
          0.949,
          0.5173
        ],
        [
          0.9591,
          0.5086
        ],
        [
          0.9695,
          0.5014
        ],
        [
          0.9795,
          0.4932
        ],
        [
          0.9895,
          0.4845
        ],
        [
          0.9995,
          0.4777
        ],
        [
          1.0,
          0.4777
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.07302788731665506,
        "difference": -0.0008760594234234772,
        "worse": 0.48,
        "kept_right": 0.07215182789323157,
        "kept_wrong": 0.0008760594234234772,
        "handed_off": 0.9269721126833453
      },
      "usd_per_1000": 0.8969642310881906
    },
    {
      "decision": 223,
      "frontier": 190,
      "frontier_accuracy": 0.9672429481346679,
      "in_sample": {
        "threshold": 0.9532922221023848,
        "answered": 0.0632393084622384,
        "accuracy": 0.9672429481346679
      },
      "curve": [
        [
          0.0,
          0.9672
        ],
        [
          0.01,
          0.9672
        ],
        [
          0.02,
          0.9672
        ],
        [
          0.03,
          0.9672
        ],
        [
          0.04,
          0.9672
        ],
        [
          0.05,
          0.9672
        ],
        [
          0.0601,
          0.9672
        ],
        [
          0.0705,
          0.9668
        ],
        [
          0.0805,
          0.9663
        ],
        [
          0.0905,
          0.9659
        ],
        [
          0.1005,
          0.9645
        ],
        [
          0.1106,
          0.9631
        ],
        [
          0.121,
          0.9636
        ],
        [
          0.131,
          0.9622
        ],
        [
          0.141,
          0.9595
        ],
        [
          0.151,
          0.9581
        ],
        [
          0.1611,
          0.9545
        ],
        [
          0.1715,
          0.9527
        ],
        [
          0.1815,
          0.95
        ],
        [
          0.1915,
          0.9481
        ],
        [
          0.2015,
          0.9468
        ],
        [
          0.2116,
          0.9445
        ],
        [
          0.222,
          0.9413
        ],
        [
          0.232,
          0.9381
        ],
        [
          0.242,
          0.9345
        ],
        [
          0.252,
          0.9299
        ],
        [
          0.2621,
          0.9258
        ],
        [
          0.2725,
          0.9236
        ],
        [
          0.2825,
          0.9177
        ],
        [
          0.2925,
          0.914
        ],
        [
          0.3025,
          0.9108
        ],
        [
          0.3126,
          0.9067
        ],
        [
          0.323,
          0.9008
        ],
        [
          0.333,
          0.8967
        ],
        [
          0.343,
          0.8931
        ],
        [
          0.353,
          0.8904
        ],
        [
          0.3631,
          0.8849
        ],
        [
          0.3735,
          0.8794
        ],
        [
          0.3835,
          0.8744
        ],
        [
          0.3935,
          0.8681
        ],
        [
          0.4035,
          0.8631
        ],
        [
          0.4136,
          0.8562
        ],
        [
          0.424,
          0.8503
        ],
        [
          0.434,
          0.8426
        ],
        [
          0.444,
          0.8371
        ],
        [
          0.454,
          0.8308
        ],
        [
          0.4641,
          0.8262
        ],
        [
          0.4745,
          0.8198
        ],
        [
          0.4845,
          0.8148
        ],
        [
          0.4945,
          0.808
        ],
        [
          0.5045,
          0.803
        ],
        [
          0.5146,
          0.7985
        ],
        [
          0.525,
          0.793
        ],
        [
          0.535,
          0.7889
        ],
        [
          0.545,
          0.783
        ],
        [
          0.5551,
          0.7771
        ],
        [
          0.5655,
          0.7712
        ],
        [
          0.5755,
          0.7657
        ],
        [
          0.5855,
          0.7584
        ],
        [
          0.5955,
          0.7511
        ],
        [
          0.6056,
          0.7457
        ],
        [
          0.616,
          0.7379
        ],
        [
          0.626,
          0.7316
        ],
        [
          0.636,
          0.7261
        ],
        [
          0.646,
          0.7216
        ],
        [
          0.6561,
          0.7129
        ],
        [
          0.6665,
          0.7075
        ],
        [
          0.6765,
          0.7006
        ],
        [
          0.6865,
          0.6929
        ],
        [
          0.6965,
          0.6865
        ],
        [
          0.7066,
          0.6806
        ],
        [
          0.717,
          0.6733
        ],
        [
          0.727,
          0.6688
        ],
        [
          0.737,
          0.6615
        ],
        [
          0.747,
          0.6542
        ],
        [
          0.7571,
          0.6506
        ],
        [
          0.7675,
          0.6442
        ],
        [
          0.7775,
          0.6388
        ],
        [
          0.7875,
          0.631
        ],
        [
          0.7975,
          0.6228
        ],
        [
          0.8076,
          0.6165
        ],
        [
          0.818,
          0.6106
        ],
        [
          0.828,
          0.6033
        ],
        [
          0.838,
          0.5965
        ],
        [
          0.848,
          0.5905
        ],
        [
          0.8581,
          0.5833
        ],
        [
          0.8685,
          0.576
        ],
        [
          0.8785,
          0.5687
        ],
        [
          0.8885,
          0.5614
        ],
        [
          0.8985,
          0.556
        ],
        [
          0.9086,
          0.5469
        ],
        [
          0.919,
          0.5387
        ],
        [
          0.929,
          0.5318
        ],
        [
          0.939,
          0.5241
        ],
        [
          0.949,
          0.5159
        ],
        [
          0.9591,
          0.5073
        ],
        [
          0.9695,
          0.5
        ],
        [
          0.9795,
          0.4927
        ],
        [
          0.9895,
          0.4841
        ],
        [
          0.9995,
          0.4777
        ],
        [
          1.0,
          0.4777
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.07439542769473545,
        "difference": -0.0010337650260678645,
        "worse": 0.48333333333333334,
        "kept_right": 0.07332227692111601,
        "kept_wrong": 0.0010731507736194707,
        "handed_off": 0.9256045723052648
      },
      "usd_per_1000": 5.158129038257264
    },
    {
      "decision": 223,
      "frontier": 124,
      "frontier_accuracy": 0.9667879890809827,
      "in_sample": {
        "threshold": 0.9532922221023848,
        "answered": 0.0632393084622384,
        "accuracy": 0.9667879890809827
      },
      "curve": [
        [
          0.0,
          0.9668
        ],
        [
          0.01,
          0.9668
        ],
        [
          0.02,
          0.9668
        ],
        [
          0.03,
          0.9668
        ],
        [
          0.04,
          0.9668
        ],
        [
          0.05,
          0.9668
        ],
        [
          0.0601,
          0.9668
        ],
        [
          0.0705,
          0.9663
        ],
        [
          0.0805,
          0.9659
        ],
        [
          0.0905,
          0.9654
        ],
        [
          0.1005,
          0.9645
        ],
        [
          0.1106,
          0.9631
        ],
        [
          0.121,
          0.9631
        ],
        [
          0.131,
          0.9618
        ],
        [
          0.141,
          0.9591
        ],
        [
          0.151,
          0.9577
        ],
        [
          0.1611,
          0.9545
        ],
        [
          0.1715,
          0.9522
        ],
        [
          0.1815,
          0.949
        ],
        [
          0.1915,
          0.9472
        ],
        [
          0.2015,
          0.9463
        ],
        [
          0.2116,
          0.944
        ],
        [
          0.222,
          0.9409
        ],
        [
          0.232,
          0.9377
        ],
        [
          0.242,
          0.934
        ],
        [
          0.252,
          0.9295
        ],
        [
          0.2621,
          0.9254
        ],
        [
          0.2725,
          0.9231
        ],
        [
          0.2825,
          0.9172
        ],
        [
          0.2925,
          0.9136
        ],
        [
          0.3025,
          0.9104
        ],
        [
          0.3126,
          0.9063
        ],
        [
          0.323,
          0.9004
        ],
        [
          0.333,
          0.8963
        ],
        [
          0.343,
          0.8931
        ],
        [
          0.353,
          0.8899
        ],
        [
          0.3631,
          0.8844
        ],
        [
          0.3735,
          0.8785
        ],
        [
          0.3835,
          0.8735
        ],
        [
          0.3935,
          0.8672
        ],
        [
          0.4035,
          0.8626
        ],
        [
          0.4136,
          0.8567
        ],
        [
          0.424,
          0.8508
        ],
        [
          0.434,
          0.843
        ],
        [
          0.444,
          0.8376
        ],
        [
          0.454,
          0.8312
        ],
        [
          0.4641,
          0.8262
        ],
        [
          0.4745,
          0.8194
        ],
        [
          0.4845,
          0.8144
        ],
        [
          0.4945,
          0.8076
        ],
        [
          0.5045,
          0.8035
        ],
        [
          0.5146,
          0.7989
        ],
        [
          0.525,
          0.7939
        ],
        [
          0.535,
          0.7903
        ],
        [
          0.545,
          0.7843
        ],
        [
          0.5551,
          0.7784
        ],
        [
          0.5655,
          0.7725
        ],
        [
          0.5755,
          0.7671
        ],
        [
          0.5855,
          0.7598
        ],
        [
          0.5955,
          0.7525
        ],
        [
          0.6056,
          0.747
        ],
        [
          0.616,
          0.7393
        ],
        [
          0.626,
          0.7334
        ],
        [
          0.636,
          0.7279
        ],
        [
          0.646,
          0.7234
        ],
        [
          0.6561,
          0.7147
        ],
        [
          0.6665,
          0.7093
        ],
        [
          0.6765,
          0.702
        ],
        [
          0.6865,
          0.6943
        ],
        [
          0.6965,
          0.6879
        ],
        [
          0.7066,
          0.682
        ],
        [
          0.717,
          0.6747
        ],
        [
          0.727,
          0.6697
        ],
        [
          0.737,
          0.6629
        ],
        [
          0.747,
          0.6561
        ],
        [
          0.7571,
          0.6524
        ],
        [
          0.7675,
          0.6451
        ],
        [
          0.7775,
          0.6401
        ],
        [
          0.7875,
          0.6324
        ],
        [
          0.7975,
          0.6242
        ],
        [
          0.8076,
          0.6178
        ],
        [
          0.818,
          0.6119
        ],
        [
          0.828,
          0.6046
        ],
        [
          0.838,
          0.5978
        ],
        [
          0.848,
          0.5924
        ],
        [
          0.8581,
          0.5842
        ],
        [
          0.8685,
          0.5769
        ],
        [
          0.8785,
          0.5701
        ],
        [
          0.8885,
          0.5628
        ],
        [
          0.8985,
          0.5564
        ],
        [
          0.9086,
          0.5473
        ],
        [
          0.919,
          0.5391
        ],
        [
          0.929,
          0.5328
        ],
        [
          0.939,
          0.525
        ],
        [
          0.949,
          0.5168
        ],
        [
          0.9591,
          0.5077
        ],
        [
          0.9695,
          0.5005
        ],
        [
          0.9795,
          0.4932
        ],
        [
          0.9895,
          0.485
        ],
        [
          0.9995,
          0.4777
        ],
        [
          1.0,
          0.4777
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.07826689032487401,
        "difference": -0.0013551843383423341,
        "worse": 0.55,
        "kept_right": 0.07683894554567815,
        "kept_wrong": 0.0014279447791959356,
        "handed_off": 0.9217331096751261
      },
      "usd_per_1000": 1.0457846564942817
    },
    {
      "decision": 223,
      "frontier": 278,
      "frontier_accuracy": 0.9499545040946314,
      "in_sample": {
        "threshold": 0.9532922221023848,
        "answered": 0.0632393084622384,
        "accuracy": 0.9499545040946314
      },
      "curve": [
        [
          0.0,
          0.95
        ],
        [
          0.01,
          0.95
        ],
        [
          0.02,
          0.95
        ],
        [
          0.03,
          0.95
        ],
        [
          0.04,
          0.95
        ],
        [
          0.05,
          0.95
        ],
        [
          0.0601,
          0.95
        ],
        [
          0.0705,
          0.9495
        ],
        [
          0.0805,
          0.949
        ],
        [
          0.0905,
          0.9486
        ],
        [
          0.1005,
          0.9477
        ],
        [
          0.1106,
          0.9463
        ],
        [
          0.121,
          0.9459
        ],
        [
          0.131,
          0.9445
        ],
        [
          0.141,
          0.9418
        ],
        [
          0.151,
          0.9404
        ],
        [
          0.1611,
          0.9377
        ],
        [
          0.1715,
          0.9359
        ],
        [
          0.1815,
          0.9327
        ],
        [
          0.1915,
          0.9308
        ],
        [
          0.2015,
          0.9295
        ],
        [
          0.2116,
          0.9272
        ],
        [
          0.222,
          0.924
        ],
        [
          0.232,
          0.9208
        ],
        [
          0.242,
          0.9167
        ],
        [
          0.252,
          0.9126
        ],
        [
          0.2621,
          0.909
        ],
        [
          0.2725,
          0.9067
        ],
        [
          0.2825,
          0.9008
        ],
        [
          0.2925,
          0.8976
        ],
        [
          0.3025,
          0.8944
        ],
        [
          0.3126,
          0.8904
        ],
        [
          0.323,
          0.8849
        ],
        [
          0.333,
          0.8813
        ],
        [
          0.343,
          0.8781
        ],
        [
          0.353,
          0.8753
        ],
        [
          0.3631,
          0.8703
        ],
        [
          0.3735,
          0.8653
        ],
        [
          0.3835,
          0.8603
        ],
        [
          0.3935,
          0.854
        ],
        [
          0.4035,
          0.8494
        ],
        [
          0.4136,
          0.843
        ],
        [
          0.424,
          0.8371
        ],
        [
          0.434,
          0.8294
        ],
        [
          0.444,
          0.8239
        ],
        [
          0.454,
          0.8176
        ],
        [
          0.4641,
          0.8126
        ],
        [
          0.4745,
          0.8066
        ],
        [
          0.4845,
          0.8021
        ],
        [
          0.4945,
          0.7953
        ],
        [
          0.5045,
          0.7907
        ],
        [
          0.5146,
          0.7862
        ],
        [
          0.525,
          0.7812
        ],
        [
          0.535,
          0.7771
        ],
        [
          0.545,
          0.7712
        ],
        [
          0.5551,
          0.7662
        ],
        [
          0.5655,
          0.7602
        ],
        [
          0.5755,
          0.7543
        ],
        [
          0.5855,
          0.7475
        ],
        [
          0.5955,
          0.7416
        ],
        [
          0.6056,
          0.7361
        ],
        [
          0.616,
          0.7284
        ],
        [
          0.626,
          0.7225
        ],
        [
          0.636,
          0.717
        ],
        [
          0.646,
          0.7125
        ],
        [
          0.6561,
          0.7047
        ],
        [
          0.6665,
          0.6993
        ],
        [
          0.6765,
          0.6929
        ],
        [
          0.6865,
          0.6852
        ],
        [
          0.6965,
          0.6793
        ],
        [
          0.7066,
          0.6733
        ],
        [
          0.717,
          0.6661
        ],
        [
          0.727,
          0.6629
        ],
        [
          0.737,
          0.6561
        ],
        [
          0.747,
          0.6492
        ],
        [
          0.7571,
          0.6451
        ],
        [
          0.7675,
          0.6374
        ],
        [
          0.7775,
          0.6328
        ],
        [
          0.7875,
          0.6256
        ],
        [
          0.7975,
          0.6178
        ],
        [
          0.8076,
          0.6115
        ],
        [
          0.818,
          0.6056
        ],
        [
          0.828,
          0.6001
        ],
        [
          0.838,
          0.5933
        ],
        [
          0.848,
          0.5874
        ],
        [
          0.8581,
          0.5801
        ],
        [
          0.8685,
          0.5723
        ],
        [
          0.8785,
          0.5651
        ],
        [
          0.8885,
          0.5587
        ],
        [
          0.8985,
          0.5528
        ],
        [
          0.9086,
          0.5446
        ],
        [
          0.919,
          0.5378
        ],
        [
          0.929,
          0.5318
        ],
        [
          0.939,
          0.5241
        ],
        [
          0.949,
          0.5159
        ],
        [
          0.9591,
          0.5068
        ],
        [
          0.9695,
          0.5
        ],
        [
          0.9795,
          0.4932
        ],
        [
          0.9895,
          0.4845
        ],
        [
          0.9995,
          0.4777
        ],
        [
          1.0,
          0.4777
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.07739611961077025,
        "difference": -0.0012580794068546611,
        "worse": 0.55,
        "kept_right": 0.07609257732635925,
        "kept_wrong": 0.0013035422844110195,
        "handed_off": 0.92260388038923
      },
      "usd_per_1000": 6.008950772939605
    },
    {
      "decision": 223,
      "frontier": 275,
      "frontier_accuracy": 0.9426751592356688,
      "in_sample": {
        "threshold": 0.9532922221023848,
        "answered": 0.0632393084622384,
        "accuracy": 0.9426751592356688
      },
      "curve": [
        [
          0.0,
          0.9427
        ],
        [
          0.01,
          0.9427
        ],
        [
          0.02,
          0.9427
        ],
        [
          0.03,
          0.9427
        ],
        [
          0.04,
          0.9427
        ],
        [
          0.05,
          0.9427
        ],
        [
          0.0601,
          0.9427
        ],
        [
          0.0705,
          0.9422
        ],
        [
          0.0805,
          0.9418
        ],
        [
          0.0905,
          0.9413
        ],
        [
          0.1005,
          0.9399
        ],
        [
          0.1106,
          0.9386
        ],
        [
          0.121,
          0.9386
        ],
        [
          0.131,
          0.9372
        ],
        [
          0.141,
          0.9345
        ],
        [
          0.151,
          0.9331
        ],
        [
          0.1611,
          0.9299
        ],
        [
          0.1715,
          0.9281
        ],
        [
          0.1815,
          0.9249
        ],
        [
          0.1915,
          0.9231
        ],
        [
          0.2015,
          0.9217
        ],
        [
          0.2116,
          0.9195
        ],
        [
          0.222,
          0.9163
        ],
        [
          0.232,
          0.9136
        ],
        [
          0.242,
          0.9095
        ],
        [
          0.252,
          0.9049
        ],
        [
          0.2621,
          0.9008
        ],
        [
          0.2725,
          0.8985
        ],
        [
          0.2825,
          0.8926
        ],
        [
          0.2925,
          0.8899
        ],
        [
          0.3025,
          0.8867
        ],
        [
          0.3126,
          0.8831
        ],
        [
          0.323,
          0.8776
        ],
        [
          0.333,
          0.8735
        ],
        [
          0.343,
          0.8703
        ],
        [
          0.353,
          0.8672
        ],
        [
          0.3631,
          0.8617
        ],
        [
          0.3735,
          0.8562
        ],
        [
          0.3835,
          0.8512
        ],
        [
          0.3935,
          0.8449
        ],
        [
          0.4035,
          0.8403
        ],
        [
          0.4136,
          0.8344
        ],
        [
          0.424,
          0.8285
        ],
        [
          0.434,
          0.8207
        ],
        [
          0.444,
          0.8153
        ],
        [
          0.454,
          0.8094
        ],
        [
          0.4641,
          0.8044
        ],
        [
          0.4745,
          0.798
        ],
        [
          0.4845,
          0.7934
        ],
        [
          0.4945,
          0.7875
        ],
        [
          0.5045,
          0.783
        ],
        [
          0.5146,
          0.7784
        ],
        [
          0.525,
          0.7739
        ],
        [
          0.535,
          0.7702
        ],
        [
          0.545,
          0.7643
        ],
        [
          0.5551,
          0.7593
        ],
        [
          0.5655,
          0.7539
        ],
        [
          0.5755,
          0.7484
        ],
        [
          0.5855,
          0.7416
        ],
        [
          0.5955,
          0.7352
        ],
        [
          0.6056,
          0.7298
        ],
        [
          0.616,
          0.7243
        ],
        [
          0.626,
          0.7188
        ],
        [
          0.636,
          0.7147
        ],
        [
          0.646,
          0.7102
        ],
        [
          0.6561,
          0.7029
        ],
        [
          0.6665,
          0.6975
        ],
        [
          0.6765,
          0.6902
        ],
        [
          0.6865,
          0.6824
        ],
        [
          0.6965,
          0.6765
        ],
        [
          0.7066,
          0.6706
        ],
        [
          0.717,
          0.6638
        ],
        [
          0.727,
          0.6592
        ],
        [
          0.737,
          0.6524
        ],
        [
          0.747,
          0.646
        ],
        [
          0.7571,
          0.6415
        ],
        [
          0.7675,
          0.6347
        ],
        [
          0.7775,
          0.6297
        ],
        [
          0.7875,
          0.6224
        ],
        [
          0.7975,
          0.6156
        ],
        [
          0.8076,
          0.6092
        ],
        [
          0.818,
          0.6042
        ],
        [
          0.828,
          0.5987
        ],
        [
          0.838,
          0.5919
        ],
        [
          0.848,
          0.5869
        ],
        [
          0.8581,
          0.5801
        ],
        [
          0.8685,
          0.5728
        ],
        [
          0.8785,
          0.5655
        ],
        [
          0.8885,
          0.5582
        ],
        [
          0.8985,
          0.5528
        ],
        [
          0.9086,
          0.5446
        ],
        [
          0.919,
          0.5373
        ],
        [
          0.929,
          0.5314
        ],
        [
          0.939,
          0.5237
        ],
        [
          0.949,
          0.5159
        ],
        [
          0.9591,
          0.5073
        ],
        [
          0.9695,
          0.5
        ],
        [
          0.9795,
          0.4932
        ],
        [
          0.9895,
          0.4845
        ],
        [
          0.9995,
          0.4777
        ],
        [
          1.0,
          0.4777
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.07382515643292784,
        "difference": -0.000985200174807337,
        "worse": 0.48,
        "kept_right": 0.07283086257166686,
        "kept_wrong": 0.0009942938612609719,
        "handed_off": 0.9261748435670725
      },
      "usd_per_1000": 0.4804115516787998
    },
    {
      "decision": 223,
      "frontier": 192,
      "frontier_accuracy": 0.9331210191082803,
      "in_sample": {
        "threshold": 0.9532922221023848,
        "answered": 0.0632393084622384,
        "accuracy": 0.9331210191082803
      },
      "curve": [
        [
          0.0,
          0.9331
        ],
        [
          0.01,
          0.9331
        ],
        [
          0.02,
          0.9331
        ],
        [
          0.03,
          0.9331
        ],
        [
          0.04,
          0.9331
        ],
        [
          0.05,
          0.9331
        ],
        [
          0.0601,
          0.9331
        ],
        [
          0.0705,
          0.9327
        ],
        [
          0.0805,
          0.9322
        ],
        [
          0.0905,
          0.9318
        ],
        [
          0.1005,
          0.9308
        ],
        [
          0.1106,
          0.9295
        ],
        [
          0.121,
          0.9295
        ],
        [
          0.131,
          0.9281
        ],
        [
          0.141,
          0.9254
        ],
        [
          0.151,
          0.9245
        ],
        [
          0.1611,
          0.9213
        ],
        [
          0.1715,
          0.9186
        ],
        [
          0.1815,
          0.9154
        ],
        [
          0.1915,
          0.9136
        ],
        [
          0.2015,
          0.9122
        ],
        [
          0.2116,
          0.9104
        ],
        [
          0.222,
          0.9076
        ],
        [
          0.232,
          0.9045
        ],
        [
          0.242,
          0.9008
        ],
        [
          0.252,
          0.8967
        ],
        [
          0.2621,
          0.8931
        ],
        [
          0.2725,
          0.8913
        ],
        [
          0.2825,
          0.8854
        ],
        [
          0.2925,
          0.8826
        ],
        [
          0.3025,
          0.8794
        ],
        [
          0.3126,
          0.8753
        ],
        [
          0.323,
          0.8699
        ],
        [
          0.333,
          0.8667
        ],
        [
          0.343,
          0.8631
        ],
        [
          0.353,
          0.8603
        ],
        [
          0.3631,
          0.8549
        ],
        [
          0.3735,
          0.8494
        ],
        [
          0.3835,
          0.8444
        ],
        [
          0.3935,
          0.838
        ],
        [
          0.4035,
          0.8339
        ],
        [
          0.4136,
          0.8285
        ],
        [
          0.424,
          0.8226
        ],
        [
          0.434,
          0.8153
        ],
        [
          0.444,
          0.8098
        ],
        [
          0.454,
          0.803
        ],
        [
          0.4641,
          0.798
        ],
        [
          0.4745,
          0.7921
        ],
        [
          0.4845,
          0.7875
        ],
        [
          0.4945,
          0.7816
        ],
        [
          0.5045,
          0.7771
        ],
        [
          0.5146,
          0.773
        ],
        [
          0.525,
          0.768
        ],
        [
          0.535,
          0.7639
        ],
        [
          0.545,
          0.758
        ],
        [
          0.5551,
          0.7525
        ],
        [
          0.5655,
          0.747
        ],
        [
          0.5755,
          0.7425
        ],
        [
          0.5855,
          0.7357
        ],
        [
          0.5955,
          0.7293
        ],
        [
          0.6056,
          0.7243
        ],
        [
          0.616,
          0.717
        ],
        [
          0.626,
          0.712
        ],
        [
          0.636,
          0.7075
        ],
        [
          0.646,
          0.7029
        ],
        [
          0.6561,
          0.6943
        ],
        [
          0.6665,
          0.6888
        ],
        [
          0.6765,
          0.682
        ],
        [
          0.6865,
          0.6738
        ],
        [
          0.6965,
          0.6683
        ],
        [
          0.7066,
          0.6629
        ],
        [
          0.717,
          0.6556
        ],
        [
          0.727,
          0.6529
        ],
        [
          0.737,
          0.6465
        ],
        [
          0.747,
          0.6397
        ],
        [
          0.7571,
          0.6356
        ],
        [
          0.7675,
          0.6283
        ],
        [
          0.7775,
          0.6233
        ],
        [
          0.7875,
          0.6165
        ],
        [
          0.7975,
          0.6092
        ],
        [
          0.8076,
          0.6028
        ],
        [
          0.818,
          0.5974
        ],
        [
          0.828,
          0.5933
        ],
        [
          0.838,
          0.5864
        ],
        [
          0.848,
          0.581
        ],
        [
          0.8581,
          0.5751
        ],
        [
          0.8685,
          0.5678
        ],
        [
          0.8785,
          0.5619
        ],
        [
          0.8885,
          0.5546
        ],
        [
          0.8985,
          0.55
        ],
        [
          0.9086,
          0.5428
        ],
        [
          0.919,
          0.535
        ],
        [
          0.929,
          0.5287
        ],
        [
          0.939,
          0.5214
        ],
        [
          0.949,
          0.5141
        ],
        [
          0.9591,
          0.5059
        ],
        [
          0.9695,
          0.4991
        ],
        [
          0.9795,
          0.4914
        ],
        [
          0.9895,
          0.4836
        ],
        [
          0.9995,
          0.4777
        ],
        [
          1.0,
          0.4777
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.07826689032487401,
        "difference": -0.0013551843383423341,
        "worse": 0.55,
        "kept_right": 0.07683894554567815,
        "kept_wrong": 0.0014279447791959356,
        "handed_off": 0.9217331096751261
      },
      "usd_per_1000": 10.704803130437865
    },
    {
      "decision": 223,
      "frontier": 123,
      "frontier_accuracy": 0.9326660600545951,
      "in_sample": {
        "threshold": 0.9532922221023848,
        "answered": 0.0632393084622384,
        "accuracy": 0.9326660600545951
      },
      "curve": [
        [
          0.0,
          0.9327
        ],
        [
          0.01,
          0.9327
        ],
        [
          0.02,
          0.9327
        ],
        [
          0.03,
          0.9327
        ],
        [
          0.04,
          0.9327
        ],
        [
          0.05,
          0.9327
        ],
        [
          0.0601,
          0.9327
        ],
        [
          0.0705,
          0.9322
        ],
        [
          0.0805,
          0.9318
        ],
        [
          0.0905,
          0.9313
        ],
        [
          0.1005,
          0.9304
        ],
        [
          0.1106,
          0.929
        ],
        [
          0.121,
          0.9286
        ],
        [
          0.131,
          0.9272
        ],
        [
          0.141,
          0.9249
        ],
        [
          0.151,
          0.9236
        ],
        [
          0.1611,
          0.9208
        ],
        [
          0.1715,
          0.919
        ],
        [
          0.1815,
          0.9158
        ],
        [
          0.1915,
          0.914
        ],
        [
          0.2015,
          0.9126
        ],
        [
          0.2116,
          0.9108
        ],
        [
          0.222,
          0.9076
        ],
        [
          0.232,
          0.9045
        ],
        [
          0.242,
          0.9004
        ],
        [
          0.252,
          0.8958
        ],
        [
          0.2621,
          0.8922
        ],
        [
          0.2725,
          0.8899
        ],
        [
          0.2825,
          0.884
        ],
        [
          0.2925,
          0.8813
        ],
        [
          0.3025,
          0.879
        ],
        [
          0.3126,
          0.8753
        ],
        [
          0.323,
          0.8694
        ],
        [
          0.333,
          0.8658
        ],
        [
          0.343,
          0.8626
        ],
        [
          0.353,
          0.8594
        ],
        [
          0.3631,
          0.8553
        ],
        [
          0.3735,
          0.8499
        ],
        [
          0.3835,
          0.8449
        ],
        [
          0.3935,
          0.8389
        ],
        [
          0.4035,
          0.8344
        ],
        [
          0.4136,
          0.828
        ],
        [
          0.424,
          0.8226
        ],
        [
          0.434,
          0.8153
        ],
        [
          0.444,
          0.8103
        ],
        [
          0.454,
          0.8035
        ],
        [
          0.4641,
          0.7985
        ],
        [
          0.4745,
          0.7925
        ],
        [
          0.4845,
          0.788
        ],
        [
          0.4945,
          0.7816
        ],
        [
          0.5045,
          0.7771
        ],
        [
          0.5146,
          0.7725
        ],
        [
          0.525,
          0.7684
        ],
        [
          0.535,
          0.7652
        ],
        [
          0.545,
          0.7593
        ],
        [
          0.5551,
          0.7548
        ],
        [
          0.5655,
          0.7493
        ],
        [
          0.5755,
          0.7443
        ],
        [
          0.5855,
          0.7375
        ],
        [
          0.5955,
          0.7311
        ],
        [
          0.6056,
          0.7261
        ],
        [
          0.616,
          0.7193
        ],
        [
          0.626,
          0.7134
        ],
        [
          0.636,
          0.7075
        ],
        [
          0.646,
          0.7034
        ],
        [
          0.6561,
          0.6956
        ],
        [
          0.6665,
          0.6906
        ],
        [
          0.6765,
          0.6843
        ],
        [
          0.6865,
          0.677
        ],
        [
          0.6965,
          0.6706
        ],
        [
          0.7066,
          0.6656
        ],
        [
          0.717,
          0.6588
        ],
        [
          0.727,
          0.6547
        ],
        [
          0.737,
          0.6474
        ],
        [
          0.747,
          0.6419
        ],
        [
          0.7571,
          0.6383
        ],
        [
          0.7675,
          0.6297
        ],
        [
          0.7775,
          0.6247
        ],
        [
          0.7875,
          0.6169
        ],
        [
          0.7975,
          0.6101
        ],
        [
          0.8076,
          0.6042
        ],
        [
          0.818,
          0.5996
        ],
        [
          0.828,
          0.5946
        ],
        [
          0.838,
          0.5887
        ],
        [
          0.848,
          0.5833
        ],
        [
          0.8581,
          0.5764
        ],
        [
          0.8685,
          0.5701
        ],
        [
          0.8785,
          0.5637
        ],
        [
          0.8885,
          0.5564
        ],
        [
          0.8985,
          0.5519
        ],
        [
          0.9086,
          0.545
        ],
        [
          0.919,
          0.5378
        ],
        [
          0.929,
          0.5314
        ],
        [
          0.939,
          0.5237
        ],
        [
          0.949,
          0.5164
        ],
        [
          0.9591,
          0.5073
        ],
        [
          0.9695,
          0.5
        ],
        [
          0.9795,
          0.4918
        ],
        [
          0.9895,
          0.4845
        ],
        [
          0.9995,
          0.4777
        ],
        [
          1.0,
          0.4777
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.07745672567137632,
        "difference": -0.0012671703159455705,
        "worse": 0.55,
        "kept_right": 0.07614409247787442,
        "kept_wrong": 0.0013126331935019284,
        "handed_off": 0.9225432743286239
      },
      "usd_per_1000": 0.2516626822755766
    },
    {
      "decision": 223,
      "frontier": 274,
      "frontier_accuracy": 0.9085532302092811,
      "in_sample": {
        "threshold": 0.9532922221023848,
        "answered": 0.0632393084622384,
        "accuracy": 0.9085532302092811
      },
      "curve": [
        [
          0.0,
          0.9086
        ],
        [
          0.01,
          0.9086
        ],
        [
          0.02,
          0.9086
        ],
        [
          0.03,
          0.9086
        ],
        [
          0.04,
          0.9086
        ],
        [
          0.05,
          0.9086
        ],
        [
          0.0601,
          0.9086
        ],
        [
          0.0705,
          0.9081
        ],
        [
          0.0805,
          0.9076
        ],
        [
          0.0905,
          0.9072
        ],
        [
          0.1005,
          0.9058
        ],
        [
          0.1106,
          0.9045
        ],
        [
          0.121,
          0.9049
        ],
        [
          0.131,
          0.9035
        ],
        [
          0.141,
          0.9008
        ],
        [
          0.151,
          0.8995
        ],
        [
          0.1611,
          0.8958
        ],
        [
          0.1715,
          0.894
        ],
        [
          0.1815,
          0.8908
        ],
        [
          0.1915,
          0.8894
        ],
        [
          0.2015,
          0.8885
        ],
        [
          0.2116,
          0.8863
        ],
        [
          0.222,
          0.884
        ],
        [
          0.232,
          0.8808
        ],
        [
          0.242,
          0.8776
        ],
        [
          0.252,
          0.8735
        ],
        [
          0.2621,
          0.8694
        ],
        [
          0.2725,
          0.8672
        ],
        [
          0.2825,
          0.8612
        ],
        [
          0.2925,
          0.859
        ],
        [
          0.3025,
          0.8558
        ],
        [
          0.3126,
          0.8526
        ],
        [
          0.323,
          0.8471
        ],
        [
          0.333,
          0.8439
        ],
        [
          0.343,
          0.8412
        ],
        [
          0.353,
          0.838
        ],
        [
          0.3631,
          0.833
        ],
        [
          0.3735,
          0.8285
        ],
        [
          0.3835,
          0.8239
        ],
        [
          0.3935,
          0.8176
        ],
        [
          0.4035,
          0.8135
        ],
        [
          0.4136,
          0.8066
        ],
        [
          0.424,
          0.8025
        ],
        [
          0.434,
          0.7957
        ],
        [
          0.444,
          0.7903
        ],
        [
          0.454,
          0.7839
        ],
        [
          0.4641,
          0.7793
        ],
        [
          0.4745,
          0.7739
        ],
        [
          0.4845,
          0.7702
        ],
        [
          0.4945,
          0.7648
        ],
        [
          0.5045,
          0.7607
        ],
        [
          0.5146,
          0.7561
        ],
        [
          0.525,
          0.752
        ],
        [
          0.535,
          0.7489
        ],
        [
          0.545,
          0.7434
        ],
        [
          0.5551,
          0.7393
        ],
        [
          0.5655,
          0.7338
        ],
        [
          0.5755,
          0.7288
        ],
        [
          0.5855,
          0.7225
        ],
        [
          0.5955,
          0.7166
        ],
        [
          0.6056,
          0.7106
        ],
        [
          0.616,
          0.7052
        ],
        [
          0.626,
          0.7002
        ],
        [
          0.636,
          0.6952
        ],
        [
          0.646,
          0.6915
        ],
        [
          0.6561,
          0.6852
        ],
        [
          0.6665,
          0.6797
        ],
        [
          0.6765,
          0.6733
        ],
        [
          0.6865,
          0.6652
        ],
        [
          0.6965,
          0.6592
        ],
        [
          0.7066,
          0.6538
        ],
        [
          0.717,
          0.6479
        ],
        [
          0.727,
          0.6447
        ],
        [
          0.737,
          0.6383
        ],
        [
          0.747,
          0.6324
        ],
        [
          0.7571,
          0.6288
        ],
        [
          0.7675,
          0.6224
        ],
        [
          0.7775,
          0.6178
        ],
        [
          0.7875,
          0.6115
        ],
        [
          0.7975,
          0.6046
        ],
        [
          0.8076,
          0.5987
        ],
        [
          0.818,
          0.5946
        ],
        [
          0.828,
          0.5887
        ],
        [
          0.838,
          0.5833
        ],
        [
          0.848,
          0.5778
        ],
        [
          0.8581,
          0.5714
        ],
        [
          0.8685,
          0.5646
        ],
        [
          0.8785,
          0.5582
        ],
        [
          0.8885,
          0.5514
        ],
        [
          0.8985,
          0.5464
        ],
        [
          0.9086,
          0.5387
        ],
        [
          0.919,
          0.5314
        ],
        [
          0.929,
          0.5259
        ],
        [
          0.939,
          0.5187
        ],
        [
          0.949,
          0.5123
        ],
        [
          0.9591,
          0.5045
        ],
        [
          0.9695,
          0.4986
        ],
        [
          0.9795,
          0.4923
        ],
        [
          0.9895,
          0.4845
        ],
        [
          0.9995,
          0.4777
        ],
        [
          1.0,
          0.4777
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.0739949637960136,
        "difference": -0.0010034012993047968,
        "worse": 0.48,
        "kept_right": 0.07297034758811642,
        "kept_wrong": 0.0010246162078971827,
        "handed_off": 0.9260050362039867
      },
      "usd_per_1000": 0.39858801927164716
    },
    {
      "decision": 223,
      "frontier": 47,
      "frontier_accuracy": 0.9044585987261147,
      "in_sample": {
        "threshold": 0.9532922221023848,
        "answered": 0.0632393084622384,
        "accuracy": 0.9044585987261147
      },
      "curve": [
        [
          0.0,
          0.9045
        ],
        [
          0.01,
          0.9045
        ],
        [
          0.02,
          0.9045
        ],
        [
          0.03,
          0.9045
        ],
        [
          0.04,
          0.9045
        ],
        [
          0.05,
          0.9045
        ],
        [
          0.0601,
          0.9045
        ],
        [
          0.0705,
          0.904
        ],
        [
          0.0805,
          0.9035
        ],
        [
          0.0905,
          0.9031
        ],
        [
          0.1005,
          0.9022
        ],
        [
          0.1106,
          0.9008
        ],
        [
          0.121,
          0.9008
        ],
        [
          0.131,
          0.8995
        ],
        [
          0.141,
          0.8972
        ],
        [
          0.151,
          0.8963
        ],
        [
          0.1611,
          0.8926
        ],
        [
          0.1715,
          0.8908
        ],
        [
          0.1815,
          0.8876
        ],
        [
          0.1915,
          0.8858
        ],
        [
          0.2015,
          0.8844
        ],
        [
          0.2116,
          0.8826
        ],
        [
          0.222,
          0.8803
        ],
        [
          0.232,
          0.8772
        ],
        [
          0.242,
          0.8731
        ],
        [
          0.252,
          0.869
        ],
        [
          0.2621,
          0.8658
        ],
        [
          0.2725,
          0.8635
        ],
        [
          0.2825,
          0.8585
        ],
        [
          0.2925,
          0.8553
        ],
        [
          0.3025,
          0.8521
        ],
        [
          0.3126,
          0.848
        ],
        [
          0.323,
          0.8421
        ],
        [
          0.333,
          0.8385
        ],
        [
          0.343,
          0.8353
        ],
        [
          0.353,
          0.8326
        ],
        [
          0.3631,
          0.828
        ],
        [
          0.3735,
          0.8235
        ],
        [
          0.3835,
          0.8185
        ],
        [
          0.3935,
          0.8126
        ],
        [
          0.4035,
          0.8089
        ],
        [
          0.4136,
          0.8048
        ],
        [
          0.424,
          0.7989
        ],
        [
          0.434,
          0.7921
        ],
        [
          0.444,
          0.7871
        ],
        [
          0.454,
          0.7821
        ],
        [
          0.4641,
          0.7771
        ],
        [
          0.4745,
          0.7712
        ],
        [
          0.4845,
          0.7666
        ],
        [
          0.4945,
          0.7607
        ],
        [
          0.5045,
          0.7561
        ],
        [
          0.5146,
          0.7525
        ],
        [
          0.525,
          0.748
        ],
        [
          0.535,
          0.7443
        ],
        [
          0.545,
          0.7384
        ],
        [
          0.5551,
          0.7338
        ],
        [
          0.5655,
          0.7279
        ],
        [
          0.5755,
          0.722
        ],
        [
          0.5855,
          0.7157
        ],
        [
          0.5955,
          0.7084
        ],
        [
          0.6056,
          0.7034
        ],
        [
          0.616,
          0.6979
        ],
        [
          0.626,
          0.6934
        ],
        [
          0.636,
          0.6884
        ],
        [
          0.646,
          0.6847
        ],
        [
          0.6561,
          0.677
        ],
        [
          0.6665,
          0.672
        ],
        [
          0.6765,
          0.6661
        ],
        [
          0.6865,
          0.6592
        ],
        [
          0.6965,
          0.6547
        ],
        [
          0.7066,
          0.6492
        ],
        [
          0.717,
          0.6429
        ],
        [
          0.727,
          0.6401
        ],
        [
          0.737,
          0.6356
        ],
        [
          0.747,
          0.6301
        ],
        [
          0.7571,
          0.6265
        ],
        [
          0.7675,
          0.6187
        ],
        [
          0.7775,
          0.6146
        ],
        [
          0.7875,
          0.6083
        ],
        [
          0.7975,
          0.6015
        ],
        [
          0.8076,
          0.5955
        ],
        [
          0.818,
          0.5905
        ],
        [
          0.828,
          0.5869
        ],
        [
          0.838,
          0.5805
        ],
        [
          0.848,
          0.5751
        ],
        [
          0.8581,
          0.5687
        ],
        [
          0.8685,
          0.5623
        ],
        [
          0.8785,
          0.5573
        ],
        [
          0.8885,
          0.5523
        ],
        [
          0.8985,
          0.5469
        ],
        [
          0.9086,
          0.5387
        ],
        [
          0.919,
          0.5323
        ],
        [
          0.929,
          0.5268
        ],
        [
          0.939,
          0.5196
        ],
        [
          0.949,
          0.5136
        ],
        [
          0.9591,
          0.5055
        ],
        [
          0.9695,
          0.4991
        ],
        [
          0.9795,
          0.4927
        ],
        [
          0.9895,
          0.4845
        ],
        [
          0.9995,
          0.4777
        ],
        [
          1.0,
          0.4777
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.07826689032487401,
        "difference": -0.0013551843383423341,
        "worse": 0.55,
        "kept_right": 0.07683894554567815,
        "kept_wrong": 0.0014279447791959356,
        "handed_off": 0.9217331096751261
      },
      "usd_per_1000": 4.5485667161624725
    },
    {
      "decision": 223,
      "frontier": 193,
      "frontier_accuracy": 0.89171974522293,
      "in_sample": {
        "threshold": 0.9435815143285365,
        "answered": 0.07825295723384895,
        "accuracy": 0.89171974522293
      },
      "curve": [
        [
          0.0,
          0.8917
        ],
        [
          0.01,
          0.8917
        ],
        [
          0.02,
          0.8917
        ],
        [
          0.03,
          0.8917
        ],
        [
          0.04,
          0.8917
        ],
        [
          0.05,
          0.8917
        ],
        [
          0.0601,
          0.8917
        ],
        [
          0.0705,
          0.8917
        ],
        [
          0.0805,
          0.8913
        ],
        [
          0.0905,
          0.8908
        ],
        [
          0.1005,
          0.8899
        ],
        [
          0.1106,
          0.8885
        ],
        [
          0.121,
          0.8885
        ],
        [
          0.131,
          0.8872
        ],
        [
          0.141,
          0.8844
        ],
        [
          0.151,
          0.8835
        ],
        [
          0.1611,
          0.8803
        ],
        [
          0.1715,
          0.8785
        ],
        [
          0.1815,
          0.8763
        ],
        [
          0.1915,
          0.8744
        ],
        [
          0.2015,
          0.8731
        ],
        [
          0.2116,
          0.8708
        ],
        [
          0.222,
          0.8681
        ],
        [
          0.232,
          0.8653
        ],
        [
          0.242,
          0.8617
        ],
        [
          0.252,
          0.8576
        ],
        [
          0.2621,
          0.854
        ],
        [
          0.2725,
          0.8521
        ],
        [
          0.2825,
          0.8467
        ],
        [
          0.2925,
          0.8439
        ],
        [
          0.3025,
          0.8408
        ],
        [
          0.3126,
          0.8371
        ],
        [
          0.323,
          0.8321
        ],
        [
          0.333,
          0.8289
        ],
        [
          0.343,
          0.8258
        ],
        [
          0.353,
          0.8226
        ],
        [
          0.3631,
          0.818
        ],
        [
          0.3735,
          0.8121
        ],
        [
          0.3835,
          0.8071
        ],
        [
          0.3935,
          0.8021
        ],
        [
          0.4035,
          0.7985
        ],
        [
          0.4136,
          0.7944
        ],
        [
          0.424,
          0.7898
        ],
        [
          0.434,
          0.7834
        ],
        [
          0.444,
          0.7789
        ],
        [
          0.454,
          0.773
        ],
        [
          0.4641,
          0.768
        ],
        [
          0.4745,
          0.7643
        ],
        [
          0.4845,
          0.7602
        ],
        [
          0.4945,
          0.7548
        ],
        [
          0.5045,
          0.7511
        ],
        [
          0.5146,
          0.7475
        ],
        [
          0.525,
          0.7429
        ],
        [
          0.535,
          0.7393
        ],
        [
          0.545,
          0.7348
        ],
        [
          0.5551,
          0.7311
        ],
        [
          0.5655,
          0.7252
        ],
        [
          0.5755,
          0.7207
        ],
        [
          0.5855,
          0.7138
        ],
        [
          0.5955,
          0.7079
        ],
        [
          0.6056,
          0.7029
        ],
        [
          0.616,
          0.6965
        ],
        [
          0.626,
          0.692
        ],
        [
          0.636,
          0.6874
        ],
        [
          0.646,
          0.6843
        ],
        [
          0.6561,
          0.6774
        ],
        [
          0.6665,
          0.672
        ],
        [
          0.6765,
          0.6656
        ],
        [
          0.6865,
          0.6583
        ],
        [
          0.6965,
          0.6533
        ],
        [
          0.7066,
          0.6479
        ],
        [
          0.717,
          0.6415
        ],
        [
          0.727,
          0.6374
        ],
        [
          0.737,
          0.6315
        ],
        [
          0.747,
          0.6256
        ],
        [
          0.7571,
          0.6224
        ],
        [
          0.7675,
          0.6151
        ],
        [
          0.7775,
          0.6106
        ],
        [
          0.7875,
          0.6028
        ],
        [
          0.7975,
          0.5955
        ],
        [
          0.8076,
          0.5901
        ],
        [
          0.818,
          0.5851
        ],
        [
          0.828,
          0.5814
        ],
        [
          0.838,
          0.5746
        ],
        [
          0.848,
          0.5701
        ],
        [
          0.8581,
          0.5646
        ],
        [
          0.8685,
          0.5582
        ],
        [
          0.8785,
          0.5528
        ],
        [
          0.8885,
          0.5469
        ],
        [
          0.8985,
          0.5419
        ],
        [
          0.9086,
          0.535
        ],
        [
          0.919,
          0.5282
        ],
        [
          0.929,
          0.5232
        ],
        [
          0.939,
          0.5177
        ],
        [
          0.949,
          0.5109
        ],
        [
          0.9591,
          0.5023
        ],
        [
          0.9695,
          0.4964
        ],
        [
          0.9795,
          0.4895
        ],
        [
          0.9895,
          0.4832
        ],
        [
          0.9995,
          0.4777
        ],
        [
          1.0,
          0.4777
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.0856221061038871,
        "difference": -0.0015401542710028779,
        "worse": 0.5833333333333334,
        "kept_right": 0.08371211649019517,
        "kept_wrong": 0.0019099896136919273,
        "handed_off": 0.9143778938961132
      },
      "usd_per_1000": 7.088107393536756
    },
    {
      "decision": 223,
      "frontier": 55,
      "frontier_accuracy": 0.8503184713375797,
      "in_sample": {
        "threshold": 0.9117430755875403,
        "answered": 0.11874431301182893,
        "accuracy": 0.8503184713375797
      },
      "curve": [
        [
          0.0,
          0.8503
        ],
        [
          0.01,
          0.8503
        ],
        [
          0.02,
          0.8508
        ],
        [
          0.03,
          0.8512
        ],
        [
          0.04,
          0.8512
        ],
        [
          0.05,
          0.8512
        ],
        [
          0.0601,
          0.8512
        ],
        [
          0.0705,
          0.8517
        ],
        [
          0.0805,
          0.8512
        ],
        [
          0.0905,
          0.8508
        ],
        [
          0.1005,
          0.8503
        ],
        [
          0.1106,
          0.8503
        ],
        [
          0.121,
          0.8499
        ],
        [
          0.131,
          0.849
        ],
        [
          0.141,
          0.8471
        ],
        [
          0.151,
          0.8458
        ],
        [
          0.1611,
          0.843
        ],
        [
          0.1715,
          0.8408
        ],
        [
          0.1815,
          0.8385
        ],
        [
          0.1915,
          0.8376
        ],
        [
          0.2015,
          0.8376
        ],
        [
          0.2116,
          0.8358
        ],
        [
          0.222,
          0.8339
        ],
        [
          0.232,
          0.833
        ],
        [
          0.242,
          0.8303
        ],
        [
          0.252,
          0.8262
        ],
        [
          0.2621,
          0.8235
        ],
        [
          0.2725,
          0.8217
        ],
        [
          0.2825,
          0.818
        ],
        [
          0.2925,
          0.8157
        ],
        [
          0.3025,
          0.8126
        ],
        [
          0.3126,
          0.8098
        ],
        [
          0.323,
          0.8062
        ],
        [
          0.333,
          0.8025
        ],
        [
          0.343,
          0.7994
        ],
        [
          0.353,
          0.7975
        ],
        [
          0.3631,
          0.7939
        ],
        [
          0.3735,
          0.7903
        ],
        [
          0.3835,
          0.7862
        ],
        [
          0.3935,
          0.7825
        ],
        [
          0.4035,
          0.7793
        ],
        [
          0.4136,
          0.7753
        ],
        [
          0.424,
          0.7698
        ],
        [
          0.434,
          0.7634
        ],
        [
          0.444,
          0.7598
        ],
        [
          0.454,
          0.7539
        ],
        [
          0.4641,
          0.7493
        ],
        [
          0.4745,
          0.7457
        ],
        [
          0.4845,
          0.7416
        ],
        [
          0.4945,
          0.7361
        ],
        [
          0.5045,
          0.7325
        ],
        [
          0.5146,
          0.7288
        ],
        [
          0.525,
          0.7243
        ],
        [
          0.535,
          0.722
        ],
        [
          0.545,
          0.7179
        ],
        [
          0.5551,
          0.7157
        ],
        [
          0.5655,
          0.7097
        ],
        [
          0.5755,
          0.7056
        ],
        [
          0.5855,
          0.6993
        ],
        [
          0.5955,
          0.6943
        ],
        [
          0.6056,
          0.6902
        ],
        [
          0.616,
          0.6838
        ],
        [
          0.626,
          0.6802
        ],
        [
          0.636,
          0.6765
        ],
        [
          0.646,
          0.6729
        ],
        [
          0.6561,
          0.6656
        ],
        [
          0.6665,
          0.6601
        ],
        [
          0.6765,
          0.6538
        ],
        [
          0.6865,
          0.6474
        ],
        [
          0.6965,
          0.6429
        ],
        [
          0.7066,
          0.6379
        ],
        [
          0.717,
          0.6315
        ],
        [
          0.727,
          0.6278
        ],
        [
          0.737,
          0.6224
        ],
        [
          0.747,
          0.6174
        ],
        [
          0.7571,
          0.6137
        ],
        [
          0.7675,
          0.6069
        ],
        [
          0.7775,
          0.6033
        ],
        [
          0.7875,
          0.5974
        ],
        [
          0.7975,
          0.5905
        ],
        [
          0.8076,
          0.5851
        ],
        [
          0.818,
          0.5805
        ],
        [
          0.828,
          0.5778
        ],
        [
          0.838,
          0.5723
        ],
        [
          0.848,
          0.5678
        ],
        [
          0.8581,
          0.561
        ],
        [
          0.8685,
          0.5551
        ],
        [
          0.8785,
          0.5505
        ],
        [
          0.8885,
          0.5446
        ],
        [
          0.8985,
          0.54
        ],
        [
          0.9086,
          0.5337
        ],
        [
          0.919,
          0.5264
        ],
        [
          0.929,
          0.5223
        ],
        [
          0.939,
          0.5155
        ],
        [
          0.949,
          0.5077
        ],
        [
          0.9591,
          0.5
        ],
        [
          0.9695,
          0.4945
        ],
        [
          0.9795,
          0.4882
        ],
        [
          0.9895,
          0.4832
        ],
        [
          0.9995,
          0.4777
        ],
        [
          1.0,
          0.4777
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.11711420849402508,
        "difference": -0.0013975639914125244,
        "worse": 0.5366666666666666,
        "kept_right": 0.11158454412258313,
        "kept_wrong": 0.005529664371441976,
        "handed_off": 0.8828857915059746
      },
      "usd_per_1000": 3.2302416053269556
    },
    {
      "decision": 205,
      "frontier": 140,
      "frontier_accuracy": 0.9808917197452229,
      "in_sample": {
        "threshold": 0.9947150916400119,
        "answered": 0.009099181073703366,
        "accuracy": 0.9808917197452229
      },
      "curve": [
        [
          0.0,
          0.9809
        ],
        [
          0.01,
          0.9804
        ],
        [
          0.02,
          0.9804
        ],
        [
          0.03,
          0.98
        ],
        [
          0.04,
          0.98
        ],
        [
          0.05,
          0.98
        ],
        [
          0.0601,
          0.98
        ],
        [
          0.0705,
          0.9795
        ],
        [
          0.0805,
          0.9791
        ],
        [
          0.0905,
          0.9777
        ],
        [
          0.1005,
          0.9768
        ],
        [
          0.1106,
          0.9768
        ],
        [
          0.121,
          0.975
        ],
        [
          0.131,
          0.9741
        ],
        [
          0.141,
          0.9732
        ],
        [
          0.151,
          0.9704
        ],
        [
          0.1611,
          0.97
        ],
        [
          0.1715,
          0.9686
        ],
        [
          0.1815,
          0.9654
        ],
        [
          0.1915,
          0.9631
        ],
        [
          0.2015,
          0.9604
        ],
        [
          0.2116,
          0.9568
        ],
        [
          0.222,
          0.9554
        ],
        [
          0.232,
          0.954
        ],
        [
          0.242,
          0.9504
        ],
        [
          0.252,
          0.9472
        ],
        [
          0.2621,
          0.9431
        ],
        [
          0.2725,
          0.9399
        ],
        [
          0.2825,
          0.9368
        ],
        [
          0.2925,
          0.9313
        ],
        [
          0.3025,
          0.9268
        ],
        [
          0.3126,
          0.9208
        ],
        [
          0.323,
          0.9154
        ],
        [
          0.333,
          0.9113
        ],
        [
          0.343,
          0.9054
        ],
        [
          0.353,
          0.8999
        ],
        [
          0.3631,
          0.8963
        ],
        [
          0.3735,
          0.8894
        ],
        [
          0.3835,
          0.8826
        ],
        [
          0.3935,
          0.8758
        ],
        [
          0.4035,
          0.8703
        ],
        [
          0.4136,
          0.864
        ],
        [
          0.424,
          0.8576
        ],
        [
          0.434,
          0.8503
        ],
        [
          0.444,
          0.8439
        ],
        [
          0.454,
          0.8371
        ],
        [
          0.4641,
          0.8294
        ],
        [
          0.4745,
          0.8226
        ],
        [
          0.4845,
          0.8162
        ],
        [
          0.4945,
          0.8089
        ],
        [
          0.5045,
          0.803
        ],
        [
          0.5146,
          0.7948
        ],
        [
          0.525,
          0.788
        ],
        [
          0.535,
          0.7834
        ],
        [
          0.545,
          0.7771
        ],
        [
          0.5551,
          0.7707
        ],
        [
          0.5655,
          0.7639
        ],
        [
          0.5755,
          0.7589
        ],
        [
          0.5855,
          0.7539
        ],
        [
          0.5955,
          0.747
        ],
        [
          0.6056,
          0.7398
        ],
        [
          0.616,
          0.732
        ],
        [
          0.626,
          0.7257
        ],
        [
          0.636,
          0.7207
        ],
        [
          0.646,
          0.7157
        ],
        [
          0.6561,
          0.7102
        ],
        [
          0.6665,
          0.7043
        ],
        [
          0.6765,
          0.6965
        ],
        [
          0.6865,
          0.6902
        ],
        [
          0.6965,
          0.6833
        ],
        [
          0.7066,
          0.6761
        ],
        [
          0.717,
          0.6683
        ],
        [
          0.727,
          0.6597
        ],
        [
          0.737,
          0.6515
        ],
        [
          0.747,
          0.646
        ],
        [
          0.7571,
          0.6397
        ],
        [
          0.7675,
          0.6324
        ],
        [
          0.7775,
          0.6247
        ],
        [
          0.7875,
          0.6169
        ],
        [
          0.7975,
          0.6087
        ],
        [
          0.8076,
          0.601
        ],
        [
          0.818,
          0.5933
        ],
        [
          0.828,
          0.5869
        ],
        [
          0.838,
          0.5792
        ],
        [
          0.848,
          0.5732
        ],
        [
          0.8581,
          0.5678
        ],
        [
          0.8685,
          0.561
        ],
        [
          0.8785,
          0.556
        ],
        [
          0.8885,
          0.5482
        ],
        [
          0.8985,
          0.5414
        ],
        [
          0.9086,
          0.535
        ],
        [
          0.919,
          0.5273
        ],
        [
          0.929,
          0.5205
        ],
        [
          0.939,
          0.5127
        ],
        [
          0.949,
          0.5041
        ],
        [
          0.9591,
          0.4973
        ],
        [
          0.9695,
          0.4895
        ],
        [
          0.9795,
          0.4823
        ],
        [
          0.9895,
          0.4736
        ],
        [
          0.9995,
          0.465
        ],
        [
          1.0,
          0.4645
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.026445360780044298,
        "difference": -0.000794345800188106,
        "worse": 0.43666666666666665,
        "kept_right": 0.025435780524148247,
        "kept_wrong": 0.0010095802558960486,
        "handed_off": 0.973554639219956
      },
      "usd_per_1000": 2.3689454627709994
    },
    {
      "decision": 205,
      "frontier": 277,
      "frontier_accuracy": 0.9795268425841674,
      "in_sample": {
        "threshold": 0.9947150916400119,
        "answered": 0.009099181073703366,
        "accuracy": 0.9795268425841674
      },
      "curve": [
        [
          0.0,
          0.9795
        ],
        [
          0.01,
          0.9791
        ],
        [
          0.02,
          0.9786
        ],
        [
          0.03,
          0.9782
        ],
        [
          0.04,
          0.9782
        ],
        [
          0.05,
          0.9782
        ],
        [
          0.0601,
          0.9782
        ],
        [
          0.0705,
          0.9777
        ],
        [
          0.0805,
          0.9773
        ],
        [
          0.0905,
          0.9759
        ],
        [
          0.1005,
          0.975
        ],
        [
          0.1106,
          0.975
        ],
        [
          0.121,
          0.9732
        ],
        [
          0.131,
          0.9722
        ],
        [
          0.141,
          0.9713
        ],
        [
          0.151,
          0.9686
        ],
        [
          0.1611,
          0.9682
        ],
        [
          0.1715,
          0.9668
        ],
        [
          0.1815,
          0.9641
        ],
        [
          0.1915,
          0.9618
        ],
        [
          0.2015,
          0.9591
        ],
        [
          0.2116,
          0.9554
        ],
        [
          0.222,
          0.9536
        ],
        [
          0.232,
          0.9522
        ],
        [
          0.242,
          0.9486
        ],
        [
          0.252,
          0.9454
        ],
        [
          0.2621,
          0.9418
        ],
        [
          0.2725,
          0.9386
        ],
        [
          0.2825,
          0.9354
        ],
        [
          0.2925,
          0.9299
        ],
        [
          0.3025,
          0.9254
        ],
        [
          0.3126,
          0.919
        ],
        [
          0.323,
          0.9136
        ],
        [
          0.333,
          0.909
        ],
        [
          0.343,
          0.9026
        ],
        [
          0.353,
          0.8972
        ],
        [
          0.3631,
          0.8935
        ],
        [
          0.3735,
          0.8872
        ],
        [
          0.3835,
          0.8803
        ],
        [
          0.3935,
          0.8735
        ],
        [
          0.4035,
          0.8681
        ],
        [
          0.4136,
          0.8617
        ],
        [
          0.424,
          0.8558
        ],
        [
          0.434,
          0.8485
        ],
        [
          0.444,
          0.8421
        ],
        [
          0.454,
          0.8358
        ],
        [
          0.4641,
          0.8276
        ],
        [
          0.4745,
          0.8207
        ],
        [
          0.4845,
          0.8144
        ],
        [
          0.4945,
          0.8071
        ],
        [
          0.5045,
          0.8012
        ],
        [
          0.5146,
          0.7934
        ],
        [
          0.525,
          0.7866
        ],
        [
          0.535,
          0.7821
        ],
        [
          0.545,
          0.7757
        ],
        [
          0.5551,
          0.7693
        ],
        [
          0.5655,
          0.7625
        ],
        [
          0.5755,
          0.7571
        ],
        [
          0.5855,
          0.752
        ],
        [
          0.5955,
          0.7452
        ],
        [
          0.6056,
          0.7379
        ],
        [
          0.616,
          0.7302
        ],
        [
          0.626,
          0.7234
        ],
        [
          0.636,
          0.7184
        ],
        [
          0.646,
          0.7134
        ],
        [
          0.6561,
          0.7079
        ],
        [
          0.6665,
          0.702
        ],
        [
          0.6765,
          0.6943
        ],
        [
          0.6865,
          0.6879
        ],
        [
          0.6965,
          0.6806
        ],
        [
          0.7066,
          0.6733
        ],
        [
          0.717,
          0.6661
        ],
        [
          0.727,
          0.6574
        ],
        [
          0.737,
          0.6497
        ],
        [
          0.747,
          0.6442
        ],
        [
          0.7571,
          0.6379
        ],
        [
          0.7675,
          0.6306
        ],
        [
          0.7775,
          0.6224
        ],
        [
          0.7875,
          0.6146
        ],
        [
          0.7975,
          0.6069
        ],
        [
          0.8076,
          0.5992
        ],
        [
          0.818,
          0.5919
        ],
        [
          0.828,
          0.5855
        ],
        [
          0.838,
          0.5778
        ],
        [
          0.848,
          0.5719
        ],
        [
          0.8581,
          0.5664
        ],
        [
          0.8685,
          0.5596
        ],
        [
          0.8785,
          0.5546
        ],
        [
          0.8885,
          0.5469
        ],
        [
          0.8985,
          0.54
        ],
        [
          0.9086,
          0.5337
        ],
        [
          0.919,
          0.5264
        ],
        [
          0.929,
          0.52
        ],
        [
          0.939,
          0.5123
        ],
        [
          0.949,
          0.5036
        ],
        [
          0.9591,
          0.4968
        ],
        [
          0.9695,
          0.4891
        ],
        [
          0.9795,
          0.4818
        ],
        [
          0.9895,
          0.4736
        ],
        [
          0.9995,
          0.465
        ],
        [
          1.0,
          0.4645
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.017825404378951144,
        "difference": -0.0007730370017187746,
        "worse": 0.43666666666666665,
        "kept_right": 0.017034182771656605,
        "kept_wrong": 0.0007912216072945426,
        "handed_off": 0.9821745956210494
      },
      "usd_per_1000": 5.971250435183543
    },
    {
      "decision": 205,
      "frontier": 276,
      "frontier_accuracy": 0.9777070063694268,
      "in_sample": {
        "threshold": 0.9947150916400119,
        "answered": 0.009099181073703366,
        "accuracy": 0.9777070063694268
      },
      "curve": [
        [
          0.0,
          0.9777
        ],
        [
          0.01,
          0.9773
        ],
        [
          0.02,
          0.9768
        ],
        [
          0.03,
          0.9763
        ],
        [
          0.04,
          0.9763
        ],
        [
          0.05,
          0.9763
        ],
        [
          0.0601,
          0.9763
        ],
        [
          0.0705,
          0.9759
        ],
        [
          0.0805,
          0.9754
        ],
        [
          0.0905,
          0.9741
        ],
        [
          0.1005,
          0.9732
        ],
        [
          0.1106,
          0.9732
        ],
        [
          0.121,
          0.9713
        ],
        [
          0.131,
          0.9704
        ],
        [
          0.141,
          0.9695
        ],
        [
          0.151,
          0.9668
        ],
        [
          0.1611,
          0.9663
        ],
        [
          0.1715,
          0.965
        ],
        [
          0.1815,
          0.9618
        ],
        [
          0.1915,
          0.9595
        ],
        [
          0.2015,
          0.9568
        ],
        [
          0.2116,
          0.9531
        ],
        [
          0.222,
          0.9513
        ],
        [
          0.232,
          0.95
        ],
        [
          0.242,
          0.9463
        ],
        [
          0.252,
          0.9431
        ],
        [
          0.2621,
          0.9395
        ],
        [
          0.2725,
          0.9363
        ],
        [
          0.2825,
          0.9331
        ],
        [
          0.2925,
          0.9281
        ],
        [
          0.3025,
          0.924
        ],
        [
          0.3126,
          0.9181
        ],
        [
          0.323,
          0.9126
        ],
        [
          0.333,
          0.9076
        ],
        [
          0.343,
          0.9013
        ],
        [
          0.353,
          0.8958
        ],
        [
          0.3631,
          0.8922
        ],
        [
          0.3735,
          0.8858
        ],
        [
          0.3835,
          0.879
        ],
        [
          0.3935,
          0.8722
        ],
        [
          0.4035,
          0.8667
        ],
        [
          0.4136,
          0.8603
        ],
        [
          0.424,
          0.8544
        ],
        [
          0.434,
          0.8471
        ],
        [
          0.444,
          0.8408
        ],
        [
          0.454,
          0.8339
        ],
        [
          0.4641,
          0.8258
        ],
        [
          0.4745,
          0.8189
        ],
        [
          0.4845,
          0.8126
        ],
        [
          0.4945,
          0.8053
        ],
        [
          0.5045,
          0.7998
        ],
        [
          0.5146,
          0.7921
        ],
        [
          0.525,
          0.7853
        ],
        [
          0.535,
          0.7807
        ],
        [
          0.545,
          0.7743
        ],
        [
          0.5551,
          0.768
        ],
        [
          0.5655,
          0.7611
        ],
        [
          0.5755,
          0.7557
        ],
        [
          0.5855,
          0.7511
        ],
        [
          0.5955,
          0.7443
        ],
        [
          0.6056,
          0.737
        ],
        [
          0.616,
          0.7293
        ],
        [
          0.626,
          0.7225
        ],
        [
          0.636,
          0.7175
        ],
        [
          0.646,
          0.7125
        ],
        [
          0.6561,
          0.707
        ],
        [
          0.6665,
          0.7011
        ],
        [
          0.6765,
          0.6934
        ],
        [
          0.6865,
          0.687
        ],
        [
          0.6965,
          0.6802
        ],
        [
          0.7066,
          0.6729
        ],
        [
          0.717,
          0.6656
        ],
        [
          0.727,
          0.657
        ],
        [
          0.737,
          0.6492
        ],
        [
          0.747,
          0.6438
        ],
        [
          0.7571,
          0.6374
        ],
        [
          0.7675,
          0.6301
        ],
        [
          0.7775,
          0.6219
        ],
        [
          0.7875,
          0.6142
        ],
        [
          0.7975,
          0.6065
        ],
        [
          0.8076,
          0.5987
        ],
        [
          0.818,
          0.5919
        ],
        [
          0.828,
          0.5855
        ],
        [
          0.838,
          0.5783
        ],
        [
          0.848,
          0.5723
        ],
        [
          0.8581,
          0.5669
        ],
        [
          0.8685,
          0.5601
        ],
        [
          0.8785,
          0.5555
        ],
        [
          0.8885,
          0.5478
        ],
        [
          0.8985,
          0.5409
        ],
        [
          0.9086,
          0.5346
        ],
        [
          0.919,
          0.5268
        ],
        [
          0.929,
          0.52
        ],
        [
          0.939,
          0.5123
        ],
        [
          0.949,
          0.5036
        ],
        [
          0.9591,
          0.4968
        ],
        [
          0.9695,
          0.4891
        ],
        [
          0.9795,
          0.4818
        ],
        [
          0.9895,
          0.4736
        ],
        [
          0.9995,
          0.465
        ],
        [
          1.0,
          0.4645
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.017825404378951144,
        "difference": -0.0007730370017187746,
        "worse": 0.43666666666666665,
        "kept_right": 0.017034182771656605,
        "kept_wrong": 0.0007912216072945426,
        "handed_off": 0.9821745956210494
      },
      "usd_per_1000": 1.6070903337611124
    },
    {
      "decision": 205,
      "frontier": 48,
      "frontier_accuracy": 0.9758871701546861,
      "in_sample": {
        "threshold": 0.9947150916400119,
        "answered": 0.009099181073703366,
        "accuracy": 0.9758871701546861
      },
      "curve": [
        [
          0.0,
          0.9759
        ],
        [
          0.01,
          0.9754
        ],
        [
          0.02,
          0.975
        ],
        [
          0.03,
          0.9745
        ],
        [
          0.04,
          0.9745
        ],
        [
          0.05,
          0.9745
        ],
        [
          0.0601,
          0.9745
        ],
        [
          0.0705,
          0.9741
        ],
        [
          0.0805,
          0.9736
        ],
        [
          0.0905,
          0.9722
        ],
        [
          0.1005,
          0.9713
        ],
        [
          0.1106,
          0.9713
        ],
        [
          0.121,
          0.9695
        ],
        [
          0.131,
          0.9686
        ],
        [
          0.141,
          0.9677
        ],
        [
          0.151,
          0.9654
        ],
        [
          0.1611,
          0.965
        ],
        [
          0.1715,
          0.9636
        ],
        [
          0.1815,
          0.9609
        ],
        [
          0.1915,
          0.9586
        ],
        [
          0.2015,
          0.9559
        ],
        [
          0.2116,
          0.9522
        ],
        [
          0.222,
          0.9504
        ],
        [
          0.232,
          0.949
        ],
        [
          0.242,
          0.9454
        ],
        [
          0.252,
          0.9422
        ],
        [
          0.2621,
          0.9386
        ],
        [
          0.2725,
          0.9354
        ],
        [
          0.2825,
          0.9322
        ],
        [
          0.2925,
          0.9268
        ],
        [
          0.3025,
          0.9227
        ],
        [
          0.3126,
          0.9167
        ],
        [
          0.323,
          0.9113
        ],
        [
          0.333,
          0.9063
        ],
        [
          0.343,
          0.8999
        ],
        [
          0.353,
          0.8954
        ],
        [
          0.3631,
          0.8917
        ],
        [
          0.3735,
          0.8854
        ],
        [
          0.3835,
          0.8785
        ],
        [
          0.3935,
          0.8717
        ],
        [
          0.4035,
          0.8667
        ],
        [
          0.4136,
          0.8603
        ],
        [
          0.424,
          0.8544
        ],
        [
          0.434,
          0.8471
        ],
        [
          0.444,
          0.8408
        ],
        [
          0.454,
          0.8344
        ],
        [
          0.4641,
          0.8262
        ],
        [
          0.4745,
          0.8194
        ],
        [
          0.4845,
          0.813
        ],
        [
          0.4945,
          0.8057
        ],
        [
          0.5045,
          0.8003
        ],
        [
          0.5146,
          0.7925
        ],
        [
          0.525,
          0.7857
        ],
        [
          0.535,
          0.7812
        ],
        [
          0.545,
          0.7753
        ],
        [
          0.5551,
          0.7689
        ],
        [
          0.5655,
          0.7616
        ],
        [
          0.5755,
          0.7561
        ],
        [
          0.5855,
          0.7511
        ],
        [
          0.5955,
          0.7443
        ],
        [
          0.6056,
          0.737
        ],
        [
          0.616,
          0.7293
        ],
        [
          0.626,
          0.7225
        ],
        [
          0.636,
          0.7179
        ],
        [
          0.646,
          0.7129
        ],
        [
          0.6561,
          0.7075
        ],
        [
          0.6665,
          0.7015
        ],
        [
          0.6765,
          0.6938
        ],
        [
          0.6865,
          0.6874
        ],
        [
          0.6965,
          0.6806
        ],
        [
          0.7066,
          0.6733
        ],
        [
          0.717,
          0.6661
        ],
        [
          0.727,
          0.6574
        ],
        [
          0.737,
          0.6497
        ],
        [
          0.747,
          0.6442
        ],
        [
          0.7571,
          0.6379
        ],
        [
          0.7675,
          0.6306
        ],
        [
          0.7775,
          0.6228
        ],
        [
          0.7875,
          0.6151
        ],
        [
          0.7975,
          0.6069
        ],
        [
          0.8076,
          0.5992
        ],
        [
          0.818,
          0.5914
        ],
        [
          0.828,
          0.5851
        ],
        [
          0.838,
          0.5773
        ],
        [
          0.848,
          0.5719
        ],
        [
          0.8581,
          0.5664
        ],
        [
          0.8685,
          0.5591
        ],
        [
          0.8785,
          0.5541
        ],
        [
          0.8885,
          0.5464
        ],
        [
          0.8985,
          0.5396
        ],
        [
          0.9086,
          0.5332
        ],
        [
          0.919,
          0.5259
        ],
        [
          0.929,
          0.5191
        ],
        [
          0.939,
          0.5118
        ],
        [
          0.949,
          0.5032
        ],
        [
          0.9591,
          0.4968
        ],
        [
          0.9695,
          0.4891
        ],
        [
          0.9795,
          0.4818
        ],
        [
          0.9895,
          0.4736
        ],
        [
          0.9995,
          0.465
        ],
        [
          1.0,
          0.4645
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.017825404378951144,
        "difference": -0.0007730370017187746,
        "worse": 0.43666666666666665,
        "kept_right": 0.017034182771656605,
        "kept_wrong": 0.0007912216072945426,
        "handed_off": 0.9821745956210494
      },
      "usd_per_1000": 5.296173543384872
    },
    {
      "decision": 205,
      "frontier": 191,
      "frontier_accuracy": 0.9740673339399454,
      "in_sample": {
        "threshold": 0.9947150916400119,
        "answered": 0.009099181073703366,
        "accuracy": 0.9740673339399454
      },
      "curve": [
        [
          0.0,
          0.9741
        ],
        [
          0.01,
          0.9736
        ],
        [
          0.02,
          0.9732
        ],
        [
          0.03,
          0.9727
        ],
        [
          0.04,
          0.9727
        ],
        [
          0.05,
          0.9727
        ],
        [
          0.0601,
          0.9727
        ],
        [
          0.0705,
          0.9722
        ],
        [
          0.0805,
          0.9718
        ],
        [
          0.0905,
          0.9704
        ],
        [
          0.1005,
          0.9695
        ],
        [
          0.1106,
          0.9695
        ],
        [
          0.121,
          0.9677
        ],
        [
          0.131,
          0.9668
        ],
        [
          0.141,
          0.9659
        ],
        [
          0.151,
          0.9636
        ],
        [
          0.1611,
          0.9631
        ],
        [
          0.1715,
          0.9618
        ],
        [
          0.1815,
          0.9591
        ],
        [
          0.1915,
          0.9568
        ],
        [
          0.2015,
          0.954
        ],
        [
          0.2116,
          0.9504
        ],
        [
          0.222,
          0.9486
        ],
        [
          0.232,
          0.9472
        ],
        [
          0.242,
          0.9436
        ],
        [
          0.252,
          0.9404
        ],
        [
          0.2621,
          0.9363
        ],
        [
          0.2725,
          0.9336
        ],
        [
          0.2825,
          0.9304
        ],
        [
          0.2925,
          0.9249
        ],
        [
          0.3025,
          0.9208
        ],
        [
          0.3126,
          0.9154
        ],
        [
          0.323,
          0.9099
        ],
        [
          0.333,
          0.9045
        ],
        [
          0.343,
          0.8985
        ],
        [
          0.353,
          0.8931
        ],
        [
          0.3631,
          0.8894
        ],
        [
          0.3735,
          0.8831
        ],
        [
          0.3835,
          0.8763
        ],
        [
          0.3935,
          0.8694
        ],
        [
          0.4035,
          0.8644
        ],
        [
          0.4136,
          0.8581
        ],
        [
          0.424,
          0.8521
        ],
        [
          0.434,
          0.8449
        ],
        [
          0.444,
          0.8385
        ],
        [
          0.454,
          0.8321
        ],
        [
          0.4641,
          0.8239
        ],
        [
          0.4745,
          0.8171
        ],
        [
          0.4845,
          0.8107
        ],
        [
          0.4945,
          0.8035
        ],
        [
          0.5045,
          0.798
        ],
        [
          0.5146,
          0.7903
        ],
        [
          0.525,
          0.7834
        ],
        [
          0.535,
          0.7789
        ],
        [
          0.545,
          0.7725
        ],
        [
          0.5551,
          0.7662
        ],
        [
          0.5655,
          0.7589
        ],
        [
          0.5755,
          0.7539
        ],
        [
          0.5855,
          0.7489
        ],
        [
          0.5955,
          0.742
        ],
        [
          0.6056,
          0.7348
        ],
        [
          0.616,
          0.727
        ],
        [
          0.626,
          0.7207
        ],
        [
          0.636,
          0.7157
        ],
        [
          0.646,
          0.7106
        ],
        [
          0.6561,
          0.7052
        ],
        [
          0.6665,
          0.6993
        ],
        [
          0.6765,
          0.6915
        ],
        [
          0.6865,
          0.6856
        ],
        [
          0.6965,
          0.6783
        ],
        [
          0.7066,
          0.6711
        ],
        [
          0.717,
          0.6642
        ],
        [
          0.727,
          0.6556
        ],
        [
          0.737,
          0.6474
        ],
        [
          0.747,
          0.6419
        ],
        [
          0.7571,
          0.636
        ],
        [
          0.7675,
          0.6283
        ],
        [
          0.7775,
          0.6201
        ],
        [
          0.7875,
          0.6124
        ],
        [
          0.7975,
          0.6051
        ],
        [
          0.8076,
          0.5974
        ],
        [
          0.818,
          0.5905
        ],
        [
          0.828,
          0.5842
        ],
        [
          0.838,
          0.5769
        ],
        [
          0.848,
          0.571
        ],
        [
          0.8581,
          0.566
        ],
        [
          0.8685,
          0.5591
        ],
        [
          0.8785,
          0.5541
        ],
        [
          0.8885,
          0.5464
        ],
        [
          0.8985,
          0.5396
        ],
        [
          0.9086,
          0.5332
        ],
        [
          0.919,
          0.5259
        ],
        [
          0.929,
          0.5191
        ],
        [
          0.939,
          0.5118
        ],
        [
          0.949,
          0.5036
        ],
        [
          0.9591,
          0.4968
        ],
        [
          0.9695,
          0.4891
        ],
        [
          0.9795,
          0.4818
        ],
        [
          0.9895,
          0.4736
        ],
        [
          0.9995,
          0.465
        ],
        [
          1.0,
          0.4645
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.017825404378951144,
        "difference": -0.0007730370017187746,
        "worse": 0.43666666666666665,
        "kept_right": 0.017034182771656605,
        "kept_wrong": 0.0007912216072945426,
        "handed_off": 0.9821745956210494
      },
      "usd_per_1000": 26.27490219912148
    },
    {
      "decision": 205,
      "frontier": 188,
      "frontier_accuracy": 0.9713375796178344,
      "in_sample": {
        "threshold": 0.9947150916400119,
        "answered": 0.009099181073703366,
        "accuracy": 0.9713375796178344
      },
      "curve": [
        [
          0.0,
          0.9713
        ],
        [
          0.01,
          0.9709
        ],
        [
          0.02,
          0.9709
        ],
        [
          0.03,
          0.9704
        ],
        [
          0.04,
          0.9704
        ],
        [
          0.05,
          0.9704
        ],
        [
          0.0601,
          0.9704
        ],
        [
          0.0705,
          0.97
        ],
        [
          0.0805,
          0.9695
        ],
        [
          0.0905,
          0.9682
        ],
        [
          0.1005,
          0.9672
        ],
        [
          0.1106,
          0.9672
        ],
        [
          0.121,
          0.9654
        ],
        [
          0.131,
          0.9645
        ],
        [
          0.141,
          0.9636
        ],
        [
          0.151,
          0.9613
        ],
        [
          0.1611,
          0.9609
        ],
        [
          0.1715,
          0.9595
        ],
        [
          0.1815,
          0.9563
        ],
        [
          0.1915,
          0.954
        ],
        [
          0.2015,
          0.9513
        ],
        [
          0.2116,
          0.9477
        ],
        [
          0.222,
          0.9463
        ],
        [
          0.232,
          0.9445
        ],
        [
          0.242,
          0.9413
        ],
        [
          0.252,
          0.9381
        ],
        [
          0.2621,
          0.934
        ],
        [
          0.2725,
          0.9308
        ],
        [
          0.2825,
          0.9277
        ],
        [
          0.2925,
          0.9222
        ],
        [
          0.3025,
          0.9177
        ],
        [
          0.3126,
          0.9122
        ],
        [
          0.323,
          0.9067
        ],
        [
          0.333,
          0.9022
        ],
        [
          0.343,
          0.8963
        ],
        [
          0.353,
          0.8908
        ],
        [
          0.3631,
          0.8872
        ],
        [
          0.3735,
          0.8808
        ],
        [
          0.3835,
          0.8749
        ],
        [
          0.3935,
          0.8685
        ],
        [
          0.4035,
          0.8631
        ],
        [
          0.4136,
          0.8567
        ],
        [
          0.424,
          0.8503
        ],
        [
          0.434,
          0.843
        ],
        [
          0.444,
          0.8367
        ],
        [
          0.454,
          0.8298
        ],
        [
          0.4641,
          0.8221
        ],
        [
          0.4745,
          0.8153
        ],
        [
          0.4845,
          0.8089
        ],
        [
          0.4945,
          0.8021
        ],
        [
          0.5045,
          0.7966
        ],
        [
          0.5146,
          0.7884
        ],
        [
          0.525,
          0.7816
        ],
        [
          0.535,
          0.7771
        ],
        [
          0.545,
          0.7707
        ],
        [
          0.5551,
          0.7643
        ],
        [
          0.5655,
          0.7575
        ],
        [
          0.5755,
          0.7534
        ],
        [
          0.5855,
          0.7484
        ],
        [
          0.5955,
          0.7416
        ],
        [
          0.6056,
          0.7343
        ],
        [
          0.616,
          0.7266
        ],
        [
          0.626,
          0.7207
        ],
        [
          0.636,
          0.7157
        ],
        [
          0.646,
          0.7106
        ],
        [
          0.6561,
          0.7052
        ],
        [
          0.6665,
          0.6997
        ],
        [
          0.6765,
          0.692
        ],
        [
          0.6865,
          0.6856
        ],
        [
          0.6965,
          0.6788
        ],
        [
          0.7066,
          0.6715
        ],
        [
          0.717,
          0.6638
        ],
        [
          0.727,
          0.6551
        ],
        [
          0.737,
          0.6474
        ],
        [
          0.747,
          0.6419
        ],
        [
          0.7571,
          0.6356
        ],
        [
          0.7675,
          0.6283
        ],
        [
          0.7775,
          0.6206
        ],
        [
          0.7875,
          0.6133
        ],
        [
          0.7975,
          0.6051
        ],
        [
          0.8076,
          0.5974
        ],
        [
          0.818,
          0.5896
        ],
        [
          0.828,
          0.5837
        ],
        [
          0.838,
          0.576
        ],
        [
          0.848,
          0.5701
        ],
        [
          0.8581,
          0.5646
        ],
        [
          0.8685,
          0.5582
        ],
        [
          0.8785,
          0.5537
        ],
        [
          0.8885,
          0.546
        ],
        [
          0.8985,
          0.5391
        ],
        [
          0.9086,
          0.5328
        ],
        [
          0.919,
          0.5255
        ],
        [
          0.929,
          0.5187
        ],
        [
          0.939,
          0.5118
        ],
        [
          0.949,
          0.5032
        ],
        [
          0.9591,
          0.4964
        ],
        [
          0.9695,
          0.4886
        ],
        [
          0.9795,
          0.4818
        ],
        [
          0.9895,
          0.4736
        ],
        [
          0.9995,
          0.465
        ],
        [
          1.0,
          0.4645
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.026445360780044298,
        "difference": -0.000794345800188106,
        "worse": 0.43666666666666665,
        "kept_right": 0.025435780524148247,
        "kept_wrong": 0.0010095802558960486,
        "handed_off": 0.973554639219956
      },
      "usd_per_1000": 0.9517776744147981
    },
    {
      "decision": 205,
      "frontier": 190,
      "frontier_accuracy": 0.9672429481346679,
      "in_sample": {
        "threshold": 0.9947150916400119,
        "answered": 0.009099181073703366,
        "accuracy": 0.9672429481346679
      },
      "curve": [
        [
          0.0,
          0.9672
        ],
        [
          0.01,
          0.9668
        ],
        [
          0.02,
          0.9663
        ],
        [
          0.03,
          0.9659
        ],
        [
          0.04,
          0.9659
        ],
        [
          0.05,
          0.9659
        ],
        [
          0.0601,
          0.9659
        ],
        [
          0.0705,
          0.9654
        ],
        [
          0.0805,
          0.965
        ],
        [
          0.0905,
          0.9641
        ],
        [
          0.1005,
          0.9631
        ],
        [
          0.1106,
          0.9631
        ],
        [
          0.121,
          0.9613
        ],
        [
          0.131,
          0.9604
        ],
        [
          0.141,
          0.9595
        ],
        [
          0.151,
          0.9572
        ],
        [
          0.1611,
          0.9568
        ],
        [
          0.1715,
          0.9554
        ],
        [
          0.1815,
          0.9527
        ],
        [
          0.1915,
          0.9504
        ],
        [
          0.2015,
          0.9477
        ],
        [
          0.2116,
          0.944
        ],
        [
          0.222,
          0.9422
        ],
        [
          0.232,
          0.9409
        ],
        [
          0.242,
          0.9377
        ],
        [
          0.252,
          0.9345
        ],
        [
          0.2621,
          0.9304
        ],
        [
          0.2725,
          0.9277
        ],
        [
          0.2825,
          0.9245
        ],
        [
          0.2925,
          0.919
        ],
        [
          0.3025,
          0.9149
        ],
        [
          0.3126,
          0.9095
        ],
        [
          0.323,
          0.904
        ],
        [
          0.333,
          0.899
        ],
        [
          0.343,
          0.8926
        ],
        [
          0.353,
          0.8876
        ],
        [
          0.3631,
          0.884
        ],
        [
          0.3735,
          0.8776
        ],
        [
          0.3835,
          0.8708
        ],
        [
          0.3935,
          0.864
        ],
        [
          0.4035,
          0.8594
        ],
        [
          0.4136,
          0.8535
        ],
        [
          0.424,
          0.8476
        ],
        [
          0.434,
          0.8403
        ],
        [
          0.444,
          0.8339
        ],
        [
          0.454,
          0.8276
        ],
        [
          0.4641,
          0.8194
        ],
        [
          0.4745,
          0.8126
        ],
        [
          0.4845,
          0.8066
        ],
        [
          0.4945,
          0.7994
        ],
        [
          0.5045,
          0.7934
        ],
        [
          0.5146,
          0.7857
        ],
        [
          0.525,
          0.7789
        ],
        [
          0.535,
          0.7743
        ],
        [
          0.545,
          0.7684
        ],
        [
          0.5551,
          0.7621
        ],
        [
          0.5655,
          0.7552
        ],
        [
          0.5755,
          0.7498
        ],
        [
          0.5855,
          0.7448
        ],
        [
          0.5955,
          0.7379
        ],
        [
          0.6056,
          0.7311
        ],
        [
          0.616,
          0.7234
        ],
        [
          0.626,
          0.7175
        ],
        [
          0.636,
          0.7125
        ],
        [
          0.646,
          0.7075
        ],
        [
          0.6561,
          0.702
        ],
        [
          0.6665,
          0.6961
        ],
        [
          0.6765,
          0.6884
        ],
        [
          0.6865,
          0.6824
        ],
        [
          0.6965,
          0.6756
        ],
        [
          0.7066,
          0.6688
        ],
        [
          0.717,
          0.6615
        ],
        [
          0.727,
          0.6529
        ],
        [
          0.737,
          0.6451
        ],
        [
          0.747,
          0.6397
        ],
        [
          0.7571,
          0.6333
        ],
        [
          0.7675,
          0.6256
        ],
        [
          0.7775,
          0.6174
        ],
        [
          0.7875,
          0.6101
        ],
        [
          0.7975,
          0.6033
        ],
        [
          0.8076,
          0.5955
        ],
        [
          0.818,
          0.5883
        ],
        [
          0.828,
          0.5819
        ],
        [
          0.838,
          0.5742
        ],
        [
          0.848,
          0.5687
        ],
        [
          0.8581,
          0.5637
        ],
        [
          0.8685,
          0.5573
        ],
        [
          0.8785,
          0.5528
        ],
        [
          0.8885,
          0.545
        ],
        [
          0.8985,
          0.5382
        ],
        [
          0.9086,
          0.5318
        ],
        [
          0.919,
          0.525
        ],
        [
          0.929,
          0.5187
        ],
        [
          0.939,
          0.5114
        ],
        [
          0.949,
          0.5032
        ],
        [
          0.9591,
          0.4964
        ],
        [
          0.9695,
          0.4886
        ],
        [
          0.9795,
          0.4813
        ],
        [
          0.9895,
          0.4732
        ],
        [
          0.9995,
          0.465
        ],
        [
          1.0,
          0.4645
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.019029135870842928,
        "difference": -0.0008488304276615346,
        "worse": 0.44,
        "kept_right": 0.01814391692559084,
        "kept_wrong": 0.0008852189452520784,
        "handed_off": 0.9809708641291573
      },
      "usd_per_1000": 5.476371268996988
    },
    {
      "decision": 205,
      "frontier": 124,
      "frontier_accuracy": 0.9667879890809827,
      "in_sample": {
        "threshold": 0.9947150916400119,
        "answered": 0.009099181073703366,
        "accuracy": 0.9667879890809827
      },
      "curve": [
        [
          0.0,
          0.9668
        ],
        [
          0.01,
          0.9663
        ],
        [
          0.02,
          0.9659
        ],
        [
          0.03,
          0.9654
        ],
        [
          0.04,
          0.9654
        ],
        [
          0.05,
          0.9654
        ],
        [
          0.0601,
          0.9654
        ],
        [
          0.0705,
          0.965
        ],
        [
          0.0805,
          0.9645
        ],
        [
          0.0905,
          0.9631
        ],
        [
          0.1005,
          0.9622
        ],
        [
          0.1106,
          0.9622
        ],
        [
          0.121,
          0.9604
        ],
        [
          0.131,
          0.9595
        ],
        [
          0.141,
          0.9586
        ],
        [
          0.151,
          0.9559
        ],
        [
          0.1611,
          0.9554
        ],
        [
          0.1715,
          0.954
        ],
        [
          0.1815,
          0.9513
        ],
        [
          0.1915,
          0.949
        ],
        [
          0.2015,
          0.9463
        ],
        [
          0.2116,
          0.9427
        ],
        [
          0.222,
          0.9409
        ],
        [
          0.232,
          0.9395
        ],
        [
          0.242,
          0.9359
        ],
        [
          0.252,
          0.9327
        ],
        [
          0.2621,
          0.929
        ],
        [
          0.2725,
          0.9263
        ],
        [
          0.2825,
          0.9231
        ],
        [
          0.2925,
          0.9177
        ],
        [
          0.3025,
          0.9136
        ],
        [
          0.3126,
          0.9076
        ],
        [
          0.323,
          0.9022
        ],
        [
          0.333,
          0.8981
        ],
        [
          0.343,
          0.8922
        ],
        [
          0.353,
          0.8872
        ],
        [
          0.3631,
          0.8835
        ],
        [
          0.3735,
          0.8772
        ],
        [
          0.3835,
          0.8703
        ],
        [
          0.3935,
          0.8635
        ],
        [
          0.4035,
          0.8585
        ],
        [
          0.4136,
          0.8526
        ],
        [
          0.424,
          0.8467
        ],
        [
          0.434,
          0.8394
        ],
        [
          0.444,
          0.833
        ],
        [
          0.454,
          0.8267
        ],
        [
          0.4641,
          0.8189
        ],
        [
          0.4745,
          0.8121
        ],
        [
          0.4845,
          0.8057
        ],
        [
          0.4945,
          0.7989
        ],
        [
          0.5045,
          0.7934
        ],
        [
          0.5146,
          0.7857
        ],
        [
          0.525,
          0.7789
        ],
        [
          0.535,
          0.7743
        ],
        [
          0.545,
          0.7684
        ],
        [
          0.5551,
          0.7621
        ],
        [
          0.5655,
          0.7552
        ],
        [
          0.5755,
          0.7502
        ],
        [
          0.5855,
          0.7457
        ],
        [
          0.5955,
          0.7389
        ],
        [
          0.6056,
          0.732
        ],
        [
          0.616,
          0.7243
        ],
        [
          0.626,
          0.7184
        ],
        [
          0.636,
          0.7138
        ],
        [
          0.646,
          0.7088
        ],
        [
          0.6561,
          0.7034
        ],
        [
          0.6665,
          0.6979
        ],
        [
          0.6765,
          0.6902
        ],
        [
          0.6865,
          0.6843
        ],
        [
          0.6965,
          0.6774
        ],
        [
          0.7066,
          0.6702
        ],
        [
          0.717,
          0.6629
        ],
        [
          0.727,
          0.6542
        ],
        [
          0.737,
          0.646
        ],
        [
          0.747,
          0.6406
        ],
        [
          0.7571,
          0.6342
        ],
        [
          0.7675,
          0.6269
        ],
        [
          0.7775,
          0.6187
        ],
        [
          0.7875,
          0.6115
        ],
        [
          0.7975,
          0.6037
        ],
        [
          0.8076,
          0.596
        ],
        [
          0.818,
          0.5883
        ],
        [
          0.828,
          0.5819
        ],
        [
          0.838,
          0.5746
        ],
        [
          0.848,
          0.5692
        ],
        [
          0.8581,
          0.5637
        ],
        [
          0.8685,
          0.5573
        ],
        [
          0.8785,
          0.5537
        ],
        [
          0.8885,
          0.546
        ],
        [
          0.8985,
          0.5391
        ],
        [
          0.9086,
          0.5328
        ],
        [
          0.919,
          0.5255
        ],
        [
          0.929,
          0.5191
        ],
        [
          0.939,
          0.5114
        ],
        [
          0.949,
          0.5027
        ],
        [
          0.9591,
          0.4959
        ],
        [
          0.9695,
          0.4882
        ],
        [
          0.9795,
          0.4813
        ],
        [
          0.9895,
          0.4732
        ],
        [
          0.9995,
          0.465
        ],
        [
          1.0,
          0.4645
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.017825404378951144,
        "difference": -0.0007730370017187746,
        "worse": 0.43666666666666665,
        "kept_right": 0.017034182771656605,
        "kept_wrong": 0.0007912216072945426,
        "handed_off": 0.9821745956210494
      },
      "usd_per_1000": 1.1240400661038368
    },
    {
      "decision": 205,
      "frontier": 278,
      "frontier_accuracy": 0.9499545040946314,
      "in_sample": {
        "threshold": 0.9947150916400119,
        "answered": 0.009099181073703366,
        "accuracy": 0.9499545040946314
      },
      "curve": [
        [
          0.0,
          0.95
        ],
        [
          0.01,
          0.9495
        ],
        [
          0.02,
          0.949
        ],
        [
          0.03,
          0.9486
        ],
        [
          0.04,
          0.9486
        ],
        [
          0.05,
          0.9486
        ],
        [
          0.0601,
          0.9486
        ],
        [
          0.0705,
          0.9481
        ],
        [
          0.0805,
          0.9477
        ],
        [
          0.0905,
          0.9468
        ],
        [
          0.1005,
          0.9459
        ],
        [
          0.1106,
          0.9459
        ],
        [
          0.121,
          0.944
        ],
        [
          0.131,
          0.9431
        ],
        [
          0.141,
          0.9422
        ],
        [
          0.151,
          0.9395
        ],
        [
          0.1611,
          0.939
        ],
        [
          0.1715,
          0.9377
        ],
        [
          0.1815,
          0.9345
        ],
        [
          0.1915,
          0.9327
        ],
        [
          0.2015,
          0.9304
        ],
        [
          0.2116,
          0.9268
        ],
        [
          0.222,
          0.9249
        ],
        [
          0.232,
          0.9236
        ],
        [
          0.242,
          0.9199
        ],
        [
          0.252,
          0.9167
        ],
        [
          0.2621,
          0.9136
        ],
        [
          0.2725,
          0.9108
        ],
        [
          0.2825,
          0.9076
        ],
        [
          0.2925,
          0.9022
        ],
        [
          0.3025,
          0.8976
        ],
        [
          0.3126,
          0.8922
        ],
        [
          0.323,
          0.8872
        ],
        [
          0.333,
          0.8831
        ],
        [
          0.343,
          0.8772
        ],
        [
          0.353,
          0.8731
        ],
        [
          0.3631,
          0.8703
        ],
        [
          0.3735,
          0.864
        ],
        [
          0.3835,
          0.8581
        ],
        [
          0.3935,
          0.8512
        ],
        [
          0.4035,
          0.8462
        ],
        [
          0.4136,
          0.8403
        ],
        [
          0.424,
          0.8339
        ],
        [
          0.434,
          0.8267
        ],
        [
          0.444,
          0.8203
        ],
        [
          0.454,
          0.8139
        ],
        [
          0.4641,
          0.8057
        ],
        [
          0.4745,
          0.7989
        ],
        [
          0.4845,
          0.793
        ],
        [
          0.4945,
          0.7866
        ],
        [
          0.5045,
          0.7816
        ],
        [
          0.5146,
          0.7743
        ],
        [
          0.525,
          0.7675
        ],
        [
          0.535,
          0.7634
        ],
        [
          0.545,
          0.7571
        ],
        [
          0.5551,
          0.7511
        ],
        [
          0.5655,
          0.7443
        ],
        [
          0.5755,
          0.7393
        ],
        [
          0.5855,
          0.7338
        ],
        [
          0.5955,
          0.7275
        ],
        [
          0.6056,
          0.7202
        ],
        [
          0.616,
          0.7125
        ],
        [
          0.626,
          0.707
        ],
        [
          0.636,
          0.7029
        ],
        [
          0.646,
          0.6984
        ],
        [
          0.6561,
          0.6938
        ],
        [
          0.6665,
          0.6884
        ],
        [
          0.6765,
          0.6806
        ],
        [
          0.6865,
          0.6752
        ],
        [
          0.6965,
          0.6683
        ],
        [
          0.7066,
          0.6611
        ],
        [
          0.717,
          0.6547
        ],
        [
          0.727,
          0.646
        ],
        [
          0.737,
          0.6383
        ],
        [
          0.747,
          0.6338
        ],
        [
          0.7571,
          0.6274
        ],
        [
          0.7675,
          0.6201
        ],
        [
          0.7775,
          0.6124
        ],
        [
          0.7875,
          0.6046
        ],
        [
          0.7975,
          0.5965
        ],
        [
          0.8076,
          0.5892
        ],
        [
          0.818,
          0.5833
        ],
        [
          0.828,
          0.5773
        ],
        [
          0.838,
          0.571
        ],
        [
          0.848,
          0.5655
        ],
        [
          0.8581,
          0.5605
        ],
        [
          0.8685,
          0.5541
        ],
        [
          0.8785,
          0.5496
        ],
        [
          0.8885,
          0.5423
        ],
        [
          0.8985,
          0.5355
        ],
        [
          0.9086,
          0.5291
        ],
        [
          0.919,
          0.5223
        ],
        [
          0.929,
          0.5164
        ],
        [
          0.939,
          0.5096
        ],
        [
          0.949,
          0.5009
        ],
        [
          0.9591,
          0.4945
        ],
        [
          0.9695,
          0.4873
        ],
        [
          0.9795,
          0.4809
        ],
        [
          0.9895,
          0.4732
        ],
        [
          0.9995,
          0.465
        ],
        [
          1.0,
          0.4645
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.01797118480697999,
        "difference": -0.0007912657497854371,
        "worse": 0.43666666666666665,
        "kept_right": 0.017158706900907313,
        "kept_wrong": 0.0008124779060726795,
        "handed_off": 0.9820288151930207
      },
      "usd_per_1000": 6.405671333067872
    },
    {
      "decision": 205,
      "frontier": 275,
      "frontier_accuracy": 0.9426751592356688,
      "in_sample": {
        "threshold": 0.9947150916400119,
        "answered": 0.009099181073703366,
        "accuracy": 0.9426751592356688
      },
      "curve": [
        [
          0.0,
          0.9427
        ],
        [
          0.01,
          0.9422
        ],
        [
          0.02,
          0.9418
        ],
        [
          0.03,
          0.9413
        ],
        [
          0.04,
          0.9413
        ],
        [
          0.05,
          0.9413
        ],
        [
          0.0601,
          0.9413
        ],
        [
          0.0705,
          0.9409
        ],
        [
          0.0805,
          0.9404
        ],
        [
          0.0905,
          0.9395
        ],
        [
          0.1005,
          0.9386
        ],
        [
          0.1106,
          0.9386
        ],
        [
          0.121,
          0.9368
        ],
        [
          0.131,
          0.9359
        ],
        [
          0.141,
          0.9349
        ],
        [
          0.151,
          0.9327
        ],
        [
          0.1611,
          0.9322
        ],
        [
          0.1715,
          0.9308
        ],
        [
          0.1815,
          0.9277
        ],
        [
          0.1915,
          0.9258
        ],
        [
          0.2015,
          0.9236
        ],
        [
          0.2116,
          0.9199
        ],
        [
          0.222,
          0.9181
        ],
        [
          0.232,
          0.9167
        ],
        [
          0.242,
          0.9131
        ],
        [
          0.252,
          0.9099
        ],
        [
          0.2621,
          0.9063
        ],
        [
          0.2725,
          0.9035
        ],
        [
          0.2825,
          0.9004
        ],
        [
          0.2925,
          0.8949
        ],
        [
          0.3025,
          0.8913
        ],
        [
          0.3126,
          0.8854
        ],
        [
          0.323,
          0.8799
        ],
        [
          0.333,
          0.8749
        ],
        [
          0.343,
          0.8694
        ],
        [
          0.353,
          0.864
        ],
        [
          0.3631,
          0.8603
        ],
        [
          0.3735,
          0.8544
        ],
        [
          0.3835,
          0.848
        ],
        [
          0.3935,
          0.8417
        ],
        [
          0.4035,
          0.8367
        ],
        [
          0.4136,
          0.8308
        ],
        [
          0.424,
          0.8248
        ],
        [
          0.434,
          0.8176
        ],
        [
          0.444,
          0.8116
        ],
        [
          0.454,
          0.8053
        ],
        [
          0.4641,
          0.798
        ],
        [
          0.4745,
          0.7912
        ],
        [
          0.4845,
          0.7857
        ],
        [
          0.4945,
          0.7789
        ],
        [
          0.5045,
          0.7743
        ],
        [
          0.5146,
          0.7675
        ],
        [
          0.525,
          0.7611
        ],
        [
          0.535,
          0.758
        ],
        [
          0.545,
          0.7516
        ],
        [
          0.5551,
          0.7457
        ],
        [
          0.5655,
          0.7389
        ],
        [
          0.5755,
          0.7338
        ],
        [
          0.5855,
          0.7288
        ],
        [
          0.5955,
          0.7225
        ],
        [
          0.6056,
          0.7152
        ],
        [
          0.616,
          0.7075
        ],
        [
          0.626,
          0.7006
        ],
        [
          0.636,
          0.6961
        ],
        [
          0.646,
          0.6911
        ],
        [
          0.6561,
          0.6865
        ],
        [
          0.6665,
          0.6811
        ],
        [
          0.6765,
          0.6733
        ],
        [
          0.6865,
          0.6674
        ],
        [
          0.6965,
          0.6611
        ],
        [
          0.7066,
          0.6542
        ],
        [
          0.717,
          0.6483
        ],
        [
          0.727,
          0.6401
        ],
        [
          0.737,
          0.6328
        ],
        [
          0.747,
          0.6288
        ],
        [
          0.7571,
          0.6228
        ],
        [
          0.7675,
          0.6156
        ],
        [
          0.7775,
          0.6087
        ],
        [
          0.7875,
          0.601
        ],
        [
          0.7975,
          0.5942
        ],
        [
          0.8076,
          0.5864
        ],
        [
          0.818,
          0.5805
        ],
        [
          0.828,
          0.5742
        ],
        [
          0.838,
          0.5669
        ],
        [
          0.848,
          0.5614
        ],
        [
          0.8581,
          0.5573
        ],
        [
          0.8685,
          0.5514
        ],
        [
          0.8785,
          0.5482
        ],
        [
          0.8885,
          0.5405
        ],
        [
          0.8985,
          0.5337
        ],
        [
          0.9086,
          0.5278
        ],
        [
          0.919,
          0.5214
        ],
        [
          0.929,
          0.515
        ],
        [
          0.939,
          0.5086
        ],
        [
          0.949,
          0.5
        ],
        [
          0.9591,
          0.4941
        ],
        [
          0.9695,
          0.4868
        ],
        [
          0.9795,
          0.4804
        ],
        [
          0.9895,
          0.4732
        ],
        [
          0.9995,
          0.465
        ],
        [
          1.0,
          0.4645
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.01797118480697999,
        "difference": -0.0007912657497854371,
        "worse": 0.43666666666666665,
        "kept_right": 0.017158706900907313,
        "kept_wrong": 0.0008124779060726795,
        "handed_off": 0.9820288151930207
      },
      "usd_per_1000": 0.51908313880882
    },
    {
      "decision": 205,
      "frontier": 192,
      "frontier_accuracy": 0.9331210191082803,
      "in_sample": {
        "threshold": 0.9947150916400119,
        "answered": 0.009099181073703366,
        "accuracy": 0.9331210191082803
      },
      "curve": [
        [
          0.0,
          0.9331
        ],
        [
          0.01,
          0.9327
        ],
        [
          0.02,
          0.9322
        ],
        [
          0.03,
          0.9318
        ],
        [
          0.04,
          0.9318
        ],
        [
          0.05,
          0.9318
        ],
        [
          0.0601,
          0.9318
        ],
        [
          0.0705,
          0.9313
        ],
        [
          0.0805,
          0.9308
        ],
        [
          0.0905,
          0.9299
        ],
        [
          0.1005,
          0.9295
        ],
        [
          0.1106,
          0.9295
        ],
        [
          0.121,
          0.9277
        ],
        [
          0.131,
          0.9268
        ],
        [
          0.141,
          0.9263
        ],
        [
          0.151,
          0.9236
        ],
        [
          0.1611,
          0.9231
        ],
        [
          0.1715,
          0.9217
        ],
        [
          0.1815,
          0.9195
        ],
        [
          0.1915,
          0.9172
        ],
        [
          0.2015,
          0.9145
        ],
        [
          0.2116,
          0.9117
        ],
        [
          0.222,
          0.9099
        ],
        [
          0.232,
          0.9081
        ],
        [
          0.242,
          0.9045
        ],
        [
          0.252,
          0.9013
        ],
        [
          0.2621,
          0.899
        ],
        [
          0.2725,
          0.8963
        ],
        [
          0.2825,
          0.8931
        ],
        [
          0.2925,
          0.8876
        ],
        [
          0.3025,
          0.8835
        ],
        [
          0.3126,
          0.8781
        ],
        [
          0.323,
          0.8731
        ],
        [
          0.333,
          0.8681
        ],
        [
          0.343,
          0.8626
        ],
        [
          0.353,
          0.8585
        ],
        [
          0.3631,
          0.8553
        ],
        [
          0.3735,
          0.849
        ],
        [
          0.3835,
          0.8435
        ],
        [
          0.3935,
          0.8367
        ],
        [
          0.4035,
          0.8321
        ],
        [
          0.4136,
          0.8271
        ],
        [
          0.424,
          0.8212
        ],
        [
          0.434,
          0.8153
        ],
        [
          0.444,
          0.8089
        ],
        [
          0.454,
          0.8025
        ],
        [
          0.4641,
          0.7948
        ],
        [
          0.4745,
          0.7884
        ],
        [
          0.4845,
          0.783
        ],
        [
          0.4945,
          0.7762
        ],
        [
          0.5045,
          0.7712
        ],
        [
          0.5146,
          0.7643
        ],
        [
          0.525,
          0.7575
        ],
        [
          0.535,
          0.7534
        ],
        [
          0.545,
          0.7475
        ],
        [
          0.5551,
          0.7416
        ],
        [
          0.5655,
          0.7348
        ],
        [
          0.5755,
          0.7302
        ],
        [
          0.5855,
          0.7266
        ],
        [
          0.5955,
          0.7197
        ],
        [
          0.6056,
          0.7129
        ],
        [
          0.616,
          0.7052
        ],
        [
          0.626,
          0.6997
        ],
        [
          0.636,
          0.6947
        ],
        [
          0.646,
          0.6902
        ],
        [
          0.6561,
          0.6856
        ],
        [
          0.6665,
          0.6815
        ],
        [
          0.6765,
          0.6742
        ],
        [
          0.6865,
          0.6683
        ],
        [
          0.6965,
          0.6611
        ],
        [
          0.7066,
          0.6538
        ],
        [
          0.717,
          0.6474
        ],
        [
          0.727,
          0.6388
        ],
        [
          0.737,
          0.6315
        ],
        [
          0.747,
          0.6269
        ],
        [
          0.7571,
          0.6219
        ],
        [
          0.7675,
          0.6151
        ],
        [
          0.7775,
          0.6069
        ],
        [
          0.7875,
          0.6001
        ],
        [
          0.7975,
          0.5928
        ],
        [
          0.8076,
          0.586
        ],
        [
          0.818,
          0.5801
        ],
        [
          0.828,
          0.5746
        ],
        [
          0.838,
          0.5678
        ],
        [
          0.848,
          0.5623
        ],
        [
          0.8581,
          0.5578
        ],
        [
          0.8685,
          0.5519
        ],
        [
          0.8785,
          0.5469
        ],
        [
          0.8885,
          0.5396
        ],
        [
          0.8985,
          0.5328
        ],
        [
          0.9086,
          0.5264
        ],
        [
          0.919,
          0.5205
        ],
        [
          0.929,
          0.5146
        ],
        [
          0.939,
          0.5077
        ],
        [
          0.949,
          0.4995
        ],
        [
          0.9591,
          0.4941
        ],
        [
          0.9695,
          0.4868
        ],
        [
          0.9795,
          0.48
        ],
        [
          0.9895,
          0.4732
        ],
        [
          0.9995,
          0.465
        ],
        [
          1.0,
          0.4645
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.018034763371920958,
        "difference": -0.000788216115258544,
        "worse": 0.43666666666666665,
        "kept_right": 0.0172192579151368,
        "kept_wrong": 0.0008155054567841538,
        "handed_off": 0.9819652366280798
      },
      "usd_per_1000": 11.414005925192363
    },
    {
      "decision": 205,
      "frontier": 123,
      "frontier_accuracy": 0.9326660600545951,
      "in_sample": {
        "threshold": 0.9947150916400119,
        "answered": 0.009099181073703366,
        "accuracy": 0.9326660600545951
      },
      "curve": [
        [
          0.0,
          0.9327
        ],
        [
          0.01,
          0.9322
        ],
        [
          0.02,
          0.9322
        ],
        [
          0.03,
          0.9318
        ],
        [
          0.04,
          0.9318
        ],
        [
          0.05,
          0.9318
        ],
        [
          0.0601,
          0.9318
        ],
        [
          0.0705,
          0.9313
        ],
        [
          0.0805,
          0.9308
        ],
        [
          0.0905,
          0.9295
        ],
        [
          0.1005,
          0.9286
        ],
        [
          0.1106,
          0.9286
        ],
        [
          0.121,
          0.9268
        ],
        [
          0.131,
          0.9263
        ],
        [
          0.141,
          0.9254
        ],
        [
          0.151,
          0.9231
        ],
        [
          0.1611,
          0.9227
        ],
        [
          0.1715,
          0.9213
        ],
        [
          0.1815,
          0.9181
        ],
        [
          0.1915,
          0.9158
        ],
        [
          0.2015,
          0.9131
        ],
        [
          0.2116,
          0.9095
        ],
        [
          0.222,
          0.9081
        ],
        [
          0.232,
          0.9067
        ],
        [
          0.242,
          0.9031
        ],
        [
          0.252,
          0.9004
        ],
        [
          0.2621,
          0.8967
        ],
        [
          0.2725,
          0.894
        ],
        [
          0.2825,
          0.8913
        ],
        [
          0.2925,
          0.8863
        ],
        [
          0.3025,
          0.8817
        ],
        [
          0.3126,
          0.8763
        ],
        [
          0.323,
          0.8712
        ],
        [
          0.333,
          0.8667
        ],
        [
          0.343,
          0.8626
        ],
        [
          0.353,
          0.8585
        ],
        [
          0.3631,
          0.8549
        ],
        [
          0.3735,
          0.849
        ],
        [
          0.3835,
          0.8426
        ],
        [
          0.3935,
          0.8362
        ],
        [
          0.4035,
          0.8312
        ],
        [
          0.4136,
          0.8258
        ],
        [
          0.424,
          0.8194
        ],
        [
          0.434,
          0.8121
        ],
        [
          0.444,
          0.8066
        ],
        [
          0.454,
          0.8012
        ],
        [
          0.4641,
          0.7934
        ],
        [
          0.4745,
          0.7871
        ],
        [
          0.4845,
          0.7812
        ],
        [
          0.4945,
          0.7743
        ],
        [
          0.5045,
          0.7693
        ],
        [
          0.5146,
          0.7621
        ],
        [
          0.525,
          0.7566
        ],
        [
          0.535,
          0.7525
        ],
        [
          0.545,
          0.7466
        ],
        [
          0.5551,
          0.7402
        ],
        [
          0.5655,
          0.7334
        ],
        [
          0.5755,
          0.7298
        ],
        [
          0.5855,
          0.7252
        ],
        [
          0.5955,
          0.7188
        ],
        [
          0.6056,
          0.7116
        ],
        [
          0.616,
          0.7038
        ],
        [
          0.626,
          0.6988
        ],
        [
          0.636,
          0.6943
        ],
        [
          0.646,
          0.6902
        ],
        [
          0.6561,
          0.6847
        ],
        [
          0.6665,
          0.6797
        ],
        [
          0.6765,
          0.6729
        ],
        [
          0.6865,
          0.6665
        ],
        [
          0.6965,
          0.6601
        ],
        [
          0.7066,
          0.6538
        ],
        [
          0.717,
          0.647
        ],
        [
          0.727,
          0.6388
        ],
        [
          0.737,
          0.6315
        ],
        [
          0.747,
          0.6265
        ],
        [
          0.7571,
          0.621
        ],
        [
          0.7675,
          0.6142
        ],
        [
          0.7775,
          0.6074
        ],
        [
          0.7875,
          0.6005
        ],
        [
          0.7975,
          0.5933
        ],
        [
          0.8076,
          0.586
        ],
        [
          0.818,
          0.5787
        ],
        [
          0.828,
          0.5737
        ],
        [
          0.838,
          0.566
        ],
        [
          0.848,
          0.561
        ],
        [
          0.8581,
          0.556
        ],
        [
          0.8685,
          0.55
        ],
        [
          0.8785,
          0.5464
        ],
        [
          0.8885,
          0.5391
        ],
        [
          0.8985,
          0.5328
        ],
        [
          0.9086,
          0.5273
        ],
        [
          0.919,
          0.52
        ],
        [
          0.929,
          0.5132
        ],
        [
          0.939,
          0.5068
        ],
        [
          0.949,
          0.4991
        ],
        [
          0.9591,
          0.4941
        ],
        [
          0.9695,
          0.4873
        ],
        [
          0.9795,
          0.4809
        ],
        [
          0.9895,
          0.4732
        ],
        [
          0.9995,
          0.465
        ],
        [
          1.0,
          0.4645
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.026445360780044298,
        "difference": -0.000794345800188106,
        "worse": 0.43666666666666665,
        "kept_right": 0.025435780524148247,
        "kept_wrong": 0.0010095802558960486,
        "handed_off": 0.973554639219956
      },
      "usd_per_1000": 0.27529749744172105
    },
    {
      "decision": 205,
      "frontier": 274,
      "frontier_accuracy": 0.9085532302092811,
      "in_sample": {
        "threshold": 0.9947150916400119,
        "answered": 0.009099181073703366,
        "accuracy": 0.9085532302092811
      },
      "curve": [
        [
          0.0,
          0.9086
        ],
        [
          0.01,
          0.9081
        ],
        [
          0.02,
          0.9076
        ],
        [
          0.03,
          0.9072
        ],
        [
          0.04,
          0.9072
        ],
        [
          0.05,
          0.9072
        ],
        [
          0.0601,
          0.9072
        ],
        [
          0.0705,
          0.9067
        ],
        [
          0.0805,
          0.9063
        ],
        [
          0.0905,
          0.9054
        ],
        [
          0.1005,
          0.9045
        ],
        [
          0.1106,
          0.9045
        ],
        [
          0.121,
          0.9031
        ],
        [
          0.131,
          0.9022
        ],
        [
          0.141,
          0.9013
        ],
        [
          0.151,
          0.899
        ],
        [
          0.1611,
          0.899
        ],
        [
          0.1715,
          0.8976
        ],
        [
          0.1815,
          0.8954
        ],
        [
          0.1915,
          0.8935
        ],
        [
          0.2015,
          0.8913
        ],
        [
          0.2116,
          0.8881
        ],
        [
          0.222,
          0.8867
        ],
        [
          0.232,
          0.8854
        ],
        [
          0.242,
          0.8817
        ],
        [
          0.252,
          0.8785
        ],
        [
          0.2621,
          0.8749
        ],
        [
          0.2725,
          0.8722
        ],
        [
          0.2825,
          0.8699
        ],
        [
          0.2925,
          0.8653
        ],
        [
          0.3025,
          0.8612
        ],
        [
          0.3126,
          0.8558
        ],
        [
          0.323,
          0.8512
        ],
        [
          0.333,
          0.8462
        ],
        [
          0.343,
          0.8408
        ],
        [
          0.353,
          0.8358
        ],
        [
          0.3631,
          0.8321
        ],
        [
          0.3735,
          0.8262
        ],
        [
          0.3835,
          0.8212
        ],
        [
          0.3935,
          0.8153
        ],
        [
          0.4035,
          0.8103
        ],
        [
          0.4136,
          0.8053
        ],
        [
          0.424,
          0.7994
        ],
        [
          0.434,
          0.7925
        ],
        [
          0.444,
          0.7862
        ],
        [
          0.454,
          0.7803
        ],
        [
          0.4641,
          0.7739
        ],
        [
          0.4745,
          0.7675
        ],
        [
          0.4845,
          0.7616
        ],
        [
          0.4945,
          0.7543
        ],
        [
          0.5045,
          0.7493
        ],
        [
          0.5146,
          0.7439
        ],
        [
          0.525,
          0.7398
        ],
        [
          0.535,
          0.737
        ],
        [
          0.545,
          0.732
        ],
        [
          0.5551,
          0.7261
        ],
        [
          0.5655,
          0.7193
        ],
        [
          0.5755,
          0.7143
        ],
        [
          0.5855,
          0.7097
        ],
        [
          0.5955,
          0.7043
        ],
        [
          0.6056,
          0.6975
        ],
        [
          0.616,
          0.6906
        ],
        [
          0.626,
          0.6847
        ],
        [
          0.636,
          0.6802
        ],
        [
          0.646,
          0.6756
        ],
        [
          0.6561,
          0.6706
        ],
        [
          0.6665,
          0.6656
        ],
        [
          0.6765,
          0.6583
        ],
        [
          0.6865,
          0.6529
        ],
        [
          0.6965,
          0.647
        ],
        [
          0.7066,
          0.641
        ],
        [
          0.717,
          0.6351
        ],
        [
          0.727,
          0.6265
        ],
        [
          0.737,
          0.6201
        ],
        [
          0.747,
          0.6169
        ],
        [
          0.7571,
          0.6115
        ],
        [
          0.7675,
          0.6042
        ],
        [
          0.7775,
          0.5974
        ],
        [
          0.7875,
          0.5919
        ],
        [
          0.7975,
          0.5855
        ],
        [
          0.8076,
          0.5778
        ],
        [
          0.818,
          0.571
        ],
        [
          0.828,
          0.5651
        ],
        [
          0.838,
          0.5591
        ],
        [
          0.848,
          0.5546
        ],
        [
          0.8581,
          0.5505
        ],
        [
          0.8685,
          0.5446
        ],
        [
          0.8785,
          0.5419
        ],
        [
          0.8885,
          0.5346
        ],
        [
          0.8985,
          0.5287
        ],
        [
          0.9086,
          0.5227
        ],
        [
          0.919,
          0.5168
        ],
        [
          0.929,
          0.5109
        ],
        [
          0.939,
          0.5055
        ],
        [
          0.949,
          0.4986
        ],
        [
          0.9591,
          0.4927
        ],
        [
          0.9695,
          0.4854
        ],
        [
          0.9795,
          0.4804
        ],
        [
          0.9895,
          0.4732
        ],
        [
          0.9995,
          0.465
        ],
        [
          1.0,
          0.4645
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.018298457534252723,
        "difference": -0.0008094420482896043,
        "worse": 0.43666666666666665,
        "kept_right": 0.017464767506967918,
        "kept_wrong": 0.0008336900272848006,
        "handed_off": 0.9817015424657478
      },
      "usd_per_1000": 0.43226239614087764
    },
    {
      "decision": 205,
      "frontier": 47,
      "frontier_accuracy": 0.9044585987261147,
      "in_sample": {
        "threshold": 0.9947150916400119,
        "answered": 0.009099181073703366,
        "accuracy": 0.9044585987261147
      },
      "curve": [
        [
          0.0,
          0.9045
        ],
        [
          0.01,
          0.904
        ],
        [
          0.02,
          0.904
        ],
        [
          0.03,
          0.9035
        ],
        [
          0.04,
          0.9035
        ],
        [
          0.05,
          0.9035
        ],
        [
          0.0601,
          0.9035
        ],
        [
          0.0705,
          0.9026
        ],
        [
          0.0805,
          0.9026
        ],
        [
          0.0905,
          0.9017
        ],
        [
          0.1005,
          0.9013
        ],
        [
          0.1106,
          0.9013
        ],
        [
          0.121,
          0.8995
        ],
        [
          0.131,
          0.8985
        ],
        [
          0.141,
          0.8976
        ],
        [
          0.151,
          0.8954
        ],
        [
          0.1611,
          0.8949
        ],
        [
          0.1715,
          0.8935
        ],
        [
          0.1815,
          0.8904
        ],
        [
          0.1915,
          0.8885
        ],
        [
          0.2015,
          0.8858
        ],
        [
          0.2116,
          0.8831
        ],
        [
          0.222,
          0.8817
        ],
        [
          0.232,
          0.8803
        ],
        [
          0.242,
          0.8772
        ],
        [
          0.252,
          0.874
        ],
        [
          0.2621,
          0.8703
        ],
        [
          0.2725,
          0.8681
        ],
        [
          0.2825,
          0.8653
        ],
        [
          0.2925,
          0.8603
        ],
        [
          0.3025,
          0.8558
        ],
        [
          0.3126,
          0.8503
        ],
        [
          0.323,
          0.8458
        ],
        [
          0.333,
          0.8412
        ],
        [
          0.343,
          0.8367
        ],
        [
          0.353,
          0.833
        ],
        [
          0.3631,
          0.8303
        ],
        [
          0.3735,
          0.8244
        ],
        [
          0.3835,
          0.8185
        ],
        [
          0.3935,
          0.8126
        ],
        [
          0.4035,
          0.8076
        ],
        [
          0.4136,
          0.8021
        ],
        [
          0.424,
          0.7962
        ],
        [
          0.434,
          0.7903
        ],
        [
          0.444,
          0.7853
        ],
        [
          0.454,
          0.7793
        ],
        [
          0.4641,
          0.7716
        ],
        [
          0.4745,
          0.7648
        ],
        [
          0.4845,
          0.7584
        ],
        [
          0.4945,
          0.7516
        ],
        [
          0.5045,
          0.7475
        ],
        [
          0.5146,
          0.7416
        ],
        [
          0.525,
          0.7348
        ],
        [
          0.535,
          0.732
        ],
        [
          0.545,
          0.7266
        ],
        [
          0.5551,
          0.7207
        ],
        [
          0.5655,
          0.7143
        ],
        [
          0.5755,
          0.7093
        ],
        [
          0.5855,
          0.7047
        ],
        [
          0.5955,
          0.6997
        ],
        [
          0.6056,
          0.6924
        ],
        [
          0.616,
          0.6861
        ],
        [
          0.626,
          0.6811
        ],
        [
          0.636,
          0.677
        ],
        [
          0.646,
          0.6738
        ],
        [
          0.6561,
          0.6697
        ],
        [
          0.6665,
          0.6647
        ],
        [
          0.6765,
          0.6574
        ],
        [
          0.6865,
          0.652
        ],
        [
          0.6965,
          0.6456
        ],
        [
          0.7066,
          0.6397
        ],
        [
          0.717,
          0.6347
        ],
        [
          0.727,
          0.6265
        ],
        [
          0.737,
          0.6197
        ],
        [
          0.747,
          0.6151
        ],
        [
          0.7571,
          0.6101
        ],
        [
          0.7675,
          0.6037
        ],
        [
          0.7775,
          0.5969
        ],
        [
          0.7875,
          0.591
        ],
        [
          0.7975,
          0.5833
        ],
        [
          0.8076,
          0.5764
        ],
        [
          0.818,
          0.5692
        ],
        [
          0.828,
          0.5641
        ],
        [
          0.838,
          0.5587
        ],
        [
          0.848,
          0.5546
        ],
        [
          0.8581,
          0.55
        ],
        [
          0.8685,
          0.5437
        ],
        [
          0.8785,
          0.5396
        ],
        [
          0.8885,
          0.5337
        ],
        [
          0.8985,
          0.5273
        ],
        [
          0.9086,
          0.5209
        ],
        [
          0.919,
          0.5146
        ],
        [
          0.929,
          0.5082
        ],
        [
          0.939,
          0.5023
        ],
        [
          0.949,
          0.4941
        ],
        [
          0.9591,
          0.4909
        ],
        [
          0.9695,
          0.485
        ],
        [
          0.9795,
          0.4791
        ],
        [
          0.9895,
          0.4727
        ],
        [
          0.9995,
          0.465
        ],
        [
          1.0,
          0.4645
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.025696380932887487,
        "difference": -0.0007639985068682066,
        "worse": 0.43666666666666665,
        "kept_right": 0.024726244404116072,
        "kept_wrong": 0.0009701365287713992,
        "handed_off": 0.9743036190671132
      },
      "usd_per_1000": 4.817704114157854
    },
    {
      "decision": 205,
      "frontier": 193,
      "frontier_accuracy": 0.89171974522293,
      "in_sample": {
        "threshold": 0.9947150916400119,
        "answered": 0.009099181073703366,
        "accuracy": 0.89171974522293
      },
      "curve": [
        [
          0.0,
          0.8917
        ],
        [
          0.01,
          0.8913
        ],
        [
          0.02,
          0.8908
        ],
        [
          0.03,
          0.8904
        ],
        [
          0.04,
          0.8904
        ],
        [
          0.05,
          0.8904
        ],
        [
          0.0601,
          0.8904
        ],
        [
          0.0705,
          0.8894
        ],
        [
          0.0805,
          0.889
        ],
        [
          0.0905,
          0.8881
        ],
        [
          0.1005,
          0.8876
        ],
        [
          0.1106,
          0.8876
        ],
        [
          0.121,
          0.8863
        ],
        [
          0.131,
          0.8854
        ],
        [
          0.141,
          0.8844
        ],
        [
          0.151,
          0.8822
        ],
        [
          0.1611,
          0.8822
        ],
        [
          0.1715,
          0.8808
        ],
        [
          0.1815,
          0.8781
        ],
        [
          0.1915,
          0.8767
        ],
        [
          0.2015,
          0.8744
        ],
        [
          0.2116,
          0.8717
        ],
        [
          0.222,
          0.8699
        ],
        [
          0.232,
          0.8685
        ],
        [
          0.242,
          0.8653
        ],
        [
          0.252,
          0.8621
        ],
        [
          0.2621,
          0.8599
        ],
        [
          0.2725,
          0.8571
        ],
        [
          0.2825,
          0.8544
        ],
        [
          0.2925,
          0.8494
        ],
        [
          0.3025,
          0.8449
        ],
        [
          0.3126,
          0.8403
        ],
        [
          0.323,
          0.8358
        ],
        [
          0.333,
          0.8317
        ],
        [
          0.343,
          0.8276
        ],
        [
          0.353,
          0.8235
        ],
        [
          0.3631,
          0.8212
        ],
        [
          0.3735,
          0.8153
        ],
        [
          0.3835,
          0.8089
        ],
        [
          0.3935,
          0.8039
        ],
        [
          0.4035,
          0.7989
        ],
        [
          0.4136,
          0.793
        ],
        [
          0.424,
          0.7875
        ],
        [
          0.434,
          0.783
        ],
        [
          0.444,
          0.7775
        ],
        [
          0.454,
          0.7716
        ],
        [
          0.4641,
          0.7643
        ],
        [
          0.4745,
          0.7584
        ],
        [
          0.4845,
          0.7539
        ],
        [
          0.4945,
          0.747
        ],
        [
          0.5045,
          0.7429
        ],
        [
          0.5146,
          0.7375
        ],
        [
          0.525,
          0.7311
        ],
        [
          0.535,
          0.7293
        ],
        [
          0.545,
          0.7238
        ],
        [
          0.5551,
          0.7184
        ],
        [
          0.5655,
          0.712
        ],
        [
          0.5755,
          0.707
        ],
        [
          0.5855,
          0.7034
        ],
        [
          0.5955,
          0.6975
        ],
        [
          0.6056,
          0.6902
        ],
        [
          0.616,
          0.6829
        ],
        [
          0.626,
          0.6779
        ],
        [
          0.636,
          0.6742
        ],
        [
          0.646,
          0.6702
        ],
        [
          0.6561,
          0.667
        ],
        [
          0.6665,
          0.6629
        ],
        [
          0.6765,
          0.6551
        ],
        [
          0.6865,
          0.6506
        ],
        [
          0.6965,
          0.6447
        ],
        [
          0.7066,
          0.6392
        ],
        [
          0.717,
          0.6333
        ],
        [
          0.727,
          0.6251
        ],
        [
          0.737,
          0.6183
        ],
        [
          0.747,
          0.6146
        ],
        [
          0.7571,
          0.6106
        ],
        [
          0.7675,
          0.6037
        ],
        [
          0.7775,
          0.596
        ],
        [
          0.7875,
          0.5896
        ],
        [
          0.7975,
          0.5833
        ],
        [
          0.8076,
          0.576
        ],
        [
          0.818,
          0.5705
        ],
        [
          0.828,
          0.5646
        ],
        [
          0.838,
          0.5587
        ],
        [
          0.848,
          0.5532
        ],
        [
          0.8581,
          0.5491
        ],
        [
          0.8685,
          0.5432
        ],
        [
          0.8785,
          0.5396
        ],
        [
          0.8885,
          0.5332
        ],
        [
          0.8985,
          0.5268
        ],
        [
          0.9086,
          0.5218
        ],
        [
          0.919,
          0.5159
        ],
        [
          0.929,
          0.5105
        ],
        [
          0.939,
          0.5041
        ],
        [
          0.949,
          0.4959
        ],
        [
          0.9591,
          0.49
        ],
        [
          0.9695,
          0.4841
        ],
        [
          0.9795,
          0.4791
        ],
        [
          0.9895,
          0.4718
        ],
        [
          0.9995,
          0.4645
        ],
        [
          1.0,
          0.4645
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.017376811260667105,
        "difference": -0.0007518165330221177,
        "worse": 0.43666666666666665,
        "kept_right": 0.01661892308222907,
        "kept_wrong": 0.0007578881784380255,
        "handed_off": 0.9826231887393336
      },
      "usd_per_1000": 7.6267779385917995
    },
    {
      "decision": 205,
      "frontier": 55,
      "frontier_accuracy": 0.8503184713375797,
      "in_sample": {
        "threshold": 0.9832574573245241,
        "answered": 0.06824385805277525,
        "accuracy": 0.8503184713375797
      },
      "curve": [
        [
          0.0,
          0.8503
        ],
        [
          0.01,
          0.8499
        ],
        [
          0.02,
          0.8499
        ],
        [
          0.03,
          0.8494
        ],
        [
          0.04,
          0.8499
        ],
        [
          0.05,
          0.8499
        ],
        [
          0.0601,
          0.8499
        ],
        [
          0.0705,
          0.8499
        ],
        [
          0.0805,
          0.8494
        ],
        [
          0.0905,
          0.8494
        ],
        [
          0.1005,
          0.8485
        ],
        [
          0.1106,
          0.849
        ],
        [
          0.121,
          0.8476
        ],
        [
          0.131,
          0.8471
        ],
        [
          0.141,
          0.8467
        ],
        [
          0.151,
          0.8453
        ],
        [
          0.1611,
          0.8453
        ],
        [
          0.1715,
          0.8449
        ],
        [
          0.1815,
          0.8435
        ],
        [
          0.1915,
          0.8412
        ],
        [
          0.2015,
          0.8394
        ],
        [
          0.2116,
          0.8367
        ],
        [
          0.222,
          0.8348
        ],
        [
          0.232,
          0.8335
        ],
        [
          0.242,
          0.8303
        ],
        [
          0.252,
          0.828
        ],
        [
          0.2621,
          0.8262
        ],
        [
          0.2725,
          0.8239
        ],
        [
          0.2825,
          0.8212
        ],
        [
          0.2925,
          0.8171
        ],
        [
          0.3025,
          0.8126
        ],
        [
          0.3126,
          0.8089
        ],
        [
          0.323,
          0.8053
        ],
        [
          0.333,
          0.8012
        ],
        [
          0.343,
          0.7966
        ],
        [
          0.353,
          0.7944
        ],
        [
          0.3631,
          0.7921
        ],
        [
          0.3735,
          0.7871
        ],
        [
          0.3835,
          0.783
        ],
        [
          0.3935,
          0.7784
        ],
        [
          0.4035,
          0.7739
        ],
        [
          0.4136,
          0.7689
        ],
        [
          0.424,
          0.7634
        ],
        [
          0.434,
          0.7584
        ],
        [
          0.444,
          0.7539
        ],
        [
          0.454,
          0.7489
        ],
        [
          0.4641,
          0.742
        ],
        [
          0.4745,
          0.7366
        ],
        [
          0.4845,
          0.732
        ],
        [
          0.4945,
          0.7257
        ],
        [
          0.5045,
          0.7216
        ],
        [
          0.5146,
          0.7157
        ],
        [
          0.525,
          0.7102
        ],
        [
          0.535,
          0.707
        ],
        [
          0.545,
          0.7015
        ],
        [
          0.5551,
          0.6956
        ],
        [
          0.5655,
          0.6911
        ],
        [
          0.5755,
          0.6861
        ],
        [
          0.5855,
          0.682
        ],
        [
          0.5955,
          0.677
        ],
        [
          0.6056,
          0.6706
        ],
        [
          0.616,
          0.6642
        ],
        [
          0.626,
          0.6611
        ],
        [
          0.636,
          0.657
        ],
        [
          0.646,
          0.6529
        ],
        [
          0.6561,
          0.6492
        ],
        [
          0.6665,
          0.6465
        ],
        [
          0.6765,
          0.6397
        ],
        [
          0.6865,
          0.6347
        ],
        [
          0.6965,
          0.6297
        ],
        [
          0.7066,
          0.6247
        ],
        [
          0.717,
          0.6192
        ],
        [
          0.727,
          0.6119
        ],
        [
          0.737,
          0.606
        ],
        [
          0.747,
          0.6037
        ],
        [
          0.7571,
          0.5996
        ],
        [
          0.7675,
          0.5933
        ],
        [
          0.7775,
          0.5869
        ],
        [
          0.7875,
          0.5819
        ],
        [
          0.7975,
          0.576
        ],
        [
          0.8076,
          0.5705
        ],
        [
          0.818,
          0.5632
        ],
        [
          0.828,
          0.5587
        ],
        [
          0.838,
          0.5528
        ],
        [
          0.848,
          0.5482
        ],
        [
          0.8581,
          0.5437
        ],
        [
          0.8685,
          0.5396
        ],
        [
          0.8785,
          0.5364
        ],
        [
          0.8885,
          0.5309
        ],
        [
          0.8985,
          0.5246
        ],
        [
          0.9086,
          0.5187
        ],
        [
          0.919,
          0.5127
        ],
        [
          0.929,
          0.5077
        ],
        [
          0.939,
          0.5027
        ],
        [
          0.949,
          0.4955
        ],
        [
          0.9591,
          0.4895
        ],
        [
          0.9695,
          0.4823
        ],
        [
          0.9795,
          0.4773
        ],
        [
          0.9895,
          0.4713
        ],
        [
          0.9995,
          0.465
        ],
        [
          1.0,
          0.4645
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.0792189206400967,
        "difference": -0.001882747056754596,
        "worse": 0.55,
        "kept_right": 0.07507139567869729,
        "kept_wrong": 0.0041475249613994,
        "handed_off": 0.9207810793599035
      },
      "usd_per_1000": 3.378657611569352
    },
    {
      "decision": 139,
      "frontier": 140,
      "frontier_accuracy": 0.9808917197452229,
      "in_sample": {
        "threshold": 0.9629,
        "answered": 0.005004549590536852,
        "accuracy": 0.9808917197452229
      },
      "curve": [
        [
          0.0,
          0.9809
        ],
        [
          0.0105,
          0.98
        ],
        [
          0.0209,
          0.9795
        ],
        [
          0.0309,
          0.9786
        ],
        [
          0.0409,
          0.9777
        ],
        [
          0.051,
          0.9777
        ],
        [
          0.0614,
          0.9768
        ],
        [
          0.0714,
          0.9763
        ],
        [
          0.0814,
          0.9754
        ],
        [
          0.0919,
          0.9732
        ],
        [
          0.1024,
          0.9704
        ],
        [
          0.1128,
          0.9672
        ],
        [
          0.1228,
          0.9645
        ],
        [
          0.1328,
          0.9604
        ],
        [
          0.1433,
          0.9572
        ],
        [
          0.1533,
          0.9536
        ],
        [
          0.1633,
          0.9522
        ],
        [
          0.1733,
          0.9481
        ],
        [
          0.1838,
          0.9459
        ],
        [
          0.1938,
          0.944
        ],
        [
          0.2038,
          0.9399
        ],
        [
          0.2138,
          0.9377
        ],
        [
          0.2238,
          0.9349
        ],
        [
          0.2338,
          0.9313
        ],
        [
          0.2439,
          0.9263
        ],
        [
          0.2543,
          0.9231
        ],
        [
          0.2643,
          0.9199
        ],
        [
          0.2743,
          0.9145
        ],
        [
          0.2843,
          0.9117
        ],
        [
          0.2944,
          0.9086
        ],
        [
          0.3048,
          0.9045
        ],
        [
          0.3148,
          0.8995
        ],
        [
          0.3248,
          0.8949
        ],
        [
          0.3348,
          0.8908
        ],
        [
          0.3449,
          0.8844
        ],
        [
          0.3562,
          0.8817
        ],
        [
          0.3662,
          0.8763
        ],
        [
          0.3763,
          0.874
        ],
        [
          0.3867,
          0.8694
        ],
        [
          0.3967,
          0.8644
        ],
        [
          0.4067,
          0.8599
        ],
        [
          0.4167,
          0.8549
        ],
        [
          0.4272,
          0.8471
        ],
        [
          0.4372,
          0.8426
        ],
        [
          0.4477,
          0.8367
        ],
        [
          0.4581,
          0.8312
        ],
        [
          0.4682,
          0.8262
        ],
        [
          0.4786,
          0.8212
        ],
        [
          0.4886,
          0.8157
        ],
        [
          0.4986,
          0.8094
        ],
        [
          0.5086,
          0.8044
        ],
        [
          0.5187,
          0.7994
        ],
        [
          0.5291,
          0.7939
        ],
        [
          0.5391,
          0.7848
        ],
        [
          0.5491,
          0.7798
        ],
        [
          0.5596,
          0.7743
        ],
        [
          0.5696,
          0.7693
        ],
        [
          0.5796,
          0.7616
        ],
        [
          0.5901,
          0.7548
        ],
        [
          0.6005,
          0.7475
        ],
        [
          0.6106,
          0.742
        ],
        [
          0.621,
          0.7352
        ],
        [
          0.631,
          0.7288
        ],
        [
          0.641,
          0.722
        ],
        [
          0.651,
          0.7143
        ],
        [
          0.6611,
          0.7084
        ],
        [
          0.6715,
          0.7029
        ],
        [
          0.682,
          0.6961
        ],
        [
          0.6924,
          0.6879
        ],
        [
          0.7025,
          0.6802
        ],
        [
          0.7129,
          0.6733
        ],
        [
          0.7229,
          0.667
        ],
        [
          0.7329,
          0.6611
        ],
        [
          0.7429,
          0.6538
        ],
        [
          0.753,
          0.647
        ],
        [
          0.7634,
          0.6392
        ],
        [
          0.7734,
          0.631
        ],
        [
          0.7843,
          0.6224
        ],
        [
          0.7944,
          0.6137
        ],
        [
          0.8048,
          0.6069
        ],
        [
          0.8148,
          0.5987
        ],
        [
          0.8248,
          0.591
        ],
        [
          0.8348,
          0.5823
        ],
        [
          0.8449,
          0.5746
        ],
        [
          0.8553,
          0.5682
        ],
        [
          0.8653,
          0.5605
        ],
        [
          0.8753,
          0.5528
        ],
        [
          0.8854,
          0.546
        ],
        [
          0.8958,
          0.5364
        ],
        [
          0.9058,
          0.5291
        ],
        [
          0.9163,
          0.5205
        ],
        [
          0.9272,
          0.5132
        ],
        [
          0.9372,
          0.5073
        ],
        [
          0.9472,
          0.4986
        ],
        [
          0.9572,
          0.4904
        ],
        [
          0.9672,
          0.4827
        ],
        [
          0.9773,
          0.475
        ],
        [
          0.9877,
          0.4668
        ],
        [
          0.9977,
          0.4591
        ],
        [
          1.0,
          0.4572
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.009209419989045014,
        "difference": -0.0008427782939492808,
        "worse": 0.4666666666666667,
        "kept_right": 0.008306005223608585,
        "kept_wrong": 0.000903414765436433,
        "handed_off": 0.9907905800109563
      },
      "usd_per_1000": 2.414191472083814
    },
    {
      "decision": 139,
      "frontier": 277,
      "frontier_accuracy": 0.9795268425841674,
      "in_sample": {
        "threshold": 0.9629,
        "answered": 0.005004549590536852,
        "accuracy": 0.9795268425841674
      },
      "curve": [
        [
          0.0,
          0.9795
        ],
        [
          0.0105,
          0.9786
        ],
        [
          0.0209,
          0.9782
        ],
        [
          0.0309,
          0.9773
        ],
        [
          0.0409,
          0.9763
        ],
        [
          0.051,
          0.9763
        ],
        [
          0.0614,
          0.9754
        ],
        [
          0.0714,
          0.975
        ],
        [
          0.0814,
          0.9741
        ],
        [
          0.0919,
          0.9718
        ],
        [
          0.1024,
          0.9691
        ],
        [
          0.1128,
          0.9659
        ],
        [
          0.1228,
          0.9636
        ],
        [
          0.1328,
          0.96
        ],
        [
          0.1433,
          0.9568
        ],
        [
          0.1533,
          0.9531
        ],
        [
          0.1633,
          0.9518
        ],
        [
          0.1733,
          0.9477
        ],
        [
          0.1838,
          0.9445
        ],
        [
          0.1938,
          0.9427
        ],
        [
          0.2038,
          0.9386
        ],
        [
          0.2138,
          0.9363
        ],
        [
          0.2238,
          0.9331
        ],
        [
          0.2338,
          0.929
        ],
        [
          0.2439,
          0.924
        ],
        [
          0.2543,
          0.9208
        ],
        [
          0.2643,
          0.9181
        ],
        [
          0.2743,
          0.9126
        ],
        [
          0.2843,
          0.9099
        ],
        [
          0.2944,
          0.9067
        ],
        [
          0.3048,
          0.9026
        ],
        [
          0.3148,
          0.8976
        ],
        [
          0.3248,
          0.8931
        ],
        [
          0.3348,
          0.889
        ],
        [
          0.3449,
          0.8826
        ],
        [
          0.3562,
          0.8794
        ],
        [
          0.3662,
          0.874
        ],
        [
          0.3763,
          0.8722
        ],
        [
          0.3867,
          0.8676
        ],
        [
          0.3967,
          0.8626
        ],
        [
          0.4067,
          0.8581
        ],
        [
          0.4167,
          0.854
        ],
        [
          0.4272,
          0.8462
        ],
        [
          0.4372,
          0.8417
        ],
        [
          0.4477,
          0.8358
        ],
        [
          0.4581,
          0.8308
        ],
        [
          0.4682,
          0.8258
        ],
        [
          0.4786,
          0.8203
        ],
        [
          0.4886,
          0.8148
        ],
        [
          0.4986,
          0.8085
        ],
        [
          0.5086,
          0.8035
        ],
        [
          0.5187,
          0.7985
        ],
        [
          0.5291,
          0.793
        ],
        [
          0.5391,
          0.7839
        ],
        [
          0.5491,
          0.7789
        ],
        [
          0.5596,
          0.7734
        ],
        [
          0.5696,
          0.7684
        ],
        [
          0.5796,
          0.7607
        ],
        [
          0.5901,
          0.7543
        ],
        [
          0.6005,
          0.747
        ],
        [
          0.6106,
          0.7416
        ],
        [
          0.621,
          0.7348
        ],
        [
          0.631,
          0.7284
        ],
        [
          0.641,
          0.7211
        ],
        [
          0.651,
          0.7134
        ],
        [
          0.6611,
          0.7075
        ],
        [
          0.6715,
          0.702
        ],
        [
          0.682,
          0.6952
        ],
        [
          0.6924,
          0.687
        ],
        [
          0.7025,
          0.6797
        ],
        [
          0.7129,
          0.6729
        ],
        [
          0.7229,
          0.6665
        ],
        [
          0.7329,
          0.6611
        ],
        [
          0.7429,
          0.6533
        ],
        [
          0.753,
          0.6465
        ],
        [
          0.7634,
          0.6388
        ],
        [
          0.7734,
          0.6306
        ],
        [
          0.7843,
          0.6224
        ],
        [
          0.7944,
          0.6137
        ],
        [
          0.8048,
          0.6069
        ],
        [
          0.8148,
          0.5987
        ],
        [
          0.8248,
          0.591
        ],
        [
          0.8348,
          0.5819
        ],
        [
          0.8449,
          0.5742
        ],
        [
          0.8553,
          0.5673
        ],
        [
          0.8653,
          0.5596
        ],
        [
          0.8753,
          0.5514
        ],
        [
          0.8854,
          0.5446
        ],
        [
          0.8958,
          0.535
        ],
        [
          0.9058,
          0.5278
        ],
        [
          0.9163,
          0.5191
        ],
        [
          0.9272,
          0.5118
        ],
        [
          0.9372,
          0.5059
        ],
        [
          0.9472,
          0.4973
        ],
        [
          0.9572,
          0.4891
        ],
        [
          0.9672,
          0.4813
        ],
        [
          0.9773,
          0.4741
        ],
        [
          0.9877,
          0.4659
        ],
        [
          0.9977,
          0.4591
        ],
        [
          1.0,
          0.4572
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.009209419989045014,
        "difference": -0.0008427782939492808,
        "worse": 0.4666666666666667,
        "kept_right": 0.008306005223608585,
        "kept_wrong": 0.000903414765436433,
        "handed_off": 0.9907905800109563
      },
      "usd_per_1000": 6.027061653256712
    },
    {
      "decision": 139,
      "frontier": 276,
      "frontier_accuracy": 0.9777070063694268,
      "in_sample": {
        "threshold": 0.9629,
        "answered": 0.005004549590536852,
        "accuracy": 0.9777070063694268
      },
      "curve": [
        [
          0.0,
          0.9777
        ],
        [
          0.0105,
          0.9768
        ],
        [
          0.0209,
          0.9763
        ],
        [
          0.0309,
          0.9754
        ],
        [
          0.0409,
          0.9745
        ],
        [
          0.051,
          0.9741
        ],
        [
          0.0614,
          0.9732
        ],
        [
          0.0714,
          0.9727
        ],
        [
          0.0814,
          0.9718
        ],
        [
          0.0919,
          0.9695
        ],
        [
          0.1024,
          0.9668
        ],
        [
          0.1128,
          0.9636
        ],
        [
          0.1228,
          0.9613
        ],
        [
          0.1328,
          0.9577
        ],
        [
          0.1433,
          0.9545
        ],
        [
          0.1533,
          0.9509
        ],
        [
          0.1633,
          0.9495
        ],
        [
          0.1733,
          0.9459
        ],
        [
          0.1838,
          0.9427
        ],
        [
          0.1938,
          0.9409
        ],
        [
          0.2038,
          0.9368
        ],
        [
          0.2138,
          0.9345
        ],
        [
          0.2238,
          0.9313
        ],
        [
          0.2338,
          0.9272
        ],
        [
          0.2439,
          0.9222
        ],
        [
          0.2543,
          0.919
        ],
        [
          0.2643,
          0.9163
        ],
        [
          0.2743,
          0.9108
        ],
        [
          0.2843,
          0.9081
        ],
        [
          0.2944,
          0.9054
        ],
        [
          0.3048,
          0.9013
        ],
        [
          0.3148,
          0.8963
        ],
        [
          0.3248,
          0.8917
        ],
        [
          0.3348,
          0.8876
        ],
        [
          0.3449,
          0.8813
        ],
        [
          0.3562,
          0.8781
        ],
        [
          0.3662,
          0.8726
        ],
        [
          0.3763,
          0.8708
        ],
        [
          0.3867,
          0.8662
        ],
        [
          0.3967,
          0.8612
        ],
        [
          0.4067,
          0.8567
        ],
        [
          0.4167,
          0.8526
        ],
        [
          0.4272,
          0.8449
        ],
        [
          0.4372,
          0.8408
        ],
        [
          0.4477,
          0.8358
        ],
        [
          0.4581,
          0.8308
        ],
        [
          0.4682,
          0.8258
        ],
        [
          0.4786,
          0.8203
        ],
        [
          0.4886,
          0.8148
        ],
        [
          0.4986,
          0.8085
        ],
        [
          0.5086,
          0.8035
        ],
        [
          0.5187,
          0.7985
        ],
        [
          0.5291,
          0.793
        ],
        [
          0.5391,
          0.7839
        ],
        [
          0.5491,
          0.7789
        ],
        [
          0.5596,
          0.7734
        ],
        [
          0.5696,
          0.7684
        ],
        [
          0.5796,
          0.7607
        ],
        [
          0.5901,
          0.7543
        ],
        [
          0.6005,
          0.747
        ],
        [
          0.6106,
          0.7416
        ],
        [
          0.621,
          0.7352
        ],
        [
          0.631,
          0.7288
        ],
        [
          0.641,
          0.7216
        ],
        [
          0.651,
          0.7138
        ],
        [
          0.6611,
          0.7079
        ],
        [
          0.6715,
          0.7025
        ],
        [
          0.682,
          0.6956
        ],
        [
          0.6924,
          0.6874
        ],
        [
          0.7025,
          0.6797
        ],
        [
          0.7129,
          0.6729
        ],
        [
          0.7229,
          0.6665
        ],
        [
          0.7329,
          0.6611
        ],
        [
          0.7429,
          0.6533
        ],
        [
          0.753,
          0.6465
        ],
        [
          0.7634,
          0.6388
        ],
        [
          0.7734,
          0.6306
        ],
        [
          0.7843,
          0.6224
        ],
        [
          0.7944,
          0.6137
        ],
        [
          0.8048,
          0.6069
        ],
        [
          0.8148,
          0.5987
        ],
        [
          0.8248,
          0.591
        ],
        [
          0.8348,
          0.5819
        ],
        [
          0.8449,
          0.5742
        ],
        [
          0.8553,
          0.5673
        ],
        [
          0.8653,
          0.5596
        ],
        [
          0.8753,
          0.5519
        ],
        [
          0.8854,
          0.545
        ],
        [
          0.8958,
          0.5355
        ],
        [
          0.9058,
          0.5278
        ],
        [
          0.9163,
          0.5191
        ],
        [
          0.9272,
          0.5118
        ],
        [
          0.9372,
          0.5059
        ],
        [
          0.9472,
          0.4973
        ],
        [
          0.9572,
          0.4895
        ],
        [
          0.9672,
          0.4818
        ],
        [
          0.9773,
          0.4741
        ],
        [
          0.9877,
          0.4659
        ],
        [
          0.9977,
          0.4591
        ],
        [
          1.0,
          0.4572
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.009188191323867305,
        "difference": -0.0008458113543071817,
        "worse": 0.4666666666666667,
        "kept_right": 0.008287809618788776,
        "kept_wrong": 0.0009003817050785319,
        "handed_off": 0.9908118086761342
      },
      "usd_per_1000": 1.6246520263460387
    },
    {
      "decision": 139,
      "frontier": 48,
      "frontier_accuracy": 0.9758871701546861,
      "in_sample": {
        "threshold": 0.9629,
        "answered": 0.005004549590536852,
        "accuracy": 0.9758871701546861
      },
      "curve": [
        [
          0.0,
          0.9759
        ],
        [
          0.0105,
          0.975
        ],
        [
          0.0209,
          0.9745
        ],
        [
          0.0309,
          0.9736
        ],
        [
          0.0409,
          0.9727
        ],
        [
          0.051,
          0.9722
        ],
        [
          0.0614,
          0.9713
        ],
        [
          0.0714,
          0.9709
        ],
        [
          0.0814,
          0.97
        ],
        [
          0.0919,
          0.9677
        ],
        [
          0.1024,
          0.965
        ],
        [
          0.1128,
          0.9618
        ],
        [
          0.1228,
          0.9595
        ],
        [
          0.1328,
          0.9559
        ],
        [
          0.1433,
          0.9527
        ],
        [
          0.1533,
          0.949
        ],
        [
          0.1633,
          0.9477
        ],
        [
          0.1733,
          0.9436
        ],
        [
          0.1838,
          0.9399
        ],
        [
          0.1938,
          0.9381
        ],
        [
          0.2038,
          0.934
        ],
        [
          0.2138,
          0.9318
        ],
        [
          0.2238,
          0.9286
        ],
        [
          0.2338,
          0.9249
        ],
        [
          0.2439,
          0.9199
        ],
        [
          0.2543,
          0.9167
        ],
        [
          0.2643,
          0.9145
        ],
        [
          0.2743,
          0.909
        ],
        [
          0.2843,
          0.9063
        ],
        [
          0.2944,
          0.9031
        ],
        [
          0.3048,
          0.899
        ],
        [
          0.3148,
          0.894
        ],
        [
          0.3248,
          0.8894
        ],
        [
          0.3348,
          0.8854
        ],
        [
          0.3449,
          0.879
        ],
        [
          0.3562,
          0.8763
        ],
        [
          0.3662,
          0.8708
        ],
        [
          0.3763,
          0.869
        ],
        [
          0.3867,
          0.8644
        ],
        [
          0.3967,
          0.8594
        ],
        [
          0.4067,
          0.8549
        ],
        [
          0.4167,
          0.8503
        ],
        [
          0.4272,
          0.8426
        ],
        [
          0.4372,
          0.838
        ],
        [
          0.4477,
          0.8326
        ],
        [
          0.4581,
          0.8276
        ],
        [
          0.4682,
          0.8226
        ],
        [
          0.4786,
          0.8176
        ],
        [
          0.4886,
          0.8121
        ],
        [
          0.4986,
          0.8057
        ],
        [
          0.5086,
          0.8003
        ],
        [
          0.5187,
          0.7953
        ],
        [
          0.5291,
          0.7898
        ],
        [
          0.5391,
          0.7807
        ],
        [
          0.5491,
          0.7757
        ],
        [
          0.5596,
          0.7707
        ],
        [
          0.5696,
          0.7662
        ],
        [
          0.5796,
          0.7584
        ],
        [
          0.5901,
          0.7525
        ],
        [
          0.6005,
          0.7457
        ],
        [
          0.6106,
          0.7402
        ],
        [
          0.621,
          0.7334
        ],
        [
          0.631,
          0.727
        ],
        [
          0.641,
          0.7197
        ],
        [
          0.651,
          0.712
        ],
        [
          0.6611,
          0.7061
        ],
        [
          0.6715,
          0.7006
        ],
        [
          0.682,
          0.6938
        ],
        [
          0.6924,
          0.6856
        ],
        [
          0.7025,
          0.6779
        ],
        [
          0.7129,
          0.6711
        ],
        [
          0.7229,
          0.6647
        ],
        [
          0.7329,
          0.6592
        ],
        [
          0.7429,
          0.6515
        ],
        [
          0.753,
          0.6447
        ],
        [
          0.7634,
          0.6369
        ],
        [
          0.7734,
          0.6288
        ],
        [
          0.7843,
          0.6206
        ],
        [
          0.7944,
          0.6119
        ],
        [
          0.8048,
          0.6051
        ],
        [
          0.8148,
          0.5969
        ],
        [
          0.8248,
          0.5892
        ],
        [
          0.8348,
          0.5801
        ],
        [
          0.8449,
          0.5723
        ],
        [
          0.8553,
          0.5655
        ],
        [
          0.8653,
          0.5578
        ],
        [
          0.8753,
          0.5505
        ],
        [
          0.8854,
          0.5437
        ],
        [
          0.8958,
          0.5341
        ],
        [
          0.9058,
          0.5268
        ],
        [
          0.9163,
          0.5182
        ],
        [
          0.9272,
          0.5109
        ],
        [
          0.9372,
          0.505
        ],
        [
          0.9472,
          0.4964
        ],
        [
          0.9572,
          0.4891
        ],
        [
          0.9672,
          0.4813
        ],
        [
          0.9773,
          0.4741
        ],
        [
          0.9877,
          0.4659
        ],
        [
          0.9977,
          0.4591
        ],
        [
          1.0,
          0.4572
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.009188191323867305,
        "difference": -0.0008458113543071817,
        "worse": 0.4666666666666667,
        "kept_right": 0.008287809618788776,
        "kept_wrong": 0.0009003817050785319,
        "handed_off": 0.9908118086761342
      },
      "usd_per_1000": 5.346176919761573
    },
    {
      "decision": 139,
      "frontier": 191,
      "frontier_accuracy": 0.9740673339399454,
      "in_sample": {
        "threshold": 0.9577,
        "answered": 0.008644222020018199,
        "accuracy": 0.9740673339399454
      },
      "curve": [
        [
          0.0,
          0.9741
        ],
        [
          0.0105,
          0.9736
        ],
        [
          0.0209,
          0.9732
        ],
        [
          0.0309,
          0.9722
        ],
        [
          0.0409,
          0.9713
        ],
        [
          0.051,
          0.9709
        ],
        [
          0.0614,
          0.97
        ],
        [
          0.0714,
          0.9695
        ],
        [
          0.0814,
          0.9686
        ],
        [
          0.0919,
          0.9663
        ],
        [
          0.1024,
          0.9636
        ],
        [
          0.1128,
          0.9609
        ],
        [
          0.1228,
          0.9581
        ],
        [
          0.1328,
          0.9545
        ],
        [
          0.1433,
          0.9513
        ],
        [
          0.1533,
          0.9477
        ],
        [
          0.1633,
          0.9463
        ],
        [
          0.1733,
          0.9422
        ],
        [
          0.1838,
          0.9386
        ],
        [
          0.1938,
          0.9368
        ],
        [
          0.2038,
          0.9327
        ],
        [
          0.2138,
          0.9308
        ],
        [
          0.2238,
          0.9277
        ],
        [
          0.2338,
          0.9236
        ],
        [
          0.2439,
          0.9186
        ],
        [
          0.2543,
          0.9154
        ],
        [
          0.2643,
          0.9126
        ],
        [
          0.2743,
          0.9076
        ],
        [
          0.2843,
          0.9049
        ],
        [
          0.2944,
          0.9017
        ],
        [
          0.3048,
          0.8981
        ],
        [
          0.3148,
          0.8931
        ],
        [
          0.3248,
          0.8885
        ],
        [
          0.3348,
          0.8844
        ],
        [
          0.3449,
          0.8776
        ],
        [
          0.3562,
          0.8744
        ],
        [
          0.3662,
          0.869
        ],
        [
          0.3763,
          0.8667
        ],
        [
          0.3867,
          0.8621
        ],
        [
          0.3967,
          0.8571
        ],
        [
          0.4067,
          0.853
        ],
        [
          0.4167,
          0.8485
        ],
        [
          0.4272,
          0.8408
        ],
        [
          0.4372,
          0.8367
        ],
        [
          0.4477,
          0.8312
        ],
        [
          0.4581,
          0.8262
        ],
        [
          0.4682,
          0.8212
        ],
        [
          0.4786,
          0.8162
        ],
        [
          0.4886,
          0.8107
        ],
        [
          0.4986,
          0.8044
        ],
        [
          0.5086,
          0.7994
        ],
        [
          0.5187,
          0.7944
        ],
        [
          0.5291,
          0.7889
        ],
        [
          0.5391,
          0.7798
        ],
        [
          0.5491,
          0.7748
        ],
        [
          0.5596,
          0.7693
        ],
        [
          0.5696,
          0.7643
        ],
        [
          0.5796,
          0.7566
        ],
        [
          0.5901,
          0.7502
        ],
        [
          0.6005,
          0.7429
        ],
        [
          0.6106,
          0.7375
        ],
        [
          0.621,
          0.7311
        ],
        [
          0.631,
          0.7247
        ],
        [
          0.641,
          0.7175
        ],
        [
          0.651,
          0.7097
        ],
        [
          0.6611,
          0.7038
        ],
        [
          0.6715,
          0.6988
        ],
        [
          0.682,
          0.692
        ],
        [
          0.6924,
          0.6838
        ],
        [
          0.7025,
          0.6761
        ],
        [
          0.7129,
          0.6697
        ],
        [
          0.7229,
          0.6633
        ],
        [
          0.7329,
          0.6579
        ],
        [
          0.7429,
          0.6501
        ],
        [
          0.753,
          0.6433
        ],
        [
          0.7634,
          0.6356
        ],
        [
          0.7734,
          0.6278
        ],
        [
          0.7843,
          0.6201
        ],
        [
          0.7944,
          0.6115
        ],
        [
          0.8048,
          0.6046
        ],
        [
          0.8148,
          0.5965
        ],
        [
          0.8248,
          0.5887
        ],
        [
          0.8348,
          0.5801
        ],
        [
          0.8449,
          0.5723
        ],
        [
          0.8553,
          0.5655
        ],
        [
          0.8653,
          0.5578
        ],
        [
          0.8753,
          0.5491
        ],
        [
          0.8854,
          0.5428
        ],
        [
          0.8958,
          0.5332
        ],
        [
          0.9058,
          0.5259
        ],
        [
          0.9163,
          0.5173
        ],
        [
          0.9272,
          0.51
        ],
        [
          0.9372,
          0.5041
        ],
        [
          0.9472,
          0.4955
        ],
        [
          0.9572,
          0.4891
        ],
        [
          0.9672,
          0.4813
        ],
        [
          0.9773,
          0.4741
        ],
        [
          0.9877,
          0.4659
        ],
        [
          0.9977,
          0.4591
        ],
        [
          1.0,
          0.4572
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.014906051020640671,
        "difference": -0.000948957806830397,
        "worse": 0.5,
        "kept_right": 0.013423502751557506,
        "kept_wrong": 0.0014825482690831707,
        "handed_off": 0.9850939489793605
      },
      "usd_per_1000": 26.35650947243344
    },
    {
      "decision": 139,
      "frontier": 188,
      "frontier_accuracy": 0.9713375796178344,
      "in_sample": {
        "threshold": 0.9629,
        "answered": 0.005004549590536852,
        "accuracy": 0.9713375796178344
      },
      "curve": [
        [
          0.0,
          0.9713
        ],
        [
          0.0105,
          0.9704
        ],
        [
          0.0209,
          0.97
        ],
        [
          0.0309,
          0.9691
        ],
        [
          0.0409,
          0.9682
        ],
        [
          0.051,
          0.9677
        ],
        [
          0.0614,
          0.9668
        ],
        [
          0.0714,
          0.9663
        ],
        [
          0.0814,
          0.9654
        ],
        [
          0.0919,
          0.9631
        ],
        [
          0.1024,
          0.9604
        ],
        [
          0.1128,
          0.9572
        ],
        [
          0.1228,
          0.9545
        ],
        [
          0.1328,
          0.9504
        ],
        [
          0.1433,
          0.9472
        ],
        [
          0.1533,
          0.9436
        ],
        [
          0.1633,
          0.9422
        ],
        [
          0.1733,
          0.9381
        ],
        [
          0.1838,
          0.9359
        ],
        [
          0.1938,
          0.9345
        ],
        [
          0.2038,
          0.9304
        ],
        [
          0.2138,
          0.9281
        ],
        [
          0.2238,
          0.9254
        ],
        [
          0.2338,
          0.9222
        ],
        [
          0.2439,
          0.9181
        ],
        [
          0.2543,
          0.9149
        ],
        [
          0.2643,
          0.9122
        ],
        [
          0.2743,
          0.9067
        ],
        [
          0.2843,
          0.904
        ],
        [
          0.2944,
          0.9008
        ],
        [
          0.3048,
          0.8967
        ],
        [
          0.3148,
          0.8917
        ],
        [
          0.3248,
          0.8872
        ],
        [
          0.3348,
          0.8831
        ],
        [
          0.3449,
          0.8763
        ],
        [
          0.3562,
          0.8735
        ],
        [
          0.3662,
          0.8681
        ],
        [
          0.3763,
          0.8662
        ],
        [
          0.3867,
          0.8617
        ],
        [
          0.3967,
          0.8567
        ],
        [
          0.4067,
          0.8526
        ],
        [
          0.4167,
          0.848
        ],
        [
          0.4272,
          0.8403
        ],
        [
          0.4372,
          0.8358
        ],
        [
          0.4477,
          0.8298
        ],
        [
          0.4581,
          0.8248
        ],
        [
          0.4682,
          0.8198
        ],
        [
          0.4786,
          0.8148
        ],
        [
          0.4886,
          0.8094
        ],
        [
          0.4986,
          0.803
        ],
        [
          0.5086,
          0.798
        ],
        [
          0.5187,
          0.793
        ],
        [
          0.5291,
          0.788
        ],
        [
          0.5391,
          0.7789
        ],
        [
          0.5491,
          0.7739
        ],
        [
          0.5596,
          0.7684
        ],
        [
          0.5696,
          0.7634
        ],
        [
          0.5796,
          0.7561
        ],
        [
          0.5901,
          0.7498
        ],
        [
          0.6005,
          0.7425
        ],
        [
          0.6106,
          0.737
        ],
        [
          0.621,
          0.7307
        ],
        [
          0.631,
          0.7247
        ],
        [
          0.641,
          0.7179
        ],
        [
          0.651,
          0.7102
        ],
        [
          0.6611,
          0.7043
        ],
        [
          0.6715,
          0.6988
        ],
        [
          0.682,
          0.692
        ],
        [
          0.6924,
          0.6838
        ],
        [
          0.7025,
          0.677
        ],
        [
          0.7129,
          0.6697
        ],
        [
          0.7229,
          0.6633
        ],
        [
          0.7329,
          0.6574
        ],
        [
          0.7429,
          0.6501
        ],
        [
          0.753,
          0.6433
        ],
        [
          0.7634,
          0.636
        ],
        [
          0.7734,
          0.6278
        ],
        [
          0.7843,
          0.6197
        ],
        [
          0.7944,
          0.6106
        ],
        [
          0.8048,
          0.6037
        ],
        [
          0.8148,
          0.5955
        ],
        [
          0.8248,
          0.5878
        ],
        [
          0.8348,
          0.5792
        ],
        [
          0.8449,
          0.5719
        ],
        [
          0.8553,
          0.5655
        ],
        [
          0.8653,
          0.5578
        ],
        [
          0.8753,
          0.5505
        ],
        [
          0.8854,
          0.5437
        ],
        [
          0.8958,
          0.5341
        ],
        [
          0.9058,
          0.5264
        ],
        [
          0.9163,
          0.5177
        ],
        [
          0.9272,
          0.5105
        ],
        [
          0.9372,
          0.5045
        ],
        [
          0.9472,
          0.4959
        ],
        [
          0.9572,
          0.4891
        ],
        [
          0.9672,
          0.4813
        ],
        [
          0.9773,
          0.4741
        ],
        [
          0.9877,
          0.4663
        ],
        [
          0.9977,
          0.4591
        ],
        [
          1.0,
          0.4572
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.008603121817443457,
        "difference": -0.000757896661118927,
        "worse": 0.46,
        "kept_right": 0.007793687865911082,
        "kept_wrong": 0.0008094339515323756,
        "handed_off": 0.9913968781825578
      },
      "usd_per_1000": 0.9725180885633155
    },
    {
      "decision": 139,
      "frontier": 190,
      "frontier_accuracy": 0.9672429481346679,
      "in_sample": {
        "threshold": 0.9577,
        "answered": 0.008644222020018199,
        "accuracy": 0.9672429481346679
      },
      "curve": [
        [
          0.0,
          0.9672
        ],
        [
          0.0105,
          0.9668
        ],
        [
          0.0209,
          0.9663
        ],
        [
          0.0309,
          0.9654
        ],
        [
          0.0409,
          0.9645
        ],
        [
          0.051,
          0.9641
        ],
        [
          0.0614,
          0.9636
        ],
        [
          0.0714,
          0.9631
        ],
        [
          0.0814,
          0.9622
        ],
        [
          0.0919,
          0.96
        ],
        [
          0.1024,
          0.9572
        ],
        [
          0.1128,
          0.9545
        ],
        [
          0.1228,
          0.9518
        ],
        [
          0.1328,
          0.9481
        ],
        [
          0.1433,
          0.9449
        ],
        [
          0.1533,
          0.9413
        ],
        [
          0.1633,
          0.9399
        ],
        [
          0.1733,
          0.9359
        ],
        [
          0.1838,
          0.9327
        ],
        [
          0.1938,
          0.9313
        ],
        [
          0.2038,
          0.9272
        ],
        [
          0.2138,
          0.9254
        ],
        [
          0.2238,
          0.9222
        ],
        [
          0.2338,
          0.9181
        ],
        [
          0.2439,
          0.9131
        ],
        [
          0.2543,
          0.9099
        ],
        [
          0.2643,
          0.9072
        ],
        [
          0.2743,
          0.9022
        ],
        [
          0.2843,
          0.8995
        ],
        [
          0.2944,
          0.8963
        ],
        [
          0.3048,
          0.8922
        ],
        [
          0.3148,
          0.8872
        ],
        [
          0.3248,
          0.8826
        ],
        [
          0.3348,
          0.879
        ],
        [
          0.3449,
          0.8726
        ],
        [
          0.3562,
          0.8694
        ],
        [
          0.3662,
          0.8644
        ],
        [
          0.3763,
          0.8626
        ],
        [
          0.3867,
          0.8581
        ],
        [
          0.3967,
          0.853
        ],
        [
          0.4067,
          0.8485
        ],
        [
          0.4167,
          0.8444
        ],
        [
          0.4272,
          0.8367
        ],
        [
          0.4372,
          0.8321
        ],
        [
          0.4477,
          0.8271
        ],
        [
          0.4581,
          0.8221
        ],
        [
          0.4682,
          0.8171
        ],
        [
          0.4786,
          0.8126
        ],
        [
          0.4886,
          0.8071
        ],
        [
          0.4986,
          0.8016
        ],
        [
          0.5086,
          0.7966
        ],
        [
          0.5187,
          0.7916
        ],
        [
          0.5291,
          0.7862
        ],
        [
          0.5391,
          0.7771
        ],
        [
          0.5491,
          0.7721
        ],
        [
          0.5596,
          0.7666
        ],
        [
          0.5696,
          0.7616
        ],
        [
          0.5796,
          0.7539
        ],
        [
          0.5901,
          0.748
        ],
        [
          0.6005,
          0.7411
        ],
        [
          0.6106,
          0.7357
        ],
        [
          0.621,
          0.7293
        ],
        [
          0.631,
          0.7234
        ],
        [
          0.641,
          0.7161
        ],
        [
          0.651,
          0.7084
        ],
        [
          0.6611,
          0.7025
        ],
        [
          0.6715,
          0.6975
        ],
        [
          0.682,
          0.6906
        ],
        [
          0.6924,
          0.6824
        ],
        [
          0.7025,
          0.6752
        ],
        [
          0.7129,
          0.6688
        ],
        [
          0.7229,
          0.6624
        ],
        [
          0.7329,
          0.657
        ],
        [
          0.7429,
          0.6492
        ],
        [
          0.753,
          0.6429
        ],
        [
          0.7634,
          0.6351
        ],
        [
          0.7734,
          0.6274
        ],
        [
          0.7843,
          0.6192
        ],
        [
          0.7944,
          0.611
        ],
        [
          0.8048,
          0.6042
        ],
        [
          0.8148,
          0.5969
        ],
        [
          0.8248,
          0.5892
        ],
        [
          0.8348,
          0.5805
        ],
        [
          0.8449,
          0.5728
        ],
        [
          0.8553,
          0.566
        ],
        [
          0.8653,
          0.5582
        ],
        [
          0.8753,
          0.5505
        ],
        [
          0.8854,
          0.5437
        ],
        [
          0.8958,
          0.5341
        ],
        [
          0.9058,
          0.5268
        ],
        [
          0.9163,
          0.5187
        ],
        [
          0.9272,
          0.5114
        ],
        [
          0.9372,
          0.5055
        ],
        [
          0.9472,
          0.4968
        ],
        [
          0.9572,
          0.4895
        ],
        [
          0.9672,
          0.4818
        ],
        [
          0.9773,
          0.4745
        ],
        [
          0.9877,
          0.4663
        ],
        [
          0.9977,
          0.4591
        ],
        [
          1.0,
          0.4572
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.015561002615657337,
        "difference": -0.0010095860012849793,
        "worse": 0.5,
        "kept_right": 0.014002663607657688,
        "kept_wrong": 0.001558339007999661,
        "handed_off": 0.9844389973843438
      },
      "usd_per_1000": 5.499234125428839
    },
    {
      "decision": 139,
      "frontier": 124,
      "frontier_accuracy": 0.9667879890809827,
      "in_sample": {
        "threshold": 0.9577,
        "answered": 0.008644222020018199,
        "accuracy": 0.9667879890809827
      },
      "curve": [
        [
          0.0,
          0.9668
        ],
        [
          0.0105,
          0.9663
        ],
        [
          0.0209,
          0.9659
        ],
        [
          0.0309,
          0.965
        ],
        [
          0.0409,
          0.9641
        ],
        [
          0.051,
          0.9641
        ],
        [
          0.0614,
          0.9631
        ],
        [
          0.0714,
          0.9627
        ],
        [
          0.0814,
          0.9618
        ],
        [
          0.0919,
          0.9595
        ],
        [
          0.1024,
          0.9568
        ],
        [
          0.1128,
          0.9536
        ],
        [
          0.1228,
          0.9513
        ],
        [
          0.1328,
          0.9477
        ],
        [
          0.1433,
          0.9445
        ],
        [
          0.1533,
          0.9409
        ],
        [
          0.1633,
          0.9395
        ],
        [
          0.1733,
          0.9359
        ],
        [
          0.1838,
          0.9327
        ],
        [
          0.1938,
          0.9308
        ],
        [
          0.2038,
          0.9268
        ],
        [
          0.2138,
          0.9245
        ],
        [
          0.2238,
          0.9213
        ],
        [
          0.2338,
          0.9172
        ],
        [
          0.2439,
          0.9122
        ],
        [
          0.2543,
          0.909
        ],
        [
          0.2643,
          0.9067
        ],
        [
          0.2743,
          0.9017
        ],
        [
          0.2843,
          0.899
        ],
        [
          0.2944,
          0.8958
        ],
        [
          0.3048,
          0.8917
        ],
        [
          0.3148,
          0.8867
        ],
        [
          0.3248,
          0.8822
        ],
        [
          0.3348,
          0.8781
        ],
        [
          0.3449,
          0.8717
        ],
        [
          0.3562,
          0.869
        ],
        [
          0.3662,
          0.8635
        ],
        [
          0.3763,
          0.8617
        ],
        [
          0.3867,
          0.8571
        ],
        [
          0.3967,
          0.8521
        ],
        [
          0.4067,
          0.848
        ],
        [
          0.4167,
          0.8439
        ],
        [
          0.4272,
          0.8362
        ],
        [
          0.4372,
          0.8321
        ],
        [
          0.4477,
          0.8271
        ],
        [
          0.4581,
          0.8221
        ],
        [
          0.4682,
          0.8171
        ],
        [
          0.4786,
          0.8126
        ],
        [
          0.4886,
          0.8071
        ],
        [
          0.4986,
          0.8012
        ],
        [
          0.5086,
          0.7962
        ],
        [
          0.5187,
          0.7912
        ],
        [
          0.5291,
          0.7857
        ],
        [
          0.5391,
          0.7766
        ],
        [
          0.5491,
          0.7716
        ],
        [
          0.5596,
          0.7666
        ],
        [
          0.5696,
          0.7616
        ],
        [
          0.5796,
          0.7543
        ],
        [
          0.5901,
          0.7489
        ],
        [
          0.6005,
          0.7416
        ],
        [
          0.6106,
          0.7361
        ],
        [
          0.621,
          0.7293
        ],
        [
          0.631,
          0.7234
        ],
        [
          0.641,
          0.7161
        ],
        [
          0.651,
          0.7084
        ],
        [
          0.6611,
          0.7025
        ],
        [
          0.6715,
          0.697
        ],
        [
          0.682,
          0.6902
        ],
        [
          0.6924,
          0.682
        ],
        [
          0.7025,
          0.6752
        ],
        [
          0.7129,
          0.6683
        ],
        [
          0.7229,
          0.6624
        ],
        [
          0.7329,
          0.657
        ],
        [
          0.7429,
          0.6492
        ],
        [
          0.753,
          0.6429
        ],
        [
          0.7634,
          0.6351
        ],
        [
          0.7734,
          0.6274
        ],
        [
          0.7843,
          0.6201
        ],
        [
          0.7944,
          0.6115
        ],
        [
          0.8048,
          0.6046
        ],
        [
          0.8148,
          0.5969
        ],
        [
          0.8248,
          0.5892
        ],
        [
          0.8348,
          0.5805
        ],
        [
          0.8449,
          0.5728
        ],
        [
          0.8553,
          0.566
        ],
        [
          0.8653,
          0.5582
        ],
        [
          0.8753,
          0.5505
        ],
        [
          0.8854,
          0.5437
        ],
        [
          0.8958,
          0.5341
        ],
        [
          0.9058,
          0.5268
        ],
        [
          0.9163,
          0.5187
        ],
        [
          0.9272,
          0.5114
        ],
        [
          0.9372,
          0.5055
        ],
        [
          0.9472,
          0.4968
        ],
        [
          0.9572,
          0.4895
        ],
        [
          0.9672,
          0.4818
        ],
        [
          0.9773,
          0.4745
        ],
        [
          0.9877,
          0.4663
        ],
        [
          0.9977,
          0.4591
        ],
        [
          1.0,
          0.4572
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.014960626820816012,
        "difference": -0.0009337786731320835,
        "worse": 0.5,
        "kept_right": 0.013475045491374947,
        "kept_wrong": 0.001485581329441072,
        "handed_off": 0.985039373179185
      },
      "usd_per_1000": 1.1308288239427087
    },
    {
      "decision": 139,
      "frontier": 278,
      "frontier_accuracy": 0.9499545040946314,
      "in_sample": {
        "threshold": 0.9577,
        "answered": 0.008644222020018199,
        "accuracy": 0.9499545040946314
      },
      "curve": [
        [
          0.0,
          0.95
        ],
        [
          0.0105,
          0.9495
        ],
        [
          0.0209,
          0.949
        ],
        [
          0.0309,
          0.9481
        ],
        [
          0.0409,
          0.9472
        ],
        [
          0.051,
          0.9472
        ],
        [
          0.0614,
          0.9463
        ],
        [
          0.0714,
          0.9459
        ],
        [
          0.0814,
          0.9449
        ],
        [
          0.0919,
          0.9427
        ],
        [
          0.1024,
          0.9399
        ],
        [
          0.1128,
          0.9377
        ],
        [
          0.1228,
          0.9354
        ],
        [
          0.1328,
          0.9313
        ],
        [
          0.1433,
          0.9286
        ],
        [
          0.1533,
          0.9249
        ],
        [
          0.1633,
          0.9236
        ],
        [
          0.1733,
          0.9195
        ],
        [
          0.1838,
          0.9167
        ],
        [
          0.1938,
          0.9149
        ],
        [
          0.2038,
          0.9108
        ],
        [
          0.2138,
          0.909
        ],
        [
          0.2238,
          0.9067
        ],
        [
          0.2338,
          0.9035
        ],
        [
          0.2439,
          0.899
        ],
        [
          0.2543,
          0.8963
        ],
        [
          0.2643,
          0.8935
        ],
        [
          0.2743,
          0.889
        ],
        [
          0.2843,
          0.8867
        ],
        [
          0.2944,
          0.8835
        ],
        [
          0.3048,
          0.8794
        ],
        [
          0.3148,
          0.8744
        ],
        [
          0.3248,
          0.8708
        ],
        [
          0.3348,
          0.8667
        ],
        [
          0.3449,
          0.8608
        ],
        [
          0.3562,
          0.8585
        ],
        [
          0.3662,
          0.854
        ],
        [
          0.3763,
          0.8517
        ],
        [
          0.3867,
          0.8467
        ],
        [
          0.3967,
          0.8417
        ],
        [
          0.4067,
          0.838
        ],
        [
          0.4167,
          0.8339
        ],
        [
          0.4272,
          0.8262
        ],
        [
          0.4372,
          0.8226
        ],
        [
          0.4477,
          0.8171
        ],
        [
          0.4581,
          0.8126
        ],
        [
          0.4682,
          0.808
        ],
        [
          0.4786,
          0.8035
        ],
        [
          0.4886,
          0.798
        ],
        [
          0.4986,
          0.7921
        ],
        [
          0.5086,
          0.7866
        ],
        [
          0.5187,
          0.7816
        ],
        [
          0.5291,
          0.7771
        ],
        [
          0.5391,
          0.7684
        ],
        [
          0.5491,
          0.7634
        ],
        [
          0.5596,
          0.7584
        ],
        [
          0.5696,
          0.7543
        ],
        [
          0.5796,
          0.7466
        ],
        [
          0.5901,
          0.7402
        ],
        [
          0.6005,
          0.7334
        ],
        [
          0.6106,
          0.7279
        ],
        [
          0.621,
          0.7216
        ],
        [
          0.631,
          0.7157
        ],
        [
          0.641,
          0.7084
        ],
        [
          0.651,
          0.7006
        ],
        [
          0.6611,
          0.6947
        ],
        [
          0.6715,
          0.6897
        ],
        [
          0.682,
          0.6833
        ],
        [
          0.6924,
          0.6752
        ],
        [
          0.7025,
          0.6683
        ],
        [
          0.7129,
          0.662
        ],
        [
          0.7229,
          0.6556
        ],
        [
          0.7329,
          0.6497
        ],
        [
          0.7429,
          0.6429
        ],
        [
          0.753,
          0.636
        ],
        [
          0.7634,
          0.6288
        ],
        [
          0.7734,
          0.6206
        ],
        [
          0.7843,
          0.6128
        ],
        [
          0.7944,
          0.6046
        ],
        [
          0.8048,
          0.5983
        ],
        [
          0.8148,
          0.5901
        ],
        [
          0.8248,
          0.5823
        ],
        [
          0.8348,
          0.5737
        ],
        [
          0.8449,
          0.5669
        ],
        [
          0.8553,
          0.561
        ],
        [
          0.8653,
          0.5532
        ],
        [
          0.8753,
          0.5464
        ],
        [
          0.8854,
          0.5396
        ],
        [
          0.8958,
          0.5314
        ],
        [
          0.9058,
          0.5241
        ],
        [
          0.9163,
          0.5155
        ],
        [
          0.9272,
          0.5086
        ],
        [
          0.9372,
          0.5027
        ],
        [
          0.9472,
          0.4941
        ],
        [
          0.9572,
          0.4877
        ],
        [
          0.9672,
          0.4804
        ],
        [
          0.9773,
          0.4736
        ],
        [
          0.9877,
          0.4654
        ],
        [
          0.9977,
          0.4591
        ],
        [
          1.0,
          0.4572
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.014960626820816012,
        "difference": -0.0009337786731320835,
        "worse": 0.5,
        "kept_right": 0.013475045491374947,
        "kept_wrong": 0.001485581329441072,
        "handed_off": 0.985039373179185
      },
      "usd_per_1000": 6.428817022501615
    },
    {
      "decision": 139,
      "frontier": 275,
      "frontier_accuracy": 0.9426751592356688,
      "in_sample": {
        "threshold": 0.9577,
        "answered": 0.008644222020018199,
        "accuracy": 0.9426751592356688
      },
      "curve": [
        [
          0.0,
          0.9427
        ],
        [
          0.0105,
          0.9422
        ],
        [
          0.0209,
          0.9418
        ],
        [
          0.0309,
          0.9409
        ],
        [
          0.0409,
          0.9399
        ],
        [
          0.051,
          0.9399
        ],
        [
          0.0614,
          0.939
        ],
        [
          0.0714,
          0.9386
        ],
        [
          0.0814,
          0.9377
        ],
        [
          0.0919,
          0.9354
        ],
        [
          0.1024,
          0.9327
        ],
        [
          0.1128,
          0.9295
        ],
        [
          0.1228,
          0.9268
        ],
        [
          0.1328,
          0.9231
        ],
        [
          0.1433,
          0.9204
        ],
        [
          0.1533,
          0.9167
        ],
        [
          0.1633,
          0.9154
        ],
        [
          0.1733,
          0.9113
        ],
        [
          0.1838,
          0.9081
        ],
        [
          0.1938,
          0.9063
        ],
        [
          0.2038,
          0.9022
        ],
        [
          0.2138,
          0.9008
        ],
        [
          0.2238,
          0.8985
        ],
        [
          0.2338,
          0.8954
        ],
        [
          0.2439,
          0.8904
        ],
        [
          0.2543,
          0.8881
        ],
        [
          0.2643,
          0.8854
        ],
        [
          0.2743,
          0.8803
        ],
        [
          0.2843,
          0.8776
        ],
        [
          0.2944,
          0.8744
        ],
        [
          0.3048,
          0.8712
        ],
        [
          0.3148,
          0.8662
        ],
        [
          0.3248,
          0.8621
        ],
        [
          0.3348,
          0.8581
        ],
        [
          0.3449,
          0.8517
        ],
        [
          0.3562,
          0.849
        ],
        [
          0.3662,
          0.8435
        ],
        [
          0.3763,
          0.8417
        ],
        [
          0.3867,
          0.8376
        ],
        [
          0.3967,
          0.8326
        ],
        [
          0.4067,
          0.8285
        ],
        [
          0.4167,
          0.8244
        ],
        [
          0.4272,
          0.8167
        ],
        [
          0.4372,
          0.813
        ],
        [
          0.4477,
          0.808
        ],
        [
          0.4581,
          0.8035
        ],
        [
          0.4682,
          0.7994
        ],
        [
          0.4786,
          0.7944
        ],
        [
          0.4886,
          0.7889
        ],
        [
          0.4986,
          0.7834
        ],
        [
          0.5086,
          0.7784
        ],
        [
          0.5187,
          0.7739
        ],
        [
          0.5291,
          0.7684
        ],
        [
          0.5391,
          0.7598
        ],
        [
          0.5491,
          0.7548
        ],
        [
          0.5596,
          0.7493
        ],
        [
          0.5696,
          0.7457
        ],
        [
          0.5796,
          0.7379
        ],
        [
          0.5901,
          0.7329
        ],
        [
          0.6005,
          0.7261
        ],
        [
          0.6106,
          0.7211
        ],
        [
          0.621,
          0.7147
        ],
        [
          0.631,
          0.7093
        ],
        [
          0.641,
          0.702
        ],
        [
          0.651,
          0.6952
        ],
        [
          0.6611,
          0.6893
        ],
        [
          0.6715,
          0.6847
        ],
        [
          0.682,
          0.6788
        ],
        [
          0.6924,
          0.6711
        ],
        [
          0.7025,
          0.6638
        ],
        [
          0.7129,
          0.6579
        ],
        [
          0.7229,
          0.6524
        ],
        [
          0.7329,
          0.647
        ],
        [
          0.7429,
          0.6401
        ],
        [
          0.753,
          0.6338
        ],
        [
          0.7634,
          0.626
        ],
        [
          0.7734,
          0.6178
        ],
        [
          0.7843,
          0.6101
        ],
        [
          0.7944,
          0.6028
        ],
        [
          0.8048,
          0.5965
        ],
        [
          0.8148,
          0.5887
        ],
        [
          0.8248,
          0.5814
        ],
        [
          0.8348,
          0.5732
        ],
        [
          0.8449,
          0.566
        ],
        [
          0.8553,
          0.5591
        ],
        [
          0.8653,
          0.5519
        ],
        [
          0.8753,
          0.5441
        ],
        [
          0.8854,
          0.5373
        ],
        [
          0.8958,
          0.5291
        ],
        [
          0.9058,
          0.5218
        ],
        [
          0.9163,
          0.5132
        ],
        [
          0.9272,
          0.5068
        ],
        [
          0.9372,
          0.5014
        ],
        [
          0.9472,
          0.4932
        ],
        [
          0.9572,
          0.4868
        ],
        [
          0.9672,
          0.4804
        ],
        [
          0.9773,
          0.4736
        ],
        [
          0.9877,
          0.4663
        ],
        [
          0.9977,
          0.4591
        ],
        [
          1.0,
          0.4572
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.014960626820816012,
        "difference": -0.0009337786731320835,
        "worse": 0.5,
        "kept_right": 0.013475045491374947,
        "kept_wrong": 0.001485581329441072,
        "handed_off": 0.985039373179185
      },
      "usd_per_1000": 0.5241826010592989
    },
    {
      "decision": 139,
      "frontier": 192,
      "frontier_accuracy": 0.9331210191082803,
      "in_sample": {
        "threshold": 0.9577,
        "answered": 0.008644222020018199,
        "accuracy": 0.9331210191082803
      },
      "curve": [
        [
          0.0,
          0.9331
        ],
        [
          0.0105,
          0.9327
        ],
        [
          0.0209,
          0.9322
        ],
        [
          0.0309,
          0.9313
        ],
        [
          0.0409,
          0.9308
        ],
        [
          0.051,
          0.9299
        ],
        [
          0.0614,
          0.929
        ],
        [
          0.0714,
          0.9286
        ],
        [
          0.0814,
          0.9277
        ],
        [
          0.0919,
          0.9254
        ],
        [
          0.1024,
          0.9227
        ],
        [
          0.1128,
          0.9204
        ],
        [
          0.1228,
          0.9177
        ],
        [
          0.1328,
          0.914
        ],
        [
          0.1433,
          0.9108
        ],
        [
          0.1533,
          0.9072
        ],
        [
          0.1633,
          0.9058
        ],
        [
          0.1733,
          0.9017
        ],
        [
          0.1838,
          0.899
        ],
        [
          0.1938,
          0.8972
        ],
        [
          0.2038,
          0.8935
        ],
        [
          0.2138,
          0.8922
        ],
        [
          0.2238,
          0.8894
        ],
        [
          0.2338,
          0.8858
        ],
        [
          0.2439,
          0.8813
        ],
        [
          0.2543,
          0.879
        ],
        [
          0.2643,
          0.8763
        ],
        [
          0.2743,
          0.8717
        ],
        [
          0.2843,
          0.8703
        ],
        [
          0.2944,
          0.8672
        ],
        [
          0.3048,
          0.8631
        ],
        [
          0.3148,
          0.8581
        ],
        [
          0.3248,
          0.854
        ],
        [
          0.3348,
          0.8499
        ],
        [
          0.3449,
          0.8435
        ],
        [
          0.3562,
          0.8408
        ],
        [
          0.3662,
          0.8371
        ],
        [
          0.3763,
          0.8353
        ],
        [
          0.3867,
          0.8317
        ],
        [
          0.3967,
          0.8267
        ],
        [
          0.4067,
          0.8226
        ],
        [
          0.4167,
          0.8185
        ],
        [
          0.4272,
          0.8112
        ],
        [
          0.4372,
          0.808
        ],
        [
          0.4477,
          0.803
        ],
        [
          0.4581,
          0.7985
        ],
        [
          0.4682,
          0.7948
        ],
        [
          0.4786,
          0.7907
        ],
        [
          0.4886,
          0.7857
        ],
        [
          0.4986,
          0.7803
        ],
        [
          0.5086,
          0.7753
        ],
        [
          0.5187,
          0.7702
        ],
        [
          0.5291,
          0.7657
        ],
        [
          0.5391,
          0.7566
        ],
        [
          0.5491,
          0.7525
        ],
        [
          0.5596,
          0.7475
        ],
        [
          0.5696,
          0.7434
        ],
        [
          0.5796,
          0.7357
        ],
        [
          0.5901,
          0.7302
        ],
        [
          0.6005,
          0.7234
        ],
        [
          0.6106,
          0.7184
        ],
        [
          0.621,
          0.7125
        ],
        [
          0.631,
          0.7066
        ],
        [
          0.641,
          0.6997
        ],
        [
          0.651,
          0.692
        ],
        [
          0.6611,
          0.6874
        ],
        [
          0.6715,
          0.6824
        ],
        [
          0.682,
          0.6761
        ],
        [
          0.6924,
          0.6683
        ],
        [
          0.7025,
          0.6611
        ],
        [
          0.7129,
          0.6547
        ],
        [
          0.7229,
          0.6488
        ],
        [
          0.7329,
          0.6438
        ],
        [
          0.7429,
          0.6365
        ],
        [
          0.753,
          0.6301
        ],
        [
          0.7634,
          0.6237
        ],
        [
          0.7734,
          0.616
        ],
        [
          0.7843,
          0.6092
        ],
        [
          0.7944,
          0.601
        ],
        [
          0.8048,
          0.5946
        ],
        [
          0.8148,
          0.5874
        ],
        [
          0.8248,
          0.5796
        ],
        [
          0.8348,
          0.5719
        ],
        [
          0.8449,
          0.5646
        ],
        [
          0.8553,
          0.5587
        ],
        [
          0.8653,
          0.551
        ],
        [
          0.8753,
          0.5432
        ],
        [
          0.8854,
          0.5369
        ],
        [
          0.8958,
          0.5287
        ],
        [
          0.9058,
          0.5227
        ],
        [
          0.9163,
          0.5146
        ],
        [
          0.9272,
          0.5077
        ],
        [
          0.9372,
          0.5018
        ],
        [
          0.9472,
          0.4932
        ],
        [
          0.9572,
          0.4868
        ],
        [
          0.9672,
          0.4795
        ],
        [
          0.9773,
          0.4732
        ],
        [
          0.9877,
          0.4654
        ],
        [
          0.9977,
          0.4591
        ],
        [
          1.0,
          0.4572
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.014377895775730024,
        "difference": -0.0008518636381460646,
        "worse": 0.49333333333333335,
        "kept_right": 0.01298335079643,
        "kept_wrong": 0.0013945449793000293,
        "handed_off": 0.9856221042242712
      },
      "usd_per_1000": 11.460011058907115
    },
    {
      "decision": 139,
      "frontier": 123,
      "frontier_accuracy": 0.9326660600545951,
      "in_sample": {
        "threshold": 0.9629,
        "answered": 0.005004549590536852,
        "accuracy": 0.9326660600545951
      },
      "curve": [
        [
          0.0,
          0.9327
        ],
        [
          0.0105,
          0.9318
        ],
        [
          0.0209,
          0.9313
        ],
        [
          0.0309,
          0.9304
        ],
        [
          0.0409,
          0.9295
        ],
        [
          0.051,
          0.9295
        ],
        [
          0.0614,
          0.9286
        ],
        [
          0.0714,
          0.9286
        ],
        [
          0.0814,
          0.9281
        ],
        [
          0.0919,
          0.9258
        ],
        [
          0.1024,
          0.9231
        ],
        [
          0.1128,
          0.9199
        ],
        [
          0.1228,
          0.9172
        ],
        [
          0.1328,
          0.9131
        ],
        [
          0.1433,
          0.9099
        ],
        [
          0.1533,
          0.9063
        ],
        [
          0.1633,
          0.9049
        ],
        [
          0.1733,
          0.9008
        ],
        [
          0.1838,
          0.8976
        ],
        [
          0.1938,
          0.8958
        ],
        [
          0.2038,
          0.8917
        ],
        [
          0.2138,
          0.8904
        ],
        [
          0.2238,
          0.8876
        ],
        [
          0.2338,
          0.8849
        ],
        [
          0.2439,
          0.8803
        ],
        [
          0.2543,
          0.8772
        ],
        [
          0.2643,
          0.8744
        ],
        [
          0.2743,
          0.8699
        ],
        [
          0.2843,
          0.8672
        ],
        [
          0.2944,
          0.8644
        ],
        [
          0.3048,
          0.8608
        ],
        [
          0.3148,
          0.8562
        ],
        [
          0.3248,
          0.8521
        ],
        [
          0.3348,
          0.848
        ],
        [
          0.3449,
          0.8417
        ],
        [
          0.3562,
          0.8394
        ],
        [
          0.3662,
          0.8344
        ],
        [
          0.3763,
          0.8326
        ],
        [
          0.3867,
          0.8285
        ],
        [
          0.3967,
          0.8239
        ],
        [
          0.4067,
          0.8203
        ],
        [
          0.4167,
          0.8171
        ],
        [
          0.4272,
          0.8112
        ],
        [
          0.4372,
          0.8071
        ],
        [
          0.4477,
          0.8016
        ],
        [
          0.4581,
          0.7962
        ],
        [
          0.4682,
          0.7921
        ],
        [
          0.4786,
          0.7871
        ],
        [
          0.4886,
          0.7825
        ],
        [
          0.4986,
          0.7762
        ],
        [
          0.5086,
          0.7712
        ],
        [
          0.5187,
          0.7666
        ],
        [
          0.5291,
          0.7625
        ],
        [
          0.5391,
          0.7539
        ],
        [
          0.5491,
          0.7489
        ],
        [
          0.5596,
          0.7443
        ],
        [
          0.5696,
          0.7402
        ],
        [
          0.5796,
          0.7329
        ],
        [
          0.5901,
          0.727
        ],
        [
          0.6005,
          0.7211
        ],
        [
          0.6106,
          0.7166
        ],
        [
          0.621,
          0.7102
        ],
        [
          0.631,
          0.7047
        ],
        [
          0.641,
          0.6979
        ],
        [
          0.651,
          0.6911
        ],
        [
          0.6611,
          0.6852
        ],
        [
          0.6715,
          0.6802
        ],
        [
          0.682,
          0.6733
        ],
        [
          0.6924,
          0.6665
        ],
        [
          0.7025,
          0.6592
        ],
        [
          0.7129,
          0.6524
        ],
        [
          0.7229,
          0.647
        ],
        [
          0.7329,
          0.6406
        ],
        [
          0.7429,
          0.6338
        ],
        [
          0.753,
          0.6283
        ],
        [
          0.7634,
          0.6219
        ],
        [
          0.7734,
          0.6142
        ],
        [
          0.7843,
          0.6069
        ],
        [
          0.7944,
          0.5992
        ],
        [
          0.8048,
          0.5928
        ],
        [
          0.8148,
          0.5855
        ],
        [
          0.8248,
          0.5783
        ],
        [
          0.8348,
          0.5705
        ],
        [
          0.8449,
          0.5632
        ],
        [
          0.8553,
          0.5573
        ],
        [
          0.8653,
          0.5505
        ],
        [
          0.8753,
          0.5437
        ],
        [
          0.8854,
          0.5373
        ],
        [
          0.8958,
          0.5291
        ],
        [
          0.9058,
          0.5218
        ],
        [
          0.9163,
          0.5132
        ],
        [
          0.9272,
          0.5064
        ],
        [
          0.9372,
          0.5005
        ],
        [
          0.9472,
          0.4923
        ],
        [
          0.9572,
          0.4868
        ],
        [
          0.9672,
          0.4804
        ],
        [
          0.9773,
          0.4741
        ],
        [
          0.9877,
          0.4659
        ],
        [
          0.9977,
          0.4591
        ],
        [
          1.0,
          0.4572
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.009209419989045014,
        "difference": -0.0008427782939492808,
        "worse": 0.4666666666666667,
        "kept_right": 0.008306005223608585,
        "kept_wrong": 0.000903414765436433,
        "handed_off": 0.9907905800109563
      },
      "usd_per_1000": 0.2834772847738933
    },
    {
      "decision": 139,
      "frontier": 274,
      "frontier_accuracy": 0.9085532302092811,
      "in_sample": {
        "threshold": 0.9577,
        "answered": 0.008644222020018199,
        "accuracy": 0.9085532302092811
      },
      "curve": [
        [
          0.0,
          0.9086
        ],
        [
          0.0105,
          0.9081
        ],
        [
          0.0209,
          0.9076
        ],
        [
          0.0309,
          0.9067
        ],
        [
          0.0409,
          0.9058
        ],
        [
          0.051,
          0.9058
        ],
        [
          0.0614,
          0.9054
        ],
        [
          0.0714,
          0.9049
        ],
        [
          0.0814,
          0.904
        ],
        [
          0.0919,
          0.9022
        ],
        [
          0.1024,
          0.8999
        ],
        [
          0.1128,
          0.8972
        ],
        [
          0.1228,
          0.8944
        ],
        [
          0.1328,
          0.8908
        ],
        [
          0.1433,
          0.8881
        ],
        [
          0.1533,
          0.8844
        ],
        [
          0.1633,
          0.8831
        ],
        [
          0.1733,
          0.879
        ],
        [
          0.1838,
          0.8758
        ],
        [
          0.1938,
          0.874
        ],
        [
          0.2038,
          0.8699
        ],
        [
          0.2138,
          0.8685
        ],
        [
          0.2238,
          0.8662
        ],
        [
          0.2338,
          0.8631
        ],
        [
          0.2439,
          0.8585
        ],
        [
          0.2543,
          0.8562
        ],
        [
          0.2643,
          0.8535
        ],
        [
          0.2743,
          0.8494
        ],
        [
          0.2843,
          0.8471
        ],
        [
          0.2944,
          0.8449
        ],
        [
          0.3048,
          0.8417
        ],
        [
          0.3148,
          0.8367
        ],
        [
          0.3248,
          0.8335
        ],
        [
          0.3348,
          0.8294
        ],
        [
          0.3449,
          0.8235
        ],
        [
          0.3562,
          0.8207
        ],
        [
          0.3662,
          0.8162
        ],
        [
          0.3763,
          0.8144
        ],
        [
          0.3867,
          0.8098
        ],
        [
          0.3967,
          0.8053
        ],
        [
          0.4067,
          0.8012
        ],
        [
          0.4167,
          0.7971
        ],
        [
          0.4272,
          0.7921
        ],
        [
          0.4372,
          0.7889
        ],
        [
          0.4477,
          0.7848
        ],
        [
          0.4581,
          0.7803
        ],
        [
          0.4682,
          0.7766
        ],
        [
          0.4786,
          0.773
        ],
        [
          0.4886,
          0.7675
        ],
        [
          0.4986,
          0.7616
        ],
        [
          0.5086,
          0.7561
        ],
        [
          0.5187,
          0.7516
        ],
        [
          0.5291,
          0.7475
        ],
        [
          0.5391,
          0.7389
        ],
        [
          0.5491,
          0.7343
        ],
        [
          0.5596,
          0.7293
        ],
        [
          0.5696,
          0.7257
        ],
        [
          0.5796,
          0.7184
        ],
        [
          0.5901,
          0.7138
        ],
        [
          0.6005,
          0.707
        ],
        [
          0.6106,
          0.7015
        ],
        [
          0.621,
          0.6952
        ],
        [
          0.631,
          0.6897
        ],
        [
          0.641,
          0.6829
        ],
        [
          0.651,
          0.6761
        ],
        [
          0.6611,
          0.6702
        ],
        [
          0.6715,
          0.6665
        ],
        [
          0.682,
          0.6611
        ],
        [
          0.6924,
          0.6533
        ],
        [
          0.7025,
          0.6465
        ],
        [
          0.7129,
          0.6406
        ],
        [
          0.7229,
          0.6365
        ],
        [
          0.7329,
          0.631
        ],
        [
          0.7429,
          0.6247
        ],
        [
          0.753,
          0.6187
        ],
        [
          0.7634,
          0.6124
        ],
        [
          0.7734,
          0.606
        ],
        [
          0.7843,
          0.5992
        ],
        [
          0.7944,
          0.591
        ],
        [
          0.8048,
          0.5851
        ],
        [
          0.8148,
          0.5773
        ],
        [
          0.8248,
          0.5705
        ],
        [
          0.8348,
          0.5628
        ],
        [
          0.8449,
          0.556
        ],
        [
          0.8553,
          0.55
        ],
        [
          0.8653,
          0.5432
        ],
        [
          0.8753,
          0.5364
        ],
        [
          0.8854,
          0.5309
        ],
        [
          0.8958,
          0.5241
        ],
        [
          0.9058,
          0.5182
        ],
        [
          0.9163,
          0.51
        ],
        [
          0.9272,
          0.5041
        ],
        [
          0.9372,
          0.4982
        ],
        [
          0.9472,
          0.49
        ],
        [
          0.9572,
          0.485
        ],
        [
          0.9672,
          0.4786
        ],
        [
          0.9773,
          0.4732
        ],
        [
          0.9877,
          0.4663
        ],
        [
          0.9977,
          0.4591
        ],
        [
          1.0,
          0.4572
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.015197338242403383,
        "difference": -0.0009459164442999833,
        "worse": 0.5,
        "kept_right": 0.013696586081436516,
        "kept_wrong": 0.0015007521609668726,
        "handed_off": 0.9848026617575977
      },
      "usd_per_1000": 0.43713472528150643
    },
    {
      "decision": 139,
      "frontier": 47,
      "frontier_accuracy": 0.9044585987261147,
      "in_sample": {
        "threshold": 0.9577,
        "answered": 0.008644222020018199,
        "accuracy": 0.9044585987261147
      },
      "curve": [
        [
          0.0,
          0.9045
        ],
        [
          0.0105,
          0.904
        ],
        [
          0.0209,
          0.9035
        ],
        [
          0.0309,
          0.9026
        ],
        [
          0.0409,
          0.9022
        ],
        [
          0.051,
          0.9022
        ],
        [
          0.0614,
          0.9013
        ],
        [
          0.0714,
          0.9008
        ],
        [
          0.0814,
          0.8999
        ],
        [
          0.0919,
          0.8985
        ],
        [
          0.1024,
          0.8967
        ],
        [
          0.1128,
          0.8944
        ],
        [
          0.1228,
          0.8922
        ],
        [
          0.1328,
          0.889
        ],
        [
          0.1433,
          0.8863
        ],
        [
          0.1533,
          0.8831
        ],
        [
          0.1633,
          0.8817
        ],
        [
          0.1733,
          0.8776
        ],
        [
          0.1838,
          0.8744
        ],
        [
          0.1938,
          0.8726
        ],
        [
          0.2038,
          0.869
        ],
        [
          0.2138,
          0.8676
        ],
        [
          0.2238,
          0.8649
        ],
        [
          0.2338,
          0.8617
        ],
        [
          0.2439,
          0.8581
        ],
        [
          0.2543,
          0.8553
        ],
        [
          0.2643,
          0.853
        ],
        [
          0.2743,
          0.849
        ],
        [
          0.2843,
          0.8471
        ],
        [
          0.2944,
          0.8444
        ],
        [
          0.3048,
          0.8408
        ],
        [
          0.3148,
          0.8362
        ],
        [
          0.3248,
          0.8335
        ],
        [
          0.3348,
          0.8298
        ],
        [
          0.3449,
          0.8235
        ],
        [
          0.3562,
          0.8203
        ],
        [
          0.3662,
          0.8162
        ],
        [
          0.3763,
          0.8153
        ],
        [
          0.3867,
          0.8112
        ],
        [
          0.3967,
          0.8062
        ],
        [
          0.4067,
          0.8021
        ],
        [
          0.4167,
          0.7985
        ],
        [
          0.4272,
          0.7916
        ],
        [
          0.4372,
          0.7884
        ],
        [
          0.4477,
          0.783
        ],
        [
          0.4581,
          0.7789
        ],
        [
          0.4682,
          0.7753
        ],
        [
          0.4786,
          0.7707
        ],
        [
          0.4886,
          0.7662
        ],
        [
          0.4986,
          0.7607
        ],
        [
          0.5086,
          0.7552
        ],
        [
          0.5187,
          0.7511
        ],
        [
          0.5291,
          0.7461
        ],
        [
          0.5391,
          0.7375
        ],
        [
          0.5491,
          0.7329
        ],
        [
          0.5596,
          0.7284
        ],
        [
          0.5696,
          0.7238
        ],
        [
          0.5796,
          0.717
        ],
        [
          0.5901,
          0.7106
        ],
        [
          0.6005,
          0.7056
        ],
        [
          0.6106,
          0.7011
        ],
        [
          0.621,
          0.6943
        ],
        [
          0.631,
          0.6884
        ],
        [
          0.641,
          0.6824
        ],
        [
          0.651,
          0.6752
        ],
        [
          0.6611,
          0.6697
        ],
        [
          0.6715,
          0.6642
        ],
        [
          0.682,
          0.6588
        ],
        [
          0.6924,
          0.6515
        ],
        [
          0.7025,
          0.6451
        ],
        [
          0.7129,
          0.6388
        ],
        [
          0.7229,
          0.6333
        ],
        [
          0.7329,
          0.6278
        ],
        [
          0.7429,
          0.6215
        ],
        [
          0.753,
          0.6146
        ],
        [
          0.7634,
          0.6087
        ],
        [
          0.7734,
          0.601
        ],
        [
          0.7843,
          0.5946
        ],
        [
          0.7944,
          0.5869
        ],
        [
          0.8048,
          0.5814
        ],
        [
          0.8148,
          0.5746
        ],
        [
          0.8248,
          0.5669
        ],
        [
          0.8348,
          0.5591
        ],
        [
          0.8449,
          0.5528
        ],
        [
          0.8553,
          0.5491
        ],
        [
          0.8653,
          0.5414
        ],
        [
          0.8753,
          0.5355
        ],
        [
          0.8854,
          0.5291
        ],
        [
          0.8958,
          0.5223
        ],
        [
          0.9058,
          0.5168
        ],
        [
          0.9163,
          0.5091
        ],
        [
          0.9272,
          0.5032
        ],
        [
          0.9372,
          0.4977
        ],
        [
          0.9472,
          0.4904
        ],
        [
          0.9572,
          0.485
        ],
        [
          0.9672,
          0.4786
        ],
        [
          0.9773,
          0.4722
        ],
        [
          0.9877,
          0.465
        ],
        [
          0.9977,
          0.4586
        ],
        [
          1.0,
          0.4572
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.016328450228691675,
        "difference": -0.0010187127004278199,
        "worse": 0.5033333333333333,
        "kept_right": 0.014685146832562185,
        "kept_wrong": 0.0016433033961295006,
        "handed_off": 0.9836715497713093
      },
      "usd_per_1000": 4.867443992328458
    },
    {
      "decision": 139,
      "frontier": 193,
      "frontier_accuracy": 0.89171974522293,
      "in_sample": {
        "threshold": 0.9577,
        "answered": 0.008644222020018199,
        "accuracy": 0.89171974522293
      },
      "curve": [
        [
          0.0,
          0.8917
        ],
        [
          0.0105,
          0.8913
        ],
        [
          0.0209,
          0.8908
        ],
        [
          0.0309,
          0.8899
        ],
        [
          0.0409,
          0.8894
        ],
        [
          0.051,
          0.889
        ],
        [
          0.0614,
          0.8881
        ],
        [
          0.0714,
          0.8881
        ],
        [
          0.0814,
          0.8872
        ],
        [
          0.0919,
          0.8849
        ],
        [
          0.1024,
          0.8822
        ],
        [
          0.1128,
          0.8808
        ],
        [
          0.1228,
          0.879
        ],
        [
          0.1328,
          0.8758
        ],
        [
          0.1433,
          0.8731
        ],
        [
          0.1533,
          0.8694
        ],
        [
          0.1633,
          0.8681
        ],
        [
          0.1733,
          0.864
        ],
        [
          0.1838,
          0.8603
        ],
        [
          0.1938,
          0.8585
        ],
        [
          0.2038,
          0.8549
        ],
        [
          0.2138,
          0.8535
        ],
        [
          0.2238,
          0.8508
        ],
        [
          0.2338,
          0.848
        ],
        [
          0.2439,
          0.8435
        ],
        [
          0.2543,
          0.8412
        ],
        [
          0.2643,
          0.8385
        ],
        [
          0.2743,
          0.8344
        ],
        [
          0.2843,
          0.833
        ],
        [
          0.2944,
          0.8308
        ],
        [
          0.3048,
          0.8276
        ],
        [
          0.3148,
          0.8235
        ],
        [
          0.3248,
          0.8198
        ],
        [
          0.3348,
          0.8162
        ],
        [
          0.3449,
          0.8107
        ],
        [
          0.3562,
          0.808
        ],
        [
          0.3662,
          0.8039
        ],
        [
          0.3763,
          0.8021
        ],
        [
          0.3867,
          0.7985
        ],
        [
          0.3967,
          0.7934
        ],
        [
          0.4067,
          0.7894
        ],
        [
          0.4167,
          0.7853
        ],
        [
          0.4272,
          0.7789
        ],
        [
          0.4372,
          0.7757
        ],
        [
          0.4477,
          0.7712
        ],
        [
          0.4581,
          0.768
        ],
        [
          0.4682,
          0.7643
        ],
        [
          0.4786,
          0.7598
        ],
        [
          0.4886,
          0.7552
        ],
        [
          0.4986,
          0.7502
        ],
        [
          0.5086,
          0.7461
        ],
        [
          0.5187,
          0.7411
        ],
        [
          0.5291,
          0.737
        ],
        [
          0.5391,
          0.7284
        ],
        [
          0.5491,
          0.7238
        ],
        [
          0.5596,
          0.7193
        ],
        [
          0.5696,
          0.7161
        ],
        [
          0.5796,
          0.7088
        ],
        [
          0.5901,
          0.7043
        ],
        [
          0.6005,
          0.6979
        ],
        [
          0.6106,
          0.6929
        ],
        [
          0.621,
          0.687
        ],
        [
          0.631,
          0.682
        ],
        [
          0.641,
          0.6752
        ],
        [
          0.651,
          0.6679
        ],
        [
          0.6611,
          0.6624
        ],
        [
          0.6715,
          0.6583
        ],
        [
          0.682,
          0.6547
        ],
        [
          0.6924,
          0.6479
        ],
        [
          0.7025,
          0.6419
        ],
        [
          0.7129,
          0.6356
        ],
        [
          0.7229,
          0.6297
        ],
        [
          0.7329,
          0.6237
        ],
        [
          0.7429,
          0.6169
        ],
        [
          0.753,
          0.6115
        ],
        [
          0.7634,
          0.6051
        ],
        [
          0.7734,
          0.5983
        ],
        [
          0.7843,
          0.5914
        ],
        [
          0.7944,
          0.5842
        ],
        [
          0.8048,
          0.5787
        ],
        [
          0.8148,
          0.5742
        ],
        [
          0.8248,
          0.5678
        ],
        [
          0.8348,
          0.5614
        ],
        [
          0.8449,
          0.5551
        ],
        [
          0.8553,
          0.5491
        ],
        [
          0.8653,
          0.5432
        ],
        [
          0.8753,
          0.5359
        ],
        [
          0.8854,
          0.53
        ],
        [
          0.8958,
          0.5237
        ],
        [
          0.9058,
          0.5182
        ],
        [
          0.9163,
          0.5109
        ],
        [
          0.9272,
          0.5045
        ],
        [
          0.9372,
          0.4995
        ],
        [
          0.9472,
          0.4914
        ],
        [
          0.9572,
          0.4841
        ],
        [
          0.9672,
          0.4777
        ],
        [
          0.9773,
          0.4718
        ],
        [
          0.9877,
          0.4645
        ],
        [
          0.9977,
          0.4591
        ],
        [
          1.0,
          0.4572
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.016022303859647223,
        "difference": -0.0010157155206093926,
        "worse": 0.5033333333333333,
        "kept_right": 0.014418410987002165,
        "kept_wrong": 0.0016038928726450762,
        "handed_off": 0.983977696140354
      },
      "usd_per_1000": 7.640822597020402
    },
    {
      "decision": 139,
      "frontier": 55,
      "frontier_accuracy": 0.8503184713375797,
      "in_sample": {
        "threshold": 0.9577,
        "answered": 0.008644222020018199,
        "accuracy": 0.8503184713375797
      },
      "curve": [
        [
          0.0,
          0.8503
        ],
        [
          0.0105,
          0.8499
        ],
        [
          0.0209,
          0.8494
        ],
        [
          0.0309,
          0.8485
        ],
        [
          0.0409,
          0.8476
        ],
        [
          0.051,
          0.8476
        ],
        [
          0.0614,
          0.8471
        ],
        [
          0.0714,
          0.8471
        ],
        [
          0.0814,
          0.8467
        ],
        [
          0.0919,
          0.8444
        ],
        [
          0.1024,
          0.843
        ],
        [
          0.1128,
          0.8421
        ],
        [
          0.1228,
          0.8403
        ],
        [
          0.1328,
          0.8371
        ],
        [
          0.1433,
          0.8353
        ],
        [
          0.1533,
          0.8326
        ],
        [
          0.1633,
          0.8321
        ],
        [
          0.1733,
          0.8303
        ],
        [
          0.1838,
          0.828
        ],
        [
          0.1938,
          0.8271
        ],
        [
          0.2038,
          0.8244
        ],
        [
          0.2138,
          0.8235
        ],
        [
          0.2238,
          0.8207
        ],
        [
          0.2338,
          0.8171
        ],
        [
          0.2439,
          0.813
        ],
        [
          0.2543,
          0.8103
        ],
        [
          0.2643,
          0.8089
        ],
        [
          0.2743,
          0.8053
        ],
        [
          0.2843,
          0.8039
        ],
        [
          0.2944,
          0.8016
        ],
        [
          0.3048,
          0.7985
        ],
        [
          0.3148,
          0.7944
        ],
        [
          0.3248,
          0.7916
        ],
        [
          0.3348,
          0.7894
        ],
        [
          0.3449,
          0.7839
        ],
        [
          0.3562,
          0.7821
        ],
        [
          0.3662,
          0.7798
        ],
        [
          0.3763,
          0.7775
        ],
        [
          0.3867,
          0.7743
        ],
        [
          0.3967,
          0.7698
        ],
        [
          0.4067,
          0.7666
        ],
        [
          0.4167,
          0.763
        ],
        [
          0.4272,
          0.758
        ],
        [
          0.4372,
          0.7561
        ],
        [
          0.4477,
          0.753
        ],
        [
          0.4581,
          0.7498
        ],
        [
          0.4682,
          0.7475
        ],
        [
          0.4786,
          0.742
        ],
        [
          0.4886,
          0.7379
        ],
        [
          0.4986,
          0.7329
        ],
        [
          0.5086,
          0.7284
        ],
        [
          0.5187,
          0.7247
        ],
        [
          0.5291,
          0.7211
        ],
        [
          0.5391,
          0.7129
        ],
        [
          0.5491,
          0.7088
        ],
        [
          0.5596,
          0.7038
        ],
        [
          0.5696,
          0.6997
        ],
        [
          0.5796,
          0.6934
        ],
        [
          0.5901,
          0.6888
        ],
        [
          0.6005,
          0.6824
        ],
        [
          0.6106,
          0.6783
        ],
        [
          0.621,
          0.6729
        ],
        [
          0.631,
          0.6679
        ],
        [
          0.641,
          0.6629
        ],
        [
          0.651,
          0.6561
        ],
        [
          0.6611,
          0.6515
        ],
        [
          0.6715,
          0.647
        ],
        [
          0.682,
          0.6433
        ],
        [
          0.6924,
          0.6374
        ],
        [
          0.7025,
          0.6315
        ],
        [
          0.7129,
          0.6251
        ],
        [
          0.7229,
          0.6201
        ],
        [
          0.7329,
          0.616
        ],
        [
          0.7429,
          0.6096
        ],
        [
          0.753,
          0.6033
        ],
        [
          0.7634,
          0.5978
        ],
        [
          0.7734,
          0.5919
        ],
        [
          0.7843,
          0.586
        ],
        [
          0.7944,
          0.5778
        ],
        [
          0.8048,
          0.5732
        ],
        [
          0.8148,
          0.5673
        ],
        [
          0.8248,
          0.5614
        ],
        [
          0.8348,
          0.5551
        ],
        [
          0.8449,
          0.5496
        ],
        [
          0.8553,
          0.545
        ],
        [
          0.8653,
          0.5396
        ],
        [
          0.8753,
          0.5341
        ],
        [
          0.8854,
          0.5282
        ],
        [
          0.8958,
          0.5223
        ],
        [
          0.9058,
          0.5168
        ],
        [
          0.9163,
          0.5096
        ],
        [
          0.9272,
          0.5041
        ],
        [
          0.9372,
          0.4982
        ],
        [
          0.9472,
          0.4904
        ],
        [
          0.9572,
          0.4845
        ],
        [
          0.9672,
          0.4786
        ],
        [
          0.9773,
          0.4727
        ],
        [
          0.9877,
          0.4659
        ],
        [
          0.9977,
          0.4591
        ],
        [
          1.0,
          0.4572
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.018808928920486193,
        "difference": -0.0013461844321488445,
        "worse": 0.5633333333333334,
        "kept_right": 0.01684726964329172,
        "kept_wrong": 0.001961659277194475,
        "handed_off": 0.981191071079515
      },
      "usd_per_1000": 3.602966365734639
    },
    {
      "decision": 126,
      "frontier": 140,
      "frontier_accuracy": 0.9808917197452229,
      "in_sample": {
        "threshold": 0.9545185565948486,
        "answered": 0.011828935395814377,
        "accuracy": 0.9808917197452229
      },
      "curve": [
        [
          0.0,
          0.9809
        ],
        [
          0.01,
          0.9809
        ],
        [
          0.02,
          0.9804
        ],
        [
          0.03,
          0.9804
        ],
        [
          0.04,
          0.9804
        ],
        [
          0.05,
          0.9795
        ],
        [
          0.0601,
          0.9786
        ],
        [
          0.0705,
          0.9786
        ],
        [
          0.0805,
          0.9777
        ],
        [
          0.0905,
          0.9759
        ],
        [
          0.1005,
          0.9741
        ],
        [
          0.1106,
          0.9727
        ],
        [
          0.121,
          0.9709
        ],
        [
          0.131,
          0.9686
        ],
        [
          0.141,
          0.9659
        ],
        [
          0.151,
          0.9631
        ],
        [
          0.1611,
          0.9609
        ],
        [
          0.1715,
          0.9554
        ],
        [
          0.1815,
          0.9522
        ],
        [
          0.1915,
          0.9486
        ],
        [
          0.2015,
          0.944
        ],
        [
          0.2116,
          0.939
        ],
        [
          0.222,
          0.9349
        ],
        [
          0.232,
          0.9295
        ],
        [
          0.242,
          0.9263
        ],
        [
          0.252,
          0.9204
        ],
        [
          0.2621,
          0.9145
        ],
        [
          0.2725,
          0.9086
        ],
        [
          0.2825,
          0.9031
        ],
        [
          0.2925,
          0.8976
        ],
        [
          0.3025,
          0.8913
        ],
        [
          0.3126,
          0.8885
        ],
        [
          0.323,
          0.8844
        ],
        [
          0.333,
          0.8781
        ],
        [
          0.343,
          0.8703
        ],
        [
          0.353,
          0.8649
        ],
        [
          0.3631,
          0.8603
        ],
        [
          0.3735,
          0.8535
        ],
        [
          0.3835,
          0.8476
        ],
        [
          0.3935,
          0.8421
        ],
        [
          0.4035,
          0.8344
        ],
        [
          0.4136,
          0.828
        ],
        [
          0.424,
          0.8194
        ],
        [
          0.434,
          0.813
        ],
        [
          0.444,
          0.8071
        ],
        [
          0.454,
          0.8007
        ],
        [
          0.4641,
          0.7939
        ],
        [
          0.4745,
          0.7871
        ],
        [
          0.4845,
          0.7803
        ],
        [
          0.4945,
          0.7734
        ],
        [
          0.5045,
          0.7657
        ],
        [
          0.5146,
          0.758
        ],
        [
          0.525,
          0.7511
        ],
        [
          0.535,
          0.7429
        ],
        [
          0.545,
          0.7343
        ],
        [
          0.5551,
          0.7284
        ],
        [
          0.5655,
          0.7211
        ],
        [
          0.5755,
          0.7147
        ],
        [
          0.5855,
          0.7084
        ],
        [
          0.5955,
          0.702
        ],
        [
          0.6056,
          0.6952
        ],
        [
          0.616,
          0.6879
        ],
        [
          0.626,
          0.6793
        ],
        [
          0.636,
          0.6706
        ],
        [
          0.646,
          0.6652
        ],
        [
          0.6561,
          0.657
        ],
        [
          0.6665,
          0.6474
        ],
        [
          0.6765,
          0.6397
        ],
        [
          0.6865,
          0.6333
        ],
        [
          0.6965,
          0.6274
        ],
        [
          0.7066,
          0.6215
        ],
        [
          0.717,
          0.616
        ],
        [
          0.727,
          0.6083
        ],
        [
          0.737,
          0.6024
        ],
        [
          0.747,
          0.5969
        ],
        [
          0.7571,
          0.5887
        ],
        [
          0.7675,
          0.5792
        ],
        [
          0.7775,
          0.5719
        ],
        [
          0.7875,
          0.5651
        ],
        [
          0.7975,
          0.5587
        ],
        [
          0.8076,
          0.5523
        ],
        [
          0.818,
          0.5446
        ],
        [
          0.828,
          0.5359
        ],
        [
          0.838,
          0.5291
        ],
        [
          0.848,
          0.5209
        ],
        [
          0.8581,
          0.5127
        ],
        [
          0.8685,
          0.5045
        ],
        [
          0.8785,
          0.4968
        ],
        [
          0.8885,
          0.4891
        ],
        [
          0.8985,
          0.4809
        ],
        [
          0.9086,
          0.4732
        ],
        [
          0.919,
          0.4677
        ],
        [
          0.929,
          0.4613
        ],
        [
          0.939,
          0.454
        ],
        [
          0.949,
          0.4477
        ],
        [
          0.9591,
          0.4386
        ],
        [
          0.9695,
          0.4304
        ],
        [
          0.9795,
          0.4227
        ],
        [
          0.9895,
          0.4158
        ],
        [
          0.9995,
          0.4081
        ],
        [
          1.0,
          0.4076
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.032028213522346236,
        "difference": -0.0011521559851722916,
        "worse": 0.49,
        "kept_right": 0.030863936300008856,
        "kept_wrong": 0.001164277222337401,
        "handed_off": 0.9679717864776543
      },
      "usd_per_1000": 2.3416236093614113
    },
    {
      "decision": 126,
      "frontier": 277,
      "frontier_accuracy": 0.9795268425841674,
      "in_sample": {
        "threshold": 0.9545185565948486,
        "answered": 0.011828935395814377,
        "accuracy": 0.9795268425841674
      },
      "curve": [
        [
          0.0,
          0.9795
        ],
        [
          0.01,
          0.9795
        ],
        [
          0.02,
          0.9791
        ],
        [
          0.03,
          0.9791
        ],
        [
          0.04,
          0.9791
        ],
        [
          0.05,
          0.9782
        ],
        [
          0.0601,
          0.9773
        ],
        [
          0.0705,
          0.9768
        ],
        [
          0.0805,
          0.9759
        ],
        [
          0.0905,
          0.9741
        ],
        [
          0.1005,
          0.9722
        ],
        [
          0.1106,
          0.9709
        ],
        [
          0.121,
          0.9691
        ],
        [
          0.131,
          0.9668
        ],
        [
          0.141,
          0.9641
        ],
        [
          0.151,
          0.9609
        ],
        [
          0.1611,
          0.9586
        ],
        [
          0.1715,
          0.9531
        ],
        [
          0.1815,
          0.95
        ],
        [
          0.1915,
          0.9463
        ],
        [
          0.2015,
          0.9418
        ],
        [
          0.2116,
          0.9368
        ],
        [
          0.222,
          0.9327
        ],
        [
          0.232,
          0.9272
        ],
        [
          0.242,
          0.9245
        ],
        [
          0.252,
          0.9186
        ],
        [
          0.2621,
          0.9126
        ],
        [
          0.2725,
          0.9072
        ],
        [
          0.2825,
          0.9013
        ],
        [
          0.2925,
          0.8958
        ],
        [
          0.3025,
          0.8899
        ],
        [
          0.3126,
          0.8872
        ],
        [
          0.323,
          0.8826
        ],
        [
          0.333,
          0.8763
        ],
        [
          0.343,
          0.8694
        ],
        [
          0.353,
          0.8644
        ],
        [
          0.3631,
          0.8599
        ],
        [
          0.3735,
          0.8526
        ],
        [
          0.3835,
          0.8462
        ],
        [
          0.3935,
          0.8408
        ],
        [
          0.4035,
          0.833
        ],
        [
          0.4136,
          0.8262
        ],
        [
          0.424,
          0.8176
        ],
        [
          0.434,
          0.8116
        ],
        [
          0.444,
          0.8057
        ],
        [
          0.454,
          0.7994
        ],
        [
          0.4641,
          0.7925
        ],
        [
          0.4745,
          0.7857
        ],
        [
          0.4845,
          0.7789
        ],
        [
          0.4945,
          0.7721
        ],
        [
          0.5045,
          0.7643
        ],
        [
          0.5146,
          0.7566
        ],
        [
          0.525,
          0.7493
        ],
        [
          0.535,
          0.7411
        ],
        [
          0.545,
          0.7325
        ],
        [
          0.5551,
          0.7266
        ],
        [
          0.5655,
          0.7193
        ],
        [
          0.5755,
          0.7129
        ],
        [
          0.5855,
          0.7066
        ],
        [
          0.5955,
          0.7002
        ],
        [
          0.6056,
          0.6934
        ],
        [
          0.616,
          0.6861
        ],
        [
          0.626,
          0.6774
        ],
        [
          0.636,
          0.6688
        ],
        [
          0.646,
          0.6633
        ],
        [
          0.6561,
          0.6551
        ],
        [
          0.6665,
          0.6456
        ],
        [
          0.6765,
          0.6383
        ],
        [
          0.6865,
          0.6324
        ],
        [
          0.6965,
          0.626
        ],
        [
          0.7066,
          0.6201
        ],
        [
          0.717,
          0.6146
        ],
        [
          0.727,
          0.6069
        ],
        [
          0.737,
          0.601
        ],
        [
          0.747,
          0.5955
        ],
        [
          0.7571,
          0.5874
        ],
        [
          0.7675,
          0.5783
        ],
        [
          0.7775,
          0.571
        ],
        [
          0.7875,
          0.5646
        ],
        [
          0.7975,
          0.5582
        ],
        [
          0.8076,
          0.5519
        ],
        [
          0.818,
          0.5441
        ],
        [
          0.828,
          0.5355
        ],
        [
          0.838,
          0.5287
        ],
        [
          0.848,
          0.5205
        ],
        [
          0.8581,
          0.5123
        ],
        [
          0.8685,
          0.5041
        ],
        [
          0.8785,
          0.4964
        ],
        [
          0.8885,
          0.4886
        ],
        [
          0.8985,
          0.4809
        ],
        [
          0.9086,
          0.4732
        ],
        [
          0.919,
          0.4682
        ],
        [
          0.929,
          0.4618
        ],
        [
          0.939,
          0.454
        ],
        [
          0.949,
          0.4477
        ],
        [
          0.9591,
          0.4386
        ],
        [
          0.9695,
          0.4304
        ],
        [
          0.9795,
          0.4222
        ],
        [
          0.9895,
          0.4158
        ],
        [
          0.9995,
          0.4081
        ],
        [
          1.0,
          0.4076
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.029004683127495063,
        "difference": -0.0008913236177816489,
        "worse": 0.4666666666666667,
        "kept_right": 0.02811335950971344,
        "kept_wrong": 0.0008913236177816489,
        "handed_off": 0.9709953168725053
      },
      "usd_per_1000": 5.889625547826548
    },
    {
      "decision": 126,
      "frontier": 276,
      "frontier_accuracy": 0.9777070063694268,
      "in_sample": {
        "threshold": 0.9545185565948486,
        "answered": 0.011828935395814377,
        "accuracy": 0.9777070063694268
      },
      "curve": [
        [
          0.0,
          0.9777
        ],
        [
          0.01,
          0.9777
        ],
        [
          0.02,
          0.9773
        ],
        [
          0.03,
          0.9773
        ],
        [
          0.04,
          0.9773
        ],
        [
          0.05,
          0.9763
        ],
        [
          0.0601,
          0.9754
        ],
        [
          0.0705,
          0.975
        ],
        [
          0.0805,
          0.9741
        ],
        [
          0.0905,
          0.9722
        ],
        [
          0.1005,
          0.9709
        ],
        [
          0.1106,
          0.9695
        ],
        [
          0.121,
          0.9677
        ],
        [
          0.131,
          0.9654
        ],
        [
          0.141,
          0.9627
        ],
        [
          0.151,
          0.9595
        ],
        [
          0.1611,
          0.9572
        ],
        [
          0.1715,
          0.9518
        ],
        [
          0.1815,
          0.9486
        ],
        [
          0.1915,
          0.9449
        ],
        [
          0.2015,
          0.9404
        ],
        [
          0.2116,
          0.9354
        ],
        [
          0.222,
          0.9313
        ],
        [
          0.232,
          0.9263
        ],
        [
          0.242,
          0.9236
        ],
        [
          0.252,
          0.9177
        ],
        [
          0.2621,
          0.9117
        ],
        [
          0.2725,
          0.9063
        ],
        [
          0.2825,
          0.9004
        ],
        [
          0.2925,
          0.8954
        ],
        [
          0.3025,
          0.8894
        ],
        [
          0.3126,
          0.8867
        ],
        [
          0.323,
          0.8822
        ],
        [
          0.333,
          0.8758
        ],
        [
          0.343,
          0.8685
        ],
        [
          0.353,
          0.8635
        ],
        [
          0.3631,
          0.859
        ],
        [
          0.3735,
          0.8517
        ],
        [
          0.3835,
          0.8453
        ],
        [
          0.3935,
          0.8399
        ],
        [
          0.4035,
          0.8321
        ],
        [
          0.4136,
          0.8253
        ],
        [
          0.424,
          0.8167
        ],
        [
          0.434,
          0.8107
        ],
        [
          0.444,
          0.8044
        ],
        [
          0.454,
          0.7985
        ],
        [
          0.4641,
          0.7916
        ],
        [
          0.4745,
          0.7853
        ],
        [
          0.4845,
          0.7784
        ],
        [
          0.4945,
          0.7716
        ],
        [
          0.5045,
          0.7639
        ],
        [
          0.5146,
          0.7561
        ],
        [
          0.525,
          0.7489
        ],
        [
          0.535,
          0.7407
        ],
        [
          0.545,
          0.732
        ],
        [
          0.5551,
          0.7266
        ],
        [
          0.5655,
          0.7193
        ],
        [
          0.5755,
          0.7129
        ],
        [
          0.5855,
          0.7066
        ],
        [
          0.5955,
          0.7002
        ],
        [
          0.6056,
          0.6934
        ],
        [
          0.616,
          0.6861
        ],
        [
          0.626,
          0.6774
        ],
        [
          0.636,
          0.6688
        ],
        [
          0.646,
          0.6633
        ],
        [
          0.6561,
          0.6551
        ],
        [
          0.6665,
          0.6456
        ],
        [
          0.6765,
          0.6383
        ],
        [
          0.6865,
          0.6324
        ],
        [
          0.6965,
          0.626
        ],
        [
          0.7066,
          0.6201
        ],
        [
          0.717,
          0.6146
        ],
        [
          0.727,
          0.6069
        ],
        [
          0.737,
          0.601
        ],
        [
          0.747,
          0.596
        ],
        [
          0.7571,
          0.5878
        ],
        [
          0.7675,
          0.5787
        ],
        [
          0.7775,
          0.5714
        ],
        [
          0.7875,
          0.5646
        ],
        [
          0.7975,
          0.5582
        ],
        [
          0.8076,
          0.5519
        ],
        [
          0.818,
          0.5441
        ],
        [
          0.828,
          0.5355
        ],
        [
          0.838,
          0.5287
        ],
        [
          0.848,
          0.5205
        ],
        [
          0.8581,
          0.5123
        ],
        [
          0.8685,
          0.5041
        ],
        [
          0.8785,
          0.4964
        ],
        [
          0.8885,
          0.4886
        ],
        [
          0.8985,
          0.4804
        ],
        [
          0.9086,
          0.4732
        ],
        [
          0.919,
          0.4682
        ],
        [
          0.929,
          0.4618
        ],
        [
          0.939,
          0.454
        ],
        [
          0.949,
          0.4477
        ],
        [
          0.9591,
          0.4386
        ],
        [
          0.9695,
          0.4304
        ],
        [
          0.9795,
          0.4222
        ],
        [
          0.9895,
          0.4158
        ],
        [
          0.9995,
          0.4081
        ],
        [
          1.0,
          0.4076
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.029004683127495063,
        "difference": -0.0008913236177816489,
        "worse": 0.4666666666666667,
        "kept_right": 0.02811335950971344,
        "kept_wrong": 0.0008913236177816489,
        "handed_off": 0.9709953168725053
      },
      "usd_per_1000": 1.5751390609473905
    },
    {
      "decision": 126,
      "frontier": 48,
      "frontier_accuracy": 0.9758871701546861,
      "in_sample": {
        "threshold": 0.9545185565948486,
        "answered": 0.011828935395814377,
        "accuracy": 0.9758871701546861
      },
      "curve": [
        [
          0.0,
          0.9759
        ],
        [
          0.01,
          0.9759
        ],
        [
          0.02,
          0.9754
        ],
        [
          0.03,
          0.9754
        ],
        [
          0.04,
          0.9754
        ],
        [
          0.05,
          0.9745
        ],
        [
          0.0601,
          0.9736
        ],
        [
          0.0705,
          0.9732
        ],
        [
          0.0805,
          0.9722
        ],
        [
          0.0905,
          0.9704
        ],
        [
          0.1005,
          0.9686
        ],
        [
          0.1106,
          0.9672
        ],
        [
          0.121,
          0.9654
        ],
        [
          0.131,
          0.9631
        ],
        [
          0.141,
          0.9604
        ],
        [
          0.151,
          0.9572
        ],
        [
          0.1611,
          0.955
        ],
        [
          0.1715,
          0.9495
        ],
        [
          0.1815,
          0.9463
        ],
        [
          0.1915,
          0.9427
        ],
        [
          0.2015,
          0.9381
        ],
        [
          0.2116,
          0.9331
        ],
        [
          0.222,
          0.929
        ],
        [
          0.232,
          0.924
        ],
        [
          0.242,
          0.9208
        ],
        [
          0.252,
          0.9149
        ],
        [
          0.2621,
          0.909
        ],
        [
          0.2725,
          0.9035
        ],
        [
          0.2825,
          0.8976
        ],
        [
          0.2925,
          0.8926
        ],
        [
          0.3025,
          0.8863
        ],
        [
          0.3126,
          0.8835
        ],
        [
          0.323,
          0.879
        ],
        [
          0.333,
          0.8726
        ],
        [
          0.343,
          0.8658
        ],
        [
          0.353,
          0.8603
        ],
        [
          0.3631,
          0.8558
        ],
        [
          0.3735,
          0.8485
        ],
        [
          0.3835,
          0.8421
        ],
        [
          0.3935,
          0.8367
        ],
        [
          0.4035,
          0.8289
        ],
        [
          0.4136,
          0.8221
        ],
        [
          0.424,
          0.8135
        ],
        [
          0.434,
          0.8076
        ],
        [
          0.444,
          0.8012
        ],
        [
          0.454,
          0.7948
        ],
        [
          0.4641,
          0.788
        ],
        [
          0.4745,
          0.7812
        ],
        [
          0.4845,
          0.7743
        ],
        [
          0.4945,
          0.7675
        ],
        [
          0.5045,
          0.7598
        ],
        [
          0.5146,
          0.752
        ],
        [
          0.525,
          0.7448
        ],
        [
          0.535,
          0.7366
        ],
        [
          0.545,
          0.7279
        ],
        [
          0.5551,
          0.7225
        ],
        [
          0.5655,
          0.7152
        ],
        [
          0.5755,
          0.7088
        ],
        [
          0.5855,
          0.7025
        ],
        [
          0.5955,
          0.6965
        ],
        [
          0.6056,
          0.6897
        ],
        [
          0.616,
          0.6829
        ],
        [
          0.626,
          0.6742
        ],
        [
          0.636,
          0.6656
        ],
        [
          0.646,
          0.6606
        ],
        [
          0.6561,
          0.6524
        ],
        [
          0.6665,
          0.6433
        ],
        [
          0.6765,
          0.636
        ],
        [
          0.6865,
          0.6301
        ],
        [
          0.6965,
          0.6242
        ],
        [
          0.7066,
          0.6183
        ],
        [
          0.717,
          0.6124
        ],
        [
          0.727,
          0.6046
        ],
        [
          0.737,
          0.5987
        ],
        [
          0.747,
          0.5937
        ],
        [
          0.7571,
          0.586
        ],
        [
          0.7675,
          0.5769
        ],
        [
          0.7775,
          0.5696
        ],
        [
          0.7875,
          0.5632
        ],
        [
          0.7975,
          0.5569
        ],
        [
          0.8076,
          0.5505
        ],
        [
          0.818,
          0.5428
        ],
        [
          0.828,
          0.5341
        ],
        [
          0.838,
          0.5273
        ],
        [
          0.848,
          0.5191
        ],
        [
          0.8581,
          0.5114
        ],
        [
          0.8685,
          0.5032
        ],
        [
          0.8785,
          0.4955
        ],
        [
          0.8885,
          0.4877
        ],
        [
          0.8985,
          0.48
        ],
        [
          0.9086,
          0.4727
        ],
        [
          0.919,
          0.4677
        ],
        [
          0.929,
          0.4613
        ],
        [
          0.939,
          0.4536
        ],
        [
          0.949,
          0.4477
        ],
        [
          0.9591,
          0.4386
        ],
        [
          0.9695,
          0.4304
        ],
        [
          0.9795,
          0.4222
        ],
        [
          0.9895,
          0.4158
        ],
        [
          0.9995,
          0.4081
        ],
        [
          1.0,
          0.4076
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.029004683127495063,
        "difference": -0.0008913236177816489,
        "worse": 0.4666666666666667,
        "kept_right": 0.02811335950971344,
        "kept_wrong": 0.0008913236177816489,
        "handed_off": 0.9709953168725053
      },
      "usd_per_1000": 5.222232496338719
    },
    {
      "decision": 126,
      "frontier": 191,
      "frontier_accuracy": 0.9740673339399454,
      "in_sample": {
        "threshold": 0.9545185565948486,
        "answered": 0.011828935395814377,
        "accuracy": 0.9740673339399454
      },
      "curve": [
        [
          0.0,
          0.9741
        ],
        [
          0.01,
          0.9741
        ],
        [
          0.02,
          0.9736
        ],
        [
          0.03,
          0.9736
        ],
        [
          0.04,
          0.9736
        ],
        [
          0.05,
          0.9727
        ],
        [
          0.0601,
          0.9718
        ],
        [
          0.0705,
          0.9713
        ],
        [
          0.0805,
          0.9704
        ],
        [
          0.0905,
          0.9686
        ],
        [
          0.1005,
          0.9668
        ],
        [
          0.1106,
          0.9654
        ],
        [
          0.121,
          0.9636
        ],
        [
          0.131,
          0.9613
        ],
        [
          0.141,
          0.9586
        ],
        [
          0.151,
          0.9554
        ],
        [
          0.1611,
          0.9531
        ],
        [
          0.1715,
          0.9481
        ],
        [
          0.1815,
          0.9449
        ],
        [
          0.1915,
          0.9413
        ],
        [
          0.2015,
          0.9368
        ],
        [
          0.2116,
          0.9318
        ],
        [
          0.222,
          0.9272
        ],
        [
          0.232,
          0.9217
        ],
        [
          0.242,
          0.9186
        ],
        [
          0.252,
          0.9126
        ],
        [
          0.2621,
          0.9063
        ],
        [
          0.2725,
          0.9008
        ],
        [
          0.2825,
          0.8949
        ],
        [
          0.2925,
          0.8904
        ],
        [
          0.3025,
          0.884
        ],
        [
          0.3126,
          0.8813
        ],
        [
          0.323,
          0.8767
        ],
        [
          0.333,
          0.8703
        ],
        [
          0.343,
          0.8631
        ],
        [
          0.353,
          0.8571
        ],
        [
          0.3631,
          0.8526
        ],
        [
          0.3735,
          0.8453
        ],
        [
          0.3835,
          0.8389
        ],
        [
          0.3935,
          0.8335
        ],
        [
          0.4035,
          0.8258
        ],
        [
          0.4136,
          0.8189
        ],
        [
          0.424,
          0.8103
        ],
        [
          0.434,
          0.8044
        ],
        [
          0.444,
          0.7985
        ],
        [
          0.454,
          0.7925
        ],
        [
          0.4641,
          0.7857
        ],
        [
          0.4745,
          0.7793
        ],
        [
          0.4845,
          0.7725
        ],
        [
          0.4945,
          0.7657
        ],
        [
          0.5045,
          0.758
        ],
        [
          0.5146,
          0.7502
        ],
        [
          0.525,
          0.7429
        ],
        [
          0.535,
          0.7348
        ],
        [
          0.545,
          0.7261
        ],
        [
          0.5551,
          0.7211
        ],
        [
          0.5655,
          0.7138
        ],
        [
          0.5755,
          0.7075
        ],
        [
          0.5855,
          0.7011
        ],
        [
          0.5955,
          0.6947
        ],
        [
          0.6056,
          0.6884
        ],
        [
          0.616,
          0.6811
        ],
        [
          0.626,
          0.6724
        ],
        [
          0.636,
          0.6642
        ],
        [
          0.646,
          0.6588
        ],
        [
          0.6561,
          0.6506
        ],
        [
          0.6665,
          0.6415
        ],
        [
          0.6765,
          0.6342
        ],
        [
          0.6865,
          0.6278
        ],
        [
          0.6965,
          0.6219
        ],
        [
          0.7066,
          0.616
        ],
        [
          0.717,
          0.6106
        ],
        [
          0.727,
          0.6028
        ],
        [
          0.737,
          0.5969
        ],
        [
          0.747,
          0.5919
        ],
        [
          0.7571,
          0.5837
        ],
        [
          0.7675,
          0.5746
        ],
        [
          0.7775,
          0.5673
        ],
        [
          0.7875,
          0.5605
        ],
        [
          0.7975,
          0.5541
        ],
        [
          0.8076,
          0.5478
        ],
        [
          0.818,
          0.5405
        ],
        [
          0.828,
          0.5318
        ],
        [
          0.838,
          0.525
        ],
        [
          0.848,
          0.5168
        ],
        [
          0.8581,
          0.5096
        ],
        [
          0.8685,
          0.5014
        ],
        [
          0.8785,
          0.4936
        ],
        [
          0.8885,
          0.4859
        ],
        [
          0.8985,
          0.4782
        ],
        [
          0.9086,
          0.4713
        ],
        [
          0.919,
          0.4668
        ],
        [
          0.929,
          0.4604
        ],
        [
          0.939,
          0.4527
        ],
        [
          0.949,
          0.4468
        ],
        [
          0.9591,
          0.4377
        ],
        [
          0.9695,
          0.4299
        ],
        [
          0.9795,
          0.4222
        ],
        [
          0.9895,
          0.4158
        ],
        [
          0.9995,
          0.4081
        ],
        [
          1.0,
          0.4076
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.029004683127495063,
        "difference": -0.0008913236177816489,
        "worse": 0.4666666666666667,
        "kept_right": 0.02811335950971344,
        "kept_wrong": 0.0008913236177816489,
        "handed_off": 0.9709953168725053
      },
      "usd_per_1000": 25.962177684779224
    },
    {
      "decision": 126,
      "frontier": 188,
      "frontier_accuracy": 0.9713375796178344,
      "in_sample": {
        "threshold": 0.911125659942627,
        "answered": 0.045950864422202004,
        "accuracy": 0.9713375796178344
      },
      "curve": [
        [
          0.0,
          0.9713
        ],
        [
          0.01,
          0.9713
        ],
        [
          0.02,
          0.9709
        ],
        [
          0.03,
          0.9709
        ],
        [
          0.04,
          0.9709
        ],
        [
          0.05,
          0.9704
        ],
        [
          0.0601,
          0.97
        ],
        [
          0.0705,
          0.97
        ],
        [
          0.0805,
          0.9691
        ],
        [
          0.0905,
          0.9672
        ],
        [
          0.1005,
          0.9654
        ],
        [
          0.1106,
          0.9645
        ],
        [
          0.121,
          0.9627
        ],
        [
          0.131,
          0.9604
        ],
        [
          0.141,
          0.9577
        ],
        [
          0.151,
          0.9554
        ],
        [
          0.1611,
          0.9531
        ],
        [
          0.1715,
          0.9477
        ],
        [
          0.1815,
          0.9445
        ],
        [
          0.1915,
          0.9409
        ],
        [
          0.2015,
          0.9363
        ],
        [
          0.2116,
          0.9318
        ],
        [
          0.222,
          0.9272
        ],
        [
          0.232,
          0.9217
        ],
        [
          0.242,
          0.9186
        ],
        [
          0.252,
          0.9126
        ],
        [
          0.2621,
          0.9067
        ],
        [
          0.2725,
          0.9008
        ],
        [
          0.2825,
          0.8954
        ],
        [
          0.2925,
          0.8894
        ],
        [
          0.3025,
          0.8835
        ],
        [
          0.3126,
          0.8808
        ],
        [
          0.323,
          0.8763
        ],
        [
          0.333,
          0.8699
        ],
        [
          0.343,
          0.8621
        ],
        [
          0.353,
          0.8571
        ],
        [
          0.3631,
          0.8526
        ],
        [
          0.3735,
          0.8458
        ],
        [
          0.3835,
          0.8399
        ],
        [
          0.3935,
          0.8344
        ],
        [
          0.4035,
          0.8267
        ],
        [
          0.4136,
          0.8198
        ],
        [
          0.424,
          0.8112
        ],
        [
          0.434,
          0.8048
        ],
        [
          0.444,
          0.7989
        ],
        [
          0.454,
          0.793
        ],
        [
          0.4641,
          0.7862
        ],
        [
          0.4745,
          0.7798
        ],
        [
          0.4845,
          0.773
        ],
        [
          0.4945,
          0.7662
        ],
        [
          0.5045,
          0.7584
        ],
        [
          0.5146,
          0.7507
        ],
        [
          0.525,
          0.7439
        ],
        [
          0.535,
          0.7357
        ],
        [
          0.545,
          0.727
        ],
        [
          0.5551,
          0.7216
        ],
        [
          0.5655,
          0.7143
        ],
        [
          0.5755,
          0.7075
        ],
        [
          0.5855,
          0.7011
        ],
        [
          0.5955,
          0.6947
        ],
        [
          0.6056,
          0.6879
        ],
        [
          0.616,
          0.6806
        ],
        [
          0.626,
          0.672
        ],
        [
          0.636,
          0.6633
        ],
        [
          0.646,
          0.6583
        ],
        [
          0.6561,
          0.6501
        ],
        [
          0.6665,
          0.6415
        ],
        [
          0.6765,
          0.6342
        ],
        [
          0.6865,
          0.6283
        ],
        [
          0.6965,
          0.6224
        ],
        [
          0.7066,
          0.6165
        ],
        [
          0.717,
          0.6106
        ],
        [
          0.727,
          0.6028
        ],
        [
          0.737,
          0.5969
        ],
        [
          0.747,
          0.5914
        ],
        [
          0.7571,
          0.5833
        ],
        [
          0.7675,
          0.5737
        ],
        [
          0.7775,
          0.5669
        ],
        [
          0.7875,
          0.5601
        ],
        [
          0.7975,
          0.5537
        ],
        [
          0.8076,
          0.5473
        ],
        [
          0.818,
          0.54
        ],
        [
          0.828,
          0.5314
        ],
        [
          0.838,
          0.525
        ],
        [
          0.848,
          0.5168
        ],
        [
          0.8581,
          0.5086
        ],
        [
          0.8685,
          0.5005
        ],
        [
          0.8785,
          0.4927
        ],
        [
          0.8885,
          0.485
        ],
        [
          0.8985,
          0.4777
        ],
        [
          0.9086,
          0.4704
        ],
        [
          0.919,
          0.4654
        ],
        [
          0.929,
          0.4595
        ],
        [
          0.939,
          0.4531
        ],
        [
          0.949,
          0.4468
        ],
        [
          0.9591,
          0.4381
        ],
        [
          0.9695,
          0.4304
        ],
        [
          0.9795,
          0.4227
        ],
        [
          0.9895,
          0.4158
        ],
        [
          0.9995,
          0.4081
        ],
        [
          1.0,
          0.4076
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.04990361527755837,
        "difference": -0.001700878666140826,
        "worse": 0.5833333333333334,
        "kept_right": 0.04765096444662371,
        "kept_wrong": 0.0022526508309346424,
        "handed_off": 0.9500963847224415
      },
      "usd_per_1000": 0.9153607009626569
    },
    {
      "decision": 126,
      "frontier": 190,
      "frontier_accuracy": 0.9672429481346679,
      "in_sample": {
        "threshold": 0.9545185565948486,
        "answered": 0.011828935395814377,
        "accuracy": 0.9672429481346679
      },
      "curve": [
        [
          0.0,
          0.9672
        ],
        [
          0.01,
          0.9672
        ],
        [
          0.02,
          0.9668
        ],
        [
          0.03,
          0.9668
        ],
        [
          0.04,
          0.9668
        ],
        [
          0.05,
          0.9659
        ],
        [
          0.0601,
          0.9654
        ],
        [
          0.0705,
          0.965
        ],
        [
          0.0805,
          0.9645
        ],
        [
          0.0905,
          0.9627
        ],
        [
          0.1005,
          0.9609
        ],
        [
          0.1106,
          0.9595
        ],
        [
          0.121,
          0.9577
        ],
        [
          0.131,
          0.9554
        ],
        [
          0.141,
          0.9527
        ],
        [
          0.151,
          0.9495
        ],
        [
          0.1611,
          0.9472
        ],
        [
          0.1715,
          0.9422
        ],
        [
          0.1815,
          0.939
        ],
        [
          0.1915,
          0.9359
        ],
        [
          0.2015,
          0.9313
        ],
        [
          0.2116,
          0.9263
        ],
        [
          0.222,
          0.9222
        ],
        [
          0.232,
          0.9167
        ],
        [
          0.242,
          0.9145
        ],
        [
          0.252,
          0.9086
        ],
        [
          0.2621,
          0.9026
        ],
        [
          0.2725,
          0.8972
        ],
        [
          0.2825,
          0.8913
        ],
        [
          0.2925,
          0.8867
        ],
        [
          0.3025,
          0.8808
        ],
        [
          0.3126,
          0.8781
        ],
        [
          0.323,
          0.874
        ],
        [
          0.333,
          0.8676
        ],
        [
          0.343,
          0.8608
        ],
        [
          0.353,
          0.8553
        ],
        [
          0.3631,
          0.8508
        ],
        [
          0.3735,
          0.8435
        ],
        [
          0.3835,
          0.8371
        ],
        [
          0.3935,
          0.8317
        ],
        [
          0.4035,
          0.8239
        ],
        [
          0.4136,
          0.8171
        ],
        [
          0.424,
          0.8085
        ],
        [
          0.434,
          0.8025
        ],
        [
          0.444,
          0.7962
        ],
        [
          0.454,
          0.7903
        ],
        [
          0.4641,
          0.7834
        ],
        [
          0.4745,
          0.7771
        ],
        [
          0.4845,
          0.7702
        ],
        [
          0.4945,
          0.7634
        ],
        [
          0.5045,
          0.7557
        ],
        [
          0.5146,
          0.748
        ],
        [
          0.525,
          0.7407
        ],
        [
          0.535,
          0.7325
        ],
        [
          0.545,
          0.7238
        ],
        [
          0.5551,
          0.7184
        ],
        [
          0.5655,
          0.7116
        ],
        [
          0.5755,
          0.7052
        ],
        [
          0.5855,
          0.6988
        ],
        [
          0.5955,
          0.6924
        ],
        [
          0.6056,
          0.6861
        ],
        [
          0.616,
          0.6788
        ],
        [
          0.626,
          0.6702
        ],
        [
          0.636,
          0.662
        ],
        [
          0.646,
          0.6565
        ],
        [
          0.6561,
          0.6483
        ],
        [
          0.6665,
          0.6388
        ],
        [
          0.6765,
          0.6315
        ],
        [
          0.6865,
          0.6256
        ],
        [
          0.6965,
          0.6201
        ],
        [
          0.7066,
          0.6142
        ],
        [
          0.717,
          0.6087
        ],
        [
          0.727,
          0.601
        ],
        [
          0.737,
          0.5951
        ],
        [
          0.747,
          0.5896
        ],
        [
          0.7571,
          0.5814
        ],
        [
          0.7675,
          0.5728
        ],
        [
          0.7775,
          0.5655
        ],
        [
          0.7875,
          0.5587
        ],
        [
          0.7975,
          0.5523
        ],
        [
          0.8076,
          0.546
        ],
        [
          0.818,
          0.5387
        ],
        [
          0.828,
          0.5305
        ],
        [
          0.838,
          0.5237
        ],
        [
          0.848,
          0.5155
        ],
        [
          0.8581,
          0.5082
        ],
        [
          0.8685,
          0.5
        ],
        [
          0.8785,
          0.4923
        ],
        [
          0.8885,
          0.485
        ],
        [
          0.8985,
          0.4773
        ],
        [
          0.9086,
          0.47
        ],
        [
          0.919,
          0.4659
        ],
        [
          0.929,
          0.4595
        ],
        [
          0.939,
          0.4522
        ],
        [
          0.949,
          0.4463
        ],
        [
          0.9591,
          0.4377
        ],
        [
          0.9695,
          0.4299
        ],
        [
          0.9795,
          0.4227
        ],
        [
          0.9895,
          0.4158
        ],
        [
          0.9995,
          0.4081
        ],
        [
          1.0,
          0.4076
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.03249582948408227,
        "difference": -0.0012371286854101213,
        "worse": 0.4866666666666667,
        "kept_right": 0.031231405982698596,
        "kept_wrong": 0.0012644235013836705,
        "handed_off": 0.967504170515918
      },
      "usd_per_1000": 5.3875653728237705
    },
    {
      "decision": 126,
      "frontier": 124,
      "frontier_accuracy": 0.9667879890809827,
      "in_sample": {
        "threshold": 0.9545185565948486,
        "answered": 0.011828935395814377,
        "accuracy": 0.9667879890809827
      },
      "curve": [
        [
          0.0,
          0.9668
        ],
        [
          0.01,
          0.9668
        ],
        [
          0.02,
          0.9663
        ],
        [
          0.03,
          0.9663
        ],
        [
          0.04,
          0.9663
        ],
        [
          0.05,
          0.9654
        ],
        [
          0.0601,
          0.9645
        ],
        [
          0.0705,
          0.9641
        ],
        [
          0.0805,
          0.9631
        ],
        [
          0.0905,
          0.9613
        ],
        [
          0.1005,
          0.96
        ],
        [
          0.1106,
          0.9586
        ],
        [
          0.121,
          0.9568
        ],
        [
          0.131,
          0.9545
        ],
        [
          0.141,
          0.9518
        ],
        [
          0.151,
          0.9486
        ],
        [
          0.1611,
          0.9463
        ],
        [
          0.1715,
          0.9409
        ],
        [
          0.1815,
          0.9377
        ],
        [
          0.1915,
          0.934
        ],
        [
          0.2015,
          0.9295
        ],
        [
          0.2116,
          0.9245
        ],
        [
          0.222,
          0.9204
        ],
        [
          0.232,
          0.9154
        ],
        [
          0.242,
          0.9131
        ],
        [
          0.252,
          0.9072
        ],
        [
          0.2621,
          0.9013
        ],
        [
          0.2725,
          0.8958
        ],
        [
          0.2825,
          0.8899
        ],
        [
          0.2925,
          0.8854
        ],
        [
          0.3025,
          0.8794
        ],
        [
          0.3126,
          0.8767
        ],
        [
          0.323,
          0.8722
        ],
        [
          0.333,
          0.8658
        ],
        [
          0.343,
          0.859
        ],
        [
          0.353,
          0.854
        ],
        [
          0.3631,
          0.8494
        ],
        [
          0.3735,
          0.8421
        ],
        [
          0.3835,
          0.8358
        ],
        [
          0.3935,
          0.8303
        ],
        [
          0.4035,
          0.8226
        ],
        [
          0.4136,
          0.8162
        ],
        [
          0.424,
          0.8076
        ],
        [
          0.434,
          0.8016
        ],
        [
          0.444,
          0.7957
        ],
        [
          0.454,
          0.7898
        ],
        [
          0.4641,
          0.783
        ],
        [
          0.4745,
          0.7762
        ],
        [
          0.4845,
          0.7693
        ],
        [
          0.4945,
          0.7625
        ],
        [
          0.5045,
          0.7548
        ],
        [
          0.5146,
          0.747
        ],
        [
          0.525,
          0.7398
        ],
        [
          0.535,
          0.732
        ],
        [
          0.545,
          0.7234
        ],
        [
          0.5551,
          0.7188
        ],
        [
          0.5655,
          0.7116
        ],
        [
          0.5755,
          0.7052
        ],
        [
          0.5855,
          0.6988
        ],
        [
          0.5955,
          0.6924
        ],
        [
          0.6056,
          0.6856
        ],
        [
          0.616,
          0.6788
        ],
        [
          0.626,
          0.6702
        ],
        [
          0.636,
          0.662
        ],
        [
          0.646,
          0.6565
        ],
        [
          0.6561,
          0.6483
        ],
        [
          0.6665,
          0.6388
        ],
        [
          0.6765,
          0.6319
        ],
        [
          0.6865,
          0.6256
        ],
        [
          0.6965,
          0.6201
        ],
        [
          0.7066,
          0.6142
        ],
        [
          0.717,
          0.6083
        ],
        [
          0.727,
          0.6005
        ],
        [
          0.737,
          0.5951
        ],
        [
          0.747,
          0.5901
        ],
        [
          0.7571,
          0.5833
        ],
        [
          0.7675,
          0.5742
        ],
        [
          0.7775,
          0.5669
        ],
        [
          0.7875,
          0.5605
        ],
        [
          0.7975,
          0.5541
        ],
        [
          0.8076,
          0.5478
        ],
        [
          0.818,
          0.5405
        ],
        [
          0.828,
          0.5318
        ],
        [
          0.838,
          0.525
        ],
        [
          0.848,
          0.5168
        ],
        [
          0.8581,
          0.5096
        ],
        [
          0.8685,
          0.5014
        ],
        [
          0.8785,
          0.4936
        ],
        [
          0.8885,
          0.4864
        ],
        [
          0.8985,
          0.4786
        ],
        [
          0.9086,
          0.4713
        ],
        [
          0.919,
          0.4663
        ],
        [
          0.929,
          0.46
        ],
        [
          0.939,
          0.4527
        ],
        [
          0.949,
          0.4468
        ],
        [
          0.9591,
          0.4377
        ],
        [
          0.9695,
          0.4299
        ],
        [
          0.9795,
          0.4222
        ],
        [
          0.9895,
          0.4154
        ],
        [
          0.9995,
          0.4081
        ],
        [
          1.0,
          0.4076
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.029004683127495063,
        "difference": -0.0008913236177816489,
        "worse": 0.4666666666666667,
        "kept_right": 0.02811335950971344,
        "kept_wrong": 0.0008913236177816489,
        "handed_off": 0.9709953168725053
      },
      "usd_per_1000": 1.0975869538165381
    },
    {
      "decision": 126,
      "frontier": 278,
      "frontier_accuracy": 0.9499545040946314,
      "in_sample": {
        "threshold": 0.9545185565948486,
        "answered": 0.011828935395814377,
        "accuracy": 0.9499545040946314
      },
      "curve": [
        [
          0.0,
          0.95
        ],
        [
          0.01,
          0.95
        ],
        [
          0.02,
          0.9495
        ],
        [
          0.03,
          0.9495
        ],
        [
          0.04,
          0.9495
        ],
        [
          0.05,
          0.949
        ],
        [
          0.0601,
          0.9481
        ],
        [
          0.0705,
          0.9481
        ],
        [
          0.0805,
          0.9472
        ],
        [
          0.0905,
          0.9454
        ],
        [
          0.1005,
          0.9436
        ],
        [
          0.1106,
          0.9422
        ],
        [
          0.121,
          0.9404
        ],
        [
          0.131,
          0.9381
        ],
        [
          0.141,
          0.9359
        ],
        [
          0.151,
          0.9327
        ],
        [
          0.1611,
          0.9304
        ],
        [
          0.1715,
          0.9249
        ],
        [
          0.1815,
          0.9217
        ],
        [
          0.1915,
          0.9181
        ],
        [
          0.2015,
          0.914
        ],
        [
          0.2116,
          0.909
        ],
        [
          0.222,
          0.9054
        ],
        [
          0.232,
          0.9004
        ],
        [
          0.242,
          0.8976
        ],
        [
          0.252,
          0.8917
        ],
        [
          0.2621,
          0.8854
        ],
        [
          0.2725,
          0.8799
        ],
        [
          0.2825,
          0.8749
        ],
        [
          0.2925,
          0.8699
        ],
        [
          0.3025,
          0.8649
        ],
        [
          0.3126,
          0.8626
        ],
        [
          0.323,
          0.8581
        ],
        [
          0.333,
          0.8517
        ],
        [
          0.343,
          0.8444
        ],
        [
          0.353,
          0.8389
        ],
        [
          0.3631,
          0.8344
        ],
        [
          0.3735,
          0.8271
        ],
        [
          0.3835,
          0.8212
        ],
        [
          0.3935,
          0.8157
        ],
        [
          0.4035,
          0.808
        ],
        [
          0.4136,
          0.8021
        ],
        [
          0.424,
          0.7939
        ],
        [
          0.434,
          0.7875
        ],
        [
          0.444,
          0.7821
        ],
        [
          0.454,
          0.7766
        ],
        [
          0.4641,
          0.7707
        ],
        [
          0.4745,
          0.7643
        ],
        [
          0.4845,
          0.7584
        ],
        [
          0.4945,
          0.7516
        ],
        [
          0.5045,
          0.7443
        ],
        [
          0.5146,
          0.7366
        ],
        [
          0.525,
          0.7298
        ],
        [
          0.535,
          0.722
        ],
        [
          0.545,
          0.7134
        ],
        [
          0.5551,
          0.7084
        ],
        [
          0.5655,
          0.7011
        ],
        [
          0.5755,
          0.6947
        ],
        [
          0.5855,
          0.6884
        ],
        [
          0.5955,
          0.682
        ],
        [
          0.6056,
          0.6752
        ],
        [
          0.616,
          0.6688
        ],
        [
          0.626,
          0.6611
        ],
        [
          0.636,
          0.6529
        ],
        [
          0.646,
          0.6483
        ],
        [
          0.6561,
          0.6406
        ],
        [
          0.6665,
          0.6315
        ],
        [
          0.6765,
          0.6251
        ],
        [
          0.6865,
          0.6187
        ],
        [
          0.6965,
          0.6128
        ],
        [
          0.7066,
          0.6069
        ],
        [
          0.717,
          0.6015
        ],
        [
          0.727,
          0.5937
        ],
        [
          0.737,
          0.5878
        ],
        [
          0.747,
          0.5828
        ],
        [
          0.7571,
          0.5755
        ],
        [
          0.7675,
          0.5664
        ],
        [
          0.7775,
          0.5596
        ],
        [
          0.7875,
          0.5532
        ],
        [
          0.7975,
          0.5473
        ],
        [
          0.8076,
          0.5409
        ],
        [
          0.818,
          0.5337
        ],
        [
          0.828,
          0.525
        ],
        [
          0.838,
          0.5182
        ],
        [
          0.848,
          0.51
        ],
        [
          0.8581,
          0.5036
        ],
        [
          0.8685,
          0.4959
        ],
        [
          0.8785,
          0.4891
        ],
        [
          0.8885,
          0.4823
        ],
        [
          0.8985,
          0.4754
        ],
        [
          0.9086,
          0.4682
        ],
        [
          0.919,
          0.4631
        ],
        [
          0.929,
          0.4577
        ],
        [
          0.939,
          0.4504
        ],
        [
          0.949,
          0.4449
        ],
        [
          0.9591,
          0.4368
        ],
        [
          0.9695,
          0.429
        ],
        [
          0.9795,
          0.4213
        ],
        [
          0.9895,
          0.4154
        ],
        [
          0.9995,
          0.4081
        ],
        [
          1.0,
          0.4076
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.03508758916566203,
        "difference": -0.0012310100549990219,
        "worse": 0.5133333333333333,
        "kept_right": 0.03366556778797654,
        "kept_wrong": 0.001422021377685497,
        "handed_off": 0.9649124108343387
      },
      "usd_per_1000": 6.2804472735863754
    },
    {
      "decision": 126,
      "frontier": 275,
      "frontier_accuracy": 0.9426751592356688,
      "in_sample": {
        "threshold": 0.9545185565948486,
        "answered": 0.011828935395814377,
        "accuracy": 0.9426751592356688
      },
      "curve": [
        [
          0.0,
          0.9427
        ],
        [
          0.01,
          0.9427
        ],
        [
          0.02,
          0.9422
        ],
        [
          0.03,
          0.9422
        ],
        [
          0.04,
          0.9422
        ],
        [
          0.05,
          0.9413
        ],
        [
          0.0601,
          0.9404
        ],
        [
          0.0705,
          0.9404
        ],
        [
          0.0805,
          0.9395
        ],
        [
          0.0905,
          0.9377
        ],
        [
          0.1005,
          0.9359
        ],
        [
          0.1106,
          0.9345
        ],
        [
          0.121,
          0.9327
        ],
        [
          0.131,
          0.9304
        ],
        [
          0.141,
          0.9277
        ],
        [
          0.151,
          0.9245
        ],
        [
          0.1611,
          0.9222
        ],
        [
          0.1715,
          0.9167
        ],
        [
          0.1815,
          0.9136
        ],
        [
          0.1915,
          0.9104
        ],
        [
          0.2015,
          0.9058
        ],
        [
          0.2116,
          0.9008
        ],
        [
          0.222,
          0.8963
        ],
        [
          0.232,
          0.8908
        ],
        [
          0.242,
          0.8881
        ],
        [
          0.252,
          0.8822
        ],
        [
          0.2621,
          0.8763
        ],
        [
          0.2725,
          0.8708
        ],
        [
          0.2825,
          0.8653
        ],
        [
          0.2925,
          0.8608
        ],
        [
          0.3025,
          0.8558
        ],
        [
          0.3126,
          0.853
        ],
        [
          0.323,
          0.8485
        ],
        [
          0.333,
          0.8421
        ],
        [
          0.343,
          0.8348
        ],
        [
          0.353,
          0.8294
        ],
        [
          0.3631,
          0.8253
        ],
        [
          0.3735,
          0.818
        ],
        [
          0.3835,
          0.8116
        ],
        [
          0.3935,
          0.8066
        ],
        [
          0.4035,
          0.7989
        ],
        [
          0.4136,
          0.7921
        ],
        [
          0.424,
          0.7834
        ],
        [
          0.434,
          0.7775
        ],
        [
          0.444,
          0.7725
        ],
        [
          0.454,
          0.7671
        ],
        [
          0.4641,
          0.7607
        ],
        [
          0.4745,
          0.7557
        ],
        [
          0.4845,
          0.7502
        ],
        [
          0.4945,
          0.7434
        ],
        [
          0.5045,
          0.7357
        ],
        [
          0.5146,
          0.7279
        ],
        [
          0.525,
          0.7211
        ],
        [
          0.535,
          0.7134
        ],
        [
          0.545,
          0.7052
        ],
        [
          0.5551,
          0.7006
        ],
        [
          0.5655,
          0.6934
        ],
        [
          0.5755,
          0.6874
        ],
        [
          0.5855,
          0.6811
        ],
        [
          0.5955,
          0.6752
        ],
        [
          0.6056,
          0.6683
        ],
        [
          0.616,
          0.662
        ],
        [
          0.626,
          0.6542
        ],
        [
          0.636,
          0.646
        ],
        [
          0.646,
          0.641
        ],
        [
          0.6561,
          0.6333
        ],
        [
          0.6665,
          0.6247
        ],
        [
          0.6765,
          0.6174
        ],
        [
          0.6865,
          0.6119
        ],
        [
          0.6965,
          0.6069
        ],
        [
          0.7066,
          0.6015
        ],
        [
          0.717,
          0.596
        ],
        [
          0.727,
          0.5892
        ],
        [
          0.737,
          0.5842
        ],
        [
          0.747,
          0.5796
        ],
        [
          0.7571,
          0.5723
        ],
        [
          0.7675,
          0.5641
        ],
        [
          0.7775,
          0.5573
        ],
        [
          0.7875,
          0.5514
        ],
        [
          0.7975,
          0.546
        ],
        [
          0.8076,
          0.54
        ],
        [
          0.818,
          0.5332
        ],
        [
          0.828,
          0.525
        ],
        [
          0.838,
          0.5187
        ],
        [
          0.848,
          0.5109
        ],
        [
          0.8581,
          0.5059
        ],
        [
          0.8685,
          0.4982
        ],
        [
          0.8785,
          0.4904
        ],
        [
          0.8885,
          0.4836
        ],
        [
          0.8985,
          0.4763
        ],
        [
          0.9086,
          0.4695
        ],
        [
          0.919,
          0.4645
        ],
        [
          0.929,
          0.4586
        ],
        [
          0.939,
          0.4509
        ],
        [
          0.949,
          0.4449
        ],
        [
          0.9591,
          0.4363
        ],
        [
          0.9695,
          0.4286
        ],
        [
          0.9795,
          0.4222
        ],
        [
          0.9895,
          0.4158
        ],
        [
          0.9995,
          0.4081
        ],
        [
          1.0,
          0.4076
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.032028213522346236,
        "difference": -0.0011521559851722916,
        "worse": 0.49,
        "kept_right": 0.030863936300008856,
        "kept_wrong": 0.001164277222337401,
        "handed_off": 0.9679717864776543
      },
      "usd_per_1000": 0.49803425538106405
    },
    {
      "decision": 126,
      "frontier": 192,
      "frontier_accuracy": 0.9331210191082803,
      "in_sample": {
        "threshold": 0.9545185565948486,
        "answered": 0.011828935395814377,
        "accuracy": 0.9331210191082803
      },
      "curve": [
        [
          0.0,
          0.9331
        ],
        [
          0.01,
          0.9331
        ],
        [
          0.02,
          0.9327
        ],
        [
          0.03,
          0.9327
        ],
        [
          0.04,
          0.9327
        ],
        [
          0.05,
          0.9322
        ],
        [
          0.0601,
          0.9313
        ],
        [
          0.0705,
          0.9308
        ],
        [
          0.0805,
          0.9299
        ],
        [
          0.0905,
          0.9281
        ],
        [
          0.1005,
          0.9263
        ],
        [
          0.1106,
          0.9249
        ],
        [
          0.121,
          0.9231
        ],
        [
          0.131,
          0.9208
        ],
        [
          0.141,
          0.9181
        ],
        [
          0.151,
          0.9149
        ],
        [
          0.1611,
          0.9126
        ],
        [
          0.1715,
          0.9072
        ],
        [
          0.1815,
          0.904
        ],
        [
          0.1915,
          0.9004
        ],
        [
          0.2015,
          0.8967
        ],
        [
          0.2116,
          0.8917
        ],
        [
          0.222,
          0.8876
        ],
        [
          0.232,
          0.8826
        ],
        [
          0.242,
          0.8799
        ],
        [
          0.252,
          0.874
        ],
        [
          0.2621,
          0.8685
        ],
        [
          0.2725,
          0.8635
        ],
        [
          0.2825,
          0.8585
        ],
        [
          0.2925,
          0.8535
        ],
        [
          0.3025,
          0.8485
        ],
        [
          0.3126,
          0.8462
        ],
        [
          0.323,
          0.8426
        ],
        [
          0.333,
          0.8367
        ],
        [
          0.343,
          0.8303
        ],
        [
          0.353,
          0.8248
        ],
        [
          0.3631,
          0.8203
        ],
        [
          0.3735,
          0.8135
        ],
        [
          0.3835,
          0.8071
        ],
        [
          0.3935,
          0.8016
        ],
        [
          0.4035,
          0.7939
        ],
        [
          0.4136,
          0.7875
        ],
        [
          0.424,
          0.7798
        ],
        [
          0.434,
          0.7734
        ],
        [
          0.444,
          0.7671
        ],
        [
          0.454,
          0.7625
        ],
        [
          0.4641,
          0.7566
        ],
        [
          0.4745,
          0.7507
        ],
        [
          0.4845,
          0.7443
        ],
        [
          0.4945,
          0.7375
        ],
        [
          0.5045,
          0.7307
        ],
        [
          0.5146,
          0.7234
        ],
        [
          0.525,
          0.7166
        ],
        [
          0.535,
          0.7084
        ],
        [
          0.545,
          0.7002
        ],
        [
          0.5551,
          0.6952
        ],
        [
          0.5655,
          0.6884
        ],
        [
          0.5755,
          0.682
        ],
        [
          0.5855,
          0.6765
        ],
        [
          0.5955,
          0.6711
        ],
        [
          0.6056,
          0.6652
        ],
        [
          0.616,
          0.6588
        ],
        [
          0.626,
          0.651
        ],
        [
          0.636,
          0.6429
        ],
        [
          0.646,
          0.6383
        ],
        [
          0.6561,
          0.631
        ],
        [
          0.6665,
          0.6219
        ],
        [
          0.6765,
          0.6151
        ],
        [
          0.6865,
          0.6092
        ],
        [
          0.6965,
          0.6037
        ],
        [
          0.7066,
          0.5983
        ],
        [
          0.717,
          0.5928
        ],
        [
          0.727,
          0.5864
        ],
        [
          0.737,
          0.5814
        ],
        [
          0.747,
          0.5764
        ],
        [
          0.7571,
          0.5696
        ],
        [
          0.7675,
          0.5605
        ],
        [
          0.7775,
          0.5537
        ],
        [
          0.7875,
          0.5478
        ],
        [
          0.7975,
          0.5423
        ],
        [
          0.8076,
          0.5364
        ],
        [
          0.818,
          0.5291
        ],
        [
          0.828,
          0.5214
        ],
        [
          0.838,
          0.5155
        ],
        [
          0.848,
          0.5073
        ],
        [
          0.8581,
          0.5018
        ],
        [
          0.8685,
          0.4941
        ],
        [
          0.8785,
          0.4868
        ],
        [
          0.8885,
          0.48
        ],
        [
          0.8985,
          0.4732
        ],
        [
          0.9086,
          0.4663
        ],
        [
          0.919,
          0.4631
        ],
        [
          0.929,
          0.4577
        ],
        [
          0.939,
          0.45
        ],
        [
          0.949,
          0.4449
        ],
        [
          0.9591,
          0.4359
        ],
        [
          0.9695,
          0.4281
        ],
        [
          0.9795,
          0.4208
        ],
        [
          0.9895,
          0.4149
        ],
        [
          0.9995,
          0.4081
        ],
        [
          1.0,
          0.4076
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.03002373731061875,
        "difference": -0.0009004834803576073,
        "worse": 0.4666666666666667,
        "kept_right": 0.02900806085998785,
        "kept_wrong": 0.0010156764506309183,
        "handed_off": 0.9699762626893814
      },
      "usd_per_1000": 11.261002792298068
    },
    {
      "decision": 126,
      "frontier": 123,
      "frontier_accuracy": 0.9326660600545951,
      "in_sample": {
        "threshold": 0.9545185565948486,
        "answered": 0.011828935395814377,
        "accuracy": 0.9326660600545951
      },
      "curve": [
        [
          0.0,
          0.9327
        ],
        [
          0.01,
          0.9327
        ],
        [
          0.02,
          0.9322
        ],
        [
          0.03,
          0.9322
        ],
        [
          0.04,
          0.9322
        ],
        [
          0.05,
          0.9313
        ],
        [
          0.0601,
          0.9304
        ],
        [
          0.0705,
          0.9304
        ],
        [
          0.0805,
          0.9295
        ],
        [
          0.0905,
          0.9277
        ],
        [
          0.1005,
          0.9258
        ],
        [
          0.1106,
          0.9249
        ],
        [
          0.121,
          0.9231
        ],
        [
          0.131,
          0.9208
        ],
        [
          0.141,
          0.9181
        ],
        [
          0.151,
          0.9154
        ],
        [
          0.1611,
          0.9136
        ],
        [
          0.1715,
          0.9081
        ],
        [
          0.1815,
          0.9049
        ],
        [
          0.1915,
          0.9013
        ],
        [
          0.2015,
          0.8967
        ],
        [
          0.2116,
          0.8917
        ],
        [
          0.222,
          0.8872
        ],
        [
          0.232,
          0.8826
        ],
        [
          0.242,
          0.8799
        ],
        [
          0.252,
          0.874
        ],
        [
          0.2621,
          0.8681
        ],
        [
          0.2725,
          0.8626
        ],
        [
          0.2825,
          0.8581
        ],
        [
          0.2925,
          0.853
        ],
        [
          0.3025,
          0.8471
        ],
        [
          0.3126,
          0.8444
        ],
        [
          0.323,
          0.8403
        ],
        [
          0.333,
          0.8344
        ],
        [
          0.343,
          0.8276
        ],
        [
          0.353,
          0.8217
        ],
        [
          0.3631,
          0.818
        ],
        [
          0.3735,
          0.8112
        ],
        [
          0.3835,
          0.8053
        ],
        [
          0.3935,
          0.7998
        ],
        [
          0.4035,
          0.7925
        ],
        [
          0.4136,
          0.7862
        ],
        [
          0.424,
          0.7775
        ],
        [
          0.434,
          0.7716
        ],
        [
          0.444,
          0.7662
        ],
        [
          0.454,
          0.7602
        ],
        [
          0.4641,
          0.7543
        ],
        [
          0.4745,
          0.7484
        ],
        [
          0.4845,
          0.7425
        ],
        [
          0.4945,
          0.7357
        ],
        [
          0.5045,
          0.7284
        ],
        [
          0.5146,
          0.7211
        ],
        [
          0.525,
          0.7143
        ],
        [
          0.535,
          0.707
        ],
        [
          0.545,
          0.6984
        ],
        [
          0.5551,
          0.6938
        ],
        [
          0.5655,
          0.687
        ],
        [
          0.5755,
          0.6811
        ],
        [
          0.5855,
          0.6756
        ],
        [
          0.5955,
          0.6702
        ],
        [
          0.6056,
          0.6633
        ],
        [
          0.616,
          0.6561
        ],
        [
          0.626,
          0.6483
        ],
        [
          0.636,
          0.6401
        ],
        [
          0.646,
          0.6356
        ],
        [
          0.6561,
          0.6283
        ],
        [
          0.6665,
          0.6192
        ],
        [
          0.6765,
          0.6128
        ],
        [
          0.6865,
          0.6078
        ],
        [
          0.6965,
          0.6024
        ],
        [
          0.7066,
          0.5969
        ],
        [
          0.717,
          0.591
        ],
        [
          0.727,
          0.5833
        ],
        [
          0.737,
          0.5778
        ],
        [
          0.747,
          0.5732
        ],
        [
          0.7571,
          0.566
        ],
        [
          0.7675,
          0.5591
        ],
        [
          0.7775,
          0.5519
        ],
        [
          0.7875,
          0.5464
        ],
        [
          0.7975,
          0.5405
        ],
        [
          0.8076,
          0.535
        ],
        [
          0.818,
          0.5287
        ],
        [
          0.828,
          0.5214
        ],
        [
          0.838,
          0.5164
        ],
        [
          0.848,
          0.5082
        ],
        [
          0.8581,
          0.5014
        ],
        [
          0.8685,
          0.4932
        ],
        [
          0.8785,
          0.4859
        ],
        [
          0.8885,
          0.4782
        ],
        [
          0.8985,
          0.4704
        ],
        [
          0.9086,
          0.4636
        ],
        [
          0.919,
          0.4595
        ],
        [
          0.929,
          0.4545
        ],
        [
          0.939,
          0.4477
        ],
        [
          0.949,
          0.4427
        ],
        [
          0.9591,
          0.4349
        ],
        [
          0.9695,
          0.4277
        ],
        [
          0.9795,
          0.4208
        ],
        [
          0.9895,
          0.4145
        ],
        [
          0.9995,
          0.4081
        ],
        [
          1.0,
          0.4076
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.032028213522346236,
        "difference": -0.0011521559851722916,
        "worse": 0.49,
        "kept_right": 0.030863936300008856,
        "kept_wrong": 0.001164277222337401,
        "handed_off": 0.9679717864776543
      },
      "usd_per_1000": 0.25998167616816054
    },
    {
      "decision": 126,
      "frontier": 274,
      "frontier_accuracy": 0.9085532302092811,
      "in_sample": {
        "threshold": 0.911125659942627,
        "answered": 0.045950864422202004,
        "accuracy": 0.9085532302092811
      },
      "curve": [
        [
          0.0,
          0.9086
        ],
        [
          0.01,
          0.9086
        ],
        [
          0.02,
          0.9081
        ],
        [
          0.03,
          0.9081
        ],
        [
          0.04,
          0.9081
        ],
        [
          0.05,
          0.9081
        ],
        [
          0.0601,
          0.9072
        ],
        [
          0.0705,
          0.9072
        ],
        [
          0.0805,
          0.9063
        ],
        [
          0.0905,
          0.9045
        ],
        [
          0.1005,
          0.9026
        ],
        [
          0.1106,
          0.9013
        ],
        [
          0.121,
          0.8995
        ],
        [
          0.131,
          0.8972
        ],
        [
          0.141,
          0.8944
        ],
        [
          0.151,
          0.8913
        ],
        [
          0.1611,
          0.889
        ],
        [
          0.1715,
          0.8835
        ],
        [
          0.1815,
          0.8808
        ],
        [
          0.1915,
          0.8781
        ],
        [
          0.2015,
          0.8735
        ],
        [
          0.2116,
          0.8685
        ],
        [
          0.222,
          0.864
        ],
        [
          0.232,
          0.859
        ],
        [
          0.242,
          0.8567
        ],
        [
          0.252,
          0.8508
        ],
        [
          0.2621,
          0.8453
        ],
        [
          0.2725,
          0.8399
        ],
        [
          0.2825,
          0.8353
        ],
        [
          0.2925,
          0.8321
        ],
        [
          0.3025,
          0.8276
        ],
        [
          0.3126,
          0.8248
        ],
        [
          0.323,
          0.8207
        ],
        [
          0.333,
          0.8144
        ],
        [
          0.343,
          0.808
        ],
        [
          0.353,
          0.8025
        ],
        [
          0.3631,
          0.798
        ],
        [
          0.3735,
          0.7907
        ],
        [
          0.3835,
          0.7843
        ],
        [
          0.3935,
          0.7793
        ],
        [
          0.4035,
          0.7716
        ],
        [
          0.4136,
          0.7652
        ],
        [
          0.424,
          0.7566
        ],
        [
          0.434,
          0.7516
        ],
        [
          0.444,
          0.747
        ],
        [
          0.454,
          0.742
        ],
        [
          0.4641,
          0.7361
        ],
        [
          0.4745,
          0.7307
        ],
        [
          0.4845,
          0.7261
        ],
        [
          0.4945,
          0.7193
        ],
        [
          0.5045,
          0.712
        ],
        [
          0.5146,
          0.7043
        ],
        [
          0.525,
          0.6975
        ],
        [
          0.535,
          0.6902
        ],
        [
          0.545,
          0.6824
        ],
        [
          0.5551,
          0.6779
        ],
        [
          0.5655,
          0.6715
        ],
        [
          0.5755,
          0.6652
        ],
        [
          0.5855,
          0.6597
        ],
        [
          0.5955,
          0.6551
        ],
        [
          0.6056,
          0.6488
        ],
        [
          0.616,
          0.6429
        ],
        [
          0.626,
          0.636
        ],
        [
          0.636,
          0.6283
        ],
        [
          0.646,
          0.6233
        ],
        [
          0.6561,
          0.6165
        ],
        [
          0.6665,
          0.6074
        ],
        [
          0.6765,
          0.601
        ],
        [
          0.6865,
          0.5969
        ],
        [
          0.6965,
          0.5928
        ],
        [
          0.7066,
          0.5883
        ],
        [
          0.717,
          0.5828
        ],
        [
          0.727,
          0.5755
        ],
        [
          0.737,
          0.571
        ],
        [
          0.747,
          0.5664
        ],
        [
          0.7571,
          0.5601
        ],
        [
          0.7675,
          0.5523
        ],
        [
          0.7775,
          0.5455
        ],
        [
          0.7875,
          0.54
        ],
        [
          0.7975,
          0.5359
        ],
        [
          0.8076,
          0.53
        ],
        [
          0.818,
          0.5237
        ],
        [
          0.828,
          0.5168
        ],
        [
          0.838,
          0.5109
        ],
        [
          0.848,
          0.5036
        ],
        [
          0.8581,
          0.4982
        ],
        [
          0.8685,
          0.4914
        ],
        [
          0.8785,
          0.4841
        ],
        [
          0.8885,
          0.4773
        ],
        [
          0.8985,
          0.47
        ],
        [
          0.9086,
          0.4631
        ],
        [
          0.919,
          0.4586
        ],
        [
          0.929,
          0.4531
        ],
        [
          0.939,
          0.4463
        ],
        [
          0.949,
          0.4418
        ],
        [
          0.9591,
          0.434
        ],
        [
          0.9695,
          0.4277
        ],
        [
          0.9795,
          0.4204
        ],
        [
          0.9895,
          0.4145
        ],
        [
          0.9995,
          0.4081
        ],
        [
          1.0,
          0.4076
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.04763606700359491,
        "difference": -0.0014583988558793841,
        "worse": 0.5566666666666666,
        "kept_right": 0.045550124545366937,
        "kept_wrong": 0.00208594245822798,
        "handed_off": 0.9523639329964045
      },
      "usd_per_1000": 0.40594101021707735
    },
    {
      "decision": 126,
      "frontier": 47,
      "frontier_accuracy": 0.9044585987261147,
      "in_sample": {
        "threshold": 0.9545185565948486,
        "answered": 0.011828935395814377,
        "accuracy": 0.9044585987261147
      },
      "curve": [
        [
          0.0,
          0.9045
        ],
        [
          0.01,
          0.9045
        ],
        [
          0.02,
          0.904
        ],
        [
          0.03,
          0.904
        ],
        [
          0.04,
          0.904
        ],
        [
          0.05,
          0.9035
        ],
        [
          0.0601,
          0.9026
        ],
        [
          0.0705,
          0.9026
        ],
        [
          0.0805,
          0.9017
        ],
        [
          0.0905,
          0.8999
        ],
        [
          0.1005,
          0.8981
        ],
        [
          0.1106,
          0.8967
        ],
        [
          0.121,
          0.8949
        ],
        [
          0.131,
          0.8926
        ],
        [
          0.141,
          0.8904
        ],
        [
          0.151,
          0.8876
        ],
        [
          0.1611,
          0.8858
        ],
        [
          0.1715,
          0.8817
        ],
        [
          0.1815,
          0.879
        ],
        [
          0.1915,
          0.8753
        ],
        [
          0.2015,
          0.8722
        ],
        [
          0.2116,
          0.8672
        ],
        [
          0.222,
          0.8635
        ],
        [
          0.232,
          0.8585
        ],
        [
          0.242,
          0.8553
        ],
        [
          0.252,
          0.8494
        ],
        [
          0.2621,
          0.8435
        ],
        [
          0.2725,
          0.8385
        ],
        [
          0.2825,
          0.8339
        ],
        [
          0.2925,
          0.8294
        ],
        [
          0.3025,
          0.8239
        ],
        [
          0.3126,
          0.8212
        ],
        [
          0.323,
          0.8171
        ],
        [
          0.333,
          0.8116
        ],
        [
          0.343,
          0.8048
        ],
        [
          0.353,
          0.7989
        ],
        [
          0.3631,
          0.7944
        ],
        [
          0.3735,
          0.7875
        ],
        [
          0.3835,
          0.7816
        ],
        [
          0.3935,
          0.7762
        ],
        [
          0.4035,
          0.7689
        ],
        [
          0.4136,
          0.7621
        ],
        [
          0.424,
          0.7543
        ],
        [
          0.434,
          0.7484
        ],
        [
          0.444,
          0.7425
        ],
        [
          0.454,
          0.7375
        ],
        [
          0.4641,
          0.7311
        ],
        [
          0.4745,
          0.7252
        ],
        [
          0.4845,
          0.7193
        ],
        [
          0.4945,
          0.7134
        ],
        [
          0.5045,
          0.707
        ],
        [
          0.5146,
          0.7002
        ],
        [
          0.525,
          0.6938
        ],
        [
          0.535,
          0.6861
        ],
        [
          0.545,
          0.6774
        ],
        [
          0.5551,
          0.6733
        ],
        [
          0.5655,
          0.6661
        ],
        [
          0.5755,
          0.6606
        ],
        [
          0.5855,
          0.6551
        ],
        [
          0.5955,
          0.6506
        ],
        [
          0.6056,
          0.6438
        ],
        [
          0.616,
          0.6374
        ],
        [
          0.626,
          0.6297
        ],
        [
          0.636,
          0.6219
        ],
        [
          0.646,
          0.6178
        ],
        [
          0.6561,
          0.6124
        ],
        [
          0.6665,
          0.6037
        ],
        [
          0.6765,
          0.5974
        ],
        [
          0.6865,
          0.5919
        ],
        [
          0.6965,
          0.5855
        ],
        [
          0.7066,
          0.581
        ],
        [
          0.717,
          0.5755
        ],
        [
          0.727,
          0.5701
        ],
        [
          0.737,
          0.5651
        ],
        [
          0.747,
          0.5605
        ],
        [
          0.7571,
          0.5528
        ],
        [
          0.7675,
          0.5446
        ],
        [
          0.7775,
          0.5387
        ],
        [
          0.7875,
          0.5341
        ],
        [
          0.7975,
          0.5296
        ],
        [
          0.8076,
          0.5237
        ],
        [
          0.818,
          0.5177
        ],
        [
          0.828,
          0.5105
        ],
        [
          0.838,
          0.5055
        ],
        [
          0.848,
          0.4982
        ],
        [
          0.8581,
          0.4932
        ],
        [
          0.8685,
          0.4859
        ],
        [
          0.8785,
          0.4791
        ],
        [
          0.8885,
          0.4722
        ],
        [
          0.8985,
          0.4654
        ],
        [
          0.9086,
          0.4581
        ],
        [
          0.919,
          0.4554
        ],
        [
          0.929,
          0.4513
        ],
        [
          0.939,
          0.4445
        ],
        [
          0.949,
          0.4395
        ],
        [
          0.9591,
          0.4318
        ],
        [
          0.9695,
          0.4254
        ],
        [
          0.9795,
          0.4186
        ],
        [
          0.9895,
          0.4131
        ],
        [
          0.9995,
          0.4081
        ],
        [
          1.0,
          0.4076
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.03508758916566203,
        "difference": -0.0012310100549990219,
        "worse": 0.5133333333333333,
        "kept_right": 0.03366556778797654,
        "kept_wrong": 0.001422021377685497,
        "handed_off": 0.9649124108343387
      },
      "usd_per_1000": 4.757583597889317
    },
    {
      "decision": 126,
      "frontier": 193,
      "frontier_accuracy": 0.89171974522293,
      "in_sample": {
        "threshold": 0.9545185565948486,
        "answered": 0.011828935395814377,
        "accuracy": 0.89171974522293
      },
      "curve": [
        [
          0.0,
          0.8917
        ],
        [
          0.01,
          0.8917
        ],
        [
          0.02,
          0.8913
        ],
        [
          0.03,
          0.8913
        ],
        [
          0.04,
          0.8913
        ],
        [
          0.05,
          0.8908
        ],
        [
          0.0601,
          0.8899
        ],
        [
          0.0705,
          0.8899
        ],
        [
          0.0805,
          0.889
        ],
        [
          0.0905,
          0.8872
        ],
        [
          0.1005,
          0.8854
        ],
        [
          0.1106,
          0.884
        ],
        [
          0.121,
          0.8822
        ],
        [
          0.131,
          0.8799
        ],
        [
          0.141,
          0.8776
        ],
        [
          0.151,
          0.8744
        ],
        [
          0.1611,
          0.8726
        ],
        [
          0.1715,
          0.8681
        ],
        [
          0.1815,
          0.8649
        ],
        [
          0.1915,
          0.8617
        ],
        [
          0.2015,
          0.8581
        ],
        [
          0.2116,
          0.853
        ],
        [
          0.222,
          0.8494
        ],
        [
          0.232,
          0.8444
        ],
        [
          0.242,
          0.8417
        ],
        [
          0.252,
          0.8362
        ],
        [
          0.2621,
          0.8321
        ],
        [
          0.2725,
          0.8271
        ],
        [
          0.2825,
          0.823
        ],
        [
          0.2925,
          0.8189
        ],
        [
          0.3025,
          0.8135
        ],
        [
          0.3126,
          0.8112
        ],
        [
          0.323,
          0.8076
        ],
        [
          0.333,
          0.8021
        ],
        [
          0.343,
          0.7957
        ],
        [
          0.353,
          0.7907
        ],
        [
          0.3631,
          0.7862
        ],
        [
          0.3735,
          0.7793
        ],
        [
          0.3835,
          0.7734
        ],
        [
          0.3935,
          0.7684
        ],
        [
          0.4035,
          0.7611
        ],
        [
          0.4136,
          0.7548
        ],
        [
          0.424,
          0.7461
        ],
        [
          0.434,
          0.7402
        ],
        [
          0.444,
          0.7343
        ],
        [
          0.454,
          0.7311
        ],
        [
          0.4641,
          0.7257
        ],
        [
          0.4745,
          0.7207
        ],
        [
          0.4845,
          0.7161
        ],
        [
          0.4945,
          0.7097
        ],
        [
          0.5045,
          0.7034
        ],
        [
          0.5146,
          0.6965
        ],
        [
          0.525,
          0.6906
        ],
        [
          0.535,
          0.6829
        ],
        [
          0.545,
          0.6752
        ],
        [
          0.5551,
          0.6702
        ],
        [
          0.5655,
          0.6638
        ],
        [
          0.5755,
          0.6579
        ],
        [
          0.5855,
          0.652
        ],
        [
          0.5955,
          0.6465
        ],
        [
          0.6056,
          0.6406
        ],
        [
          0.616,
          0.6347
        ],
        [
          0.626,
          0.6274
        ],
        [
          0.636,
          0.6197
        ],
        [
          0.646,
          0.6146
        ],
        [
          0.6561,
          0.6092
        ],
        [
          0.6665,
          0.5996
        ],
        [
          0.6765,
          0.5933
        ],
        [
          0.6865,
          0.5878
        ],
        [
          0.6965,
          0.5823
        ],
        [
          0.7066,
          0.5769
        ],
        [
          0.717,
          0.5723
        ],
        [
          0.727,
          0.5655
        ],
        [
          0.737,
          0.5614
        ],
        [
          0.747,
          0.5564
        ],
        [
          0.7571,
          0.5505
        ],
        [
          0.7675,
          0.5428
        ],
        [
          0.7775,
          0.5373
        ],
        [
          0.7875,
          0.5309
        ],
        [
          0.7975,
          0.5259
        ],
        [
          0.8076,
          0.5205
        ],
        [
          0.818,
          0.5136
        ],
        [
          0.828,
          0.5055
        ],
        [
          0.838,
          0.4995
        ],
        [
          0.848,
          0.4914
        ],
        [
          0.8581,
          0.4877
        ],
        [
          0.8685,
          0.4809
        ],
        [
          0.8785,
          0.4754
        ],
        [
          0.8885,
          0.4695
        ],
        [
          0.8985,
          0.4636
        ],
        [
          0.9086,
          0.4577
        ],
        [
          0.919,
          0.4545
        ],
        [
          0.929,
          0.4513
        ],
        [
          0.939,
          0.444
        ],
        [
          0.949,
          0.4395
        ],
        [
          0.9591,
          0.4322
        ],
        [
          0.9695,
          0.4249
        ],
        [
          0.9795,
          0.419
        ],
        [
          0.9895,
          0.4136
        ],
        [
          0.9995,
          0.4081
        ],
        [
          1.0,
          0.4076
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.03508758916566203,
        "difference": -0.0012310100549990219,
        "worse": 0.5133333333333333,
        "kept_right": 0.03366556778797654,
        "kept_wrong": 0.001422021377685497,
        "handed_off": 0.9649124108343387
      },
      "usd_per_1000": 7.475745736276096
    },
    {
      "decision": 126,
      "frontier": 55,
      "frontier_accuracy": 0.8503184713375797,
      "in_sample": {
        "threshold": 0.8997169137001038,
        "answered": 0.055505004549590536,
        "accuracy": 0.8503184713375797
      },
      "curve": [
        [
          0.0,
          0.8503
        ],
        [
          0.01,
          0.8503
        ],
        [
          0.02,
          0.8503
        ],
        [
          0.03,
          0.8503
        ],
        [
          0.04,
          0.8508
        ],
        [
          0.05,
          0.8503
        ],
        [
          0.0601,
          0.8499
        ],
        [
          0.0705,
          0.8499
        ],
        [
          0.0805,
          0.8494
        ],
        [
          0.0905,
          0.848
        ],
        [
          0.1005,
          0.8485
        ],
        [
          0.1106,
          0.8476
        ],
        [
          0.121,
          0.8458
        ],
        [
          0.131,
          0.8439
        ],
        [
          0.141,
          0.8417
        ],
        [
          0.151,
          0.8394
        ],
        [
          0.1611,
          0.8376
        ],
        [
          0.1715,
          0.833
        ],
        [
          0.1815,
          0.8312
        ],
        [
          0.1915,
          0.828
        ],
        [
          0.2015,
          0.8239
        ],
        [
          0.2116,
          0.8189
        ],
        [
          0.222,
          0.8162
        ],
        [
          0.232,
          0.8121
        ],
        [
          0.242,
          0.8098
        ],
        [
          0.252,
          0.8044
        ],
        [
          0.2621,
          0.7989
        ],
        [
          0.2725,
          0.7953
        ],
        [
          0.2825,
          0.7912
        ],
        [
          0.2925,
          0.7871
        ],
        [
          0.3025,
          0.7825
        ],
        [
          0.3126,
          0.7812
        ],
        [
          0.323,
          0.7784
        ],
        [
          0.333,
          0.7739
        ],
        [
          0.343,
          0.7689
        ],
        [
          0.353,
          0.7634
        ],
        [
          0.3631,
          0.7593
        ],
        [
          0.3735,
          0.7525
        ],
        [
          0.3835,
          0.748
        ],
        [
          0.3935,
          0.7439
        ],
        [
          0.4035,
          0.7393
        ],
        [
          0.4136,
          0.7334
        ],
        [
          0.424,
          0.7266
        ],
        [
          0.434,
          0.7211
        ],
        [
          0.444,
          0.717
        ],
        [
          0.454,
          0.7143
        ],
        [
          0.4641,
          0.7088
        ],
        [
          0.4745,
          0.7034
        ],
        [
          0.4845,
          0.7002
        ],
        [
          0.4945,
          0.6938
        ],
        [
          0.5045,
          0.6879
        ],
        [
          0.5146,
          0.6824
        ],
        [
          0.525,
          0.6765
        ],
        [
          0.535,
          0.6688
        ],
        [
          0.545,
          0.6624
        ],
        [
          0.5551,
          0.6579
        ],
        [
          0.5655,
          0.6529
        ],
        [
          0.5755,
          0.647
        ],
        [
          0.5855,
          0.6415
        ],
        [
          0.5955,
          0.6365
        ],
        [
          0.6056,
          0.631
        ],
        [
          0.616,
          0.6247
        ],
        [
          0.626,
          0.6169
        ],
        [
          0.636,
          0.6092
        ],
        [
          0.646,
          0.6046
        ],
        [
          0.6561,
          0.5992
        ],
        [
          0.6665,
          0.5919
        ],
        [
          0.6765,
          0.5855
        ],
        [
          0.6865,
          0.5796
        ],
        [
          0.6965,
          0.5742
        ],
        [
          0.7066,
          0.5692
        ],
        [
          0.717,
          0.5646
        ],
        [
          0.727,
          0.5582
        ],
        [
          0.737,
          0.5532
        ],
        [
          0.747,
          0.5487
        ],
        [
          0.7571,
          0.5432
        ],
        [
          0.7675,
          0.5359
        ],
        [
          0.7775,
          0.5305
        ],
        [
          0.7875,
          0.525
        ],
        [
          0.7975,
          0.52
        ],
        [
          0.8076,
          0.5141
        ],
        [
          0.818,
          0.5077
        ],
        [
          0.828,
          0.5005
        ],
        [
          0.838,
          0.4959
        ],
        [
          0.848,
          0.4877
        ],
        [
          0.8581,
          0.4836
        ],
        [
          0.8685,
          0.4773
        ],
        [
          0.8785,
          0.4718
        ],
        [
          0.8885,
          0.4663
        ],
        [
          0.8985,
          0.4604
        ],
        [
          0.9086,
          0.4536
        ],
        [
          0.919,
          0.4509
        ],
        [
          0.929,
          0.4486
        ],
        [
          0.939,
          0.4422
        ],
        [
          0.949,
          0.4386
        ],
        [
          0.9591,
          0.4313
        ],
        [
          0.9695,
          0.4254
        ],
        [
          0.9795,
          0.4199
        ],
        [
          0.9895,
          0.4136
        ],
        [
          0.9995,
          0.4081
        ],
        [
          1.0,
          0.4076
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.07249187506235133,
        "difference": -0.0019496503168061885,
        "worse": 0.5866666666666667,
        "kept_right": 0.06779850715449384,
        "kept_wrong": 0.004693367907857494,
        "handed_off": 0.9275081249376482
      },
      "usd_per_1000": 3.3894241236988725
    },
    {
      "decision": 340,
      "frontier": 140,
      "frontier_accuracy": 0.9808917197452229,
      "in_sample": {
        "threshold": 0.9105755929840016,
        "answered": 0.006369426751592357,
        "accuracy": 0.9808917197452229
      },
      "curve": [
        [
          0.0,
          0.9809
        ],
        [
          0.01,
          0.9804
        ],
        [
          0.02,
          0.9759
        ],
        [
          0.03,
          0.9709
        ],
        [
          0.04,
          0.9677
        ],
        [
          0.05,
          0.9641
        ],
        [
          0.0601,
          0.96
        ],
        [
          0.0705,
          0.9568
        ],
        [
          0.0805,
          0.9513
        ],
        [
          0.0905,
          0.9463
        ],
        [
          0.1005,
          0.9427
        ],
        [
          0.1106,
          0.9359
        ],
        [
          0.121,
          0.9308
        ],
        [
          0.131,
          0.9254
        ],
        [
          0.141,
          0.9199
        ],
        [
          0.151,
          0.9122
        ],
        [
          0.1611,
          0.9067
        ],
        [
          0.1715,
          0.9022
        ],
        [
          0.1815,
          0.8954
        ],
        [
          0.1915,
          0.8885
        ],
        [
          0.2015,
          0.8813
        ],
        [
          0.2116,
          0.8749
        ],
        [
          0.222,
          0.869
        ],
        [
          0.232,
          0.8626
        ],
        [
          0.242,
          0.8553
        ],
        [
          0.252,
          0.8508
        ],
        [
          0.2621,
          0.8439
        ],
        [
          0.2725,
          0.8362
        ],
        [
          0.2825,
          0.8289
        ],
        [
          0.2925,
          0.8217
        ],
        [
          0.3025,
          0.8144
        ],
        [
          0.3126,
          0.8066
        ],
        [
          0.323,
          0.7994
        ],
        [
          0.333,
          0.7934
        ],
        [
          0.343,
          0.7884
        ],
        [
          0.353,
          0.7812
        ],
        [
          0.3631,
          0.7743
        ],
        [
          0.3735,
          0.7675
        ],
        [
          0.3835,
          0.7598
        ],
        [
          0.3935,
          0.7539
        ],
        [
          0.4035,
          0.747
        ],
        [
          0.4136,
          0.7398
        ],
        [
          0.424,
          0.7329
        ],
        [
          0.434,
          0.7261
        ],
        [
          0.444,
          0.7188
        ],
        [
          0.454,
          0.7125
        ],
        [
          0.4641,
          0.7056
        ],
        [
          0.4745,
          0.6997
        ],
        [
          0.4845,
          0.6924
        ],
        [
          0.4945,
          0.6847
        ],
        [
          0.5045,
          0.6783
        ],
        [
          0.5146,
          0.6702
        ],
        [
          0.525,
          0.6624
        ],
        [
          0.535,
          0.6561
        ],
        [
          0.545,
          0.6479
        ],
        [
          0.5551,
          0.6415
        ],
        [
          0.5655,
          0.6328
        ],
        [
          0.5755,
          0.6242
        ],
        [
          0.5855,
          0.6156
        ],
        [
          0.5955,
          0.6069
        ],
        [
          0.6056,
          0.5992
        ],
        [
          0.616,
          0.5914
        ],
        [
          0.626,
          0.5842
        ],
        [
          0.636,
          0.5764
        ],
        [
          0.646,
          0.5682
        ],
        [
          0.6561,
          0.5605
        ],
        [
          0.6665,
          0.5523
        ],
        [
          0.6765,
          0.545
        ],
        [
          0.6865,
          0.5369
        ],
        [
          0.6965,
          0.5305
        ],
        [
          0.7066,
          0.5232
        ],
        [
          0.717,
          0.5146
        ],
        [
          0.727,
          0.5077
        ],
        [
          0.737,
          0.4991
        ],
        [
          0.747,
          0.4904
        ],
        [
          0.7571,
          0.4845
        ],
        [
          0.7675,
          0.4777
        ],
        [
          0.7775,
          0.47
        ],
        [
          0.7875,
          0.4622
        ],
        [
          0.7975,
          0.4536
        ],
        [
          0.8076,
          0.4454
        ],
        [
          0.818,
          0.4349
        ],
        [
          0.828,
          0.4272
        ],
        [
          0.838,
          0.4204
        ],
        [
          0.848,
          0.4117
        ],
        [
          0.8581,
          0.404
        ],
        [
          0.8685,
          0.3963
        ],
        [
          0.8785,
          0.3881
        ],
        [
          0.8885,
          0.3831
        ],
        [
          0.8985,
          0.3749
        ],
        [
          0.9086,
          0.3672
        ],
        [
          0.919,
          0.3594
        ],
        [
          0.929,
          0.3503
        ],
        [
          0.939,
          0.3421
        ],
        [
          0.949,
          0.3348
        ],
        [
          0.9591,
          0.3262
        ],
        [
          0.9695,
          0.318
        ],
        [
          0.9795,
          0.3103
        ],
        [
          0.9895,
          0.3035
        ],
        [
          0.9995,
          0.2944
        ],
        [
          1.0,
          0.2939
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.008030627143942811,
        "difference": -0.001064059388393343,
        "worse": 0.5366666666666666,
        "kept_right": 0.006781633462308525,
        "kept_wrong": 0.0012489936816342901,
        "handed_off": 0.9919693728560579
      },
      "usd_per_1000": 2.399676246448945
    },
    {
      "decision": 340,
      "frontier": 277,
      "frontier_accuracy": 0.9795268425841674,
      "in_sample": {
        "threshold": 0.9105755929840016,
        "answered": 0.006369426751592357,
        "accuracy": 0.9795268425841674
      },
      "curve": [
        [
          0.0,
          0.9795
        ],
        [
          0.01,
          0.9791
        ],
        [
          0.02,
          0.9745
        ],
        [
          0.03,
          0.97
        ],
        [
          0.04,
          0.9668
        ],
        [
          0.05,
          0.9631
        ],
        [
          0.0601,
          0.9591
        ],
        [
          0.0705,
          0.9559
        ],
        [
          0.0805,
          0.95
        ],
        [
          0.0905,
          0.9449
        ],
        [
          0.1005,
          0.9409
        ],
        [
          0.1106,
          0.9336
        ],
        [
          0.121,
          0.9286
        ],
        [
          0.131,
          0.9227
        ],
        [
          0.141,
          0.9172
        ],
        [
          0.151,
          0.9095
        ],
        [
          0.1611,
          0.904
        ],
        [
          0.1715,
          0.899
        ],
        [
          0.1815,
          0.8922
        ],
        [
          0.1915,
          0.8854
        ],
        [
          0.2015,
          0.8781
        ],
        [
          0.2116,
          0.8717
        ],
        [
          0.222,
          0.8658
        ],
        [
          0.232,
          0.8594
        ],
        [
          0.242,
          0.8521
        ],
        [
          0.252,
          0.8476
        ],
        [
          0.2621,
          0.8412
        ],
        [
          0.2725,
          0.833
        ],
        [
          0.2825,
          0.8258
        ],
        [
          0.2925,
          0.8185
        ],
        [
          0.3025,
          0.8112
        ],
        [
          0.3126,
          0.8035
        ],
        [
          0.323,
          0.7962
        ],
        [
          0.333,
          0.7903
        ],
        [
          0.343,
          0.7857
        ],
        [
          0.353,
          0.7784
        ],
        [
          0.3631,
          0.7716
        ],
        [
          0.3735,
          0.7648
        ],
        [
          0.3835,
          0.7575
        ],
        [
          0.3935,
          0.7516
        ],
        [
          0.4035,
          0.7452
        ],
        [
          0.4136,
          0.7379
        ],
        [
          0.424,
          0.7311
        ],
        [
          0.434,
          0.7243
        ],
        [
          0.444,
          0.7175
        ],
        [
          0.454,
          0.7106
        ],
        [
          0.4641,
          0.7038
        ],
        [
          0.4745,
          0.6975
        ],
        [
          0.4845,
          0.6902
        ],
        [
          0.4945,
          0.6824
        ],
        [
          0.5045,
          0.6761
        ],
        [
          0.5146,
          0.6674
        ],
        [
          0.525,
          0.6597
        ],
        [
          0.535,
          0.6538
        ],
        [
          0.545,
          0.6456
        ],
        [
          0.5551,
          0.6392
        ],
        [
          0.5655,
          0.6306
        ],
        [
          0.5755,
          0.6219
        ],
        [
          0.5855,
          0.6133
        ],
        [
          0.5955,
          0.6051
        ],
        [
          0.6056,
          0.5978
        ],
        [
          0.616,
          0.5901
        ],
        [
          0.626,
          0.5823
        ],
        [
          0.636,
          0.5742
        ],
        [
          0.646,
          0.566
        ],
        [
          0.6561,
          0.5582
        ],
        [
          0.6665,
          0.5505
        ],
        [
          0.6765,
          0.5432
        ],
        [
          0.6865,
          0.535
        ],
        [
          0.6965,
          0.5287
        ],
        [
          0.7066,
          0.5214
        ],
        [
          0.717,
          0.5127
        ],
        [
          0.727,
          0.5059
        ],
        [
          0.737,
          0.4973
        ],
        [
          0.747,
          0.4891
        ],
        [
          0.7571,
          0.4832
        ],
        [
          0.7675,
          0.4763
        ],
        [
          0.7775,
          0.4686
        ],
        [
          0.7875,
          0.4613
        ],
        [
          0.7975,
          0.4527
        ],
        [
          0.8076,
          0.4445
        ],
        [
          0.818,
          0.434
        ],
        [
          0.828,
          0.4263
        ],
        [
          0.838,
          0.4195
        ],
        [
          0.848,
          0.4108
        ],
        [
          0.8581,
          0.4031
        ],
        [
          0.8685,
          0.3954
        ],
        [
          0.8785,
          0.3876
        ],
        [
          0.8885,
          0.3822
        ],
        [
          0.8985,
          0.374
        ],
        [
          0.9086,
          0.3667
        ],
        [
          0.919,
          0.359
        ],
        [
          0.929,
          0.3499
        ],
        [
          0.939,
          0.3417
        ],
        [
          0.949,
          0.3344
        ],
        [
          0.9591,
          0.3258
        ],
        [
          0.9695,
          0.3176
        ],
        [
          0.9795,
          0.3098
        ],
        [
          0.9895,
          0.3035
        ],
        [
          0.9995,
          0.2944
        ],
        [
          1.0,
          0.2939
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.008030627143942811,
        "difference": -0.001064059388393343,
        "worse": 0.5366666666666666,
        "kept_right": 0.006781633462308525,
        "kept_wrong": 0.0012489936816342901,
        "handed_off": 0.9919693728560579
      },
      "usd_per_1000": 6.016844839017521
    },
    {
      "decision": 340,
      "frontier": 276,
      "frontier_accuracy": 0.9777070063694268,
      "in_sample": {
        "threshold": 0.9105755929840016,
        "answered": 0.006369426751592357,
        "accuracy": 0.9777070063694268
      },
      "curve": [
        [
          0.0,
          0.9777
        ],
        [
          0.01,
          0.9773
        ],
        [
          0.02,
          0.9727
        ],
        [
          0.03,
          0.9682
        ],
        [
          0.04,
          0.965
        ],
        [
          0.05,
          0.9613
        ],
        [
          0.0601,
          0.9577
        ],
        [
          0.0705,
          0.9545
        ],
        [
          0.0805,
          0.9486
        ],
        [
          0.0905,
          0.944
        ],
        [
          0.1005,
          0.9399
        ],
        [
          0.1106,
          0.9327
        ],
        [
          0.121,
          0.9277
        ],
        [
          0.131,
          0.9217
        ],
        [
          0.141,
          0.9163
        ],
        [
          0.151,
          0.9086
        ],
        [
          0.1611,
          0.9031
        ],
        [
          0.1715,
          0.8981
        ],
        [
          0.1815,
          0.8917
        ],
        [
          0.1915,
          0.8849
        ],
        [
          0.2015,
          0.8776
        ],
        [
          0.2116,
          0.8712
        ],
        [
          0.222,
          0.8653
        ],
        [
          0.232,
          0.859
        ],
        [
          0.242,
          0.8517
        ],
        [
          0.252,
          0.8471
        ],
        [
          0.2621,
          0.8408
        ],
        [
          0.2725,
          0.8326
        ],
        [
          0.2825,
          0.8253
        ],
        [
          0.2925,
          0.818
        ],
        [
          0.3025,
          0.8103
        ],
        [
          0.3126,
          0.803
        ],
        [
          0.323,
          0.7957
        ],
        [
          0.333,
          0.7898
        ],
        [
          0.343,
          0.7853
        ],
        [
          0.353,
          0.7784
        ],
        [
          0.3631,
          0.7716
        ],
        [
          0.3735,
          0.7648
        ],
        [
          0.3835,
          0.7575
        ],
        [
          0.3935,
          0.7516
        ],
        [
          0.4035,
          0.7452
        ],
        [
          0.4136,
          0.7379
        ],
        [
          0.424,
          0.7311
        ],
        [
          0.434,
          0.7243
        ],
        [
          0.444,
          0.717
        ],
        [
          0.454,
          0.7102
        ],
        [
          0.4641,
          0.7034
        ],
        [
          0.4745,
          0.697
        ],
        [
          0.4845,
          0.6897
        ],
        [
          0.4945,
          0.682
        ],
        [
          0.5045,
          0.6756
        ],
        [
          0.5146,
          0.667
        ],
        [
          0.525,
          0.6592
        ],
        [
          0.535,
          0.6533
        ],
        [
          0.545,
          0.6451
        ],
        [
          0.5551,
          0.6388
        ],
        [
          0.5655,
          0.6301
        ],
        [
          0.5755,
          0.6215
        ],
        [
          0.5855,
          0.6128
        ],
        [
          0.5955,
          0.6046
        ],
        [
          0.6056,
          0.5969
        ],
        [
          0.616,
          0.5892
        ],
        [
          0.626,
          0.5819
        ],
        [
          0.636,
          0.5737
        ],
        [
          0.646,
          0.5655
        ],
        [
          0.6561,
          0.5578
        ],
        [
          0.6665,
          0.55
        ],
        [
          0.6765,
          0.5428
        ],
        [
          0.6865,
          0.5346
        ],
        [
          0.6965,
          0.5282
        ],
        [
          0.7066,
          0.5209
        ],
        [
          0.717,
          0.5123
        ],
        [
          0.727,
          0.5059
        ],
        [
          0.737,
          0.4973
        ],
        [
          0.747,
          0.4891
        ],
        [
          0.7571,
          0.4836
        ],
        [
          0.7675,
          0.4768
        ],
        [
          0.7775,
          0.4691
        ],
        [
          0.7875,
          0.4618
        ],
        [
          0.7975,
          0.4531
        ],
        [
          0.8076,
          0.4449
        ],
        [
          0.818,
          0.4345
        ],
        [
          0.828,
          0.4268
        ],
        [
          0.838,
          0.4199
        ],
        [
          0.848,
          0.4113
        ],
        [
          0.8581,
          0.4035
        ],
        [
          0.8685,
          0.3958
        ],
        [
          0.8785,
          0.3885
        ],
        [
          0.8885,
          0.3831
        ],
        [
          0.8985,
          0.3749
        ],
        [
          0.9086,
          0.3672
        ],
        [
          0.919,
          0.3594
        ],
        [
          0.929,
          0.3503
        ],
        [
          0.939,
          0.3421
        ],
        [
          0.949,
          0.3348
        ],
        [
          0.9591,
          0.3262
        ],
        [
          0.9695,
          0.318
        ],
        [
          0.9795,
          0.3103
        ],
        [
          0.9895,
          0.3035
        ],
        [
          0.9995,
          0.2944
        ],
        [
          1.0,
          0.2939
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.008030627143942811,
        "difference": -0.001064059388393343,
        "worse": 0.5366666666666666,
        "kept_right": 0.006781633462308525,
        "kept_wrong": 0.0012489936816342901,
        "handed_off": 0.9919693728560579
      },
      "usd_per_1000": 1.609162968449437
    },
    {
      "decision": 340,
      "frontier": 48,
      "frontier_accuracy": 0.9758871701546861,
      "in_sample": {
        "threshold": 0.9105755929840016,
        "answered": 0.006369426751592357,
        "accuracy": 0.9758871701546861
      },
      "curve": [
        [
          0.0,
          0.9759
        ],
        [
          0.01,
          0.9754
        ],
        [
          0.02,
          0.9709
        ],
        [
          0.03,
          0.9663
        ],
        [
          0.04,
          0.9631
        ],
        [
          0.05,
          0.9595
        ],
        [
          0.0601,
          0.9554
        ],
        [
          0.0705,
          0.9522
        ],
        [
          0.0805,
          0.9463
        ],
        [
          0.0905,
          0.9413
        ],
        [
          0.1005,
          0.9372
        ],
        [
          0.1106,
          0.9299
        ],
        [
          0.121,
          0.9249
        ],
        [
          0.131,
          0.9195
        ],
        [
          0.141,
          0.9145
        ],
        [
          0.151,
          0.9067
        ],
        [
          0.1611,
          0.9013
        ],
        [
          0.1715,
          0.8963
        ],
        [
          0.1815,
          0.8899
        ],
        [
          0.1915,
          0.8831
        ],
        [
          0.2015,
          0.8763
        ],
        [
          0.2116,
          0.8699
        ],
        [
          0.222,
          0.864
        ],
        [
          0.232,
          0.8576
        ],
        [
          0.242,
          0.8508
        ],
        [
          0.252,
          0.8462
        ],
        [
          0.2621,
          0.8399
        ],
        [
          0.2725,
          0.8317
        ],
        [
          0.2825,
          0.8244
        ],
        [
          0.2925,
          0.8171
        ],
        [
          0.3025,
          0.8094
        ],
        [
          0.3126,
          0.8016
        ],
        [
          0.323,
          0.7944
        ],
        [
          0.333,
          0.7884
        ],
        [
          0.343,
          0.7839
        ],
        [
          0.353,
          0.7771
        ],
        [
          0.3631,
          0.7702
        ],
        [
          0.3735,
          0.7634
        ],
        [
          0.3835,
          0.7557
        ],
        [
          0.3935,
          0.7498
        ],
        [
          0.4035,
          0.7434
        ],
        [
          0.4136,
          0.7366
        ],
        [
          0.424,
          0.7298
        ],
        [
          0.434,
          0.7229
        ],
        [
          0.444,
          0.7166
        ],
        [
          0.454,
          0.7093
        ],
        [
          0.4641,
          0.7025
        ],
        [
          0.4745,
          0.6961
        ],
        [
          0.4845,
          0.6888
        ],
        [
          0.4945,
          0.6811
        ],
        [
          0.5045,
          0.6742
        ],
        [
          0.5146,
          0.6656
        ],
        [
          0.525,
          0.6579
        ],
        [
          0.535,
          0.652
        ],
        [
          0.545,
          0.6438
        ],
        [
          0.5551,
          0.6374
        ],
        [
          0.5655,
          0.6288
        ],
        [
          0.5755,
          0.6201
        ],
        [
          0.5855,
          0.6115
        ],
        [
          0.5955,
          0.6033
        ],
        [
          0.6056,
          0.596
        ],
        [
          0.616,
          0.5883
        ],
        [
          0.626,
          0.5805
        ],
        [
          0.636,
          0.5723
        ],
        [
          0.646,
          0.5641
        ],
        [
          0.6561,
          0.5564
        ],
        [
          0.6665,
          0.5487
        ],
        [
          0.6765,
          0.5414
        ],
        [
          0.6865,
          0.5332
        ],
        [
          0.6965,
          0.5268
        ],
        [
          0.7066,
          0.5196
        ],
        [
          0.717,
          0.5109
        ],
        [
          0.727,
          0.5045
        ],
        [
          0.737,
          0.4959
        ],
        [
          0.747,
          0.4877
        ],
        [
          0.7571,
          0.4818
        ],
        [
          0.7675,
          0.475
        ],
        [
          0.7775,
          0.4672
        ],
        [
          0.7875,
          0.46
        ],
        [
          0.7975,
          0.4513
        ],
        [
          0.8076,
          0.4431
        ],
        [
          0.818,
          0.4327
        ],
        [
          0.828,
          0.4249
        ],
        [
          0.838,
          0.4186
        ],
        [
          0.848,
          0.4099
        ],
        [
          0.8581,
          0.4022
        ],
        [
          0.8685,
          0.3944
        ],
        [
          0.8785,
          0.3867
        ],
        [
          0.8885,
          0.3822
        ],
        [
          0.8985,
          0.374
        ],
        [
          0.9086,
          0.3667
        ],
        [
          0.919,
          0.359
        ],
        [
          0.929,
          0.3499
        ],
        [
          0.939,
          0.3417
        ],
        [
          0.949,
          0.3344
        ],
        [
          0.9591,
          0.3258
        ],
        [
          0.9695,
          0.3176
        ],
        [
          0.9795,
          0.3098
        ],
        [
          0.9895,
          0.303
        ],
        [
          0.9995,
          0.2944
        ],
        [
          1.0,
          0.2939
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.008030627143942811,
        "difference": -0.001064059388393343,
        "worse": 0.5366666666666666,
        "kept_right": 0.006781633462308525,
        "kept_wrong": 0.0012489936816342901,
        "handed_off": 0.9919693728560579
      },
      "usd_per_1000": 5.335035714679798
    },
    {
      "decision": 340,
      "frontier": 191,
      "frontier_accuracy": 0.9740673339399454,
      "in_sample": {
        "threshold": 0.9105755929840016,
        "answered": 0.006369426751592357,
        "accuracy": 0.9740673339399454
      },
      "curve": [
        [
          0.0,
          0.9741
        ],
        [
          0.01,
          0.9736
        ],
        [
          0.02,
          0.9691
        ],
        [
          0.03,
          0.9641
        ],
        [
          0.04,
          0.9609
        ],
        [
          0.05,
          0.9572
        ],
        [
          0.0601,
          0.9531
        ],
        [
          0.0705,
          0.95
        ],
        [
          0.0805,
          0.944
        ],
        [
          0.0905,
          0.939
        ],
        [
          0.1005,
          0.9349
        ],
        [
          0.1106,
          0.9277
        ],
        [
          0.121,
          0.9227
        ],
        [
          0.131,
          0.9172
        ],
        [
          0.141,
          0.9117
        ],
        [
          0.151,
          0.904
        ],
        [
          0.1611,
          0.8985
        ],
        [
          0.1715,
          0.8935
        ],
        [
          0.1815,
          0.8872
        ],
        [
          0.1915,
          0.8799
        ],
        [
          0.2015,
          0.8726
        ],
        [
          0.2116,
          0.8662
        ],
        [
          0.222,
          0.8603
        ],
        [
          0.232,
          0.854
        ],
        [
          0.242,
          0.8471
        ],
        [
          0.252,
          0.8426
        ],
        [
          0.2621,
          0.8362
        ],
        [
          0.2725,
          0.828
        ],
        [
          0.2825,
          0.8207
        ],
        [
          0.2925,
          0.8139
        ],
        [
          0.3025,
          0.8062
        ],
        [
          0.3126,
          0.7985
        ],
        [
          0.323,
          0.7912
        ],
        [
          0.333,
          0.7853
        ],
        [
          0.343,
          0.7807
        ],
        [
          0.353,
          0.7739
        ],
        [
          0.3631,
          0.7671
        ],
        [
          0.3735,
          0.7607
        ],
        [
          0.3835,
          0.753
        ],
        [
          0.3935,
          0.747
        ],
        [
          0.4035,
          0.7407
        ],
        [
          0.4136,
          0.7334
        ],
        [
          0.424,
          0.7266
        ],
        [
          0.434,
          0.7197
        ],
        [
          0.444,
          0.7129
        ],
        [
          0.454,
          0.7061
        ],
        [
          0.4641,
          0.6993
        ],
        [
          0.4745,
          0.6934
        ],
        [
          0.4845,
          0.6861
        ],
        [
          0.4945,
          0.6783
        ],
        [
          0.5045,
          0.6715
        ],
        [
          0.5146,
          0.6629
        ],
        [
          0.525,
          0.6551
        ],
        [
          0.535,
          0.6492
        ],
        [
          0.545,
          0.641
        ],
        [
          0.5551,
          0.6347
        ],
        [
          0.5655,
          0.626
        ],
        [
          0.5755,
          0.6174
        ],
        [
          0.5855,
          0.6087
        ],
        [
          0.5955,
          0.6005
        ],
        [
          0.6056,
          0.5928
        ],
        [
          0.616,
          0.5855
        ],
        [
          0.626,
          0.5783
        ],
        [
          0.636,
          0.5701
        ],
        [
          0.646,
          0.5619
        ],
        [
          0.6561,
          0.5541
        ],
        [
          0.6665,
          0.5473
        ],
        [
          0.6765,
          0.54
        ],
        [
          0.6865,
          0.5318
        ],
        [
          0.6965,
          0.5255
        ],
        [
          0.7066,
          0.5182
        ],
        [
          0.717,
          0.51
        ],
        [
          0.727,
          0.5041
        ],
        [
          0.737,
          0.4955
        ],
        [
          0.747,
          0.4868
        ],
        [
          0.7571,
          0.4813
        ],
        [
          0.7675,
          0.4745
        ],
        [
          0.7775,
          0.4668
        ],
        [
          0.7875,
          0.4595
        ],
        [
          0.7975,
          0.4513
        ],
        [
          0.8076,
          0.4431
        ],
        [
          0.818,
          0.4327
        ],
        [
          0.828,
          0.4249
        ],
        [
          0.838,
          0.4181
        ],
        [
          0.848,
          0.4095
        ],
        [
          0.8581,
          0.4017
        ],
        [
          0.8685,
          0.3944
        ],
        [
          0.8785,
          0.3867
        ],
        [
          0.8885,
          0.3813
        ],
        [
          0.8985,
          0.3731
        ],
        [
          0.9086,
          0.3658
        ],
        [
          0.919,
          0.3581
        ],
        [
          0.929,
          0.3494
        ],
        [
          0.939,
          0.3412
        ],
        [
          0.949,
          0.3335
        ],
        [
          0.9591,
          0.3248
        ],
        [
          0.9695,
          0.3171
        ],
        [
          0.9795,
          0.3098
        ],
        [
          0.9895,
          0.303
        ],
        [
          0.9995,
          0.2944
        ],
        [
          1.0,
          0.2939
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.008030627143942811,
        "difference": -0.001064059388393343,
        "worse": 0.5366666666666666,
        "kept_right": 0.006781633462308525,
        "kept_wrong": 0.0012489936816342901,
        "handed_off": 0.9919693728560579
      },
      "usd_per_1000": 26.522975619386557
    },
    {
      "decision": 340,
      "frontier": 188,
      "frontier_accuracy": 0.9713375796178344,
      "in_sample": {
        "threshold": 0.9105755929840016,
        "answered": 0.006369426751592357,
        "accuracy": 0.9713375796178344
      },
      "curve": [
        [
          0.0,
          0.9713
        ],
        [
          0.01,
          0.9709
        ],
        [
          0.02,
          0.9663
        ],
        [
          0.03,
          0.9613
        ],
        [
          0.04,
          0.9581
        ],
        [
          0.05,
          0.9545
        ],
        [
          0.0601,
          0.9504
        ],
        [
          0.0705,
          0.9468
        ],
        [
          0.0805,
          0.9413
        ],
        [
          0.0905,
          0.9363
        ],
        [
          0.1005,
          0.9327
        ],
        [
          0.1106,
          0.9258
        ],
        [
          0.121,
          0.9208
        ],
        [
          0.131,
          0.9154
        ],
        [
          0.141,
          0.9099
        ],
        [
          0.151,
          0.9022
        ],
        [
          0.1611,
          0.8967
        ],
        [
          0.1715,
          0.8917
        ],
        [
          0.1815,
          0.8844
        ],
        [
          0.1915,
          0.8772
        ],
        [
          0.2015,
          0.8699
        ],
        [
          0.2116,
          0.8635
        ],
        [
          0.222,
          0.8576
        ],
        [
          0.232,
          0.8512
        ],
        [
          0.242,
          0.8444
        ],
        [
          0.252,
          0.8399
        ],
        [
          0.2621,
          0.833
        ],
        [
          0.2725,
          0.8253
        ],
        [
          0.2825,
          0.8185
        ],
        [
          0.2925,
          0.8112
        ],
        [
          0.3025,
          0.8039
        ],
        [
          0.3126,
          0.7962
        ],
        [
          0.323,
          0.7889
        ],
        [
          0.333,
          0.783
        ],
        [
          0.343,
          0.778
        ],
        [
          0.353,
          0.7716
        ],
        [
          0.3631,
          0.7657
        ],
        [
          0.3735,
          0.7589
        ],
        [
          0.3835,
          0.752
        ],
        [
          0.3935,
          0.7461
        ],
        [
          0.4035,
          0.7398
        ],
        [
          0.4136,
          0.7325
        ],
        [
          0.424,
          0.7257
        ],
        [
          0.434,
          0.7188
        ],
        [
          0.444,
          0.7116
        ],
        [
          0.454,
          0.7047
        ],
        [
          0.4641,
          0.6979
        ],
        [
          0.4745,
          0.692
        ],
        [
          0.4845,
          0.6847
        ],
        [
          0.4945,
          0.677
        ],
        [
          0.5045,
          0.6706
        ],
        [
          0.5146,
          0.6624
        ],
        [
          0.525,
          0.6551
        ],
        [
          0.535,
          0.6488
        ],
        [
          0.545,
          0.6406
        ],
        [
          0.5551,
          0.6342
        ],
        [
          0.5655,
          0.6256
        ],
        [
          0.5755,
          0.6169
        ],
        [
          0.5855,
          0.6083
        ],
        [
          0.5955,
          0.6005
        ],
        [
          0.6056,
          0.5928
        ],
        [
          0.616,
          0.586
        ],
        [
          0.626,
          0.5796
        ],
        [
          0.636,
          0.5719
        ],
        [
          0.646,
          0.5637
        ],
        [
          0.6561,
          0.5564
        ],
        [
          0.6665,
          0.5487
        ],
        [
          0.6765,
          0.5414
        ],
        [
          0.6865,
          0.5332
        ],
        [
          0.6965,
          0.5268
        ],
        [
          0.7066,
          0.52
        ],
        [
          0.717,
          0.5118
        ],
        [
          0.727,
          0.505
        ],
        [
          0.737,
          0.4964
        ],
        [
          0.747,
          0.4882
        ],
        [
          0.7571,
          0.4827
        ],
        [
          0.7675,
          0.4759
        ],
        [
          0.7775,
          0.4682
        ],
        [
          0.7875,
          0.4604
        ],
        [
          0.7975,
          0.4518
        ],
        [
          0.8076,
          0.4436
        ],
        [
          0.818,
          0.4331
        ],
        [
          0.828,
          0.4254
        ],
        [
          0.838,
          0.4186
        ],
        [
          0.848,
          0.4099
        ],
        [
          0.8581,
          0.4026
        ],
        [
          0.8685,
          0.3949
        ],
        [
          0.8785,
          0.3867
        ],
        [
          0.8885,
          0.3817
        ],
        [
          0.8985,
          0.374
        ],
        [
          0.9086,
          0.3667
        ],
        [
          0.919,
          0.359
        ],
        [
          0.929,
          0.3499
        ],
        [
          0.939,
          0.3417
        ],
        [
          0.949,
          0.3344
        ],
        [
          0.9591,
          0.3258
        ],
        [
          0.9695,
          0.318
        ],
        [
          0.9795,
          0.3103
        ],
        [
          0.9895,
          0.3035
        ],
        [
          0.9995,
          0.2944
        ],
        [
          1.0,
          0.2939
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.008030627143942811,
        "difference": -0.001064059388393343,
        "worse": 0.5366666666666666,
        "kept_right": 0.006781633462308525,
        "kept_wrong": 0.0012489936816342901,
        "handed_off": 0.9919693728560579
      },
      "usd_per_1000": 0.9557028056014243
    },
    {
      "decision": 340,
      "frontier": 190,
      "frontier_accuracy": 0.9672429481346679,
      "in_sample": {
        "threshold": 0.9105755929840016,
        "answered": 0.006369426751592357,
        "accuracy": 0.9672429481346679
      },
      "curve": [
        [
          0.0,
          0.9672
        ],
        [
          0.01,
          0.9668
        ],
        [
          0.02,
          0.9622
        ],
        [
          0.03,
          0.9577
        ],
        [
          0.04,
          0.9545
        ],
        [
          0.05,
          0.9509
        ],
        [
          0.0601,
          0.9468
        ],
        [
          0.0705,
          0.9436
        ],
        [
          0.0805,
          0.9377
        ],
        [
          0.0905,
          0.9327
        ],
        [
          0.1005,
          0.9286
        ],
        [
          0.1106,
          0.9213
        ],
        [
          0.121,
          0.9167
        ],
        [
          0.131,
          0.9113
        ],
        [
          0.141,
          0.9058
        ],
        [
          0.151,
          0.8981
        ],
        [
          0.1611,
          0.8926
        ],
        [
          0.1715,
          0.8876
        ],
        [
          0.1815,
          0.8813
        ],
        [
          0.1915,
          0.8744
        ],
        [
          0.2015,
          0.8672
        ],
        [
          0.2116,
          0.8608
        ],
        [
          0.222,
          0.8549
        ],
        [
          0.232,
          0.8485
        ],
        [
          0.242,
          0.8412
        ],
        [
          0.252,
          0.8367
        ],
        [
          0.2621,
          0.8303
        ],
        [
          0.2725,
          0.8221
        ],
        [
          0.2825,
          0.8148
        ],
        [
          0.2925,
          0.808
        ],
        [
          0.3025,
          0.8003
        ],
        [
          0.3126,
          0.793
        ],
        [
          0.323,
          0.7857
        ],
        [
          0.333,
          0.7798
        ],
        [
          0.343,
          0.7748
        ],
        [
          0.353,
          0.7675
        ],
        [
          0.3631,
          0.7611
        ],
        [
          0.3735,
          0.7548
        ],
        [
          0.3835,
          0.7475
        ],
        [
          0.3935,
          0.7416
        ],
        [
          0.4035,
          0.7352
        ],
        [
          0.4136,
          0.7279
        ],
        [
          0.424,
          0.7211
        ],
        [
          0.434,
          0.7143
        ],
        [
          0.444,
          0.7075
        ],
        [
          0.454,
          0.7011
        ],
        [
          0.4641,
          0.6943
        ],
        [
          0.4745,
          0.6888
        ],
        [
          0.4845,
          0.6815
        ],
        [
          0.4945,
          0.6742
        ],
        [
          0.5045,
          0.6679
        ],
        [
          0.5146,
          0.6592
        ],
        [
          0.525,
          0.652
        ],
        [
          0.535,
          0.646
        ],
        [
          0.545,
          0.6379
        ],
        [
          0.5551,
          0.6315
        ],
        [
          0.5655,
          0.6228
        ],
        [
          0.5755,
          0.6142
        ],
        [
          0.5855,
          0.6056
        ],
        [
          0.5955,
          0.5974
        ],
        [
          0.6056,
          0.5901
        ],
        [
          0.616,
          0.5828
        ],
        [
          0.626,
          0.5755
        ],
        [
          0.636,
          0.5678
        ],
        [
          0.646,
          0.5596
        ],
        [
          0.6561,
          0.5519
        ],
        [
          0.6665,
          0.5446
        ],
        [
          0.6765,
          0.5373
        ],
        [
          0.6865,
          0.5291
        ],
        [
          0.6965,
          0.5227
        ],
        [
          0.7066,
          0.5155
        ],
        [
          0.717,
          0.5068
        ],
        [
          0.727,
          0.5009
        ],
        [
          0.737,
          0.4923
        ],
        [
          0.747,
          0.4841
        ],
        [
          0.7571,
          0.4786
        ],
        [
          0.7675,
          0.4718
        ],
        [
          0.7775,
          0.4641
        ],
        [
          0.7875,
          0.4563
        ],
        [
          0.7975,
          0.4481
        ],
        [
          0.8076,
          0.4404
        ],
        [
          0.818,
          0.4299
        ],
        [
          0.828,
          0.4227
        ],
        [
          0.838,
          0.4163
        ],
        [
          0.848,
          0.4081
        ],
        [
          0.8581,
          0.4004
        ],
        [
          0.8685,
          0.3931
        ],
        [
          0.8785,
          0.3858
        ],
        [
          0.8885,
          0.3808
        ],
        [
          0.8985,
          0.3726
        ],
        [
          0.9086,
          0.3653
        ],
        [
          0.919,
          0.3576
        ],
        [
          0.929,
          0.349
        ],
        [
          0.939,
          0.3408
        ],
        [
          0.949,
          0.333
        ],
        [
          0.9591,
          0.3248
        ],
        [
          0.9695,
          0.3171
        ],
        [
          0.9795,
          0.3094
        ],
        [
          0.9895,
          0.303
        ],
        [
          0.9995,
          0.2944
        ],
        [
          1.0,
          0.2939
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.008030627143942811,
        "difference": -0.001064059388393343,
        "worse": 0.5366666666666666,
        "kept_right": 0.006781633462308525,
        "kept_wrong": 0.0012489936816342901,
        "handed_off": 0.9919693728560579
      },
      "usd_per_1000": 5.523800317316646
    },
    {
      "decision": 340,
      "frontier": 124,
      "frontier_accuracy": 0.9667879890809827,
      "in_sample": {
        "threshold": 0.9105755929840016,
        "answered": 0.006369426751592357,
        "accuracy": 0.9667879890809827
      },
      "curve": [
        [
          0.0,
          0.9668
        ],
        [
          0.01,
          0.9663
        ],
        [
          0.02,
          0.9618
        ],
        [
          0.03,
          0.9572
        ],
        [
          0.04,
          0.954
        ],
        [
          0.05,
          0.9504
        ],
        [
          0.0601,
          0.9468
        ],
        [
          0.0705,
          0.9436
        ],
        [
          0.0805,
          0.9377
        ],
        [
          0.0905,
          0.9327
        ],
        [
          0.1005,
          0.9286
        ],
        [
          0.1106,
          0.9213
        ],
        [
          0.121,
          0.9163
        ],
        [
          0.131,
          0.9108
        ],
        [
          0.141,
          0.9058
        ],
        [
          0.151,
          0.8981
        ],
        [
          0.1611,
          0.8926
        ],
        [
          0.1715,
          0.8876
        ],
        [
          0.1815,
          0.8813
        ],
        [
          0.1915,
          0.8744
        ],
        [
          0.2015,
          0.8681
        ],
        [
          0.2116,
          0.8617
        ],
        [
          0.222,
          0.8558
        ],
        [
          0.232,
          0.8494
        ],
        [
          0.242,
          0.8421
        ],
        [
          0.252,
          0.8376
        ],
        [
          0.2621,
          0.8312
        ],
        [
          0.2725,
          0.823
        ],
        [
          0.2825,
          0.8157
        ],
        [
          0.2925,
          0.8085
        ],
        [
          0.3025,
          0.8012
        ],
        [
          0.3126,
          0.7939
        ],
        [
          0.323,
          0.7866
        ],
        [
          0.333,
          0.7807
        ],
        [
          0.343,
          0.7757
        ],
        [
          0.353,
          0.7689
        ],
        [
          0.3631,
          0.7621
        ],
        [
          0.3735,
          0.7552
        ],
        [
          0.3835,
          0.748
        ],
        [
          0.3935,
          0.742
        ],
        [
          0.4035,
          0.7357
        ],
        [
          0.4136,
          0.7284
        ],
        [
          0.424,
          0.7216
        ],
        [
          0.434,
          0.7147
        ],
        [
          0.444,
          0.7088
        ],
        [
          0.454,
          0.7025
        ],
        [
          0.4641,
          0.6956
        ],
        [
          0.4745,
          0.6897
        ],
        [
          0.4845,
          0.6824
        ],
        [
          0.4945,
          0.6752
        ],
        [
          0.5045,
          0.6688
        ],
        [
          0.5146,
          0.6601
        ],
        [
          0.525,
          0.6524
        ],
        [
          0.535,
          0.6465
        ],
        [
          0.545,
          0.6383
        ],
        [
          0.5551,
          0.6319
        ],
        [
          0.5655,
          0.6233
        ],
        [
          0.5755,
          0.6146
        ],
        [
          0.5855,
          0.606
        ],
        [
          0.5955,
          0.5978
        ],
        [
          0.6056,
          0.5905
        ],
        [
          0.616,
          0.5837
        ],
        [
          0.626,
          0.576
        ],
        [
          0.636,
          0.5678
        ],
        [
          0.646,
          0.5596
        ],
        [
          0.6561,
          0.5519
        ],
        [
          0.6665,
          0.5446
        ],
        [
          0.6765,
          0.5373
        ],
        [
          0.6865,
          0.5291
        ],
        [
          0.6965,
          0.5227
        ],
        [
          0.7066,
          0.5155
        ],
        [
          0.717,
          0.5068
        ],
        [
          0.727,
          0.5009
        ],
        [
          0.737,
          0.4923
        ],
        [
          0.747,
          0.4841
        ],
        [
          0.7571,
          0.4782
        ],
        [
          0.7675,
          0.4713
        ],
        [
          0.7775,
          0.4636
        ],
        [
          0.7875,
          0.4563
        ],
        [
          0.7975,
          0.4481
        ],
        [
          0.8076,
          0.4404
        ],
        [
          0.818,
          0.4299
        ],
        [
          0.828,
          0.4227
        ],
        [
          0.838,
          0.4158
        ],
        [
          0.848,
          0.4072
        ],
        [
          0.8581,
          0.3999
        ],
        [
          0.8685,
          0.3922
        ],
        [
          0.8785,
          0.3849
        ],
        [
          0.8885,
          0.3794
        ],
        [
          0.8985,
          0.3712
        ],
        [
          0.9086,
          0.364
        ],
        [
          0.919,
          0.3567
        ],
        [
          0.929,
          0.3476
        ],
        [
          0.939,
          0.3394
        ],
        [
          0.949,
          0.333
        ],
        [
          0.9591,
          0.3244
        ],
        [
          0.9695,
          0.3171
        ],
        [
          0.9795,
          0.3094
        ],
        [
          0.9895,
          0.303
        ],
        [
          0.9995,
          0.2944
        ],
        [
          1.0,
          0.2939
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.008030627143942811,
        "difference": -0.001064059388393343,
        "worse": 0.5366666666666666,
        "kept_right": 0.006781633462308525,
        "kept_wrong": 0.0012489936816342901,
        "handed_off": 0.9919693728560579
      },
      "usd_per_1000": 1.1212954617940154
    },
    {
      "decision": 340,
      "frontier": 278,
      "frontier_accuracy": 0.9499545040946314,
      "in_sample": {
        "threshold": 0.9105755929840016,
        "answered": 0.006369426751592357,
        "accuracy": 0.9499545040946314
      },
      "curve": [
        [
          0.0,
          0.95
        ],
        [
          0.01,
          0.9495
        ],
        [
          0.02,
          0.9449
        ],
        [
          0.03,
          0.9399
        ],
        [
          0.04,
          0.9372
        ],
        [
          0.05,
          0.934
        ],
        [
          0.0601,
          0.9304
        ],
        [
          0.0705,
          0.9272
        ],
        [
          0.0805,
          0.9213
        ],
        [
          0.0905,
          0.9172
        ],
        [
          0.1005,
          0.9136
        ],
        [
          0.1106,
          0.9063
        ],
        [
          0.121,
          0.9013
        ],
        [
          0.131,
          0.8958
        ],
        [
          0.141,
          0.8908
        ],
        [
          0.151,
          0.8835
        ],
        [
          0.1611,
          0.8785
        ],
        [
          0.1715,
          0.874
        ],
        [
          0.1815,
          0.8672
        ],
        [
          0.1915,
          0.8603
        ],
        [
          0.2015,
          0.853
        ],
        [
          0.2116,
          0.8467
        ],
        [
          0.222,
          0.8408
        ],
        [
          0.232,
          0.8344
        ],
        [
          0.242,
          0.8285
        ],
        [
          0.252,
          0.8239
        ],
        [
          0.2621,
          0.8176
        ],
        [
          0.2725,
          0.8098
        ],
        [
          0.2825,
          0.8025
        ],
        [
          0.2925,
          0.7953
        ],
        [
          0.3025,
          0.788
        ],
        [
          0.3126,
          0.7807
        ],
        [
          0.323,
          0.7739
        ],
        [
          0.333,
          0.768
        ],
        [
          0.343,
          0.7639
        ],
        [
          0.353,
          0.7571
        ],
        [
          0.3631,
          0.7507
        ],
        [
          0.3735,
          0.7439
        ],
        [
          0.3835,
          0.7366
        ],
        [
          0.3935,
          0.7311
        ],
        [
          0.4035,
          0.7247
        ],
        [
          0.4136,
          0.7179
        ],
        [
          0.424,
          0.7116
        ],
        [
          0.434,
          0.7061
        ],
        [
          0.444,
          0.6997
        ],
        [
          0.454,
          0.6938
        ],
        [
          0.4641,
          0.687
        ],
        [
          0.4745,
          0.6811
        ],
        [
          0.4845,
          0.6738
        ],
        [
          0.4945,
          0.6674
        ],
        [
          0.5045,
          0.6615
        ],
        [
          0.5146,
          0.6538
        ],
        [
          0.525,
          0.646
        ],
        [
          0.535,
          0.6397
        ],
        [
          0.545,
          0.6319
        ],
        [
          0.5551,
          0.626
        ],
        [
          0.5655,
          0.6174
        ],
        [
          0.5755,
          0.6092
        ],
        [
          0.5855,
          0.601
        ],
        [
          0.5955,
          0.5928
        ],
        [
          0.6056,
          0.5851
        ],
        [
          0.616,
          0.5783
        ],
        [
          0.626,
          0.571
        ],
        [
          0.636,
          0.5632
        ],
        [
          0.646,
          0.5551
        ],
        [
          0.6561,
          0.5473
        ],
        [
          0.6665,
          0.5396
        ],
        [
          0.6765,
          0.5323
        ],
        [
          0.6865,
          0.5241
        ],
        [
          0.6965,
          0.5177
        ],
        [
          0.7066,
          0.5109
        ],
        [
          0.717,
          0.5023
        ],
        [
          0.727,
          0.4964
        ],
        [
          0.737,
          0.4891
        ],
        [
          0.747,
          0.4809
        ],
        [
          0.7571,
          0.4768
        ],
        [
          0.7675,
          0.47
        ],
        [
          0.7775,
          0.4622
        ],
        [
          0.7875,
          0.4545
        ],
        [
          0.7975,
          0.4468
        ],
        [
          0.8076,
          0.439
        ],
        [
          0.818,
          0.4286
        ],
        [
          0.828,
          0.4208
        ],
        [
          0.838,
          0.414
        ],
        [
          0.848,
          0.4054
        ],
        [
          0.8581,
          0.3981
        ],
        [
          0.8685,
          0.3904
        ],
        [
          0.8785,
          0.3826
        ],
        [
          0.8885,
          0.3785
        ],
        [
          0.8985,
          0.3703
        ],
        [
          0.9086,
          0.3631
        ],
        [
          0.919,
          0.3558
        ],
        [
          0.929,
          0.3467
        ],
        [
          0.939,
          0.3385
        ],
        [
          0.949,
          0.3317
        ],
        [
          0.9591,
          0.323
        ],
        [
          0.9695,
          0.3157
        ],
        [
          0.9795,
          0.3094
        ],
        [
          0.9895,
          0.303
        ],
        [
          0.9995,
          0.2944
        ],
        [
          1.0,
          0.2939
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.008030627143942811,
        "difference": -0.001064059388393343,
        "worse": 0.5366666666666666,
        "kept_right": 0.006781633462308525,
        "kept_wrong": 0.0012489936816342901,
        "handed_off": 0.9919693728560579
      },
      "usd_per_1000": 6.4565563395003505
    },
    {
      "decision": 340,
      "frontier": 275,
      "frontier_accuracy": 0.9426751592356688,
      "in_sample": {
        "threshold": 0.9105755929840016,
        "answered": 0.006369426751592357,
        "accuracy": 0.9426751592356688
      },
      "curve": [
        [
          0.0,
          0.9427
        ],
        [
          0.01,
          0.9422
        ],
        [
          0.02,
          0.9377
        ],
        [
          0.03,
          0.9331
        ],
        [
          0.04,
          0.9299
        ],
        [
          0.05,
          0.9268
        ],
        [
          0.0601,
          0.9227
        ],
        [
          0.0705,
          0.9195
        ],
        [
          0.0805,
          0.9136
        ],
        [
          0.0905,
          0.9086
        ],
        [
          0.1005,
          0.9045
        ],
        [
          0.1106,
          0.8972
        ],
        [
          0.121,
          0.8926
        ],
        [
          0.131,
          0.8867
        ],
        [
          0.141,
          0.8817
        ],
        [
          0.151,
          0.874
        ],
        [
          0.1611,
          0.8685
        ],
        [
          0.1715,
          0.8635
        ],
        [
          0.1815,
          0.8571
        ],
        [
          0.1915,
          0.8499
        ],
        [
          0.2015,
          0.8435
        ],
        [
          0.2116,
          0.8371
        ],
        [
          0.222,
          0.8312
        ],
        [
          0.232,
          0.8248
        ],
        [
          0.242,
          0.8189
        ],
        [
          0.252,
          0.8153
        ],
        [
          0.2621,
          0.8089
        ],
        [
          0.2725,
          0.8012
        ],
        [
          0.2825,
          0.7939
        ],
        [
          0.2925,
          0.7866
        ],
        [
          0.3025,
          0.7793
        ],
        [
          0.3126,
          0.7716
        ],
        [
          0.323,
          0.7648
        ],
        [
          0.333,
          0.7593
        ],
        [
          0.343,
          0.7552
        ],
        [
          0.353,
          0.7489
        ],
        [
          0.3631,
          0.742
        ],
        [
          0.3735,
          0.7357
        ],
        [
          0.3835,
          0.7284
        ],
        [
          0.3935,
          0.7225
        ],
        [
          0.4035,
          0.7166
        ],
        [
          0.4136,
          0.7102
        ],
        [
          0.424,
          0.7038
        ],
        [
          0.434,
          0.697
        ],
        [
          0.444,
          0.6906
        ],
        [
          0.454,
          0.6843
        ],
        [
          0.4641,
          0.6774
        ],
        [
          0.4745,
          0.6715
        ],
        [
          0.4845,
          0.6642
        ],
        [
          0.4945,
          0.6579
        ],
        [
          0.5045,
          0.652
        ],
        [
          0.5146,
          0.6438
        ],
        [
          0.525,
          0.636
        ],
        [
          0.535,
          0.6301
        ],
        [
          0.545,
          0.6224
        ],
        [
          0.5551,
          0.616
        ],
        [
          0.5655,
          0.6074
        ],
        [
          0.5755,
          0.5987
        ],
        [
          0.5855,
          0.5905
        ],
        [
          0.5955,
          0.5837
        ],
        [
          0.6056,
          0.5764
        ],
        [
          0.616,
          0.5692
        ],
        [
          0.626,
          0.5623
        ],
        [
          0.636,
          0.5541
        ],
        [
          0.646,
          0.5464
        ],
        [
          0.6561,
          0.5387
        ],
        [
          0.6665,
          0.5318
        ],
        [
          0.6765,
          0.5246
        ],
        [
          0.6865,
          0.5164
        ],
        [
          0.6965,
          0.51
        ],
        [
          0.7066,
          0.5032
        ],
        [
          0.717,
          0.4964
        ],
        [
          0.727,
          0.4909
        ],
        [
          0.737,
          0.4832
        ],
        [
          0.747,
          0.475
        ],
        [
          0.7571,
          0.4704
        ],
        [
          0.7675,
          0.4636
        ],
        [
          0.7775,
          0.4559
        ],
        [
          0.7875,
          0.4481
        ],
        [
          0.7975,
          0.4413
        ],
        [
          0.8076,
          0.4345
        ],
        [
          0.818,
          0.424
        ],
        [
          0.828,
          0.4172
        ],
        [
          0.838,
          0.4108
        ],
        [
          0.848,
          0.4026
        ],
        [
          0.8581,
          0.3958
        ],
        [
          0.8685,
          0.3894
        ],
        [
          0.8785,
          0.3822
        ],
        [
          0.8885,
          0.3772
        ],
        [
          0.8985,
          0.3694
        ],
        [
          0.9086,
          0.3621
        ],
        [
          0.919,
          0.3544
        ],
        [
          0.929,
          0.3467
        ],
        [
          0.939,
          0.3385
        ],
        [
          0.949,
          0.3321
        ],
        [
          0.9591,
          0.3239
        ],
        [
          0.9695,
          0.3162
        ],
        [
          0.9795,
          0.3094
        ],
        [
          0.9895,
          0.303
        ],
        [
          0.9995,
          0.2944
        ],
        [
          1.0,
          0.2939
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.008030627143942811,
        "difference": -0.001064059388393343,
        "worse": 0.5366666666666666,
        "kept_right": 0.006781633462308525,
        "kept_wrong": 0.0012489936816342901,
        "handed_off": 0.9919693728560579
      },
      "usd_per_1000": 0.510381330192409
    },
    {
      "decision": 340,
      "frontier": 192,
      "frontier_accuracy": 0.9331210191082803,
      "in_sample": {
        "threshold": 0.9105755929840016,
        "answered": 0.006369426751592357,
        "accuracy": 0.9331210191082803
      },
      "curve": [
        [
          0.0,
          0.9331
        ],
        [
          0.01,
          0.9327
        ],
        [
          0.02,
          0.9281
        ],
        [
          0.03,
          0.9236
        ],
        [
          0.04,
          0.9213
        ],
        [
          0.05,
          0.9181
        ],
        [
          0.0601,
          0.9145
        ],
        [
          0.0705,
          0.9108
        ],
        [
          0.0805,
          0.9049
        ],
        [
          0.0905,
          0.9004
        ],
        [
          0.1005,
          0.8963
        ],
        [
          0.1106,
          0.889
        ],
        [
          0.121,
          0.8849
        ],
        [
          0.131,
          0.8794
        ],
        [
          0.141,
          0.8744
        ],
        [
          0.151,
          0.8667
        ],
        [
          0.1611,
          0.8617
        ],
        [
          0.1715,
          0.8571
        ],
        [
          0.1815,
          0.8508
        ],
        [
          0.1915,
          0.8435
        ],
        [
          0.2015,
          0.8371
        ],
        [
          0.2116,
          0.8312
        ],
        [
          0.222,
          0.8253
        ],
        [
          0.232,
          0.8189
        ],
        [
          0.242,
          0.8144
        ],
        [
          0.252,
          0.8098
        ],
        [
          0.2621,
          0.8035
        ],
        [
          0.2725,
          0.7957
        ],
        [
          0.2825,
          0.7889
        ],
        [
          0.2925,
          0.7816
        ],
        [
          0.3025,
          0.7739
        ],
        [
          0.3126,
          0.7666
        ],
        [
          0.323,
          0.7598
        ],
        [
          0.333,
          0.7548
        ],
        [
          0.343,
          0.7502
        ],
        [
          0.353,
          0.7429
        ],
        [
          0.3631,
          0.737
        ],
        [
          0.3735,
          0.7307
        ],
        [
          0.3835,
          0.7234
        ],
        [
          0.3935,
          0.7175
        ],
        [
          0.4035,
          0.712
        ],
        [
          0.4136,
          0.7052
        ],
        [
          0.424,
          0.6984
        ],
        [
          0.434,
          0.692
        ],
        [
          0.444,
          0.6861
        ],
        [
          0.454,
          0.6811
        ],
        [
          0.4641,
          0.6747
        ],
        [
          0.4745,
          0.6683
        ],
        [
          0.4845,
          0.6615
        ],
        [
          0.4945,
          0.6547
        ],
        [
          0.5045,
          0.6483
        ],
        [
          0.5146,
          0.6406
        ],
        [
          0.525,
          0.6333
        ],
        [
          0.535,
          0.6274
        ],
        [
          0.545,
          0.621
        ],
        [
          0.5551,
          0.6151
        ],
        [
          0.5655,
          0.6069
        ],
        [
          0.5755,
          0.5992
        ],
        [
          0.5855,
          0.591
        ],
        [
          0.5955,
          0.5828
        ],
        [
          0.6056,
          0.5755
        ],
        [
          0.616,
          0.5696
        ],
        [
          0.626,
          0.5623
        ],
        [
          0.636,
          0.5546
        ],
        [
          0.646,
          0.5469
        ],
        [
          0.6561,
          0.5391
        ],
        [
          0.6665,
          0.5328
        ],
        [
          0.6765,
          0.5259
        ],
        [
          0.6865,
          0.5177
        ],
        [
          0.6965,
          0.5114
        ],
        [
          0.7066,
          0.5041
        ],
        [
          0.717,
          0.4955
        ],
        [
          0.727,
          0.49
        ],
        [
          0.737,
          0.4813
        ],
        [
          0.747,
          0.4736
        ],
        [
          0.7571,
          0.4704
        ],
        [
          0.7675,
          0.4641
        ],
        [
          0.7775,
          0.4568
        ],
        [
          0.7875,
          0.4495
        ],
        [
          0.7975,
          0.4422
        ],
        [
          0.8076,
          0.4354
        ],
        [
          0.818,
          0.4254
        ],
        [
          0.828,
          0.4181
        ],
        [
          0.838,
          0.4122
        ],
        [
          0.848,
          0.4035
        ],
        [
          0.8581,
          0.3963
        ],
        [
          0.8685,
          0.3899
        ],
        [
          0.8785,
          0.3817
        ],
        [
          0.8885,
          0.3776
        ],
        [
          0.8985,
          0.3703
        ],
        [
          0.9086,
          0.3631
        ],
        [
          0.919,
          0.3558
        ],
        [
          0.929,
          0.3471
        ],
        [
          0.939,
          0.3389
        ],
        [
          0.949,
          0.3326
        ],
        [
          0.9591,
          0.3239
        ],
        [
          0.9695,
          0.3167
        ],
        [
          0.9795,
          0.3094
        ],
        [
          0.9895,
          0.3035
        ],
        [
          0.9995,
          0.2944
        ],
        [
          1.0,
          0.2939
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.008030627143942811,
        "difference": -0.001064059388393343,
        "worse": 0.5366666666666666,
        "kept_right": 0.006781633462308525,
        "kept_wrong": 0.0012489936816342901,
        "handed_off": 0.9919693728560579
      },
      "usd_per_1000": 11.516333241634609
    },
    {
      "decision": 340,
      "frontier": 123,
      "frontier_accuracy": 0.9326660600545951,
      "in_sample": {
        "threshold": 0.9105755929840016,
        "answered": 0.006369426751592357,
        "accuracy": 0.9326660600545951
      },
      "curve": [
        [
          0.0,
          0.9327
        ],
        [
          0.01,
          0.9322
        ],
        [
          0.02,
          0.9277
        ],
        [
          0.03,
          0.9222
        ],
        [
          0.04,
          0.919
        ],
        [
          0.05,
          0.9163
        ],
        [
          0.0601,
          0.9122
        ],
        [
          0.0705,
          0.909
        ],
        [
          0.0805,
          0.9035
        ],
        [
          0.0905,
          0.899
        ],
        [
          0.1005,
          0.8963
        ],
        [
          0.1106,
          0.8894
        ],
        [
          0.121,
          0.8849
        ],
        [
          0.131,
          0.8794
        ],
        [
          0.141,
          0.8744
        ],
        [
          0.151,
          0.8667
        ],
        [
          0.1611,
          0.8612
        ],
        [
          0.1715,
          0.8571
        ],
        [
          0.1815,
          0.8503
        ],
        [
          0.1915,
          0.843
        ],
        [
          0.2015,
          0.8367
        ],
        [
          0.2116,
          0.8308
        ],
        [
          0.222,
          0.8244
        ],
        [
          0.232,
          0.818
        ],
        [
          0.242,
          0.8112
        ],
        [
          0.252,
          0.8066
        ],
        [
          0.2621,
          0.8003
        ],
        [
          0.2725,
          0.7934
        ],
        [
          0.2825,
          0.7866
        ],
        [
          0.2925,
          0.7793
        ],
        [
          0.3025,
          0.7725
        ],
        [
          0.3126,
          0.7648
        ],
        [
          0.323,
          0.7575
        ],
        [
          0.333,
          0.752
        ],
        [
          0.343,
          0.747
        ],
        [
          0.353,
          0.7402
        ],
        [
          0.3631,
          0.7343
        ],
        [
          0.3735,
          0.7279
        ],
        [
          0.3835,
          0.7207
        ],
        [
          0.3935,
          0.7161
        ],
        [
          0.4035,
          0.7097
        ],
        [
          0.4136,
          0.7029
        ],
        [
          0.424,
          0.6965
        ],
        [
          0.434,
          0.6897
        ],
        [
          0.444,
          0.6829
        ],
        [
          0.454,
          0.6774
        ],
        [
          0.4641,
          0.6706
        ],
        [
          0.4745,
          0.6647
        ],
        [
          0.4845,
          0.6574
        ],
        [
          0.4945,
          0.6515
        ],
        [
          0.5045,
          0.6447
        ],
        [
          0.5146,
          0.6365
        ],
        [
          0.525,
          0.6288
        ],
        [
          0.535,
          0.6242
        ],
        [
          0.545,
          0.616
        ],
        [
          0.5551,
          0.6101
        ],
        [
          0.5655,
          0.6015
        ],
        [
          0.5755,
          0.5933
        ],
        [
          0.5855,
          0.5851
        ],
        [
          0.5955,
          0.5773
        ],
        [
          0.6056,
          0.5705
        ],
        [
          0.616,
          0.5637
        ],
        [
          0.626,
          0.5569
        ],
        [
          0.636,
          0.5505
        ],
        [
          0.646,
          0.5441
        ],
        [
          0.6561,
          0.5378
        ],
        [
          0.6665,
          0.5318
        ],
        [
          0.6765,
          0.5246
        ],
        [
          0.6865,
          0.5168
        ],
        [
          0.6965,
          0.5105
        ],
        [
          0.7066,
          0.5041
        ],
        [
          0.717,
          0.4968
        ],
        [
          0.727,
          0.4904
        ],
        [
          0.737,
          0.4827
        ],
        [
          0.747,
          0.4754
        ],
        [
          0.7571,
          0.4704
        ],
        [
          0.7675,
          0.4641
        ],
        [
          0.7775,
          0.4568
        ],
        [
          0.7875,
          0.449
        ],
        [
          0.7975,
          0.4409
        ],
        [
          0.8076,
          0.4336
        ],
        [
          0.818,
          0.4236
        ],
        [
          0.828,
          0.4158
        ],
        [
          0.838,
          0.4095
        ],
        [
          0.848,
          0.4013
        ],
        [
          0.8581,
          0.3949
        ],
        [
          0.8685,
          0.3881
        ],
        [
          0.8785,
          0.3808
        ],
        [
          0.8885,
          0.3767
        ],
        [
          0.8985,
          0.3685
        ],
        [
          0.9086,
          0.3612
        ],
        [
          0.919,
          0.3535
        ],
        [
          0.929,
          0.3444
        ],
        [
          0.939,
          0.3371
        ],
        [
          0.949,
          0.3308
        ],
        [
          0.9591,
          0.323
        ],
        [
          0.9695,
          0.3167
        ],
        [
          0.9795,
          0.3098
        ],
        [
          0.9895,
          0.303
        ],
        [
          0.9995,
          0.2944
        ],
        [
          1.0,
          0.2939
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.008030627143942811,
        "difference": -0.001064059388393343,
        "worse": 0.5366666666666666,
        "kept_right": 0.006781633462308525,
        "kept_wrong": 0.0012489936816342901,
        "handed_off": 0.9919693728560579
      },
      "usd_per_1000": 0.2664270424668523
    },
    {
      "decision": 340,
      "frontier": 274,
      "frontier_accuracy": 0.9085532302092811,
      "in_sample": {
        "threshold": 0.9105755929840016,
        "answered": 0.006369426751592357,
        "accuracy": 0.9085532302092811
      },
      "curve": [
        [
          0.0,
          0.9086
        ],
        [
          0.01,
          0.9081
        ],
        [
          0.02,
          0.9035
        ],
        [
          0.03,
          0.8985
        ],
        [
          0.04,
          0.8954
        ],
        [
          0.05,
          0.8922
        ],
        [
          0.0601,
          0.8881
        ],
        [
          0.0705,
          0.8849
        ],
        [
          0.0805,
          0.879
        ],
        [
          0.0905,
          0.8744
        ],
        [
          0.1005,
          0.8708
        ],
        [
          0.1106,
          0.8635
        ],
        [
          0.121,
          0.859
        ],
        [
          0.131,
          0.8535
        ],
        [
          0.141,
          0.8485
        ],
        [
          0.151,
          0.8408
        ],
        [
          0.1611,
          0.8362
        ],
        [
          0.1715,
          0.8312
        ],
        [
          0.1815,
          0.8253
        ],
        [
          0.1915,
          0.8185
        ],
        [
          0.2015,
          0.8121
        ],
        [
          0.2116,
          0.8057
        ],
        [
          0.222,
          0.7998
        ],
        [
          0.232,
          0.7934
        ],
        [
          0.242,
          0.788
        ],
        [
          0.252,
          0.7843
        ],
        [
          0.2621,
          0.778
        ],
        [
          0.2725,
          0.7702
        ],
        [
          0.2825,
          0.7634
        ],
        [
          0.2925,
          0.7566
        ],
        [
          0.3025,
          0.7493
        ],
        [
          0.3126,
          0.742
        ],
        [
          0.323,
          0.7352
        ],
        [
          0.333,
          0.7302
        ],
        [
          0.343,
          0.7261
        ],
        [
          0.353,
          0.7188
        ],
        [
          0.3631,
          0.712
        ],
        [
          0.3735,
          0.7056
        ],
        [
          0.3835,
          0.6984
        ],
        [
          0.3935,
          0.6934
        ],
        [
          0.4035,
          0.6874
        ],
        [
          0.4136,
          0.6811
        ],
        [
          0.424,
          0.6756
        ],
        [
          0.434,
          0.6692
        ],
        [
          0.444,
          0.6633
        ],
        [
          0.454,
          0.657
        ],
        [
          0.4641,
          0.6501
        ],
        [
          0.4745,
          0.6438
        ],
        [
          0.4845,
          0.6365
        ],
        [
          0.4945,
          0.6306
        ],
        [
          0.5045,
          0.6247
        ],
        [
          0.5146,
          0.6174
        ],
        [
          0.525,
          0.6101
        ],
        [
          0.535,
          0.6046
        ],
        [
          0.545,
          0.5978
        ],
        [
          0.5551,
          0.5919
        ],
        [
          0.5655,
          0.5842
        ],
        [
          0.5755,
          0.576
        ],
        [
          0.5855,
          0.5696
        ],
        [
          0.5955,
          0.5632
        ],
        [
          0.6056,
          0.556
        ],
        [
          0.616,
          0.55
        ],
        [
          0.626,
          0.5446
        ],
        [
          0.636,
          0.5369
        ],
        [
          0.646,
          0.5296
        ],
        [
          0.6561,
          0.5227
        ],
        [
          0.6665,
          0.5168
        ],
        [
          0.6765,
          0.5109
        ],
        [
          0.6865,
          0.5032
        ],
        [
          0.6965,
          0.4968
        ],
        [
          0.7066,
          0.49
        ],
        [
          0.717,
          0.4836
        ],
        [
          0.727,
          0.4782
        ],
        [
          0.737,
          0.4713
        ],
        [
          0.747,
          0.4636
        ],
        [
          0.7571,
          0.46
        ],
        [
          0.7675,
          0.454
        ],
        [
          0.7775,
          0.4463
        ],
        [
          0.7875,
          0.439
        ],
        [
          0.7975,
          0.4327
        ],
        [
          0.8076,
          0.4258
        ],
        [
          0.818,
          0.4158
        ],
        [
          0.828,
          0.4086
        ],
        [
          0.838,
          0.4031
        ],
        [
          0.848,
          0.3949
        ],
        [
          0.8581,
          0.389
        ],
        [
          0.8685,
          0.3826
        ],
        [
          0.8785,
          0.3763
        ],
        [
          0.8885,
          0.3708
        ],
        [
          0.8985,
          0.3626
        ],
        [
          0.9086,
          0.3553
        ],
        [
          0.919,
          0.348
        ],
        [
          0.929,
          0.3408
        ],
        [
          0.939,
          0.3326
        ],
        [
          0.949,
          0.3262
        ],
        [
          0.9591,
          0.3198
        ],
        [
          0.9695,
          0.3148
        ],
        [
          0.9795,
          0.3085
        ],
        [
          0.9895,
          0.3025
        ],
        [
          0.9995,
          0.2944
        ],
        [
          1.0,
          0.2939
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.008030627143942811,
        "difference": -0.001064059388393343,
        "worse": 0.5366666666666666,
        "kept_right": 0.006781633462308525,
        "kept_wrong": 0.0012489936816342901,
        "handed_off": 0.9919693728560579
      },
      "usd_per_1000": 0.42282265777814637
    },
    {
      "decision": 340,
      "frontier": 47,
      "frontier_accuracy": 0.9044585987261147,
      "in_sample": {
        "threshold": 0.9105755929840016,
        "answered": 0.006369426751592357,
        "accuracy": 0.9044585987261147
      },
      "curve": [
        [
          0.0,
          0.9045
        ],
        [
          0.01,
          0.904
        ],
        [
          0.02,
          0.8995
        ],
        [
          0.03,
          0.8944
        ],
        [
          0.04,
          0.8917
        ],
        [
          0.05,
          0.8885
        ],
        [
          0.0601,
          0.8844
        ],
        [
          0.0705,
          0.8813
        ],
        [
          0.0805,
          0.8753
        ],
        [
          0.0905,
          0.8712
        ],
        [
          0.1005,
          0.8672
        ],
        [
          0.1106,
          0.8599
        ],
        [
          0.121,
          0.8558
        ],
        [
          0.131,
          0.8503
        ],
        [
          0.141,
          0.8458
        ],
        [
          0.151,
          0.8385
        ],
        [
          0.1611,
          0.8335
        ],
        [
          0.1715,
          0.8294
        ],
        [
          0.1815,
          0.8221
        ],
        [
          0.1915,
          0.8153
        ],
        [
          0.2015,
          0.8089
        ],
        [
          0.2116,
          0.8025
        ],
        [
          0.222,
          0.7966
        ],
        [
          0.232,
          0.7903
        ],
        [
          0.242,
          0.7843
        ],
        [
          0.252,
          0.7798
        ],
        [
          0.2621,
          0.7739
        ],
        [
          0.2725,
          0.7666
        ],
        [
          0.2825,
          0.7593
        ],
        [
          0.2925,
          0.752
        ],
        [
          0.3025,
          0.7448
        ],
        [
          0.3126,
          0.7379
        ],
        [
          0.323,
          0.7311
        ],
        [
          0.333,
          0.7261
        ],
        [
          0.343,
          0.7211
        ],
        [
          0.353,
          0.7143
        ],
        [
          0.3631,
          0.7079
        ],
        [
          0.3735,
          0.7025
        ],
        [
          0.3835,
          0.6956
        ],
        [
          0.3935,
          0.6902
        ],
        [
          0.4035,
          0.6843
        ],
        [
          0.4136,
          0.6788
        ],
        [
          0.424,
          0.672
        ],
        [
          0.434,
          0.6661
        ],
        [
          0.444,
          0.6588
        ],
        [
          0.454,
          0.6524
        ],
        [
          0.4641,
          0.6456
        ],
        [
          0.4745,
          0.6392
        ],
        [
          0.4845,
          0.6328
        ],
        [
          0.4945,
          0.626
        ],
        [
          0.5045,
          0.6201
        ],
        [
          0.5146,
          0.6128
        ],
        [
          0.525,
          0.6065
        ],
        [
          0.535,
          0.6005
        ],
        [
          0.545,
          0.5933
        ],
        [
          0.5551,
          0.5887
        ],
        [
          0.5655,
          0.581
        ],
        [
          0.5755,
          0.5728
        ],
        [
          0.5855,
          0.566
        ],
        [
          0.5955,
          0.5582
        ],
        [
          0.6056,
          0.5514
        ],
        [
          0.616,
          0.5441
        ],
        [
          0.626,
          0.5373
        ],
        [
          0.636,
          0.5296
        ],
        [
          0.646,
          0.5223
        ],
        [
          0.6561,
          0.5164
        ],
        [
          0.6665,
          0.51
        ],
        [
          0.6765,
          0.5041
        ],
        [
          0.6865,
          0.4973
        ],
        [
          0.6965,
          0.4918
        ],
        [
          0.7066,
          0.4859
        ],
        [
          0.717,
          0.4777
        ],
        [
          0.727,
          0.4718
        ],
        [
          0.737,
          0.465
        ],
        [
          0.747,
          0.4572
        ],
        [
          0.7571,
          0.4536
        ],
        [
          0.7675,
          0.4472
        ],
        [
          0.7775,
          0.4399
        ],
        [
          0.7875,
          0.4327
        ],
        [
          0.7975,
          0.4258
        ],
        [
          0.8076,
          0.4195
        ],
        [
          0.818,
          0.4099
        ],
        [
          0.828,
          0.404
        ],
        [
          0.838,
          0.399
        ],
        [
          0.848,
          0.3913
        ],
        [
          0.8581,
          0.3858
        ],
        [
          0.8685,
          0.3803
        ],
        [
          0.8785,
          0.3731
        ],
        [
          0.8885,
          0.3685
        ],
        [
          0.8985,
          0.3608
        ],
        [
          0.9086,
          0.3544
        ],
        [
          0.919,
          0.3476
        ],
        [
          0.929,
          0.3403
        ],
        [
          0.939,
          0.3339
        ],
        [
          0.949,
          0.3271
        ],
        [
          0.9591,
          0.3198
        ],
        [
          0.9695,
          0.3148
        ],
        [
          0.9795,
          0.308
        ],
        [
          0.9895,
          0.303
        ],
        [
          0.9995,
          0.2944
        ],
        [
          1.0,
          0.2939
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.008030627143942811,
        "difference": -0.001064059388393343,
        "worse": 0.5366666666666666,
        "kept_right": 0.006781633462308525,
        "kept_wrong": 0.0012489936816342901,
        "handed_off": 0.9919693728560579
      },
      "usd_per_1000": 4.89099027530156
    },
    {
      "decision": 340,
      "frontier": 193,
      "frontier_accuracy": 0.89171974522293,
      "in_sample": {
        "threshold": 0.9105755929840016,
        "answered": 0.006369426751592357,
        "accuracy": 0.89171974522293
      },
      "curve": [
        [
          0.0,
          0.8917
        ],
        [
          0.01,
          0.8913
        ],
        [
          0.02,
          0.8867
        ],
        [
          0.03,
          0.8822
        ],
        [
          0.04,
          0.879
        ],
        [
          0.05,
          0.8763
        ],
        [
          0.0601,
          0.8726
        ],
        [
          0.0705,
          0.8694
        ],
        [
          0.0805,
          0.864
        ],
        [
          0.0905,
          0.8594
        ],
        [
          0.1005,
          0.8562
        ],
        [
          0.1106,
          0.849
        ],
        [
          0.121,
          0.8453
        ],
        [
          0.131,
          0.8403
        ],
        [
          0.141,
          0.8353
        ],
        [
          0.151,
          0.8285
        ],
        [
          0.1611,
          0.8235
        ],
        [
          0.1715,
          0.8194
        ],
        [
          0.1815,
          0.8126
        ],
        [
          0.1915,
          0.8066
        ],
        [
          0.2015,
          0.7998
        ],
        [
          0.2116,
          0.7934
        ],
        [
          0.222,
          0.7875
        ],
        [
          0.232,
          0.7812
        ],
        [
          0.242,
          0.7757
        ],
        [
          0.252,
          0.7716
        ],
        [
          0.2621,
          0.7648
        ],
        [
          0.2725,
          0.7571
        ],
        [
          0.2825,
          0.7502
        ],
        [
          0.2925,
          0.7434
        ],
        [
          0.3025,
          0.7361
        ],
        [
          0.3126,
          0.7293
        ],
        [
          0.323,
          0.7229
        ],
        [
          0.333,
          0.7197
        ],
        [
          0.343,
          0.7161
        ],
        [
          0.353,
          0.7097
        ],
        [
          0.3631,
          0.7043
        ],
        [
          0.3735,
          0.6988
        ],
        [
          0.3835,
          0.692
        ],
        [
          0.3935,
          0.6861
        ],
        [
          0.4035,
          0.6811
        ],
        [
          0.4136,
          0.6747
        ],
        [
          0.424,
          0.6683
        ],
        [
          0.434,
          0.6624
        ],
        [
          0.444,
          0.6574
        ],
        [
          0.454,
          0.6515
        ],
        [
          0.4641,
          0.6456
        ],
        [
          0.4745,
          0.6397
        ],
        [
          0.4845,
          0.6333
        ],
        [
          0.4945,
          0.6278
        ],
        [
          0.5045,
          0.6224
        ],
        [
          0.5146,
          0.616
        ],
        [
          0.525,
          0.6087
        ],
        [
          0.535,
          0.6037
        ],
        [
          0.545,
          0.5969
        ],
        [
          0.5551,
          0.5919
        ],
        [
          0.5655,
          0.5842
        ],
        [
          0.5755,
          0.5764
        ],
        [
          0.5855,
          0.5682
        ],
        [
          0.5955,
          0.561
        ],
        [
          0.6056,
          0.5532
        ],
        [
          0.616,
          0.5478
        ],
        [
          0.626,
          0.5409
        ],
        [
          0.636,
          0.535
        ],
        [
          0.646,
          0.5278
        ],
        [
          0.6561,
          0.5205
        ],
        [
          0.6665,
          0.5141
        ],
        [
          0.6765,
          0.5082
        ],
        [
          0.6865,
          0.5
        ],
        [
          0.6965,
          0.4941
        ],
        [
          0.7066,
          0.4877
        ],
        [
          0.717,
          0.4791
        ],
        [
          0.727,
          0.4736
        ],
        [
          0.737,
          0.4668
        ],
        [
          0.747,
          0.4591
        ],
        [
          0.7571,
          0.455
        ],
        [
          0.7675,
          0.4495
        ],
        [
          0.7775,
          0.4431
        ],
        [
          0.7875,
          0.4363
        ],
        [
          0.7975,
          0.4295
        ],
        [
          0.8076,
          0.4231
        ],
        [
          0.818,
          0.4131
        ],
        [
          0.828,
          0.4081
        ],
        [
          0.838,
          0.4017
        ],
        [
          0.848,
          0.3935
        ],
        [
          0.8581,
          0.3867
        ],
        [
          0.8685,
          0.3817
        ],
        [
          0.8785,
          0.3749
        ],
        [
          0.8885,
          0.3712
        ],
        [
          0.8985,
          0.3644
        ],
        [
          0.9086,
          0.3585
        ],
        [
          0.919,
          0.3517
        ],
        [
          0.929,
          0.3439
        ],
        [
          0.939,
          0.3362
        ],
        [
          0.949,
          0.3298
        ],
        [
          0.9591,
          0.3226
        ],
        [
          0.9695,
          0.3148
        ],
        [
          0.9795,
          0.3085
        ],
        [
          0.9895,
          0.3025
        ],
        [
          0.9995,
          0.2944
        ],
        [
          1.0,
          0.2939
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.008030627143942811,
        "difference": -0.001064059388393343,
        "worse": 0.5366666666666666,
        "kept_right": 0.006781633462308525,
        "kept_wrong": 0.0012489936816342901,
        "handed_off": 0.9919693728560579
      },
      "usd_per_1000": 7.685371984419751
    },
    {
      "decision": 340,
      "frontier": 55,
      "frontier_accuracy": 0.8503184713375797,
      "in_sample": {
        "threshold": 0.9105755929840016,
        "answered": 0.006369426751592357,
        "accuracy": 0.8503184713375797
      },
      "curve": [
        [
          0.0,
          0.8503
        ],
        [
          0.01,
          0.8499
        ],
        [
          0.02,
          0.8453
        ],
        [
          0.03,
          0.8408
        ],
        [
          0.04,
          0.838
        ],
        [
          0.05,
          0.8348
        ],
        [
          0.0601,
          0.8317
        ],
        [
          0.0705,
          0.8285
        ],
        [
          0.0805,
          0.8239
        ],
        [
          0.0905,
          0.8198
        ],
        [
          0.1005,
          0.8167
        ],
        [
          0.1106,
          0.8098
        ],
        [
          0.121,
          0.8066
        ],
        [
          0.131,
          0.8016
        ],
        [
          0.141,
          0.7985
        ],
        [
          0.151,
          0.7907
        ],
        [
          0.1611,
          0.7857
        ],
        [
          0.1715,
          0.7821
        ],
        [
          0.1815,
          0.7757
        ],
        [
          0.1915,
          0.7693
        ],
        [
          0.2015,
          0.7634
        ],
        [
          0.2116,
          0.7575
        ],
        [
          0.222,
          0.7525
        ],
        [
          0.232,
          0.7466
        ],
        [
          0.242,
          0.742
        ],
        [
          0.252,
          0.7384
        ],
        [
          0.2621,
          0.7316
        ],
        [
          0.2725,
          0.7252
        ],
        [
          0.2825,
          0.7197
        ],
        [
          0.2925,
          0.7143
        ],
        [
          0.3025,
          0.7084
        ],
        [
          0.3126,
          0.702
        ],
        [
          0.323,
          0.6956
        ],
        [
          0.333,
          0.6924
        ],
        [
          0.343,
          0.6884
        ],
        [
          0.353,
          0.682
        ],
        [
          0.3631,
          0.6756
        ],
        [
          0.3735,
          0.6702
        ],
        [
          0.3835,
          0.6638
        ],
        [
          0.3935,
          0.6597
        ],
        [
          0.4035,
          0.6533
        ],
        [
          0.4136,
          0.647
        ],
        [
          0.424,
          0.6429
        ],
        [
          0.434,
          0.6374
        ],
        [
          0.444,
          0.631
        ],
        [
          0.454,
          0.6247
        ],
        [
          0.4641,
          0.6178
        ],
        [
          0.4745,
          0.6128
        ],
        [
          0.4845,
          0.6078
        ],
        [
          0.4945,
          0.6019
        ],
        [
          0.5045,
          0.596
        ],
        [
          0.5146,
          0.5887
        ],
        [
          0.525,
          0.5823
        ],
        [
          0.535,
          0.5773
        ],
        [
          0.545,
          0.5701
        ],
        [
          0.5551,
          0.566
        ],
        [
          0.5655,
          0.5587
        ],
        [
          0.5755,
          0.5505
        ],
        [
          0.5855,
          0.5432
        ],
        [
          0.5955,
          0.5359
        ],
        [
          0.6056,
          0.5287
        ],
        [
          0.616,
          0.5232
        ],
        [
          0.626,
          0.5164
        ],
        [
          0.636,
          0.5105
        ],
        [
          0.646,
          0.5032
        ],
        [
          0.6561,
          0.4959
        ],
        [
          0.6665,
          0.4909
        ],
        [
          0.6765,
          0.4864
        ],
        [
          0.6865,
          0.4795
        ],
        [
          0.6965,
          0.4741
        ],
        [
          0.7066,
          0.4686
        ],
        [
          0.717,
          0.4613
        ],
        [
          0.727,
          0.4577
        ],
        [
          0.737,
          0.4504
        ],
        [
          0.747,
          0.4431
        ],
        [
          0.7571,
          0.4381
        ],
        [
          0.7675,
          0.434
        ],
        [
          0.7775,
          0.4281
        ],
        [
          0.7875,
          0.4222
        ],
        [
          0.7975,
          0.4158
        ],
        [
          0.8076,
          0.4108
        ],
        [
          0.818,
          0.404
        ],
        [
          0.828,
          0.3981
        ],
        [
          0.838,
          0.3926
        ],
        [
          0.848,
          0.3854
        ],
        [
          0.8581,
          0.3794
        ],
        [
          0.8685,
          0.3735
        ],
        [
          0.8785,
          0.3681
        ],
        [
          0.8885,
          0.3649
        ],
        [
          0.8985,
          0.3585
        ],
        [
          0.9086,
          0.3521
        ],
        [
          0.919,
          0.3462
        ],
        [
          0.929,
          0.338
        ],
        [
          0.939,
          0.3317
        ],
        [
          0.949,
          0.3271
        ],
        [
          0.9591,
          0.3212
        ],
        [
          0.9695,
          0.3148
        ],
        [
          0.9795,
          0.308
        ],
        [
          0.9895,
          0.3021
        ],
        [
          0.9995,
          0.2944
        ],
        [
          1.0,
          0.2939
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.008030627143942811,
        "difference": -0.001064059388393343,
        "worse": 0.5366666666666666,
        "kept_right": 0.006781633462308525,
        "kept_wrong": 0.0012489936816342901,
        "handed_off": 0.9919693728560579
      },
      "usd_per_1000": 3.6249870291484343
    },
    {
      "decision": 141,
      "frontier": 140,
      "frontier_accuracy": 0.9808917197452229,
      "in_sample": {
        "threshold": 0.9578,
        "answered": 0.0031847133757961785,
        "accuracy": 0.9808917197452229
      },
      "curve": [
        [
          0.0,
          0.9809
        ],
        [
          0.01,
          0.9804
        ],
        [
          0.02,
          0.9786
        ],
        [
          0.03,
          0.9763
        ],
        [
          0.04,
          0.9727
        ],
        [
          0.05,
          0.9686
        ],
        [
          0.0601,
          0.9631
        ],
        [
          0.0705,
          0.9577
        ],
        [
          0.0805,
          0.9509
        ],
        [
          0.091,
          0.9454
        ],
        [
          0.101,
          0.9386
        ],
        [
          0.111,
          0.9336
        ],
        [
          0.121,
          0.929
        ],
        [
          0.131,
          0.9231
        ],
        [
          0.141,
          0.9167
        ],
        [
          0.151,
          0.9104
        ],
        [
          0.1611,
          0.9049
        ],
        [
          0.1715,
          0.8967
        ],
        [
          0.1815,
          0.8926
        ],
        [
          0.1915,
          0.8867
        ],
        [
          0.2015,
          0.8803
        ],
        [
          0.2116,
          0.8749
        ],
        [
          0.222,
          0.8676
        ],
        [
          0.2325,
          0.8594
        ],
        [
          0.2429,
          0.8526
        ],
        [
          0.253,
          0.8449
        ],
        [
          0.2634,
          0.838
        ],
        [
          0.2734,
          0.8321
        ],
        [
          0.2834,
          0.8271
        ],
        [
          0.2944,
          0.8207
        ],
        [
          0.3048,
          0.8126
        ],
        [
          0.3148,
          0.8057
        ],
        [
          0.3248,
          0.7985
        ],
        [
          0.3348,
          0.7916
        ],
        [
          0.3449,
          0.7853
        ],
        [
          0.3553,
          0.7771
        ],
        [
          0.3658,
          0.7693
        ],
        [
          0.3763,
          0.7621
        ],
        [
          0.3867,
          0.7539
        ],
        [
          0.3967,
          0.7457
        ],
        [
          0.4067,
          0.7384
        ],
        [
          0.4172,
          0.7302
        ],
        [
          0.4272,
          0.7211
        ],
        [
          0.4372,
          0.7157
        ],
        [
          0.4477,
          0.7079
        ],
        [
          0.4581,
          0.7002
        ],
        [
          0.4682,
          0.6924
        ],
        [
          0.4786,
          0.6843
        ],
        [
          0.4886,
          0.6783
        ],
        [
          0.4986,
          0.6711
        ],
        [
          0.5086,
          0.6624
        ],
        [
          0.5191,
          0.6561
        ],
        [
          0.5291,
          0.6488
        ],
        [
          0.5391,
          0.6401
        ],
        [
          0.5491,
          0.6328
        ],
        [
          0.5591,
          0.6256
        ],
        [
          0.5692,
          0.616
        ],
        [
          0.5801,
          0.6056
        ],
        [
          0.5905,
          0.5996
        ],
        [
          0.6005,
          0.591
        ],
        [
          0.6115,
          0.581
        ],
        [
          0.6219,
          0.5728
        ],
        [
          0.6319,
          0.5669
        ],
        [
          0.6419,
          0.5582
        ],
        [
          0.652,
          0.5505
        ],
        [
          0.6624,
          0.5437
        ],
        [
          0.6724,
          0.5355
        ],
        [
          0.6824,
          0.5264
        ],
        [
          0.6934,
          0.5187
        ],
        [
          0.7043,
          0.5105
        ],
        [
          0.7147,
          0.5018
        ],
        [
          0.7252,
          0.4927
        ],
        [
          0.7352,
          0.485
        ],
        [
          0.747,
          0.4759
        ],
        [
          0.7571,
          0.4663
        ],
        [
          0.7675,
          0.4586
        ],
        [
          0.778,
          0.4513
        ],
        [
          0.7884,
          0.4427
        ],
        [
          0.7985,
          0.434
        ],
        [
          0.8094,
          0.424
        ],
        [
          0.8203,
          0.4163
        ],
        [
          0.8308,
          0.4072
        ],
        [
          0.8417,
          0.3976
        ],
        [
          0.8544,
          0.3872
        ],
        [
          0.8644,
          0.3781
        ],
        [
          0.8753,
          0.3699
        ],
        [
          0.8854,
          0.3608
        ],
        [
          0.8972,
          0.3526
        ],
        [
          0.9076,
          0.3458
        ],
        [
          0.9186,
          0.3394
        ],
        [
          0.9295,
          0.3298
        ],
        [
          0.9404,
          0.3212
        ],
        [
          0.9504,
          0.313
        ],
        [
          0.9609,
          0.3057
        ],
        [
          0.9713,
          0.2975
        ],
        [
          0.9813,
          0.2898
        ],
        [
          0.9918,
          0.2821
        ],
        [
          1.0,
          0.2748
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.008024528241911754,
        "difference": -0.0009488307284619542,
        "worse": 0.53,
        "kept_right": 0.0070756975134497945,
        "kept_wrong": 0.0009488307284619542,
        "handed_off": 0.9919754717580893
      },
      "usd_per_1000": 2.399691000322133
    },
    {
      "decision": 141,
      "frontier": 277,
      "frontier_accuracy": 0.9795268425841674,
      "in_sample": {
        "threshold": 0.9578,
        "answered": 0.0031847133757961785,
        "accuracy": 0.9795268425841674
      },
      "curve": [
        [
          0.0,
          0.9795
        ],
        [
          0.01,
          0.9791
        ],
        [
          0.02,
          0.9773
        ],
        [
          0.03,
          0.975
        ],
        [
          0.04,
          0.9713
        ],
        [
          0.05,
          0.9668
        ],
        [
          0.0601,
          0.9613
        ],
        [
          0.0705,
          0.9554
        ],
        [
          0.0805,
          0.9486
        ],
        [
          0.091,
          0.9431
        ],
        [
          0.101,
          0.9368
        ],
        [
          0.111,
          0.9318
        ],
        [
          0.121,
          0.9272
        ],
        [
          0.131,
          0.9213
        ],
        [
          0.141,
          0.9149
        ],
        [
          0.151,
          0.9086
        ],
        [
          0.1611,
          0.9031
        ],
        [
          0.1715,
          0.8944
        ],
        [
          0.1815,
          0.8904
        ],
        [
          0.1915,
          0.8844
        ],
        [
          0.2015,
          0.8781
        ],
        [
          0.2116,
          0.8722
        ],
        [
          0.222,
          0.8649
        ],
        [
          0.2325,
          0.8567
        ],
        [
          0.2429,
          0.8494
        ],
        [
          0.253,
          0.8417
        ],
        [
          0.2634,
          0.8348
        ],
        [
          0.2734,
          0.8289
        ],
        [
          0.2834,
          0.8239
        ],
        [
          0.2944,
          0.818
        ],
        [
          0.3048,
          0.8098
        ],
        [
          0.3148,
          0.803
        ],
        [
          0.3248,
          0.7957
        ],
        [
          0.3348,
          0.7894
        ],
        [
          0.3449,
          0.783
        ],
        [
          0.3553,
          0.7753
        ],
        [
          0.3658,
          0.7675
        ],
        [
          0.3763,
          0.7607
        ],
        [
          0.3867,
          0.7525
        ],
        [
          0.3967,
          0.7443
        ],
        [
          0.4067,
          0.7361
        ],
        [
          0.4172,
          0.7279
        ],
        [
          0.4272,
          0.7188
        ],
        [
          0.4372,
          0.7134
        ],
        [
          0.4477,
          0.7056
        ],
        [
          0.4581,
          0.6979
        ],
        [
          0.4682,
          0.6906
        ],
        [
          0.4786,
          0.6824
        ],
        [
          0.4886,
          0.6765
        ],
        [
          0.4986,
          0.6692
        ],
        [
          0.5086,
          0.6606
        ],
        [
          0.5191,
          0.6538
        ],
        [
          0.5291,
          0.6456
        ],
        [
          0.5391,
          0.6369
        ],
        [
          0.5491,
          0.6292
        ],
        [
          0.5591,
          0.6219
        ],
        [
          0.5692,
          0.6128
        ],
        [
          0.5801,
          0.6024
        ],
        [
          0.5905,
          0.5969
        ],
        [
          0.6005,
          0.5883
        ],
        [
          0.6115,
          0.5783
        ],
        [
          0.6219,
          0.5701
        ],
        [
          0.6319,
          0.5646
        ],
        [
          0.6419,
          0.5564
        ],
        [
          0.652,
          0.5487
        ],
        [
          0.6624,
          0.5414
        ],
        [
          0.6724,
          0.5337
        ],
        [
          0.6824,
          0.5246
        ],
        [
          0.6934,
          0.5173
        ],
        [
          0.7043,
          0.5091
        ],
        [
          0.7147,
          0.5005
        ],
        [
          0.7252,
          0.4914
        ],
        [
          0.7352,
          0.4836
        ],
        [
          0.747,
          0.4745
        ],
        [
          0.7571,
          0.4654
        ],
        [
          0.7675,
          0.4586
        ],
        [
          0.778,
          0.4513
        ],
        [
          0.7884,
          0.4427
        ],
        [
          0.7985,
          0.434
        ],
        [
          0.8094,
          0.424
        ],
        [
          0.8203,
          0.4163
        ],
        [
          0.8308,
          0.4072
        ],
        [
          0.8417,
          0.3976
        ],
        [
          0.8544,
          0.3867
        ],
        [
          0.8644,
          0.3776
        ],
        [
          0.8753,
          0.3694
        ],
        [
          0.8854,
          0.3603
        ],
        [
          0.8972,
          0.3517
        ],
        [
          0.9076,
          0.3449
        ],
        [
          0.9186,
          0.3385
        ],
        [
          0.9295,
          0.3289
        ],
        [
          0.9404,
          0.3207
        ],
        [
          0.9504,
          0.3126
        ],
        [
          0.9609,
          0.3053
        ],
        [
          0.9713,
          0.2971
        ],
        [
          0.9813,
          0.2894
        ],
        [
          0.9918,
          0.2821
        ],
        [
          1.0,
          0.2748
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.008024528241911754,
        "difference": -0.0009488307284619542,
        "worse": 0.53,
        "kept_right": 0.0070756975134497945,
        "kept_wrong": 0.0009488307284619542,
        "handed_off": 0.9919754717580893
      },
      "usd_per_1000": 6.016881832243535
    },
    {
      "decision": 141,
      "frontier": 276,
      "frontier_accuracy": 0.9777070063694268,
      "in_sample": {
        "threshold": 0.9578,
        "answered": 0.0031847133757961785,
        "accuracy": 0.9777070063694268
      },
      "curve": [
        [
          0.0,
          0.9777
        ],
        [
          0.01,
          0.9773
        ],
        [
          0.02,
          0.9754
        ],
        [
          0.03,
          0.9732
        ],
        [
          0.04,
          0.9695
        ],
        [
          0.05,
          0.9654
        ],
        [
          0.0601,
          0.96
        ],
        [
          0.0705,
          0.954
        ],
        [
          0.0805,
          0.9472
        ],
        [
          0.091,
          0.9418
        ],
        [
          0.101,
          0.9359
        ],
        [
          0.111,
          0.9308
        ],
        [
          0.121,
          0.9263
        ],
        [
          0.131,
          0.9204
        ],
        [
          0.141,
          0.914
        ],
        [
          0.151,
          0.9076
        ],
        [
          0.1611,
          0.9022
        ],
        [
          0.1715,
          0.8935
        ],
        [
          0.1815,
          0.8894
        ],
        [
          0.1915,
          0.8844
        ],
        [
          0.2015,
          0.8781
        ],
        [
          0.2116,
          0.8722
        ],
        [
          0.222,
          0.8649
        ],
        [
          0.2325,
          0.8567
        ],
        [
          0.2429,
          0.8494
        ],
        [
          0.253,
          0.8417
        ],
        [
          0.2634,
          0.8348
        ],
        [
          0.2734,
          0.8285
        ],
        [
          0.2834,
          0.8235
        ],
        [
          0.2944,
          0.8176
        ],
        [
          0.3048,
          0.8094
        ],
        [
          0.3148,
          0.8025
        ],
        [
          0.3248,
          0.7953
        ],
        [
          0.3348,
          0.7889
        ],
        [
          0.3449,
          0.7825
        ],
        [
          0.3553,
          0.7743
        ],
        [
          0.3658,
          0.7666
        ],
        [
          0.3763,
          0.7598
        ],
        [
          0.3867,
          0.7516
        ],
        [
          0.3967,
          0.7434
        ],
        [
          0.4067,
          0.7352
        ],
        [
          0.4172,
          0.727
        ],
        [
          0.4272,
          0.7179
        ],
        [
          0.4372,
          0.7125
        ],
        [
          0.4477,
          0.7047
        ],
        [
          0.4581,
          0.697
        ],
        [
          0.4682,
          0.6897
        ],
        [
          0.4786,
          0.6815
        ],
        [
          0.4886,
          0.6756
        ],
        [
          0.4986,
          0.6683
        ],
        [
          0.5086,
          0.6597
        ],
        [
          0.5191,
          0.6533
        ],
        [
          0.5291,
          0.6451
        ],
        [
          0.5391,
          0.6365
        ],
        [
          0.5491,
          0.6288
        ],
        [
          0.5591,
          0.6215
        ],
        [
          0.5692,
          0.6119
        ],
        [
          0.5801,
          0.6015
        ],
        [
          0.5905,
          0.596
        ],
        [
          0.6005,
          0.5874
        ],
        [
          0.6115,
          0.5773
        ],
        [
          0.6219,
          0.5692
        ],
        [
          0.6319,
          0.5637
        ],
        [
          0.6419,
          0.556
        ],
        [
          0.652,
          0.5482
        ],
        [
          0.6624,
          0.5409
        ],
        [
          0.6724,
          0.5332
        ],
        [
          0.6824,
          0.5241
        ],
        [
          0.6934,
          0.5168
        ],
        [
          0.7043,
          0.5086
        ],
        [
          0.7147,
          0.5
        ],
        [
          0.7252,
          0.4909
        ],
        [
          0.7352,
          0.4832
        ],
        [
          0.747,
          0.4741
        ],
        [
          0.7571,
          0.465
        ],
        [
          0.7675,
          0.4581
        ],
        [
          0.778,
          0.4509
        ],
        [
          0.7884,
          0.4427
        ],
        [
          0.7985,
          0.434
        ],
        [
          0.8094,
          0.4245
        ],
        [
          0.8203,
          0.4167
        ],
        [
          0.8308,
          0.4076
        ],
        [
          0.8417,
          0.3981
        ],
        [
          0.8544,
          0.3872
        ],
        [
          0.8644,
          0.3781
        ],
        [
          0.8753,
          0.3699
        ],
        [
          0.8854,
          0.3608
        ],
        [
          0.8972,
          0.3521
        ],
        [
          0.9076,
          0.3453
        ],
        [
          0.9186,
          0.3389
        ],
        [
          0.9295,
          0.3294
        ],
        [
          0.9404,
          0.3212
        ],
        [
          0.9504,
          0.313
        ],
        [
          0.9609,
          0.3057
        ],
        [
          0.9713,
          0.2975
        ],
        [
          0.9813,
          0.2898
        ],
        [
          0.9918,
          0.2821
        ],
        [
          1.0,
          0.2748
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.008024528241911754,
        "difference": -0.0009488307284619542,
        "worse": 0.53,
        "kept_right": 0.0070756975134497945,
        "kept_wrong": 0.0009488307284619542,
        "handed_off": 0.9919754717580893
      },
      "usd_per_1000": 1.6091728620283772
    },
    {
      "decision": 141,
      "frontier": 48,
      "frontier_accuracy": 0.9758871701546861,
      "in_sample": {
        "threshold": 0.9578,
        "answered": 0.0031847133757961785,
        "accuracy": 0.9758871701546861
      },
      "curve": [
        [
          0.0,
          0.9759
        ],
        [
          0.01,
          0.9754
        ],
        [
          0.02,
          0.9736
        ],
        [
          0.03,
          0.9713
        ],
        [
          0.04,
          0.9682
        ],
        [
          0.05,
          0.9636
        ],
        [
          0.0601,
          0.9581
        ],
        [
          0.0705,
          0.9522
        ],
        [
          0.0805,
          0.9454
        ],
        [
          0.091,
          0.9399
        ],
        [
          0.101,
          0.9331
        ],
        [
          0.111,
          0.9281
        ],
        [
          0.121,
          0.9236
        ],
        [
          0.131,
          0.9172
        ],
        [
          0.141,
          0.9108
        ],
        [
          0.151,
          0.9045
        ],
        [
          0.1611,
          0.899
        ],
        [
          0.1715,
          0.8904
        ],
        [
          0.1815,
          0.8863
        ],
        [
          0.1915,
          0.8803
        ],
        [
          0.2015,
          0.874
        ],
        [
          0.2116,
          0.8681
        ],
        [
          0.222,
          0.8608
        ],
        [
          0.2325,
          0.8526
        ],
        [
          0.2429,
          0.8453
        ],
        [
          0.253,
          0.8376
        ],
        [
          0.2634,
          0.8308
        ],
        [
          0.2734,
          0.8244
        ],
        [
          0.2834,
          0.8194
        ],
        [
          0.2944,
          0.8135
        ],
        [
          0.3048,
          0.8053
        ],
        [
          0.3148,
          0.7985
        ],
        [
          0.3248,
          0.7912
        ],
        [
          0.3348,
          0.7853
        ],
        [
          0.3449,
          0.7789
        ],
        [
          0.3553,
          0.7712
        ],
        [
          0.3658,
          0.7634
        ],
        [
          0.3763,
          0.7566
        ],
        [
          0.3867,
          0.7484
        ],
        [
          0.3967,
          0.7407
        ],
        [
          0.4067,
          0.7325
        ],
        [
          0.4172,
          0.7243
        ],
        [
          0.4272,
          0.7152
        ],
        [
          0.4372,
          0.7097
        ],
        [
          0.4477,
          0.702
        ],
        [
          0.4581,
          0.6943
        ],
        [
          0.4682,
          0.6874
        ],
        [
          0.4786,
          0.6797
        ],
        [
          0.4886,
          0.6738
        ],
        [
          0.4986,
          0.6665
        ],
        [
          0.5086,
          0.6579
        ],
        [
          0.5191,
          0.651
        ],
        [
          0.5291,
          0.6429
        ],
        [
          0.5391,
          0.6342
        ],
        [
          0.5491,
          0.6269
        ],
        [
          0.5591,
          0.6197
        ],
        [
          0.5692,
          0.6106
        ],
        [
          0.5801,
          0.6001
        ],
        [
          0.5905,
          0.5942
        ],
        [
          0.6005,
          0.5855
        ],
        [
          0.6115,
          0.5755
        ],
        [
          0.6219,
          0.5673
        ],
        [
          0.6319,
          0.5623
        ],
        [
          0.6419,
          0.5541
        ],
        [
          0.652,
          0.5464
        ],
        [
          0.6624,
          0.5391
        ],
        [
          0.6724,
          0.5314
        ],
        [
          0.6824,
          0.5227
        ],
        [
          0.6934,
          0.5155
        ],
        [
          0.7043,
          0.5073
        ],
        [
          0.7147,
          0.4986
        ],
        [
          0.7252,
          0.4895
        ],
        [
          0.7352,
          0.4818
        ],
        [
          0.747,
          0.4727
        ],
        [
          0.7571,
          0.4636
        ],
        [
          0.7675,
          0.4568
        ],
        [
          0.778,
          0.4495
        ],
        [
          0.7884,
          0.4413
        ],
        [
          0.7985,
          0.4327
        ],
        [
          0.8094,
          0.4231
        ],
        [
          0.8203,
          0.4158
        ],
        [
          0.8308,
          0.4067
        ],
        [
          0.8417,
          0.3972
        ],
        [
          0.8544,
          0.3867
        ],
        [
          0.8644,
          0.3776
        ],
        [
          0.8753,
          0.3694
        ],
        [
          0.8854,
          0.3603
        ],
        [
          0.8972,
          0.3517
        ],
        [
          0.9076,
          0.3449
        ],
        [
          0.9186,
          0.3385
        ],
        [
          0.9295,
          0.3289
        ],
        [
          0.9404,
          0.3207
        ],
        [
          0.9504,
          0.3126
        ],
        [
          0.9609,
          0.3057
        ],
        [
          0.9713,
          0.2975
        ],
        [
          0.9813,
          0.2898
        ],
        [
          0.9918,
          0.2821
        ],
        [
          1.0,
          0.2748
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.008024528241911754,
        "difference": -0.0009488307284619542,
        "worse": 0.53,
        "kept_right": 0.0070756975134497945,
        "kept_wrong": 0.0009488307284619542,
        "handed_off": 0.9919754717580893
      },
      "usd_per_1000": 5.335068515954763
    },
    {
      "decision": 141,
      "frontier": 191,
      "frontier_accuracy": 0.9740673339399454,
      "in_sample": {
        "threshold": 0.9578,
        "answered": 0.0031847133757961785,
        "accuracy": 0.9740673339399454
      },
      "curve": [
        [
          0.0,
          0.9741
        ],
        [
          0.01,
          0.9736
        ],
        [
          0.02,
          0.9718
        ],
        [
          0.03,
          0.9695
        ],
        [
          0.04,
          0.9659
        ],
        [
          0.05,
          0.9613
        ],
        [
          0.0601,
          0.9563
        ],
        [
          0.0705,
          0.95
        ],
        [
          0.0805,
          0.9431
        ],
        [
          0.091,
          0.9377
        ],
        [
          0.101,
          0.9308
        ],
        [
          0.111,
          0.9258
        ],
        [
          0.121,
          0.9213
        ],
        [
          0.131,
          0.9149
        ],
        [
          0.141,
          0.9086
        ],
        [
          0.151,
          0.9022
        ],
        [
          0.1611,
          0.8967
        ],
        [
          0.1715,
          0.8881
        ],
        [
          0.1815,
          0.884
        ],
        [
          0.1915,
          0.8785
        ],
        [
          0.2015,
          0.8722
        ],
        [
          0.2116,
          0.8662
        ],
        [
          0.222,
          0.859
        ],
        [
          0.2325,
          0.8508
        ],
        [
          0.2429,
          0.8435
        ],
        [
          0.253,
          0.8362
        ],
        [
          0.2634,
          0.8294
        ],
        [
          0.2734,
          0.823
        ],
        [
          0.2834,
          0.818
        ],
        [
          0.2944,
          0.8116
        ],
        [
          0.3048,
          0.8035
        ],
        [
          0.3148,
          0.7966
        ],
        [
          0.3248,
          0.7894
        ],
        [
          0.3348,
          0.783
        ],
        [
          0.3449,
          0.7766
        ],
        [
          0.3553,
          0.7689
        ],
        [
          0.3658,
          0.7611
        ],
        [
          0.3763,
          0.7548
        ],
        [
          0.3867,
          0.7466
        ],
        [
          0.3967,
          0.7384
        ],
        [
          0.4067,
          0.7307
        ],
        [
          0.4172,
          0.7225
        ],
        [
          0.4272,
          0.7134
        ],
        [
          0.4372,
          0.7079
        ],
        [
          0.4477,
          0.7002
        ],
        [
          0.4581,
          0.6924
        ],
        [
          0.4682,
          0.6852
        ],
        [
          0.4786,
          0.6774
        ],
        [
          0.4886,
          0.6715
        ],
        [
          0.4986,
          0.6642
        ],
        [
          0.5086,
          0.6556
        ],
        [
          0.5191,
          0.6497
        ],
        [
          0.5291,
          0.6415
        ],
        [
          0.5391,
          0.6328
        ],
        [
          0.5491,
          0.6251
        ],
        [
          0.5591,
          0.6178
        ],
        [
          0.5692,
          0.6087
        ],
        [
          0.5801,
          0.5983
        ],
        [
          0.5905,
          0.5928
        ],
        [
          0.6005,
          0.5842
        ],
        [
          0.6115,
          0.5742
        ],
        [
          0.6219,
          0.566
        ],
        [
          0.6319,
          0.5605
        ],
        [
          0.6419,
          0.5523
        ],
        [
          0.652,
          0.5446
        ],
        [
          0.6624,
          0.5373
        ],
        [
          0.6724,
          0.5296
        ],
        [
          0.6824,
          0.5209
        ],
        [
          0.6934,
          0.5132
        ],
        [
          0.7043,
          0.5055
        ],
        [
          0.7147,
          0.4968
        ],
        [
          0.7252,
          0.4877
        ],
        [
          0.7352,
          0.48
        ],
        [
          0.747,
          0.4709
        ],
        [
          0.7571,
          0.4622
        ],
        [
          0.7675,
          0.4554
        ],
        [
          0.778,
          0.4481
        ],
        [
          0.7884,
          0.4404
        ],
        [
          0.7985,
          0.4318
        ],
        [
          0.8094,
          0.4222
        ],
        [
          0.8203,
          0.4145
        ],
        [
          0.8308,
          0.4054
        ],
        [
          0.8417,
          0.3958
        ],
        [
          0.8544,
          0.3854
        ],
        [
          0.8644,
          0.3763
        ],
        [
          0.8753,
          0.3681
        ],
        [
          0.8854,
          0.359
        ],
        [
          0.8972,
          0.3508
        ],
        [
          0.9076,
          0.3439
        ],
        [
          0.9186,
          0.338
        ],
        [
          0.9295,
          0.3285
        ],
        [
          0.9404,
          0.3203
        ],
        [
          0.9504,
          0.3121
        ],
        [
          0.9609,
          0.3053
        ],
        [
          0.9713,
          0.2971
        ],
        [
          0.9813,
          0.2894
        ],
        [
          0.9918,
          0.2821
        ],
        [
          1.0,
          0.2748
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.008024528241911754,
        "difference": -0.0009488307284619542,
        "worse": 0.53,
        "kept_right": 0.0070756975134497945,
        "kept_wrong": 0.0009488307284619542,
        "handed_off": 0.9919754717580893
      },
      "usd_per_1000": 26.523138689975525
    },
    {
      "decision": 141,
      "frontier": 188,
      "frontier_accuracy": 0.9713375796178344,
      "in_sample": {
        "threshold": 0.9578,
        "answered": 0.0031847133757961785,
        "accuracy": 0.9713375796178344
      },
      "curve": [
        [
          0.0,
          0.9713
        ],
        [
          0.01,
          0.9709
        ],
        [
          0.02,
          0.9691
        ],
        [
          0.03,
          0.9668
        ],
        [
          0.04,
          0.9631
        ],
        [
          0.05,
          0.9591
        ],
        [
          0.0601,
          0.954
        ],
        [
          0.0705,
          0.9486
        ],
        [
          0.0805,
          0.9418
        ],
        [
          0.091,
          0.9368
        ],
        [
          0.101,
          0.9304
        ],
        [
          0.111,
          0.9254
        ],
        [
          0.121,
          0.9208
        ],
        [
          0.131,
          0.9149
        ],
        [
          0.141,
          0.9086
        ],
        [
          0.151,
          0.9022
        ],
        [
          0.1611,
          0.8967
        ],
        [
          0.1715,
          0.8885
        ],
        [
          0.1815,
          0.8844
        ],
        [
          0.1915,
          0.879
        ],
        [
          0.2015,
          0.8726
        ],
        [
          0.2116,
          0.8672
        ],
        [
          0.222,
          0.8599
        ],
        [
          0.2325,
          0.8517
        ],
        [
          0.2429,
          0.8449
        ],
        [
          0.253,
          0.8371
        ],
        [
          0.2634,
          0.8303
        ],
        [
          0.2734,
          0.8244
        ],
        [
          0.2834,
          0.8194
        ],
        [
          0.2944,
          0.813
        ],
        [
          0.3048,
          0.8048
        ],
        [
          0.3148,
          0.798
        ],
        [
          0.3248,
          0.7907
        ],
        [
          0.3348,
          0.7843
        ],
        [
          0.3449,
          0.778
        ],
        [
          0.3553,
          0.7698
        ],
        [
          0.3658,
          0.7621
        ],
        [
          0.3763,
          0.7552
        ],
        [
          0.3867,
          0.747
        ],
        [
          0.3967,
          0.7389
        ],
        [
          0.4067,
          0.7316
        ],
        [
          0.4172,
          0.7234
        ],
        [
          0.4272,
          0.7143
        ],
        [
          0.4372,
          0.7093
        ],
        [
          0.4477,
          0.702
        ],
        [
          0.4581,
          0.6943
        ],
        [
          0.4682,
          0.6865
        ],
        [
          0.4786,
          0.6783
        ],
        [
          0.4886,
          0.6724
        ],
        [
          0.4986,
          0.6656
        ],
        [
          0.5086,
          0.6574
        ],
        [
          0.5191,
          0.651
        ],
        [
          0.5291,
          0.6433
        ],
        [
          0.5391,
          0.6351
        ],
        [
          0.5491,
          0.6288
        ],
        [
          0.5591,
          0.6215
        ],
        [
          0.5692,
          0.6124
        ],
        [
          0.5801,
          0.6019
        ],
        [
          0.5905,
          0.596
        ],
        [
          0.6005,
          0.5874
        ],
        [
          0.6115,
          0.5773
        ],
        [
          0.6219,
          0.5692
        ],
        [
          0.6319,
          0.5637
        ],
        [
          0.6419,
          0.5551
        ],
        [
          0.652,
          0.5473
        ],
        [
          0.6624,
          0.5405
        ],
        [
          0.6724,
          0.5323
        ],
        [
          0.6824,
          0.5237
        ],
        [
          0.6934,
          0.5159
        ],
        [
          0.7043,
          0.5077
        ],
        [
          0.7147,
          0.4995
        ],
        [
          0.7252,
          0.4904
        ],
        [
          0.7352,
          0.4827
        ],
        [
          0.747,
          0.4736
        ],
        [
          0.7571,
          0.4645
        ],
        [
          0.7675,
          0.4568
        ],
        [
          0.778,
          0.45
        ],
        [
          0.7884,
          0.4413
        ],
        [
          0.7985,
          0.4327
        ],
        [
          0.8094,
          0.4236
        ],
        [
          0.8203,
          0.4158
        ],
        [
          0.8308,
          0.4067
        ],
        [
          0.8417,
          0.3972
        ],
        [
          0.8544,
          0.3867
        ],
        [
          0.8644,
          0.3776
        ],
        [
          0.8753,
          0.3694
        ],
        [
          0.8854,
          0.3603
        ],
        [
          0.8972,
          0.3526
        ],
        [
          0.9076,
          0.3458
        ],
        [
          0.9186,
          0.3394
        ],
        [
          0.9295,
          0.3298
        ],
        [
          0.9404,
          0.3212
        ],
        [
          0.9504,
          0.313
        ],
        [
          0.9609,
          0.3057
        ],
        [
          0.9713,
          0.2975
        ],
        [
          0.9813,
          0.2898
        ],
        [
          0.9918,
          0.2821
        ],
        [
          1.0,
          0.2748
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.008024528241911754,
        "difference": -0.0009488307284619542,
        "worse": 0.53,
        "kept_right": 0.0070756975134497945,
        "kept_wrong": 0.0009488307284619542,
        "handed_off": 0.9919754717580893
      },
      "usd_per_1000": 0.9557086815265704
    },
    {
      "decision": 141,
      "frontier": 190,
      "frontier_accuracy": 0.9672429481346679,
      "in_sample": {
        "threshold": 0.9578,
        "answered": 0.0031847133757961785,
        "accuracy": 0.9672429481346679
      },
      "curve": [
        [
          0.0,
          0.9672
        ],
        [
          0.01,
          0.9668
        ],
        [
          0.02,
          0.965
        ],
        [
          0.03,
          0.9627
        ],
        [
          0.04,
          0.9591
        ],
        [
          0.05,
          0.9545
        ],
        [
          0.0601,
          0.9495
        ],
        [
          0.0705,
          0.944
        ],
        [
          0.0805,
          0.9377
        ],
        [
          0.091,
          0.9322
        ],
        [
          0.101,
          0.9258
        ],
        [
          0.111,
          0.9208
        ],
        [
          0.121,
          0.9163
        ],
        [
          0.131,
          0.9104
        ],
        [
          0.141,
          0.904
        ],
        [
          0.151,
          0.8976
        ],
        [
          0.1611,
          0.8922
        ],
        [
          0.1715,
          0.8835
        ],
        [
          0.1815,
          0.8794
        ],
        [
          0.1915,
          0.8749
        ],
        [
          0.2015,
          0.8685
        ],
        [
          0.2116,
          0.8626
        ],
        [
          0.222,
          0.8553
        ],
        [
          0.2325,
          0.8471
        ],
        [
          0.2429,
          0.8399
        ],
        [
          0.253,
          0.8326
        ],
        [
          0.2634,
          0.8262
        ],
        [
          0.2734,
          0.8198
        ],
        [
          0.2834,
          0.8148
        ],
        [
          0.2944,
          0.8089
        ],
        [
          0.3048,
          0.8007
        ],
        [
          0.3148,
          0.7939
        ],
        [
          0.3248,
          0.7871
        ],
        [
          0.3348,
          0.7812
        ],
        [
          0.3449,
          0.7748
        ],
        [
          0.3553,
          0.7671
        ],
        [
          0.3658,
          0.7593
        ],
        [
          0.3763,
          0.753
        ],
        [
          0.3867,
          0.7448
        ],
        [
          0.3967,
          0.7366
        ],
        [
          0.4067,
          0.7288
        ],
        [
          0.4172,
          0.7207
        ],
        [
          0.4272,
          0.7116
        ],
        [
          0.4372,
          0.7061
        ],
        [
          0.4477,
          0.6984
        ],
        [
          0.4581,
          0.6906
        ],
        [
          0.4682,
          0.6838
        ],
        [
          0.4786,
          0.6761
        ],
        [
          0.4886,
          0.6702
        ],
        [
          0.4986,
          0.6633
        ],
        [
          0.5086,
          0.6547
        ],
        [
          0.5191,
          0.6483
        ],
        [
          0.5291,
          0.6401
        ],
        [
          0.5391,
          0.6315
        ],
        [
          0.5491,
          0.6237
        ],
        [
          0.5591,
          0.6169
        ],
        [
          0.5692,
          0.6078
        ],
        [
          0.5801,
          0.5974
        ],
        [
          0.5905,
          0.5924
        ],
        [
          0.6005,
          0.5837
        ],
        [
          0.6115,
          0.5737
        ],
        [
          0.6219,
          0.5655
        ],
        [
          0.6319,
          0.5601
        ],
        [
          0.6419,
          0.5523
        ],
        [
          0.652,
          0.5446
        ],
        [
          0.6624,
          0.5373
        ],
        [
          0.6724,
          0.5296
        ],
        [
          0.6824,
          0.5205
        ],
        [
          0.6934,
          0.5127
        ],
        [
          0.7043,
          0.5045
        ],
        [
          0.7147,
          0.4959
        ],
        [
          0.7252,
          0.4868
        ],
        [
          0.7352,
          0.4791
        ],
        [
          0.747,
          0.4709
        ],
        [
          0.7571,
          0.4622
        ],
        [
          0.7675,
          0.4554
        ],
        [
          0.778,
          0.4481
        ],
        [
          0.7884,
          0.4404
        ],
        [
          0.7985,
          0.4318
        ],
        [
          0.8094,
          0.4217
        ],
        [
          0.8203,
          0.414
        ],
        [
          0.8308,
          0.4054
        ],
        [
          0.8417,
          0.3958
        ],
        [
          0.8544,
          0.3854
        ],
        [
          0.8644,
          0.3763
        ],
        [
          0.8753,
          0.3681
        ],
        [
          0.8854,
          0.359
        ],
        [
          0.8972,
          0.3503
        ],
        [
          0.9076,
          0.3435
        ],
        [
          0.9186,
          0.3376
        ],
        [
          0.9295,
          0.328
        ],
        [
          0.9404,
          0.3194
        ],
        [
          0.9504,
          0.3112
        ],
        [
          0.9609,
          0.3044
        ],
        [
          0.9713,
          0.2966
        ],
        [
          0.9813,
          0.2889
        ],
        [
          0.9918,
          0.2816
        ],
        [
          1.0,
          0.2748
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.008024528241911754,
        "difference": -0.0009488307284619542,
        "worse": 0.53,
        "kept_right": 0.0070756975134497945,
        "kept_wrong": 0.0009488307284619542,
        "handed_off": 0.9919754717580893
      },
      "usd_per_1000": 5.52383427916859
    },
    {
      "decision": 141,
      "frontier": 124,
      "frontier_accuracy": 0.9667879890809827,
      "in_sample": {
        "threshold": 0.9578,
        "answered": 0.0031847133757961785,
        "accuracy": 0.9667879890809827
      },
      "curve": [
        [
          0.0,
          0.9668
        ],
        [
          0.01,
          0.9663
        ],
        [
          0.02,
          0.9645
        ],
        [
          0.03,
          0.9622
        ],
        [
          0.04,
          0.9595
        ],
        [
          0.05,
          0.955
        ],
        [
          0.0601,
          0.95
        ],
        [
          0.0705,
          0.944
        ],
        [
          0.0805,
          0.9372
        ],
        [
          0.091,
          0.9318
        ],
        [
          0.101,
          0.9258
        ],
        [
          0.111,
          0.9208
        ],
        [
          0.121,
          0.9163
        ],
        [
          0.131,
          0.9104
        ],
        [
          0.141,
          0.904
        ],
        [
          0.151,
          0.8976
        ],
        [
          0.1611,
          0.8922
        ],
        [
          0.1715,
          0.8835
        ],
        [
          0.1815,
          0.8794
        ],
        [
          0.1915,
          0.874
        ],
        [
          0.2015,
          0.8676
        ],
        [
          0.2116,
          0.8617
        ],
        [
          0.222,
          0.8544
        ],
        [
          0.2325,
          0.8462
        ],
        [
          0.2429,
          0.8389
        ],
        [
          0.253,
          0.8317
        ],
        [
          0.2634,
          0.8248
        ],
        [
          0.2734,
          0.8189
        ],
        [
          0.2834,
          0.8139
        ],
        [
          0.2944,
          0.808
        ],
        [
          0.3048,
          0.8003
        ],
        [
          0.3148,
          0.7934
        ],
        [
          0.3248,
          0.7862
        ],
        [
          0.3348,
          0.7803
        ],
        [
          0.3449,
          0.7739
        ],
        [
          0.3553,
          0.7662
        ],
        [
          0.3658,
          0.7589
        ],
        [
          0.3763,
          0.753
        ],
        [
          0.3867,
          0.7448
        ],
        [
          0.3967,
          0.7375
        ],
        [
          0.4067,
          0.7298
        ],
        [
          0.4172,
          0.7216
        ],
        [
          0.4272,
          0.7125
        ],
        [
          0.4372,
          0.707
        ],
        [
          0.4477,
          0.6997
        ],
        [
          0.4581,
          0.692
        ],
        [
          0.4682,
          0.6852
        ],
        [
          0.4786,
          0.6774
        ],
        [
          0.4886,
          0.672
        ],
        [
          0.4986,
          0.6652
        ],
        [
          0.5086,
          0.6565
        ],
        [
          0.5191,
          0.6506
        ],
        [
          0.5291,
          0.6424
        ],
        [
          0.5391,
          0.6338
        ],
        [
          0.5491,
          0.626
        ],
        [
          0.5591,
          0.6187
        ],
        [
          0.5692,
          0.6092
        ],
        [
          0.5801,
          0.5987
        ],
        [
          0.5905,
          0.5924
        ],
        [
          0.6005,
          0.5837
        ],
        [
          0.6115,
          0.5737
        ],
        [
          0.6219,
          0.5655
        ],
        [
          0.6319,
          0.5601
        ],
        [
          0.6419,
          0.5523
        ],
        [
          0.652,
          0.5446
        ],
        [
          0.6624,
          0.5373
        ],
        [
          0.6724,
          0.5296
        ],
        [
          0.6824,
          0.5205
        ],
        [
          0.6934,
          0.5132
        ],
        [
          0.7043,
          0.505
        ],
        [
          0.7147,
          0.4964
        ],
        [
          0.7252,
          0.4873
        ],
        [
          0.7352,
          0.4795
        ],
        [
          0.747,
          0.4704
        ],
        [
          0.7571,
          0.4613
        ],
        [
          0.7675,
          0.4545
        ],
        [
          0.778,
          0.4472
        ],
        [
          0.7884,
          0.439
        ],
        [
          0.7985,
          0.4304
        ],
        [
          0.8094,
          0.4208
        ],
        [
          0.8203,
          0.4131
        ],
        [
          0.8308,
          0.4049
        ],
        [
          0.8417,
          0.3954
        ],
        [
          0.8544,
          0.3849
        ],
        [
          0.8644,
          0.3758
        ],
        [
          0.8753,
          0.3676
        ],
        [
          0.8854,
          0.3585
        ],
        [
          0.8972,
          0.3499
        ],
        [
          0.9076,
          0.343
        ],
        [
          0.9186,
          0.3371
        ],
        [
          0.9295,
          0.3276
        ],
        [
          0.9404,
          0.3189
        ],
        [
          0.9504,
          0.3107
        ],
        [
          0.9609,
          0.3039
        ],
        [
          0.9713,
          0.2962
        ],
        [
          0.9813,
          0.2884
        ],
        [
          0.9918,
          0.2816
        ],
        [
          1.0,
          0.2748
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.008024528241911754,
        "difference": -0.0009488307284619542,
        "worse": 0.53,
        "kept_right": 0.0070756975134497945,
        "kept_wrong": 0.0009488307284619542,
        "handed_off": 0.9919754717580893
      },
      "usd_per_1000": 1.121302355828606
    },
    {
      "decision": 141,
      "frontier": 278,
      "frontier_accuracy": 0.9499545040946314,
      "in_sample": {
        "threshold": 0.9578,
        "answered": 0.0031847133757961785,
        "accuracy": 0.9499545040946314
      },
      "curve": [
        [
          0.0,
          0.95
        ],
        [
          0.01,
          0.9495
        ],
        [
          0.02,
          0.9477
        ],
        [
          0.03,
          0.9454
        ],
        [
          0.04,
          0.9422
        ],
        [
          0.05,
          0.9381
        ],
        [
          0.0601,
          0.9336
        ],
        [
          0.0705,
          0.9277
        ],
        [
          0.0805,
          0.9208
        ],
        [
          0.091,
          0.9154
        ],
        [
          0.101,
          0.909
        ],
        [
          0.111,
          0.9045
        ],
        [
          0.121,
          0.8999
        ],
        [
          0.131,
          0.8944
        ],
        [
          0.141,
          0.8881
        ],
        [
          0.151,
          0.8826
        ],
        [
          0.1611,
          0.8772
        ],
        [
          0.1715,
          0.8685
        ],
        [
          0.1815,
          0.8644
        ],
        [
          0.1915,
          0.859
        ],
        [
          0.2015,
          0.8526
        ],
        [
          0.2116,
          0.8471
        ],
        [
          0.222,
          0.8399
        ],
        [
          0.2325,
          0.8317
        ],
        [
          0.2429,
          0.8244
        ],
        [
          0.253,
          0.8171
        ],
        [
          0.2634,
          0.8103
        ],
        [
          0.2734,
          0.8048
        ],
        [
          0.2834,
          0.8003
        ],
        [
          0.2944,
          0.7939
        ],
        [
          0.3048,
          0.7866
        ],
        [
          0.3148,
          0.7798
        ],
        [
          0.3248,
          0.7734
        ],
        [
          0.3348,
          0.7666
        ],
        [
          0.3449,
          0.7607
        ],
        [
          0.3553,
          0.753
        ],
        [
          0.3658,
          0.7457
        ],
        [
          0.3763,
          0.7393
        ],
        [
          0.3867,
          0.7311
        ],
        [
          0.3967,
          0.7229
        ],
        [
          0.4067,
          0.7152
        ],
        [
          0.4172,
          0.707
        ],
        [
          0.4272,
          0.6979
        ],
        [
          0.4372,
          0.6924
        ],
        [
          0.4477,
          0.6852
        ],
        [
          0.4581,
          0.6779
        ],
        [
          0.4682,
          0.6715
        ],
        [
          0.4786,
          0.6647
        ],
        [
          0.4886,
          0.6597
        ],
        [
          0.4986,
          0.6529
        ],
        [
          0.5086,
          0.6442
        ],
        [
          0.5191,
          0.6383
        ],
        [
          0.5291,
          0.6306
        ],
        [
          0.5391,
          0.6219
        ],
        [
          0.5491,
          0.6146
        ],
        [
          0.5591,
          0.6074
        ],
        [
          0.5692,
          0.5992
        ],
        [
          0.5801,
          0.5887
        ],
        [
          0.5905,
          0.5833
        ],
        [
          0.6005,
          0.5755
        ],
        [
          0.6115,
          0.5655
        ],
        [
          0.6219,
          0.5573
        ],
        [
          0.6319,
          0.5519
        ],
        [
          0.6419,
          0.5432
        ],
        [
          0.652,
          0.5364
        ],
        [
          0.6624,
          0.5291
        ],
        [
          0.6724,
          0.5214
        ],
        [
          0.6824,
          0.5136
        ],
        [
          0.6934,
          0.5055
        ],
        [
          0.7043,
          0.4977
        ],
        [
          0.7147,
          0.4891
        ],
        [
          0.7252,
          0.4804
        ],
        [
          0.7352,
          0.4727
        ],
        [
          0.747,
          0.4636
        ],
        [
          0.7571,
          0.455
        ],
        [
          0.7675,
          0.4481
        ],
        [
          0.778,
          0.4413
        ],
        [
          0.7884,
          0.4345
        ],
        [
          0.7985,
          0.4263
        ],
        [
          0.8094,
          0.4181
        ],
        [
          0.8203,
          0.4108
        ],
        [
          0.8308,
          0.4017
        ],
        [
          0.8417,
          0.3926
        ],
        [
          0.8544,
          0.3826
        ],
        [
          0.8644,
          0.374
        ],
        [
          0.8753,
          0.3658
        ],
        [
          0.8854,
          0.3576
        ],
        [
          0.8972,
          0.349
        ],
        [
          0.9076,
          0.3426
        ],
        [
          0.9186,
          0.3367
        ],
        [
          0.9295,
          0.328
        ],
        [
          0.9404,
          0.3203
        ],
        [
          0.9504,
          0.3121
        ],
        [
          0.9609,
          0.3053
        ],
        [
          0.9713,
          0.2971
        ],
        [
          0.9813,
          0.2894
        ],
        [
          0.9918,
          0.2821
        ],
        [
          1.0,
          0.2748
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.008024528241911754,
        "difference": -0.0009488307284619542,
        "worse": 0.53,
        "kept_right": 0.0070756975134497945,
        "kept_wrong": 0.0009488307284619542,
        "handed_off": 0.9919754717580893
      },
      "usd_per_1000": 6.45659603619427
    },
    {
      "decision": 141,
      "frontier": 275,
      "frontier_accuracy": 0.9426751592356688,
      "in_sample": {
        "threshold": 0.9578,
        "answered": 0.0031847133757961785,
        "accuracy": 0.9426751592356688
      },
      "curve": [
        [
          0.0,
          0.9427
        ],
        [
          0.01,
          0.9422
        ],
        [
          0.02,
          0.9404
        ],
        [
          0.03,
          0.9381
        ],
        [
          0.04,
          0.9345
        ],
        [
          0.05,
          0.9304
        ],
        [
          0.0601,
          0.9258
        ],
        [
          0.0705,
          0.9195
        ],
        [
          0.0805,
          0.9126
        ],
        [
          0.091,
          0.9072
        ],
        [
          0.101,
          0.9008
        ],
        [
          0.111,
          0.8967
        ],
        [
          0.121,
          0.8926
        ],
        [
          0.131,
          0.8872
        ],
        [
          0.141,
          0.8808
        ],
        [
          0.151,
          0.8749
        ],
        [
          0.1611,
          0.8694
        ],
        [
          0.1715,
          0.8612
        ],
        [
          0.1815,
          0.8576
        ],
        [
          0.1915,
          0.8526
        ],
        [
          0.2015,
          0.8462
        ],
        [
          0.2116,
          0.8408
        ],
        [
          0.222,
          0.8335
        ],
        [
          0.2325,
          0.8258
        ],
        [
          0.2429,
          0.8185
        ],
        [
          0.253,
          0.8112
        ],
        [
          0.2634,
          0.8044
        ],
        [
          0.2734,
          0.7989
        ],
        [
          0.2834,
          0.7944
        ],
        [
          0.2944,
          0.7884
        ],
        [
          0.3048,
          0.7816
        ],
        [
          0.3148,
          0.7748
        ],
        [
          0.3248,
          0.7684
        ],
        [
          0.3348,
          0.7621
        ],
        [
          0.3449,
          0.7561
        ],
        [
          0.3553,
          0.7484
        ],
        [
          0.3658,
          0.7411
        ],
        [
          0.3763,
          0.7348
        ],
        [
          0.3867,
          0.727
        ],
        [
          0.3967,
          0.7197
        ],
        [
          0.4067,
          0.7125
        ],
        [
          0.4172,
          0.7043
        ],
        [
          0.4272,
          0.6961
        ],
        [
          0.4372,
          0.6906
        ],
        [
          0.4477,
          0.6829
        ],
        [
          0.4581,
          0.6752
        ],
        [
          0.4682,
          0.6679
        ],
        [
          0.4786,
          0.6611
        ],
        [
          0.4886,
          0.6551
        ],
        [
          0.4986,
          0.6483
        ],
        [
          0.5086,
          0.6401
        ],
        [
          0.5191,
          0.6342
        ],
        [
          0.5291,
          0.626
        ],
        [
          0.5391,
          0.6174
        ],
        [
          0.5491,
          0.6101
        ],
        [
          0.5591,
          0.6037
        ],
        [
          0.5692,
          0.5951
        ],
        [
          0.5801,
          0.5846
        ],
        [
          0.5905,
          0.5796
        ],
        [
          0.6005,
          0.571
        ],
        [
          0.6115,
          0.5619
        ],
        [
          0.6219,
          0.5537
        ],
        [
          0.6319,
          0.5491
        ],
        [
          0.6419,
          0.5414
        ],
        [
          0.652,
          0.5341
        ],
        [
          0.6624,
          0.5273
        ],
        [
          0.6724,
          0.5196
        ],
        [
          0.6824,
          0.5109
        ],
        [
          0.6934,
          0.5032
        ],
        [
          0.7043,
          0.4955
        ],
        [
          0.7147,
          0.4868
        ],
        [
          0.7252,
          0.4777
        ],
        [
          0.7352,
          0.4704
        ],
        [
          0.747,
          0.4618
        ],
        [
          0.7571,
          0.4536
        ],
        [
          0.7675,
          0.4472
        ],
        [
          0.778,
          0.4399
        ],
        [
          0.7884,
          0.4331
        ],
        [
          0.7985,
          0.4245
        ],
        [
          0.8094,
          0.4154
        ],
        [
          0.8203,
          0.4086
        ],
        [
          0.8308,
          0.4004
        ],
        [
          0.8417,
          0.3908
        ],
        [
          0.8544,
          0.3799
        ],
        [
          0.8644,
          0.3708
        ],
        [
          0.8753,
          0.3635
        ],
        [
          0.8854,
          0.3544
        ],
        [
          0.8972,
          0.3462
        ],
        [
          0.9076,
          0.3399
        ],
        [
          0.9186,
          0.3339
        ],
        [
          0.9295,
          0.3248
        ],
        [
          0.9404,
          0.3162
        ],
        [
          0.9504,
          0.3089
        ],
        [
          0.9609,
          0.3025
        ],
        [
          0.9713,
          0.2953
        ],
        [
          0.9813,
          0.2875
        ],
        [
          0.9918,
          0.2807
        ],
        [
          1.0,
          0.2748
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.008024528241911754,
        "difference": -0.0009488307284619542,
        "worse": 0.53,
        "kept_right": 0.0070756975134497945,
        "kept_wrong": 0.0009488307284619542,
        "handed_off": 0.9919754717580893
      },
      "usd_per_1000": 0.5103844681579718
    },
    {
      "decision": 141,
      "frontier": 192,
      "frontier_accuracy": 0.9331210191082803,
      "in_sample": {
        "threshold": 0.9578,
        "answered": 0.0031847133757961785,
        "accuracy": 0.9331210191082803
      },
      "curve": [
        [
          0.0,
          0.9331
        ],
        [
          0.01,
          0.9327
        ],
        [
          0.02,
          0.9308
        ],
        [
          0.03,
          0.9286
        ],
        [
          0.04,
          0.9254
        ],
        [
          0.05,
          0.9208
        ],
        [
          0.0601,
          0.9167
        ],
        [
          0.0705,
          0.9104
        ],
        [
          0.0805,
          0.904
        ],
        [
          0.091,
          0.8985
        ],
        [
          0.101,
          0.8922
        ],
        [
          0.111,
          0.8876
        ],
        [
          0.121,
          0.8831
        ],
        [
          0.131,
          0.8776
        ],
        [
          0.141,
          0.8712
        ],
        [
          0.151,
          0.8653
        ],
        [
          0.1611,
          0.8599
        ],
        [
          0.1715,
          0.8512
        ],
        [
          0.1815,
          0.8471
        ],
        [
          0.1915,
          0.8417
        ],
        [
          0.2015,
          0.8353
        ],
        [
          0.2116,
          0.8294
        ],
        [
          0.222,
          0.823
        ],
        [
          0.2325,
          0.8148
        ],
        [
          0.2429,
          0.8076
        ],
        [
          0.253,
          0.8003
        ],
        [
          0.2634,
          0.7939
        ],
        [
          0.2734,
          0.7884
        ],
        [
          0.2834,
          0.7839
        ],
        [
          0.2944,
          0.7784
        ],
        [
          0.3048,
          0.7712
        ],
        [
          0.3148,
          0.7652
        ],
        [
          0.3248,
          0.7589
        ],
        [
          0.3348,
          0.7539
        ],
        [
          0.3449,
          0.7484
        ],
        [
          0.3553,
          0.7407
        ],
        [
          0.3658,
          0.7334
        ],
        [
          0.3763,
          0.727
        ],
        [
          0.3867,
          0.7188
        ],
        [
          0.3967,
          0.7111
        ],
        [
          0.4067,
          0.7038
        ],
        [
          0.4172,
          0.6956
        ],
        [
          0.4272,
          0.687
        ],
        [
          0.4372,
          0.6815
        ],
        [
          0.4477,
          0.6738
        ],
        [
          0.4581,
          0.6665
        ],
        [
          0.4682,
          0.6592
        ],
        [
          0.4786,
          0.6524
        ],
        [
          0.4886,
          0.6474
        ],
        [
          0.4986,
          0.641
        ],
        [
          0.5086,
          0.6338
        ],
        [
          0.5191,
          0.6292
        ],
        [
          0.5291,
          0.6219
        ],
        [
          0.5391,
          0.6133
        ],
        [
          0.5491,
          0.606
        ],
        [
          0.5591,
          0.5987
        ],
        [
          0.5692,
          0.591
        ],
        [
          0.5801,
          0.5814
        ],
        [
          0.5905,
          0.5764
        ],
        [
          0.6005,
          0.5687
        ],
        [
          0.6115,
          0.5587
        ],
        [
          0.6219,
          0.5505
        ],
        [
          0.6319,
          0.545
        ],
        [
          0.6419,
          0.5364
        ],
        [
          0.652,
          0.5291
        ],
        [
          0.6624,
          0.5218
        ],
        [
          0.6724,
          0.5146
        ],
        [
          0.6824,
          0.5068
        ],
        [
          0.6934,
          0.4991
        ],
        [
          0.7043,
          0.4918
        ],
        [
          0.7147,
          0.4832
        ],
        [
          0.7252,
          0.4741
        ],
        [
          0.7352,
          0.4668
        ],
        [
          0.747,
          0.4581
        ],
        [
          0.7571,
          0.4495
        ],
        [
          0.7675,
          0.4422
        ],
        [
          0.778,
          0.4349
        ],
        [
          0.7884,
          0.4281
        ],
        [
          0.7985,
          0.4204
        ],
        [
          0.8094,
          0.4122
        ],
        [
          0.8203,
          0.4049
        ],
        [
          0.8308,
          0.3963
        ],
        [
          0.8417,
          0.3876
        ],
        [
          0.8544,
          0.3772
        ],
        [
          0.8644,
          0.3685
        ],
        [
          0.8753,
          0.3608
        ],
        [
          0.8854,
          0.3521
        ],
        [
          0.8972,
          0.3444
        ],
        [
          0.9076,
          0.3376
        ],
        [
          0.9186,
          0.333
        ],
        [
          0.9295,
          0.3248
        ],
        [
          0.9404,
          0.3176
        ],
        [
          0.9504,
          0.3094
        ],
        [
          0.9609,
          0.3039
        ],
        [
          0.9713,
          0.2957
        ],
        [
          0.9813,
          0.288
        ],
        [
          0.9918,
          0.2816
        ],
        [
          1.0,
          0.2748
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.008024528241911754,
        "difference": -0.0009488307284619542,
        "worse": 0.53,
        "kept_right": 0.0070756975134497945,
        "kept_wrong": 0.0009488307284619542,
        "handed_off": 0.9919754717580893
      },
      "usd_per_1000": 11.516404047236191
    },
    {
      "decision": 141,
      "frontier": 123,
      "frontier_accuracy": 0.9326660600545951,
      "in_sample": {
        "threshold": 0.9578,
        "answered": 0.0031847133757961785,
        "accuracy": 0.9326660600545951
      },
      "curve": [
        [
          0.0,
          0.9327
        ],
        [
          0.01,
          0.9322
        ],
        [
          0.02,
          0.9304
        ],
        [
          0.03,
          0.9281
        ],
        [
          0.04,
          0.9249
        ],
        [
          0.05,
          0.9208
        ],
        [
          0.0601,
          0.9158
        ],
        [
          0.0705,
          0.9099
        ],
        [
          0.0805,
          0.9035
        ],
        [
          0.091,
          0.8985
        ],
        [
          0.101,
          0.8926
        ],
        [
          0.111,
          0.8876
        ],
        [
          0.121,
          0.8831
        ],
        [
          0.131,
          0.8776
        ],
        [
          0.141,
          0.8712
        ],
        [
          0.151,
          0.8653
        ],
        [
          0.1611,
          0.8603
        ],
        [
          0.1715,
          0.8526
        ],
        [
          0.1815,
          0.8485
        ],
        [
          0.1915,
          0.843
        ],
        [
          0.2015,
          0.8367
        ],
        [
          0.2116,
          0.8312
        ],
        [
          0.222,
          0.8244
        ],
        [
          0.2325,
          0.8162
        ],
        [
          0.2429,
          0.8098
        ],
        [
          0.253,
          0.8025
        ],
        [
          0.2634,
          0.7957
        ],
        [
          0.2734,
          0.7903
        ],
        [
          0.2834,
          0.7853
        ],
        [
          0.2944,
          0.7789
        ],
        [
          0.3048,
          0.7712
        ],
        [
          0.3148,
          0.7643
        ],
        [
          0.3248,
          0.7575
        ],
        [
          0.3348,
          0.7511
        ],
        [
          0.3449,
          0.7461
        ],
        [
          0.3553,
          0.7379
        ],
        [
          0.3658,
          0.7302
        ],
        [
          0.3763,
          0.7234
        ],
        [
          0.3867,
          0.7157
        ],
        [
          0.3967,
          0.7075
        ],
        [
          0.4067,
          0.7011
        ],
        [
          0.4172,
          0.6929
        ],
        [
          0.4272,
          0.6856
        ],
        [
          0.4372,
          0.6802
        ],
        [
          0.4477,
          0.6738
        ],
        [
          0.4581,
          0.667
        ],
        [
          0.4682,
          0.6597
        ],
        [
          0.4786,
          0.6515
        ],
        [
          0.4886,
          0.6456
        ],
        [
          0.4986,
          0.6401
        ],
        [
          0.5086,
          0.6319
        ],
        [
          0.5191,
          0.6274
        ],
        [
          0.5291,
          0.6206
        ],
        [
          0.5391,
          0.6119
        ],
        [
          0.5491,
          0.6051
        ],
        [
          0.5591,
          0.5983
        ],
        [
          0.5692,
          0.5901
        ],
        [
          0.5801,
          0.5801
        ],
        [
          0.5905,
          0.5746
        ],
        [
          0.6005,
          0.566
        ],
        [
          0.6115,
          0.556
        ],
        [
          0.6219,
          0.5482
        ],
        [
          0.6319,
          0.5441
        ],
        [
          0.6419,
          0.5359
        ],
        [
          0.652,
          0.5282
        ],
        [
          0.6624,
          0.5218
        ],
        [
          0.6724,
          0.5141
        ],
        [
          0.6824,
          0.5055
        ],
        [
          0.6934,
          0.4977
        ],
        [
          0.7043,
          0.4918
        ],
        [
          0.7147,
          0.4836
        ],
        [
          0.7252,
          0.4759
        ],
        [
          0.7352,
          0.4695
        ],
        [
          0.747,
          0.4613
        ],
        [
          0.7571,
          0.4531
        ],
        [
          0.7675,
          0.4454
        ],
        [
          0.778,
          0.4395
        ],
        [
          0.7884,
          0.4318
        ],
        [
          0.7985,
          0.4236
        ],
        [
          0.8094,
          0.4145
        ],
        [
          0.8203,
          0.4081
        ],
        [
          0.8308,
          0.4004
        ],
        [
          0.8417,
          0.3917
        ],
        [
          0.8544,
          0.3822
        ],
        [
          0.8644,
          0.3731
        ],
        [
          0.8753,
          0.3662
        ],
        [
          0.8854,
          0.3571
        ],
        [
          0.8972,
          0.3499
        ],
        [
          0.9076,
          0.3435
        ],
        [
          0.9186,
          0.3376
        ],
        [
          0.9295,
          0.3289
        ],
        [
          0.9404,
          0.3198
        ],
        [
          0.9504,
          0.3121
        ],
        [
          0.9609,
          0.3048
        ],
        [
          0.9713,
          0.2962
        ],
        [
          0.9813,
          0.2884
        ],
        [
          0.9918,
          0.2821
        ],
        [
          1.0,
          0.2748
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.008024528241911754,
        "difference": -0.0009488307284619542,
        "worse": 0.53,
        "kept_right": 0.0070756975134497945,
        "kept_wrong": 0.0009488307284619542,
        "handed_off": 0.9919754717580893
      },
      "usd_per_1000": 0.2664286805339891
    },
    {
      "decision": 141,
      "frontier": 274,
      "frontier_accuracy": 0.9085532302092811,
      "in_sample": {
        "threshold": 0.9578,
        "answered": 0.0031847133757961785,
        "accuracy": 0.9085532302092811
      },
      "curve": [
        [
          0.0,
          0.9086
        ],
        [
          0.01,
          0.9081
        ],
        [
          0.02,
          0.9063
        ],
        [
          0.03,
          0.904
        ],
        [
          0.04,
          0.9008
        ],
        [
          0.05,
          0.8972
        ],
        [
          0.0601,
          0.8926
        ],
        [
          0.0705,
          0.8872
        ],
        [
          0.0805,
          0.8808
        ],
        [
          0.091,
          0.8753
        ],
        [
          0.101,
          0.869
        ],
        [
          0.111,
          0.8644
        ],
        [
          0.121,
          0.8603
        ],
        [
          0.131,
          0.8553
        ],
        [
          0.141,
          0.8494
        ],
        [
          0.151,
          0.8435
        ],
        [
          0.1611,
          0.838
        ],
        [
          0.1715,
          0.8298
        ],
        [
          0.1815,
          0.8267
        ],
        [
          0.1915,
          0.8221
        ],
        [
          0.2015,
          0.8162
        ],
        [
          0.2116,
          0.8107
        ],
        [
          0.222,
          0.8039
        ],
        [
          0.2325,
          0.7962
        ],
        [
          0.2429,
          0.7889
        ],
        [
          0.253,
          0.7816
        ],
        [
          0.2634,
          0.7748
        ],
        [
          0.2734,
          0.7698
        ],
        [
          0.2834,
          0.7662
        ],
        [
          0.2944,
          0.7607
        ],
        [
          0.3048,
          0.7539
        ],
        [
          0.3148,
          0.747
        ],
        [
          0.3248,
          0.7402
        ],
        [
          0.3348,
          0.7352
        ],
        [
          0.3449,
          0.7293
        ],
        [
          0.3553,
          0.7216
        ],
        [
          0.3658,
          0.7143
        ],
        [
          0.3763,
          0.7084
        ],
        [
          0.3867,
          0.7002
        ],
        [
          0.3967,
          0.6929
        ],
        [
          0.4067,
          0.6861
        ],
        [
          0.4172,
          0.6788
        ],
        [
          0.4272,
          0.6706
        ],
        [
          0.4372,
          0.6661
        ],
        [
          0.4477,
          0.6597
        ],
        [
          0.4581,
          0.6538
        ],
        [
          0.4682,
          0.6465
        ],
        [
          0.4786,
          0.6397
        ],
        [
          0.4886,
          0.6342
        ],
        [
          0.4986,
          0.6288
        ],
        [
          0.5086,
          0.621
        ],
        [
          0.5191,
          0.616
        ],
        [
          0.5291,
          0.6083
        ],
        [
          0.5391,
          0.6001
        ],
        [
          0.5491,
          0.5924
        ],
        [
          0.5591,
          0.5869
        ],
        [
          0.5692,
          0.5805
        ],
        [
          0.5801,
          0.5701
        ],
        [
          0.5905,
          0.5651
        ],
        [
          0.6005,
          0.5573
        ],
        [
          0.6115,
          0.5478
        ],
        [
          0.6219,
          0.5405
        ],
        [
          0.6319,
          0.5369
        ],
        [
          0.6419,
          0.5291
        ],
        [
          0.652,
          0.5218
        ],
        [
          0.6624,
          0.5155
        ],
        [
          0.6724,
          0.5082
        ],
        [
          0.6824,
          0.5
        ],
        [
          0.6934,
          0.4923
        ],
        [
          0.7043,
          0.4854
        ],
        [
          0.7147,
          0.4777
        ],
        [
          0.7252,
          0.4695
        ],
        [
          0.7352,
          0.4618
        ],
        [
          0.747,
          0.4531
        ],
        [
          0.7571,
          0.4454
        ],
        [
          0.7675,
          0.439
        ],
        [
          0.778,
          0.4322
        ],
        [
          0.7884,
          0.4268
        ],
        [
          0.7985,
          0.419
        ],
        [
          0.8094,
          0.4095
        ],
        [
          0.8203,
          0.4031
        ],
        [
          0.8308,
          0.3958
        ],
        [
          0.8417,
          0.3876
        ],
        [
          0.8544,
          0.3776
        ],
        [
          0.8644,
          0.3694
        ],
        [
          0.8753,
          0.3621
        ],
        [
          0.8854,
          0.3535
        ],
        [
          0.8972,
          0.3453
        ],
        [
          0.9076,
          0.3389
        ],
        [
          0.9186,
          0.3335
        ],
        [
          0.9295,
          0.3244
        ],
        [
          0.9404,
          0.3162
        ],
        [
          0.9504,
          0.3089
        ],
        [
          0.9609,
          0.3025
        ],
        [
          0.9713,
          0.2948
        ],
        [
          0.9813,
          0.2875
        ],
        [
          0.9918,
          0.2807
        ],
        [
          1.0,
          0.2748
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.008024528241911754,
        "difference": -0.0009488307284619542,
        "worse": 0.53,
        "kept_right": 0.0070756975134497945,
        "kept_wrong": 0.0009488307284619542,
        "handed_off": 0.9919754717580893
      },
      "usd_per_1000": 0.42282525740877697
    },
    {
      "decision": 141,
      "frontier": 47,
      "frontier_accuracy": 0.9044585987261147,
      "in_sample": {
        "threshold": 0.9578,
        "answered": 0.0031847133757961785,
        "accuracy": 0.9044585987261147
      },
      "curve": [
        [
          0.0,
          0.9045
        ],
        [
          0.01,
          0.904
        ],
        [
          0.02,
          0.9022
        ],
        [
          0.03,
          0.8999
        ],
        [
          0.04,
          0.8967
        ],
        [
          0.05,
          0.8926
        ],
        [
          0.0601,
          0.8876
        ],
        [
          0.0705,
          0.8822
        ],
        [
          0.0805,
          0.8758
        ],
        [
          0.091,
          0.8708
        ],
        [
          0.101,
          0.8649
        ],
        [
          0.111,
          0.8594
        ],
        [
          0.121,
          0.8549
        ],
        [
          0.131,
          0.8503
        ],
        [
          0.141,
          0.8439
        ],
        [
          0.151,
          0.838
        ],
        [
          0.1611,
          0.8326
        ],
        [
          0.1715,
          0.8244
        ],
        [
          0.1815,
          0.8203
        ],
        [
          0.1915,
          0.8148
        ],
        [
          0.2015,
          0.8085
        ],
        [
          0.2116,
          0.8025
        ],
        [
          0.222,
          0.7962
        ],
        [
          0.2325,
          0.7884
        ],
        [
          0.2429,
          0.7816
        ],
        [
          0.253,
          0.7743
        ],
        [
          0.2634,
          0.768
        ],
        [
          0.2734,
          0.7634
        ],
        [
          0.2834,
          0.7593
        ],
        [
          0.2944,
          0.7539
        ],
        [
          0.3048,
          0.7466
        ],
        [
          0.3148,
          0.7402
        ],
        [
          0.3248,
          0.7343
        ],
        [
          0.3348,
          0.7284
        ],
        [
          0.3449,
          0.7225
        ],
        [
          0.3553,
          0.7143
        ],
        [
          0.3658,
          0.7079
        ],
        [
          0.3763,
          0.7025
        ],
        [
          0.3867,
          0.6952
        ],
        [
          0.3967,
          0.687
        ],
        [
          0.4067,
          0.6793
        ],
        [
          0.4172,
          0.6715
        ],
        [
          0.4272,
          0.6633
        ],
        [
          0.4372,
          0.6583
        ],
        [
          0.4477,
          0.651
        ],
        [
          0.4581,
          0.6451
        ],
        [
          0.4682,
          0.6388
        ],
        [
          0.4786,
          0.6319
        ],
        [
          0.4886,
          0.6274
        ],
        [
          0.4986,
          0.6224
        ],
        [
          0.5086,
          0.6142
        ],
        [
          0.5191,
          0.6083
        ],
        [
          0.5291,
          0.601
        ],
        [
          0.5391,
          0.5933
        ],
        [
          0.5491,
          0.5869
        ],
        [
          0.5591,
          0.5796
        ],
        [
          0.5692,
          0.5723
        ],
        [
          0.5801,
          0.5628
        ],
        [
          0.5905,
          0.5573
        ],
        [
          0.6005,
          0.5505
        ],
        [
          0.6115,
          0.5405
        ],
        [
          0.6219,
          0.5337
        ],
        [
          0.6319,
          0.5287
        ],
        [
          0.6419,
          0.5214
        ],
        [
          0.652,
          0.5146
        ],
        [
          0.6624,
          0.5077
        ],
        [
          0.6724,
          0.5005
        ],
        [
          0.6824,
          0.4932
        ],
        [
          0.6934,
          0.4868
        ],
        [
          0.7043,
          0.4791
        ],
        [
          0.7147,
          0.4718
        ],
        [
          0.7252,
          0.4627
        ],
        [
          0.7352,
          0.4563
        ],
        [
          0.747,
          0.4481
        ],
        [
          0.7571,
          0.439
        ],
        [
          0.7675,
          0.4322
        ],
        [
          0.778,
          0.4263
        ],
        [
          0.7884,
          0.4199
        ],
        [
          0.7985,
          0.4122
        ],
        [
          0.8094,
          0.4045
        ],
        [
          0.8203,
          0.3981
        ],
        [
          0.8308,
          0.3894
        ],
        [
          0.8417,
          0.3813
        ],
        [
          0.8544,
          0.3717
        ],
        [
          0.8644,
          0.3644
        ],
        [
          0.8753,
          0.3585
        ],
        [
          0.8854,
          0.3503
        ],
        [
          0.8972,
          0.3426
        ],
        [
          0.9076,
          0.3362
        ],
        [
          0.9186,
          0.3303
        ],
        [
          0.9295,
          0.3226
        ],
        [
          0.9404,
          0.3148
        ],
        [
          0.9504,
          0.3066
        ],
        [
          0.9609,
          0.3007
        ],
        [
          0.9713,
          0.293
        ],
        [
          0.9813,
          0.2862
        ],
        [
          0.9918,
          0.2803
        ],
        [
          1.0,
          0.2748
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.008024528241911754,
        "difference": -0.0009488307284619542,
        "worse": 0.53,
        "kept_right": 0.0070756975134497945,
        "kept_wrong": 0.0009488307284619542,
        "handed_off": 0.9919754717580893
      },
      "usd_per_1000": 4.891020346462364
    },
    {
      "decision": 141,
      "frontier": 193,
      "frontier_accuracy": 0.89171974522293,
      "in_sample": {
        "threshold": 0.9578,
        "answered": 0.0031847133757961785,
        "accuracy": 0.89171974522293
      },
      "curve": [
        [
          0.0,
          0.8917
        ],
        [
          0.01,
          0.8913
        ],
        [
          0.02,
          0.8894
        ],
        [
          0.03,
          0.8872
        ],
        [
          0.04,
          0.884
        ],
        [
          0.05,
          0.8799
        ],
        [
          0.0601,
          0.8749
        ],
        [
          0.0705,
          0.8694
        ],
        [
          0.0805,
          0.8631
        ],
        [
          0.091,
          0.8576
        ],
        [
          0.101,
          0.8512
        ],
        [
          0.111,
          0.8471
        ],
        [
          0.121,
          0.8426
        ],
        [
          0.131,
          0.8385
        ],
        [
          0.141,
          0.8335
        ],
        [
          0.151,
          0.828
        ],
        [
          0.1611,
          0.8226
        ],
        [
          0.1715,
          0.8148
        ],
        [
          0.1815,
          0.8112
        ],
        [
          0.1915,
          0.8062
        ],
        [
          0.2015,
          0.8007
        ],
        [
          0.2116,
          0.7957
        ],
        [
          0.222,
          0.7894
        ],
        [
          0.2325,
          0.7816
        ],
        [
          0.2429,
          0.7753
        ],
        [
          0.253,
          0.768
        ],
        [
          0.2634,
          0.7616
        ],
        [
          0.2734,
          0.7566
        ],
        [
          0.2834,
          0.7534
        ],
        [
          0.2944,
          0.748
        ],
        [
          0.3048,
          0.7407
        ],
        [
          0.3148,
          0.7348
        ],
        [
          0.3248,
          0.7288
        ],
        [
          0.3348,
          0.7234
        ],
        [
          0.3449,
          0.7175
        ],
        [
          0.3553,
          0.7102
        ],
        [
          0.3658,
          0.7029
        ],
        [
          0.3763,
          0.6965
        ],
        [
          0.3867,
          0.6888
        ],
        [
          0.3967,
          0.6811
        ],
        [
          0.4067,
          0.6733
        ],
        [
          0.4172,
          0.6656
        ],
        [
          0.4272,
          0.6574
        ],
        [
          0.4372,
          0.6529
        ],
        [
          0.4477,
          0.6465
        ],
        [
          0.4581,
          0.6406
        ],
        [
          0.4682,
          0.6342
        ],
        [
          0.4786,
          0.6274
        ],
        [
          0.4886,
          0.6224
        ],
        [
          0.4986,
          0.6165
        ],
        [
          0.5086,
          0.6096
        ],
        [
          0.5191,
          0.6056
        ],
        [
          0.5291,
          0.5983
        ],
        [
          0.5391,
          0.5905
        ],
        [
          0.5491,
          0.5833
        ],
        [
          0.5591,
          0.5769
        ],
        [
          0.5692,
          0.5692
        ],
        [
          0.5801,
          0.5596
        ],
        [
          0.5905,
          0.5541
        ],
        [
          0.6005,
          0.5478
        ],
        [
          0.6115,
          0.5387
        ],
        [
          0.6219,
          0.5318
        ],
        [
          0.6319,
          0.5268
        ],
        [
          0.6419,
          0.5191
        ],
        [
          0.652,
          0.5123
        ],
        [
          0.6624,
          0.5077
        ],
        [
          0.6724,
          0.5005
        ],
        [
          0.6824,
          0.4936
        ],
        [
          0.6934,
          0.4864
        ],
        [
          0.7043,
          0.4786
        ],
        [
          0.7147,
          0.4713
        ],
        [
          0.7252,
          0.4627
        ],
        [
          0.7352,
          0.4559
        ],
        [
          0.747,
          0.4472
        ],
        [
          0.7571,
          0.4395
        ],
        [
          0.7675,
          0.4327
        ],
        [
          0.778,
          0.4263
        ],
        [
          0.7884,
          0.4208
        ],
        [
          0.7985,
          0.4136
        ],
        [
          0.8094,
          0.4058
        ],
        [
          0.8203,
          0.3999
        ],
        [
          0.8308,
          0.3922
        ],
        [
          0.8417,
          0.384
        ],
        [
          0.8544,
          0.374
        ],
        [
          0.8644,
          0.3649
        ],
        [
          0.8753,
          0.3571
        ],
        [
          0.8854,
          0.3494
        ],
        [
          0.8972,
          0.3421
        ],
        [
          0.9076,
          0.3358
        ],
        [
          0.9186,
          0.3303
        ],
        [
          0.9295,
          0.3217
        ],
        [
          0.9404,
          0.3153
        ],
        [
          0.9504,
          0.308
        ],
        [
          0.9609,
          0.303
        ],
        [
          0.9713,
          0.2957
        ],
        [
          0.9813,
          0.2884
        ],
        [
          0.9918,
          0.2816
        ],
        [
          1.0,
          0.2748
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.008024528241911754,
        "difference": -0.0009488307284619542,
        "worse": 0.53,
        "kept_right": 0.0070756975134497945,
        "kept_wrong": 0.0009488307284619542,
        "handed_off": 0.9919754717580893
      },
      "usd_per_1000": 7.685419236212082
    },
    {
      "decision": 141,
      "frontier": 55,
      "frontier_accuracy": 0.8503184713375797,
      "in_sample": {
        "threshold": 0.9578,
        "answered": 0.0031847133757961785,
        "accuracy": 0.8503184713375797
      },
      "curve": [
        [
          0.0,
          0.8503
        ],
        [
          0.01,
          0.8499
        ],
        [
          0.02,
          0.848
        ],
        [
          0.03,
          0.8458
        ],
        [
          0.04,
          0.8426
        ],
        [
          0.05,
          0.8385
        ],
        [
          0.0601,
          0.8335
        ],
        [
          0.0705,
          0.8285
        ],
        [
          0.0805,
          0.8226
        ],
        [
          0.091,
          0.8185
        ],
        [
          0.101,
          0.813
        ],
        [
          0.111,
          0.808
        ],
        [
          0.121,
          0.8039
        ],
        [
          0.131,
          0.8003
        ],
        [
          0.141,
          0.7953
        ],
        [
          0.151,
          0.7894
        ],
        [
          0.1611,
          0.7843
        ],
        [
          0.1715,
          0.7766
        ],
        [
          0.1815,
          0.7739
        ],
        [
          0.1915,
          0.7698
        ],
        [
          0.2015,
          0.7648
        ],
        [
          0.2116,
          0.7598
        ],
        [
          0.222,
          0.7543
        ],
        [
          0.2325,
          0.7475
        ],
        [
          0.2429,
          0.7411
        ],
        [
          0.253,
          0.7338
        ],
        [
          0.2634,
          0.7279
        ],
        [
          0.2734,
          0.7229
        ],
        [
          0.2834,
          0.7193
        ],
        [
          0.2944,
          0.7134
        ],
        [
          0.3048,
          0.707
        ],
        [
          0.3148,
          0.7011
        ],
        [
          0.3248,
          0.6947
        ],
        [
          0.3348,
          0.6911
        ],
        [
          0.3449,
          0.6856
        ],
        [
          0.3553,
          0.6783
        ],
        [
          0.3658,
          0.6715
        ],
        [
          0.3763,
          0.6665
        ],
        [
          0.3867,
          0.6597
        ],
        [
          0.3967,
          0.6529
        ],
        [
          0.4067,
          0.6479
        ],
        [
          0.4172,
          0.6401
        ],
        [
          0.4272,
          0.6324
        ],
        [
          0.4372,
          0.6274
        ],
        [
          0.4477,
          0.6224
        ],
        [
          0.4581,
          0.616
        ],
        [
          0.4682,
          0.6106
        ],
        [
          0.4786,
          0.6037
        ],
        [
          0.4886,
          0.5996
        ],
        [
          0.4986,
          0.5942
        ],
        [
          0.5086,
          0.5883
        ],
        [
          0.5191,
          0.5837
        ],
        [
          0.5291,
          0.5764
        ],
        [
          0.5391,
          0.5696
        ],
        [
          0.5491,
          0.5641
        ],
        [
          0.5591,
          0.5582
        ],
        [
          0.5692,
          0.5528
        ],
        [
          0.5801,
          0.5437
        ],
        [
          0.5905,
          0.5387
        ],
        [
          0.6005,
          0.5318
        ],
        [
          0.6115,
          0.5241
        ],
        [
          0.6219,
          0.5164
        ],
        [
          0.6319,
          0.5114
        ],
        [
          0.6419,
          0.5041
        ],
        [
          0.652,
          0.4977
        ],
        [
          0.6624,
          0.4941
        ],
        [
          0.6724,
          0.4873
        ],
        [
          0.6824,
          0.4809
        ],
        [
          0.6934,
          0.4741
        ],
        [
          0.7043,
          0.4672
        ],
        [
          0.7147,
          0.4613
        ],
        [
          0.7252,
          0.4536
        ],
        [
          0.7352,
          0.4472
        ],
        [
          0.747,
          0.4404
        ],
        [
          0.7571,
          0.4322
        ],
        [
          0.7675,
          0.4263
        ],
        [
          0.778,
          0.4195
        ],
        [
          0.7884,
          0.4126
        ],
        [
          0.7985,
          0.4058
        ],
        [
          0.8094,
          0.3985
        ],
        [
          0.8203,
          0.3917
        ],
        [
          0.8308,
          0.384
        ],
        [
          0.8417,
          0.3763
        ],
        [
          0.8544,
          0.3658
        ],
        [
          0.8644,
          0.3581
        ],
        [
          0.8753,
          0.3508
        ],
        [
          0.8854,
          0.3435
        ],
        [
          0.8972,
          0.3371
        ],
        [
          0.9076,
          0.3312
        ],
        [
          0.9186,
          0.3262
        ],
        [
          0.9295,
          0.3194
        ],
        [
          0.9404,
          0.3121
        ],
        [
          0.9504,
          0.3053
        ],
        [
          0.9609,
          0.2994
        ],
        [
          0.9713,
          0.2925
        ],
        [
          0.9813,
          0.2862
        ],
        [
          0.9918,
          0.2798
        ],
        [
          1.0,
          0.2748
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.008024528241911754,
        "difference": -0.0009488307284619542,
        "worse": 0.53,
        "kept_right": 0.0070756975134497945,
        "kept_wrong": 0.0009488307284619542,
        "handed_off": 0.9919754717580893
      },
      "usd_per_1000": 3.6250093165711705
    },
    {
      "decision": 243,
      "frontier": 140,
      "frontier_accuracy": 0.9808917197452229,
      "in_sample": {
        "threshold": 0.9256,
        "answered": 0.0018198362147406734,
        "accuracy": 0.9808917197452229
      },
      "curve": [
        [
          0.0,
          0.9809
        ],
        [
          0.01,
          0.9777
        ],
        [
          0.02,
          0.9741
        ],
        [
          0.03,
          0.9709
        ],
        [
          0.04,
          0.965
        ],
        [
          0.05,
          0.96
        ],
        [
          0.0601,
          0.955
        ],
        [
          0.0705,
          0.9472
        ],
        [
          0.0805,
          0.9418
        ],
        [
          0.091,
          0.9359
        ],
        [
          0.101,
          0.9308
        ],
        [
          0.111,
          0.924
        ],
        [
          0.121,
          0.9186
        ],
        [
          0.131,
          0.9131
        ],
        [
          0.141,
          0.9054
        ],
        [
          0.151,
          0.8981
        ],
        [
          0.1611,
          0.8908
        ],
        [
          0.1715,
          0.8844
        ],
        [
          0.1815,
          0.8767
        ],
        [
          0.1915,
          0.8708
        ],
        [
          0.2015,
          0.864
        ],
        [
          0.2116,
          0.859
        ],
        [
          0.222,
          0.8512
        ],
        [
          0.232,
          0.843
        ],
        [
          0.242,
          0.8362
        ],
        [
          0.252,
          0.8294
        ],
        [
          0.2621,
          0.8217
        ],
        [
          0.2725,
          0.8144
        ],
        [
          0.2825,
          0.8071
        ],
        [
          0.2925,
          0.7998
        ],
        [
          0.3025,
          0.7939
        ],
        [
          0.3126,
          0.7862
        ],
        [
          0.3235,
          0.7766
        ],
        [
          0.3339,
          0.7698
        ],
        [
          0.3444,
          0.7621
        ],
        [
          0.3544,
          0.7539
        ],
        [
          0.3644,
          0.7461
        ],
        [
          0.3744,
          0.7407
        ],
        [
          0.3844,
          0.7325
        ],
        [
          0.3944,
          0.7257
        ],
        [
          0.4045,
          0.7184
        ],
        [
          0.4149,
          0.7102
        ],
        [
          0.4249,
          0.7052
        ],
        [
          0.4354,
          0.6975
        ],
        [
          0.4454,
          0.6902
        ],
        [
          0.4554,
          0.6843
        ],
        [
          0.4654,
          0.6774
        ],
        [
          0.4759,
          0.6697
        ],
        [
          0.4864,
          0.6638
        ],
        [
          0.4973,
          0.6556
        ],
        [
          0.5077,
          0.646
        ],
        [
          0.5177,
          0.6397
        ],
        [
          0.5278,
          0.631
        ],
        [
          0.5387,
          0.6228
        ],
        [
          0.5491,
          0.6142
        ],
        [
          0.5591,
          0.606
        ],
        [
          0.5692,
          0.5974
        ],
        [
          0.5796,
          0.5901
        ],
        [
          0.5896,
          0.5823
        ],
        [
          0.5996,
          0.5728
        ],
        [
          0.6096,
          0.5655
        ],
        [
          0.6197,
          0.5596
        ],
        [
          0.6306,
          0.5537
        ],
        [
          0.641,
          0.5469
        ],
        [
          0.6515,
          0.5373
        ],
        [
          0.6624,
          0.5309
        ],
        [
          0.6724,
          0.5241
        ],
        [
          0.6824,
          0.5146
        ],
        [
          0.6924,
          0.5045
        ],
        [
          0.7029,
          0.4968
        ],
        [
          0.7129,
          0.4914
        ],
        [
          0.7229,
          0.4841
        ],
        [
          0.7334,
          0.4768
        ],
        [
          0.7434,
          0.4682
        ],
        [
          0.7539,
          0.4609
        ],
        [
          0.7643,
          0.4531
        ],
        [
          0.7743,
          0.4445
        ],
        [
          0.7848,
          0.4363
        ],
        [
          0.7957,
          0.4272
        ],
        [
          0.8071,
          0.4208
        ],
        [
          0.8171,
          0.4145
        ],
        [
          0.8276,
          0.4067
        ],
        [
          0.838,
          0.399
        ],
        [
          0.848,
          0.3913
        ],
        [
          0.8581,
          0.3835
        ],
        [
          0.8685,
          0.3744
        ],
        [
          0.8794,
          0.3662
        ],
        [
          0.8894,
          0.3576
        ],
        [
          0.8995,
          0.3499
        ],
        [
          0.9104,
          0.3426
        ],
        [
          0.9208,
          0.3335
        ],
        [
          0.9308,
          0.3258
        ],
        [
          0.9418,
          0.3162
        ],
        [
          0.9522,
          0.3089
        ],
        [
          0.9622,
          0.3016
        ],
        [
          0.9722,
          0.293
        ],
        [
          0.9823,
          0.2848
        ],
        [
          0.9927,
          0.2771
        ],
        [
          1.0,
          0.2712
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.0017280412206164926,
        "difference": -0.00047906128561831424,
        "worse": 0.17666666666666667,
        "kept_right": 0.0012489799349981788,
        "kept_wrong": 0.00047906128561831424,
        "handed_off": 0.9982719587793846
      },
      "usd_per_1000": 2.429685818317653
    },
    {
      "decision": 243,
      "frontier": 277,
      "frontier_accuracy": 0.9795268425841674,
      "in_sample": {
        "threshold": 0.9256,
        "answered": 0.0018198362147406734,
        "accuracy": 0.9795268425841674
      },
      "curve": [
        [
          0.0,
          0.9795
        ],
        [
          0.01,
          0.9763
        ],
        [
          0.02,
          0.9732
        ],
        [
          0.03,
          0.97
        ],
        [
          0.04,
          0.9636
        ],
        [
          0.05,
          0.9586
        ],
        [
          0.0601,
          0.9536
        ],
        [
          0.0705,
          0.9459
        ],
        [
          0.0805,
          0.9404
        ],
        [
          0.091,
          0.9345
        ],
        [
          0.101,
          0.9295
        ],
        [
          0.111,
          0.9227
        ],
        [
          0.121,
          0.9172
        ],
        [
          0.131,
          0.9122
        ],
        [
          0.141,
          0.904
        ],
        [
          0.151,
          0.8972
        ],
        [
          0.1611,
          0.8899
        ],
        [
          0.1715,
          0.8826
        ],
        [
          0.1815,
          0.8749
        ],
        [
          0.1915,
          0.869
        ],
        [
          0.2015,
          0.8621
        ],
        [
          0.2116,
          0.8571
        ],
        [
          0.222,
          0.8494
        ],
        [
          0.232,
          0.8412
        ],
        [
          0.242,
          0.8344
        ],
        [
          0.252,
          0.8276
        ],
        [
          0.2621,
          0.8203
        ],
        [
          0.2725,
          0.813
        ],
        [
          0.2825,
          0.8057
        ],
        [
          0.2925,
          0.7985
        ],
        [
          0.3025,
          0.7916
        ],
        [
          0.3126,
          0.7839
        ],
        [
          0.3235,
          0.7743
        ],
        [
          0.3339,
          0.7675
        ],
        [
          0.3444,
          0.7598
        ],
        [
          0.3544,
          0.752
        ],
        [
          0.3644,
          0.7439
        ],
        [
          0.3744,
          0.7384
        ],
        [
          0.3844,
          0.7298
        ],
        [
          0.3944,
          0.7229
        ],
        [
          0.4045,
          0.7157
        ],
        [
          0.4149,
          0.7075
        ],
        [
          0.4249,
          0.7025
        ],
        [
          0.4354,
          0.6952
        ],
        [
          0.4454,
          0.6884
        ],
        [
          0.4554,
          0.6824
        ],
        [
          0.4654,
          0.6756
        ],
        [
          0.4759,
          0.6679
        ],
        [
          0.4864,
          0.662
        ],
        [
          0.4973,
          0.6538
        ],
        [
          0.5077,
          0.6442
        ],
        [
          0.5177,
          0.6379
        ],
        [
          0.5278,
          0.6292
        ],
        [
          0.5387,
          0.621
        ],
        [
          0.5491,
          0.6128
        ],
        [
          0.5591,
          0.6046
        ],
        [
          0.5692,
          0.596
        ],
        [
          0.5796,
          0.5883
        ],
        [
          0.5896,
          0.5805
        ],
        [
          0.5996,
          0.571
        ],
        [
          0.6096,
          0.5637
        ],
        [
          0.6197,
          0.5578
        ],
        [
          0.6306,
          0.5519
        ],
        [
          0.641,
          0.545
        ],
        [
          0.6515,
          0.5355
        ],
        [
          0.6624,
          0.5296
        ],
        [
          0.6724,
          0.5227
        ],
        [
          0.6824,
          0.5132
        ],
        [
          0.6924,
          0.5032
        ],
        [
          0.7029,
          0.4955
        ],
        [
          0.7129,
          0.4891
        ],
        [
          0.7229,
          0.4818
        ],
        [
          0.7334,
          0.4745
        ],
        [
          0.7434,
          0.4659
        ],
        [
          0.7539,
          0.4586
        ],
        [
          0.7643,
          0.4509
        ],
        [
          0.7743,
          0.4427
        ],
        [
          0.7848,
          0.4345
        ],
        [
          0.7957,
          0.4254
        ],
        [
          0.8071,
          0.419
        ],
        [
          0.8171,
          0.4126
        ],
        [
          0.8276,
          0.4054
        ],
        [
          0.838,
          0.3976
        ],
        [
          0.848,
          0.3899
        ],
        [
          0.8581,
          0.3822
        ],
        [
          0.8685,
          0.3735
        ],
        [
          0.8794,
          0.3653
        ],
        [
          0.8894,
          0.3567
        ],
        [
          0.8995,
          0.3485
        ],
        [
          0.9104,
          0.3412
        ],
        [
          0.9208,
          0.3321
        ],
        [
          0.9308,
          0.3244
        ],
        [
          0.9418,
          0.3153
        ],
        [
          0.9522,
          0.3085
        ],
        [
          0.9622,
          0.3012
        ],
        [
          0.9722,
          0.2925
        ],
        [
          0.9823,
          0.2843
        ],
        [
          0.9927,
          0.2771
        ],
        [
          1.0,
          0.2712
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.0017765701863429102,
        "difference": -0.0005063588288394243,
        "worse": 0.17666666666666667,
        "kept_right": 0.0012702113575034867,
        "kept_wrong": 0.0005063588288394243,
        "handed_off": 0.9982234298136581
      },
      "usd_per_1000": 6.069542132112993
    },
    {
      "decision": 243,
      "frontier": 276,
      "frontier_accuracy": 0.9777070063694268,
      "in_sample": {
        "threshold": 0.9256,
        "answered": 0.0018198362147406734,
        "accuracy": 0.9777070063694268
      },
      "curve": [
        [
          0.0,
          0.9777
        ],
        [
          0.01,
          0.975
        ],
        [
          0.02,
          0.9718
        ],
        [
          0.03,
          0.9686
        ],
        [
          0.04,
          0.9622
        ],
        [
          0.05,
          0.9572
        ],
        [
          0.0601,
          0.9522
        ],
        [
          0.0705,
          0.9445
        ],
        [
          0.0805,
          0.939
        ],
        [
          0.091,
          0.9331
        ],
        [
          0.101,
          0.9281
        ],
        [
          0.111,
          0.9213
        ],
        [
          0.121,
          0.9158
        ],
        [
          0.131,
          0.9104
        ],
        [
          0.141,
          0.9022
        ],
        [
          0.151,
          0.8954
        ],
        [
          0.1611,
          0.8881
        ],
        [
          0.1715,
          0.8808
        ],
        [
          0.1815,
          0.8731
        ],
        [
          0.1915,
          0.8672
        ],
        [
          0.2015,
          0.8603
        ],
        [
          0.2116,
          0.8553
        ],
        [
          0.222,
          0.8476
        ],
        [
          0.232,
          0.8394
        ],
        [
          0.242,
          0.833
        ],
        [
          0.252,
          0.8262
        ],
        [
          0.2621,
          0.8185
        ],
        [
          0.2725,
          0.8112
        ],
        [
          0.2825,
          0.8039
        ],
        [
          0.2925,
          0.7966
        ],
        [
          0.3025,
          0.7903
        ],
        [
          0.3126,
          0.7825
        ],
        [
          0.3235,
          0.773
        ],
        [
          0.3339,
          0.7662
        ],
        [
          0.3444,
          0.7584
        ],
        [
          0.3544,
          0.7507
        ],
        [
          0.3644,
          0.7425
        ],
        [
          0.3744,
          0.737
        ],
        [
          0.3844,
          0.7284
        ],
        [
          0.3944,
          0.7216
        ],
        [
          0.4045,
          0.7143
        ],
        [
          0.4149,
          0.7061
        ],
        [
          0.4249,
          0.7011
        ],
        [
          0.4354,
          0.6938
        ],
        [
          0.4454,
          0.687
        ],
        [
          0.4554,
          0.6811
        ],
        [
          0.4654,
          0.6742
        ],
        [
          0.4759,
          0.6665
        ],
        [
          0.4864,
          0.6611
        ],
        [
          0.4973,
          0.6538
        ],
        [
          0.5077,
          0.6447
        ],
        [
          0.5177,
          0.6383
        ],
        [
          0.5278,
          0.6297
        ],
        [
          0.5387,
          0.6215
        ],
        [
          0.5491,
          0.6133
        ],
        [
          0.5591,
          0.6051
        ],
        [
          0.5692,
          0.5965
        ],
        [
          0.5796,
          0.5887
        ],
        [
          0.5896,
          0.581
        ],
        [
          0.5996,
          0.5714
        ],
        [
          0.6096,
          0.5641
        ],
        [
          0.6197,
          0.5587
        ],
        [
          0.6306,
          0.5528
        ],
        [
          0.641,
          0.546
        ],
        [
          0.6515,
          0.5369
        ],
        [
          0.6624,
          0.5309
        ],
        [
          0.6724,
          0.5241
        ],
        [
          0.6824,
          0.5146
        ],
        [
          0.6924,
          0.5045
        ],
        [
          0.7029,
          0.4968
        ],
        [
          0.7129,
          0.4904
        ],
        [
          0.7229,
          0.4832
        ],
        [
          0.7334,
          0.4759
        ],
        [
          0.7434,
          0.4672
        ],
        [
          0.7539,
          0.46
        ],
        [
          0.7643,
          0.4522
        ],
        [
          0.7743,
          0.444
        ],
        [
          0.7848,
          0.4359
        ],
        [
          0.7957,
          0.4268
        ],
        [
          0.8071,
          0.4204
        ],
        [
          0.8171,
          0.414
        ],
        [
          0.8276,
          0.4063
        ],
        [
          0.838,
          0.3985
        ],
        [
          0.848,
          0.3908
        ],
        [
          0.8581,
          0.3831
        ],
        [
          0.8685,
          0.374
        ],
        [
          0.8794,
          0.3658
        ],
        [
          0.8894,
          0.3571
        ],
        [
          0.8995,
          0.349
        ],
        [
          0.9104,
          0.3417
        ],
        [
          0.9208,
          0.3326
        ],
        [
          0.9308,
          0.3248
        ],
        [
          0.9418,
          0.3153
        ],
        [
          0.9522,
          0.3085
        ],
        [
          0.9622,
          0.3012
        ],
        [
          0.9722,
          0.2925
        ],
        [
          0.9823,
          0.2843
        ],
        [
          0.9927,
          0.2771
        ],
        [
          1.0,
          0.2712
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.0025800452773563952,
        "difference": -0.0007458688570455144,
        "worse": 0.41333333333333333,
        "kept_right": 0.0016734819797481577,
        "kept_wrong": 0.0009065632976082409,
        "handed_off": 0.9974199547226448
      },
      "usd_per_1000": 1.6327678151229457
    },
    {
      "decision": 243,
      "frontier": 48,
      "frontier_accuracy": 0.9758871701546861,
      "in_sample": {
        "threshold": 0.9256,
        "answered": 0.0018198362147406734,
        "accuracy": 0.9758871701546861
      },
      "curve": [
        [
          0.0,
          0.9759
        ],
        [
          0.01,
          0.9727
        ],
        [
          0.02,
          0.9695
        ],
        [
          0.03,
          0.9663
        ],
        [
          0.04,
          0.96
        ],
        [
          0.05,
          0.955
        ],
        [
          0.0601,
          0.95
        ],
        [
          0.0705,
          0.9422
        ],
        [
          0.0805,
          0.9368
        ],
        [
          0.091,
          0.9308
        ],
        [
          0.101,
          0.9258
        ],
        [
          0.111,
          0.919
        ],
        [
          0.121,
          0.9136
        ],
        [
          0.131,
          0.9086
        ],
        [
          0.141,
          0.9004
        ],
        [
          0.151,
          0.8935
        ],
        [
          0.1611,
          0.8863
        ],
        [
          0.1715,
          0.879
        ],
        [
          0.1815,
          0.8712
        ],
        [
          0.1915,
          0.8653
        ],
        [
          0.2015,
          0.8585
        ],
        [
          0.2116,
          0.853
        ],
        [
          0.222,
          0.8453
        ],
        [
          0.232,
          0.8371
        ],
        [
          0.242,
          0.8308
        ],
        [
          0.252,
          0.8239
        ],
        [
          0.2621,
          0.8167
        ],
        [
          0.2725,
          0.8094
        ],
        [
          0.2825,
          0.8021
        ],
        [
          0.2925,
          0.7948
        ],
        [
          0.3025,
          0.7889
        ],
        [
          0.3126,
          0.7816
        ],
        [
          0.3235,
          0.7721
        ],
        [
          0.3339,
          0.7652
        ],
        [
          0.3444,
          0.7575
        ],
        [
          0.3544,
          0.7493
        ],
        [
          0.3644,
          0.7411
        ],
        [
          0.3744,
          0.7357
        ],
        [
          0.3844,
          0.727
        ],
        [
          0.3944,
          0.7202
        ],
        [
          0.4045,
          0.7129
        ],
        [
          0.4149,
          0.7047
        ],
        [
          0.4249,
          0.6997
        ],
        [
          0.4354,
          0.692
        ],
        [
          0.4454,
          0.6852
        ],
        [
          0.4554,
          0.6793
        ],
        [
          0.4654,
          0.6724
        ],
        [
          0.4759,
          0.6647
        ],
        [
          0.4864,
          0.6592
        ],
        [
          0.4973,
          0.6515
        ],
        [
          0.5077,
          0.6419
        ],
        [
          0.5177,
          0.6356
        ],
        [
          0.5278,
          0.6269
        ],
        [
          0.5387,
          0.6187
        ],
        [
          0.5491,
          0.6106
        ],
        [
          0.5591,
          0.6024
        ],
        [
          0.5692,
          0.5937
        ],
        [
          0.5796,
          0.5864
        ],
        [
          0.5896,
          0.5787
        ],
        [
          0.5996,
          0.5692
        ],
        [
          0.6096,
          0.5619
        ],
        [
          0.6197,
          0.556
        ],
        [
          0.6306,
          0.5505
        ],
        [
          0.641,
          0.5437
        ],
        [
          0.6515,
          0.5341
        ],
        [
          0.6624,
          0.5282
        ],
        [
          0.6724,
          0.5214
        ],
        [
          0.6824,
          0.5118
        ],
        [
          0.6924,
          0.5027
        ],
        [
          0.7029,
          0.495
        ],
        [
          0.7129,
          0.4895
        ],
        [
          0.7229,
          0.4823
        ],
        [
          0.7334,
          0.475
        ],
        [
          0.7434,
          0.4663
        ],
        [
          0.7539,
          0.4591
        ],
        [
          0.7643,
          0.4513
        ],
        [
          0.7743,
          0.4431
        ],
        [
          0.7848,
          0.4349
        ],
        [
          0.7957,
          0.4258
        ],
        [
          0.8071,
          0.4195
        ],
        [
          0.8171,
          0.4131
        ],
        [
          0.8276,
          0.4058
        ],
        [
          0.838,
          0.3981
        ],
        [
          0.848,
          0.3904
        ],
        [
          0.8581,
          0.3826
        ],
        [
          0.8685,
          0.3735
        ],
        [
          0.8794,
          0.3653
        ],
        [
          0.8894,
          0.3567
        ],
        [
          0.8995,
          0.3485
        ],
        [
          0.9104,
          0.3412
        ],
        [
          0.9208,
          0.3321
        ],
        [
          0.9308,
          0.3244
        ],
        [
          0.9418,
          0.3148
        ],
        [
          0.9522,
          0.3076
        ],
        [
          0.9622,
          0.3003
        ],
        [
          0.9722,
          0.2921
        ],
        [
          0.9823,
          0.2839
        ],
        [
          0.9927,
          0.2766
        ],
        [
          1.0,
          0.2712
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.0017765701863429102,
        "difference": -0.0005063588288394243,
        "worse": 0.17666666666666667,
        "kept_right": 0.0012702113575034867,
        "kept_wrong": 0.0005063588288394243,
        "handed_off": 0.9982234298136581
      },
      "usd_per_1000": 5.383434414275813
    },
    {
      "decision": 243,
      "frontier": 191,
      "frontier_accuracy": 0.9740673339399454,
      "in_sample": {
        "threshold": 0.9256,
        "answered": 0.0018198362147406734,
        "accuracy": 0.9740673339399454
      },
      "curve": [
        [
          0.0,
          0.9741
        ],
        [
          0.01,
          0.9709
        ],
        [
          0.02,
          0.9677
        ],
        [
          0.03,
          0.9641
        ],
        [
          0.04,
          0.9577
        ],
        [
          0.05,
          0.9527
        ],
        [
          0.0601,
          0.9477
        ],
        [
          0.0705,
          0.9399
        ],
        [
          0.0805,
          0.9345
        ],
        [
          0.091,
          0.9286
        ],
        [
          0.101,
          0.9236
        ],
        [
          0.111,
          0.9167
        ],
        [
          0.121,
          0.9113
        ],
        [
          0.131,
          0.9063
        ],
        [
          0.141,
          0.8981
        ],
        [
          0.151,
          0.8908
        ],
        [
          0.1611,
          0.8835
        ],
        [
          0.1715,
          0.8767
        ],
        [
          0.1815,
          0.869
        ],
        [
          0.1915,
          0.8631
        ],
        [
          0.2015,
          0.8562
        ],
        [
          0.2116,
          0.8508
        ],
        [
          0.222,
          0.843
        ],
        [
          0.232,
          0.8348
        ],
        [
          0.242,
          0.8285
        ],
        [
          0.252,
          0.8217
        ],
        [
          0.2621,
          0.8144
        ],
        [
          0.2725,
          0.8071
        ],
        [
          0.2825,
          0.7998
        ],
        [
          0.2925,
          0.7925
        ],
        [
          0.3025,
          0.7857
        ],
        [
          0.3126,
          0.778
        ],
        [
          0.3235,
          0.7684
        ],
        [
          0.3339,
          0.7616
        ],
        [
          0.3444,
          0.7539
        ],
        [
          0.3544,
          0.7461
        ],
        [
          0.3644,
          0.7379
        ],
        [
          0.3744,
          0.7325
        ],
        [
          0.3844,
          0.7238
        ],
        [
          0.3944,
          0.717
        ],
        [
          0.4045,
          0.7097
        ],
        [
          0.4149,
          0.7015
        ],
        [
          0.4249,
          0.6965
        ],
        [
          0.4354,
          0.6888
        ],
        [
          0.4454,
          0.682
        ],
        [
          0.4554,
          0.6761
        ],
        [
          0.4654,
          0.6692
        ],
        [
          0.4759,
          0.6615
        ],
        [
          0.4864,
          0.6561
        ],
        [
          0.4973,
          0.6483
        ],
        [
          0.5077,
          0.6392
        ],
        [
          0.5177,
          0.6328
        ],
        [
          0.5278,
          0.6242
        ],
        [
          0.5387,
          0.616
        ],
        [
          0.5491,
          0.6078
        ],
        [
          0.5591,
          0.5996
        ],
        [
          0.5692,
          0.591
        ],
        [
          0.5796,
          0.5833
        ],
        [
          0.5896,
          0.5751
        ],
        [
          0.5996,
          0.566
        ],
        [
          0.6096,
          0.5591
        ],
        [
          0.6197,
          0.5537
        ],
        [
          0.6306,
          0.5482
        ],
        [
          0.641,
          0.5414
        ],
        [
          0.6515,
          0.5318
        ],
        [
          0.6624,
          0.5259
        ],
        [
          0.6724,
          0.5191
        ],
        [
          0.6824,
          0.5096
        ],
        [
          0.6924,
          0.5
        ],
        [
          0.7029,
          0.4923
        ],
        [
          0.7129,
          0.4868
        ],
        [
          0.7229,
          0.4795
        ],
        [
          0.7334,
          0.4722
        ],
        [
          0.7434,
          0.4636
        ],
        [
          0.7539,
          0.4568
        ],
        [
          0.7643,
          0.449
        ],
        [
          0.7743,
          0.4404
        ],
        [
          0.7848,
          0.4322
        ],
        [
          0.7957,
          0.4231
        ],
        [
          0.8071,
          0.4167
        ],
        [
          0.8171,
          0.4108
        ],
        [
          0.8276,
          0.4031
        ],
        [
          0.838,
          0.3954
        ],
        [
          0.848,
          0.3876
        ],
        [
          0.8581,
          0.3803
        ],
        [
          0.8685,
          0.3717
        ],
        [
          0.8794,
          0.364
        ],
        [
          0.8894,
          0.3558
        ],
        [
          0.8995,
          0.3476
        ],
        [
          0.9104,
          0.3403
        ],
        [
          0.9208,
          0.3317
        ],
        [
          0.9308,
          0.3244
        ],
        [
          0.9418,
          0.3148
        ],
        [
          0.9522,
          0.308
        ],
        [
          0.9622,
          0.3012
        ],
        [
          0.9722,
          0.293
        ],
        [
          0.9823,
          0.2848
        ],
        [
          0.9927,
          0.2771
        ],
        [
          1.0,
          0.2712
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.0017280412206164926,
        "difference": -0.00047906128561831424,
        "worse": 0.17666666666666667,
        "kept_right": 0.0012489799349981788,
        "kept_wrong": 0.00047906128561831424,
        "handed_off": 0.9982719587793846
      },
      "usd_per_1000": 26.706255212701905
    },
    {
      "decision": 243,
      "frontier": 188,
      "frontier_accuracy": 0.9713375796178344,
      "in_sample": {
        "threshold": 0.9256,
        "answered": 0.0018198362147406734,
        "accuracy": 0.9713375796178344
      },
      "curve": [
        [
          0.0,
          0.9713
        ],
        [
          0.01,
          0.9682
        ],
        [
          0.02,
          0.965
        ],
        [
          0.03,
          0.9613
        ],
        [
          0.04,
          0.9554
        ],
        [
          0.05,
          0.9504
        ],
        [
          0.0601,
          0.9454
        ],
        [
          0.0705,
          0.9381
        ],
        [
          0.0805,
          0.9327
        ],
        [
          0.091,
          0.9268
        ],
        [
          0.101,
          0.9217
        ],
        [
          0.111,
          0.9149
        ],
        [
          0.121,
          0.9095
        ],
        [
          0.131,
          0.904
        ],
        [
          0.141,
          0.8963
        ],
        [
          0.151,
          0.889
        ],
        [
          0.1611,
          0.8817
        ],
        [
          0.1715,
          0.8753
        ],
        [
          0.1815,
          0.8676
        ],
        [
          0.1915,
          0.8617
        ],
        [
          0.2015,
          0.8549
        ],
        [
          0.2116,
          0.8499
        ],
        [
          0.222,
          0.8421
        ],
        [
          0.232,
          0.8339
        ],
        [
          0.242,
          0.8276
        ],
        [
          0.252,
          0.8207
        ],
        [
          0.2621,
          0.8135
        ],
        [
          0.2725,
          0.8057
        ],
        [
          0.2825,
          0.7985
        ],
        [
          0.2925,
          0.7912
        ],
        [
          0.3025,
          0.7848
        ],
        [
          0.3126,
          0.7771
        ],
        [
          0.3235,
          0.7675
        ],
        [
          0.3339,
          0.7607
        ],
        [
          0.3444,
          0.7534
        ],
        [
          0.3544,
          0.7448
        ],
        [
          0.3644,
          0.7375
        ],
        [
          0.3744,
          0.732
        ],
        [
          0.3844,
          0.7243
        ],
        [
          0.3944,
          0.7175
        ],
        [
          0.4045,
          0.7102
        ],
        [
          0.4149,
          0.702
        ],
        [
          0.4249,
          0.697
        ],
        [
          0.4354,
          0.6897
        ],
        [
          0.4454,
          0.6824
        ],
        [
          0.4554,
          0.6765
        ],
        [
          0.4654,
          0.6697
        ],
        [
          0.4759,
          0.662
        ],
        [
          0.4864,
          0.6561
        ],
        [
          0.4973,
          0.6479
        ],
        [
          0.5077,
          0.6388
        ],
        [
          0.5177,
          0.6324
        ],
        [
          0.5278,
          0.6237
        ],
        [
          0.5387,
          0.616
        ],
        [
          0.5491,
          0.6074
        ],
        [
          0.5591,
          0.5996
        ],
        [
          0.5692,
          0.591
        ],
        [
          0.5796,
          0.5837
        ],
        [
          0.5896,
          0.576
        ],
        [
          0.5996,
          0.5664
        ],
        [
          0.6096,
          0.5591
        ],
        [
          0.6197,
          0.5537
        ],
        [
          0.6306,
          0.5482
        ],
        [
          0.641,
          0.5414
        ],
        [
          0.6515,
          0.5323
        ],
        [
          0.6624,
          0.5264
        ],
        [
          0.6724,
          0.5196
        ],
        [
          0.6824,
          0.5105
        ],
        [
          0.6924,
          0.5014
        ],
        [
          0.7029,
          0.4936
        ],
        [
          0.7129,
          0.4877
        ],
        [
          0.7229,
          0.4804
        ],
        [
          0.7334,
          0.4732
        ],
        [
          0.7434,
          0.465
        ],
        [
          0.7539,
          0.4577
        ],
        [
          0.7643,
          0.45
        ],
        [
          0.7743,
          0.4413
        ],
        [
          0.7848,
          0.4331
        ],
        [
          0.7957,
          0.424
        ],
        [
          0.8071,
          0.4177
        ],
        [
          0.8171,
          0.4113
        ],
        [
          0.8276,
          0.404
        ],
        [
          0.838,
          0.3963
        ],
        [
          0.848,
          0.3885
        ],
        [
          0.8581,
          0.3808
        ],
        [
          0.8685,
          0.3722
        ],
        [
          0.8794,
          0.364
        ],
        [
          0.8894,
          0.3558
        ],
        [
          0.8995,
          0.3485
        ],
        [
          0.9104,
          0.3412
        ],
        [
          0.9208,
          0.3326
        ],
        [
          0.9308,
          0.3248
        ],
        [
          0.9418,
          0.3148
        ],
        [
          0.9522,
          0.308
        ],
        [
          0.9622,
          0.3007
        ],
        [
          0.9722,
          0.2925
        ],
        [
          0.9823,
          0.2843
        ],
        [
          0.9927,
          0.2771
        ],
        [
          1.0,
          0.2712
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.0017280412206164926,
        "difference": -0.00047906128561831424,
        "worse": 0.17666666666666667,
        "kept_right": 0.0012489799349981788,
        "kept_wrong": 0.00047906128561831424,
        "handed_off": 0.9982719587793846
      },
      "usd_per_1000": 0.9765379342554789
    },
    {
      "decision": 243,
      "frontier": 190,
      "frontier_accuracy": 0.9672429481346679,
      "in_sample": {
        "threshold": 0.9256,
        "answered": 0.0018198362147406734,
        "accuracy": 0.9672429481346679
      },
      "curve": [
        [
          0.0,
          0.9672
        ],
        [
          0.01,
          0.9641
        ],
        [
          0.02,
          0.9609
        ],
        [
          0.03,
          0.9577
        ],
        [
          0.04,
          0.9513
        ],
        [
          0.05,
          0.9463
        ],
        [
          0.0601,
          0.9413
        ],
        [
          0.0705,
          0.9336
        ],
        [
          0.0805,
          0.9281
        ],
        [
          0.091,
          0.9222
        ],
        [
          0.101,
          0.9172
        ],
        [
          0.111,
          0.9104
        ],
        [
          0.121,
          0.9049
        ],
        [
          0.131,
          0.8999
        ],
        [
          0.141,
          0.8917
        ],
        [
          0.151,
          0.8849
        ],
        [
          0.1611,
          0.8776
        ],
        [
          0.1715,
          0.8708
        ],
        [
          0.1815,
          0.8631
        ],
        [
          0.1915,
          0.8571
        ],
        [
          0.2015,
          0.8503
        ],
        [
          0.2116,
          0.8453
        ],
        [
          0.222,
          0.8376
        ],
        [
          0.232,
          0.8294
        ],
        [
          0.242,
          0.8226
        ],
        [
          0.252,
          0.8157
        ],
        [
          0.2621,
          0.8085
        ],
        [
          0.2725,
          0.8016
        ],
        [
          0.2825,
          0.7944
        ],
        [
          0.2925,
          0.7871
        ],
        [
          0.3025,
          0.7807
        ],
        [
          0.3126,
          0.773
        ],
        [
          0.3235,
          0.7634
        ],
        [
          0.3339,
          0.7566
        ],
        [
          0.3444,
          0.7493
        ],
        [
          0.3544,
          0.7416
        ],
        [
          0.3644,
          0.7334
        ],
        [
          0.3744,
          0.7279
        ],
        [
          0.3844,
          0.7193
        ],
        [
          0.3944,
          0.7125
        ],
        [
          0.4045,
          0.7052
        ],
        [
          0.4149,
          0.697
        ],
        [
          0.4249,
          0.6915
        ],
        [
          0.4354,
          0.6843
        ],
        [
          0.4454,
          0.6774
        ],
        [
          0.4554,
          0.6715
        ],
        [
          0.4654,
          0.6647
        ],
        [
          0.4759,
          0.657
        ],
        [
          0.4864,
          0.6515
        ],
        [
          0.4973,
          0.6438
        ],
        [
          0.5077,
          0.6347
        ],
        [
          0.5177,
          0.6288
        ],
        [
          0.5278,
          0.6201
        ],
        [
          0.5387,
          0.6119
        ],
        [
          0.5491,
          0.6037
        ],
        [
          0.5591,
          0.596
        ],
        [
          0.5692,
          0.5874
        ],
        [
          0.5796,
          0.5796
        ],
        [
          0.5896,
          0.5714
        ],
        [
          0.5996,
          0.5623
        ],
        [
          0.6096,
          0.5555
        ],
        [
          0.6197,
          0.55
        ],
        [
          0.6306,
          0.5441
        ],
        [
          0.641,
          0.5373
        ],
        [
          0.6515,
          0.5282
        ],
        [
          0.6624,
          0.5223
        ],
        [
          0.6724,
          0.5155
        ],
        [
          0.6824,
          0.5059
        ],
        [
          0.6924,
          0.4964
        ],
        [
          0.7029,
          0.4886
        ],
        [
          0.7129,
          0.4836
        ],
        [
          0.7229,
          0.4763
        ],
        [
          0.7334,
          0.4691
        ],
        [
          0.7434,
          0.4604
        ],
        [
          0.7539,
          0.4531
        ],
        [
          0.7643,
          0.4454
        ],
        [
          0.7743,
          0.4368
        ],
        [
          0.7848,
          0.4286
        ],
        [
          0.7957,
          0.4199
        ],
        [
          0.8071,
          0.4136
        ],
        [
          0.8171,
          0.4086
        ],
        [
          0.8276,
          0.4017
        ],
        [
          0.838,
          0.394
        ],
        [
          0.848,
          0.3863
        ],
        [
          0.8581,
          0.379
        ],
        [
          0.8685,
          0.3699
        ],
        [
          0.8794,
          0.3626
        ],
        [
          0.8894,
          0.3544
        ],
        [
          0.8995,
          0.3462
        ],
        [
          0.9104,
          0.3389
        ],
        [
          0.9208,
          0.3308
        ],
        [
          0.9308,
          0.323
        ],
        [
          0.9418,
          0.3139
        ],
        [
          0.9522,
          0.3071
        ],
        [
          0.9622,
          0.3003
        ],
        [
          0.9722,
          0.2916
        ],
        [
          0.9823,
          0.2834
        ],
        [
          0.9927,
          0.2766
        ],
        [
          1.0,
          0.2712
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.0017765701863429102,
        "difference": -0.0005063588288394243,
        "worse": 0.17666666666666667,
        "kept_right": 0.0012702113575034867,
        "kept_wrong": 0.0005063588288394243,
        "handed_off": 0.9982234298136581
      },
      "usd_per_1000": 5.573389118753276
    },
    {
      "decision": 243,
      "frontier": 124,
      "frontier_accuracy": 0.9667879890809827,
      "in_sample": {
        "threshold": 0.9256,
        "answered": 0.0018198362147406734,
        "accuracy": 0.9667879890809827
      },
      "curve": [
        [
          0.0,
          0.9668
        ],
        [
          0.01,
          0.9641
        ],
        [
          0.02,
          0.9609
        ],
        [
          0.03,
          0.9577
        ],
        [
          0.04,
          0.9513
        ],
        [
          0.05,
          0.9468
        ],
        [
          0.0601,
          0.9422
        ],
        [
          0.0705,
          0.9345
        ],
        [
          0.0805,
          0.929
        ],
        [
          0.091,
          0.9231
        ],
        [
          0.101,
          0.9181
        ],
        [
          0.111,
          0.9113
        ],
        [
          0.121,
          0.9058
        ],
        [
          0.131,
          0.9008
        ],
        [
          0.141,
          0.8926
        ],
        [
          0.151,
          0.8858
        ],
        [
          0.1611,
          0.879
        ],
        [
          0.1715,
          0.8722
        ],
        [
          0.1815,
          0.8644
        ],
        [
          0.1915,
          0.8585
        ],
        [
          0.2015,
          0.8517
        ],
        [
          0.2116,
          0.8467
        ],
        [
          0.222,
          0.8389
        ],
        [
          0.232,
          0.8308
        ],
        [
          0.242,
          0.8244
        ],
        [
          0.252,
          0.8176
        ],
        [
          0.2621,
          0.8098
        ],
        [
          0.2725,
          0.8025
        ],
        [
          0.2825,
          0.7953
        ],
        [
          0.2925,
          0.788
        ],
        [
          0.3025,
          0.7825
        ],
        [
          0.3126,
          0.7753
        ],
        [
          0.3235,
          0.7657
        ],
        [
          0.3339,
          0.7589
        ],
        [
          0.3444,
          0.7511
        ],
        [
          0.3544,
          0.7429
        ],
        [
          0.3644,
          0.7348
        ],
        [
          0.3744,
          0.7293
        ],
        [
          0.3844,
          0.7207
        ],
        [
          0.3944,
          0.7138
        ],
        [
          0.4045,
          0.7066
        ],
        [
          0.4149,
          0.6984
        ],
        [
          0.4249,
          0.6929
        ],
        [
          0.4354,
          0.6856
        ],
        [
          0.4454,
          0.6788
        ],
        [
          0.4554,
          0.6729
        ],
        [
          0.4654,
          0.6661
        ],
        [
          0.4759,
          0.6583
        ],
        [
          0.4864,
          0.6529
        ],
        [
          0.4973,
          0.6456
        ],
        [
          0.5077,
          0.636
        ],
        [
          0.5177,
          0.6297
        ],
        [
          0.5278,
          0.621
        ],
        [
          0.5387,
          0.6133
        ],
        [
          0.5491,
          0.6051
        ],
        [
          0.5591,
          0.5969
        ],
        [
          0.5692,
          0.5883
        ],
        [
          0.5796,
          0.5805
        ],
        [
          0.5896,
          0.5728
        ],
        [
          0.5996,
          0.5641
        ],
        [
          0.6096,
          0.5569
        ],
        [
          0.6197,
          0.551
        ],
        [
          0.6306,
          0.545
        ],
        [
          0.641,
          0.5382
        ],
        [
          0.6515,
          0.5291
        ],
        [
          0.6624,
          0.5232
        ],
        [
          0.6724,
          0.5168
        ],
        [
          0.6824,
          0.5077
        ],
        [
          0.6924,
          0.4986
        ],
        [
          0.7029,
          0.4909
        ],
        [
          0.7129,
          0.4854
        ],
        [
          0.7229,
          0.4782
        ],
        [
          0.7334,
          0.4709
        ],
        [
          0.7434,
          0.4627
        ],
        [
          0.7539,
          0.4554
        ],
        [
          0.7643,
          0.4477
        ],
        [
          0.7743,
          0.4399
        ],
        [
          0.7848,
          0.4318
        ],
        [
          0.7957,
          0.4227
        ],
        [
          0.8071,
          0.4163
        ],
        [
          0.8171,
          0.4099
        ],
        [
          0.8276,
          0.4026
        ],
        [
          0.838,
          0.3949
        ],
        [
          0.848,
          0.3872
        ],
        [
          0.8581,
          0.3794
        ],
        [
          0.8685,
          0.3712
        ],
        [
          0.8794,
          0.3635
        ],
        [
          0.8894,
          0.3549
        ],
        [
          0.8995,
          0.3467
        ],
        [
          0.9104,
          0.3399
        ],
        [
          0.9208,
          0.3312
        ],
        [
          0.9308,
          0.3235
        ],
        [
          0.9418,
          0.3144
        ],
        [
          0.9522,
          0.3076
        ],
        [
          0.9622,
          0.3003
        ],
        [
          0.9722,
          0.2925
        ],
        [
          0.9823,
          0.2843
        ],
        [
          0.9927,
          0.2771
        ],
        [
          1.0,
          0.2712
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.0025800452773563952,
        "difference": -0.0007458688570455144,
        "worse": 0.41333333333333333,
        "kept_right": 0.0016734819797481577,
        "kept_wrong": 0.0009065632976082409,
        "handed_off": 0.9974199547226448
      },
      "usd_per_1000": 1.1422196190653737
    },
    {
      "decision": 243,
      "frontier": 278,
      "frontier_accuracy": 0.9499545040946314,
      "in_sample": {
        "threshold": 0.9256,
        "answered": 0.0018198362147406734,
        "accuracy": 0.9499545040946314
      },
      "curve": [
        [
          0.0,
          0.95
        ],
        [
          0.01,
          0.9468
        ],
        [
          0.02,
          0.9436
        ],
        [
          0.03,
          0.9404
        ],
        [
          0.04,
          0.9345
        ],
        [
          0.05,
          0.9295
        ],
        [
          0.0601,
          0.9245
        ],
        [
          0.0705,
          0.9167
        ],
        [
          0.0805,
          0.9113
        ],
        [
          0.091,
          0.9054
        ],
        [
          0.101,
          0.9013
        ],
        [
          0.111,
          0.8944
        ],
        [
          0.121,
          0.889
        ],
        [
          0.131,
          0.884
        ],
        [
          0.141,
          0.8763
        ],
        [
          0.151,
          0.8694
        ],
        [
          0.1611,
          0.8626
        ],
        [
          0.1715,
          0.8567
        ],
        [
          0.1815,
          0.849
        ],
        [
          0.1915,
          0.843
        ],
        [
          0.2015,
          0.8367
        ],
        [
          0.2116,
          0.8321
        ],
        [
          0.222,
          0.8248
        ],
        [
          0.232,
          0.8171
        ],
        [
          0.242,
          0.8107
        ],
        [
          0.252,
          0.8039
        ],
        [
          0.2621,
          0.7966
        ],
        [
          0.2725,
          0.7898
        ],
        [
          0.2825,
          0.7834
        ],
        [
          0.2925,
          0.7762
        ],
        [
          0.3025,
          0.7702
        ],
        [
          0.3126,
          0.7625
        ],
        [
          0.3235,
          0.7534
        ],
        [
          0.3339,
          0.7466
        ],
        [
          0.3444,
          0.7389
        ],
        [
          0.3544,
          0.7311
        ],
        [
          0.3644,
          0.7229
        ],
        [
          0.3744,
          0.7175
        ],
        [
          0.3844,
          0.7097
        ],
        [
          0.3944,
          0.7029
        ],
        [
          0.4045,
          0.6961
        ],
        [
          0.4149,
          0.6879
        ],
        [
          0.4249,
          0.6838
        ],
        [
          0.4354,
          0.6765
        ],
        [
          0.4454,
          0.6692
        ],
        [
          0.4554,
          0.6638
        ],
        [
          0.4654,
          0.6574
        ],
        [
          0.4759,
          0.6501
        ],
        [
          0.4864,
          0.6438
        ],
        [
          0.4973,
          0.636
        ],
        [
          0.5077,
          0.6269
        ],
        [
          0.5177,
          0.6206
        ],
        [
          0.5278,
          0.6119
        ],
        [
          0.5387,
          0.6042
        ],
        [
          0.5491,
          0.5955
        ],
        [
          0.5591,
          0.5878
        ],
        [
          0.5692,
          0.5792
        ],
        [
          0.5796,
          0.5728
        ],
        [
          0.5896,
          0.5651
        ],
        [
          0.5996,
          0.556
        ],
        [
          0.6096,
          0.5491
        ],
        [
          0.6197,
          0.5432
        ],
        [
          0.6306,
          0.5382
        ],
        [
          0.641,
          0.5314
        ],
        [
          0.6515,
          0.5223
        ],
        [
          0.6624,
          0.5164
        ],
        [
          0.6724,
          0.51
        ],
        [
          0.6824,
          0.5009
        ],
        [
          0.6924,
          0.4918
        ],
        [
          0.7029,
          0.4841
        ],
        [
          0.7129,
          0.4791
        ],
        [
          0.7229,
          0.4727
        ],
        [
          0.7334,
          0.4659
        ],
        [
          0.7434,
          0.4581
        ],
        [
          0.7539,
          0.4509
        ],
        [
          0.7643,
          0.4436
        ],
        [
          0.7743,
          0.4359
        ],
        [
          0.7848,
          0.4277
        ],
        [
          0.7957,
          0.4186
        ],
        [
          0.8071,
          0.4122
        ],
        [
          0.8171,
          0.4058
        ],
        [
          0.8276,
          0.3985
        ],
        [
          0.838,
          0.3908
        ],
        [
          0.848,
          0.3835
        ],
        [
          0.8581,
          0.3758
        ],
        [
          0.8685,
          0.3676
        ],
        [
          0.8794,
          0.3599
        ],
        [
          0.8894,
          0.3512
        ],
        [
          0.8995,
          0.3435
        ],
        [
          0.9104,
          0.3371
        ],
        [
          0.9208,
          0.3285
        ],
        [
          0.9308,
          0.3207
        ],
        [
          0.9418,
          0.3116
        ],
        [
          0.9522,
          0.3053
        ],
        [
          0.9622,
          0.2985
        ],
        [
          0.9722,
          0.2907
        ],
        [
          0.9823,
          0.283
        ],
        [
          0.9927,
          0.2757
        ],
        [
          1.0,
          0.2712
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.0017765701863429102,
        "difference": -0.0005063588288394243,
        "worse": 0.17666666666666667,
        "kept_right": 0.0012702113575034867,
        "kept_wrong": 0.0005063588288394243,
        "handed_off": 0.9982234298136581
      },
      "usd_per_1000": 6.512025876219648
    },
    {
      "decision": 243,
      "frontier": 275,
      "frontier_accuracy": 0.9426751592356688,
      "in_sample": {
        "threshold": 0.9256,
        "answered": 0.0018198362147406734,
        "accuracy": 0.9426751592356688
      },
      "curve": [
        [
          0.0,
          0.9427
        ],
        [
          0.01,
          0.9395
        ],
        [
          0.02,
          0.9363
        ],
        [
          0.03,
          0.9327
        ],
        [
          0.04,
          0.9268
        ],
        [
          0.05,
          0.9217
        ],
        [
          0.0601,
          0.9167
        ],
        [
          0.0705,
          0.9099
        ],
        [
          0.0805,
          0.9045
        ],
        [
          0.091,
          0.8985
        ],
        [
          0.101,
          0.894
        ],
        [
          0.111,
          0.8876
        ],
        [
          0.121,
          0.8822
        ],
        [
          0.131,
          0.8772
        ],
        [
          0.141,
          0.869
        ],
        [
          0.151,
          0.8626
        ],
        [
          0.1611,
          0.8562
        ],
        [
          0.1715,
          0.8503
        ],
        [
          0.1815,
          0.8426
        ],
        [
          0.1915,
          0.8367
        ],
        [
          0.2015,
          0.8303
        ],
        [
          0.2116,
          0.8253
        ],
        [
          0.222,
          0.8176
        ],
        [
          0.232,
          0.8098
        ],
        [
          0.242,
          0.8035
        ],
        [
          0.252,
          0.7971
        ],
        [
          0.2621,
          0.7903
        ],
        [
          0.2725,
          0.7834
        ],
        [
          0.2825,
          0.7766
        ],
        [
          0.2925,
          0.7693
        ],
        [
          0.3025,
          0.763
        ],
        [
          0.3126,
          0.7557
        ],
        [
          0.3235,
          0.7466
        ],
        [
          0.3339,
          0.7398
        ],
        [
          0.3444,
          0.732
        ],
        [
          0.3544,
          0.7243
        ],
        [
          0.3644,
          0.717
        ],
        [
          0.3744,
          0.7116
        ],
        [
          0.3844,
          0.7034
        ],
        [
          0.3944,
          0.697
        ],
        [
          0.4045,
          0.6902
        ],
        [
          0.4149,
          0.6824
        ],
        [
          0.4249,
          0.677
        ],
        [
          0.4354,
          0.6702
        ],
        [
          0.4454,
          0.6633
        ],
        [
          0.4554,
          0.6574
        ],
        [
          0.4654,
          0.6506
        ],
        [
          0.4759,
          0.6433
        ],
        [
          0.4864,
          0.6379
        ],
        [
          0.4973,
          0.6301
        ],
        [
          0.5077,
          0.621
        ],
        [
          0.5177,
          0.6156
        ],
        [
          0.5278,
          0.6074
        ],
        [
          0.5387,
          0.5992
        ],
        [
          0.5491,
          0.5914
        ],
        [
          0.5591,
          0.5837
        ],
        [
          0.5692,
          0.5751
        ],
        [
          0.5796,
          0.5673
        ],
        [
          0.5896,
          0.5605
        ],
        [
          0.5996,
          0.5514
        ],
        [
          0.6096,
          0.545
        ],
        [
          0.6197,
          0.54
        ],
        [
          0.6306,
          0.5346
        ],
        [
          0.641,
          0.5291
        ],
        [
          0.6515,
          0.52
        ],
        [
          0.6624,
          0.515
        ],
        [
          0.6724,
          0.5086
        ],
        [
          0.6824,
          0.4991
        ],
        [
          0.6924,
          0.4909
        ],
        [
          0.7029,
          0.4836
        ],
        [
          0.7129,
          0.4777
        ],
        [
          0.7229,
          0.4709
        ],
        [
          0.7334,
          0.4641
        ],
        [
          0.7434,
          0.4559
        ],
        [
          0.7539,
          0.4486
        ],
        [
          0.7643,
          0.4409
        ],
        [
          0.7743,
          0.4331
        ],
        [
          0.7848,
          0.4249
        ],
        [
          0.7957,
          0.4163
        ],
        [
          0.8071,
          0.4099
        ],
        [
          0.8171,
          0.4045
        ],
        [
          0.8276,
          0.3967
        ],
        [
          0.838,
          0.3904
        ],
        [
          0.848,
          0.3826
        ],
        [
          0.8581,
          0.3753
        ],
        [
          0.8685,
          0.3672
        ],
        [
          0.8794,
          0.3594
        ],
        [
          0.8894,
          0.3508
        ],
        [
          0.8995,
          0.343
        ],
        [
          0.9104,
          0.3358
        ],
        [
          0.9208,
          0.3267
        ],
        [
          0.9308,
          0.3194
        ],
        [
          0.9418,
          0.3103
        ],
        [
          0.9522,
          0.3039
        ],
        [
          0.9622,
          0.298
        ],
        [
          0.9722,
          0.2898
        ],
        [
          0.9823,
          0.2821
        ],
        [
          0.9927,
          0.2762
        ],
        [
          1.0,
          0.2712
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.0017280412206164926,
        "difference": -0.00047906128561831424,
        "worse": 0.17666666666666667,
        "kept_right": 0.0012489799349981788,
        "kept_wrong": 0.00047906128561831424,
        "handed_off": 0.9982719587793846
      },
      "usd_per_1000": 0.5283870601381278
    },
    {
      "decision": 243,
      "frontier": 192,
      "frontier_accuracy": 0.9331210191082803,
      "in_sample": {
        "threshold": 0.9256,
        "answered": 0.0018198362147406734,
        "accuracy": 0.9331210191082803
      },
      "curve": [
        [
          0.0,
          0.9331
        ],
        [
          0.01,
          0.9299
        ],
        [
          0.02,
          0.9272
        ],
        [
          0.03,
          0.9236
        ],
        [
          0.04,
          0.9172
        ],
        [
          0.05,
          0.9122
        ],
        [
          0.0601,
          0.9072
        ],
        [
          0.0705,
          0.8995
        ],
        [
          0.0805,
          0.894
        ],
        [
          0.091,
          0.8881
        ],
        [
          0.101,
          0.884
        ],
        [
          0.111,
          0.8781
        ],
        [
          0.121,
          0.8726
        ],
        [
          0.131,
          0.8676
        ],
        [
          0.141,
          0.8599
        ],
        [
          0.151,
          0.8535
        ],
        [
          0.1611,
          0.8462
        ],
        [
          0.1715,
          0.8408
        ],
        [
          0.1815,
          0.8335
        ],
        [
          0.1915,
          0.828
        ],
        [
          0.2015,
          0.8221
        ],
        [
          0.2116,
          0.8176
        ],
        [
          0.222,
          0.8116
        ],
        [
          0.232,
          0.8035
        ],
        [
          0.242,
          0.7975
        ],
        [
          0.252,
          0.7907
        ],
        [
          0.2621,
          0.7839
        ],
        [
          0.2725,
          0.7766
        ],
        [
          0.2825,
          0.7707
        ],
        [
          0.2925,
          0.7639
        ],
        [
          0.3025,
          0.7584
        ],
        [
          0.3126,
          0.7507
        ],
        [
          0.3235,
          0.742
        ],
        [
          0.3339,
          0.7357
        ],
        [
          0.3444,
          0.7279
        ],
        [
          0.3544,
          0.7207
        ],
        [
          0.3644,
          0.7129
        ],
        [
          0.3744,
          0.7075
        ],
        [
          0.3844,
          0.6993
        ],
        [
          0.3944,
          0.6924
        ],
        [
          0.4045,
          0.6856
        ],
        [
          0.4149,
          0.6779
        ],
        [
          0.4249,
          0.6738
        ],
        [
          0.4354,
          0.6665
        ],
        [
          0.4454,
          0.6592
        ],
        [
          0.4554,
          0.6533
        ],
        [
          0.4654,
          0.6474
        ],
        [
          0.4759,
          0.6397
        ],
        [
          0.4864,
          0.6342
        ],
        [
          0.4973,
          0.6269
        ],
        [
          0.5077,
          0.6183
        ],
        [
          0.5177,
          0.6124
        ],
        [
          0.5278,
          0.6037
        ],
        [
          0.5387,
          0.5969
        ],
        [
          0.5491,
          0.5887
        ],
        [
          0.5591,
          0.5805
        ],
        [
          0.5692,
          0.5719
        ],
        [
          0.5796,
          0.566
        ],
        [
          0.5896,
          0.5587
        ],
        [
          0.5996,
          0.5496
        ],
        [
          0.6096,
          0.5428
        ],
        [
          0.6197,
          0.5373
        ],
        [
          0.6306,
          0.5318
        ],
        [
          0.641,
          0.525
        ],
        [
          0.6515,
          0.5155
        ],
        [
          0.6624,
          0.5091
        ],
        [
          0.6724,
          0.5032
        ],
        [
          0.6824,
          0.4936
        ],
        [
          0.6924,
          0.4854
        ],
        [
          0.7029,
          0.4777
        ],
        [
          0.7129,
          0.4718
        ],
        [
          0.7229,
          0.4645
        ],
        [
          0.7334,
          0.4586
        ],
        [
          0.7434,
          0.4509
        ],
        [
          0.7539,
          0.4449
        ],
        [
          0.7643,
          0.4377
        ],
        [
          0.7743,
          0.4295
        ],
        [
          0.7848,
          0.4222
        ],
        [
          0.7957,
          0.414
        ],
        [
          0.8071,
          0.4076
        ],
        [
          0.8171,
          0.4031
        ],
        [
          0.8276,
          0.3958
        ],
        [
          0.838,
          0.389
        ],
        [
          0.848,
          0.3817
        ],
        [
          0.8581,
          0.3749
        ],
        [
          0.8685,
          0.3672
        ],
        [
          0.8794,
          0.3594
        ],
        [
          0.8894,
          0.3517
        ],
        [
          0.8995,
          0.3435
        ],
        [
          0.9104,
          0.3367
        ],
        [
          0.9208,
          0.328
        ],
        [
          0.9308,
          0.3207
        ],
        [
          0.9418,
          0.3116
        ],
        [
          0.9522,
          0.3048
        ],
        [
          0.9622,
          0.2975
        ],
        [
          0.9722,
          0.2898
        ],
        [
          0.9823,
          0.2825
        ],
        [
          0.9927,
          0.2762
        ],
        [
          1.0,
          0.2712
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.0017280412206164926,
        "difference": -0.00047906128561831424,
        "worse": 0.17666666666666667,
        "kept_right": 0.0012489799349981788,
        "kept_wrong": 0.00047906128561831424,
        "handed_off": 0.9982719587793846
      },
      "usd_per_1000": 11.604266491005795
    },
    {
      "decision": 243,
      "frontier": 123,
      "frontier_accuracy": 0.9326660600545951,
      "in_sample": {
        "threshold": 0.9256,
        "answered": 0.0018198362147406734,
        "accuracy": 0.9326660600545951
      },
      "curve": [
        [
          0.0,
          0.9327
        ],
        [
          0.01,
          0.9295
        ],
        [
          0.02,
          0.9263
        ],
        [
          0.03,
          0.9227
        ],
        [
          0.04,
          0.9167
        ],
        [
          0.05,
          0.9117
        ],
        [
          0.0601,
          0.9072
        ],
        [
          0.0705,
          0.9004
        ],
        [
          0.0805,
          0.8949
        ],
        [
          0.091,
          0.889
        ],
        [
          0.101,
          0.884
        ],
        [
          0.111,
          0.8776
        ],
        [
          0.121,
          0.8726
        ],
        [
          0.131,
          0.8676
        ],
        [
          0.141,
          0.8608
        ],
        [
          0.151,
          0.8535
        ],
        [
          0.1611,
          0.8462
        ],
        [
          0.1715,
          0.8403
        ],
        [
          0.1815,
          0.833
        ],
        [
          0.1915,
          0.8276
        ],
        [
          0.2015,
          0.8212
        ],
        [
          0.2116,
          0.8157
        ],
        [
          0.222,
          0.8089
        ],
        [
          0.232,
          0.8012
        ],
        [
          0.242,
          0.7948
        ],
        [
          0.252,
          0.788
        ],
        [
          0.2621,
          0.7807
        ],
        [
          0.2725,
          0.7748
        ],
        [
          0.2825,
          0.7684
        ],
        [
          0.2925,
          0.7611
        ],
        [
          0.3025,
          0.7561
        ],
        [
          0.3126,
          0.7493
        ],
        [
          0.3235,
          0.7398
        ],
        [
          0.3339,
          0.7329
        ],
        [
          0.3444,
          0.7252
        ],
        [
          0.3544,
          0.717
        ],
        [
          0.3644,
          0.7093
        ],
        [
          0.3744,
          0.7038
        ],
        [
          0.3844,
          0.6965
        ],
        [
          0.3944,
          0.6897
        ],
        [
          0.4045,
          0.6824
        ],
        [
          0.4149,
          0.6738
        ],
        [
          0.4249,
          0.6688
        ],
        [
          0.4354,
          0.6624
        ],
        [
          0.4454,
          0.6556
        ],
        [
          0.4554,
          0.6501
        ],
        [
          0.4654,
          0.6433
        ],
        [
          0.4759,
          0.6365
        ],
        [
          0.4864,
          0.6306
        ],
        [
          0.4973,
          0.6233
        ],
        [
          0.5077,
          0.6137
        ],
        [
          0.5177,
          0.6078
        ],
        [
          0.5278,
          0.5996
        ],
        [
          0.5387,
          0.5924
        ],
        [
          0.5491,
          0.5837
        ],
        [
          0.5591,
          0.576
        ],
        [
          0.5692,
          0.5678
        ],
        [
          0.5796,
          0.5614
        ],
        [
          0.5896,
          0.5541
        ],
        [
          0.5996,
          0.5446
        ],
        [
          0.6096,
          0.5382
        ],
        [
          0.6197,
          0.5332
        ],
        [
          0.6306,
          0.5282
        ],
        [
          0.641,
          0.5218
        ],
        [
          0.6515,
          0.5123
        ],
        [
          0.6624,
          0.5064
        ],
        [
          0.6724,
          0.5005
        ],
        [
          0.6824,
          0.4923
        ],
        [
          0.6924,
          0.4832
        ],
        [
          0.7029,
          0.4759
        ],
        [
          0.7129,
          0.4709
        ],
        [
          0.7229,
          0.4645
        ],
        [
          0.7334,
          0.4581
        ],
        [
          0.7434,
          0.45
        ],
        [
          0.7539,
          0.4427
        ],
        [
          0.7643,
          0.4363
        ],
        [
          0.7743,
          0.4277
        ],
        [
          0.7848,
          0.4204
        ],
        [
          0.7957,
          0.4113
        ],
        [
          0.8071,
          0.4054
        ],
        [
          0.8171,
          0.4004
        ],
        [
          0.8276,
          0.3931
        ],
        [
          0.838,
          0.3858
        ],
        [
          0.848,
          0.3781
        ],
        [
          0.8581,
          0.3717
        ],
        [
          0.8685,
          0.364
        ],
        [
          0.8794,
          0.3562
        ],
        [
          0.8894,
          0.3494
        ],
        [
          0.8995,
          0.3426
        ],
        [
          0.9104,
          0.3358
        ],
        [
          0.9208,
          0.3276
        ],
        [
          0.9308,
          0.3212
        ],
        [
          0.9418,
          0.3116
        ],
        [
          0.9522,
          0.3057
        ],
        [
          0.9622,
          0.2985
        ],
        [
          0.9722,
          0.2912
        ],
        [
          0.9823,
          0.283
        ],
        [
          0.9927,
          0.2762
        ],
        [
          1.0,
          0.2712
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.0017280412206164926,
        "difference": -0.00047906128561831424,
        "worse": 0.17666666666666667,
        "kept_right": 0.0012489799349981788,
        "kept_wrong": 0.00047906128561831424,
        "handed_off": 0.9982719587793846
      },
      "usd_per_1000": 0.28288278215901375
    },
    {
      "decision": 243,
      "frontier": 274,
      "frontier_accuracy": 0.9085532302092811,
      "in_sample": {
        "threshold": 0.9256,
        "answered": 0.0018198362147406734,
        "accuracy": 0.9085532302092811
      },
      "curve": [
        [
          0.0,
          0.9086
        ],
        [
          0.01,
          0.9054
        ],
        [
          0.02,
          0.9022
        ],
        [
          0.03,
          0.8985
        ],
        [
          0.04,
          0.8926
        ],
        [
          0.05,
          0.8881
        ],
        [
          0.0601,
          0.8831
        ],
        [
          0.0705,
          0.8763
        ],
        [
          0.0805,
          0.8712
        ],
        [
          0.091,
          0.8653
        ],
        [
          0.101,
          0.8608
        ],
        [
          0.111,
          0.8549
        ],
        [
          0.121,
          0.8494
        ],
        [
          0.131,
          0.8453
        ],
        [
          0.141,
          0.838
        ],
        [
          0.151,
          0.8317
        ],
        [
          0.1611,
          0.8258
        ],
        [
          0.1715,
          0.8203
        ],
        [
          0.1815,
          0.813
        ],
        [
          0.1915,
          0.8076
        ],
        [
          0.2015,
          0.8012
        ],
        [
          0.2116,
          0.7971
        ],
        [
          0.222,
          0.7898
        ],
        [
          0.232,
          0.7821
        ],
        [
          0.242,
          0.7766
        ],
        [
          0.252,
          0.7702
        ],
        [
          0.2621,
          0.7634
        ],
        [
          0.2725,
          0.7566
        ],
        [
          0.2825,
          0.7502
        ],
        [
          0.2925,
          0.7434
        ],
        [
          0.3025,
          0.737
        ],
        [
          0.3126,
          0.7307
        ],
        [
          0.3235,
          0.722
        ],
        [
          0.3339,
          0.7152
        ],
        [
          0.3444,
          0.7079
        ],
        [
          0.3544,
          0.7006
        ],
        [
          0.3644,
          0.6934
        ],
        [
          0.3744,
          0.6888
        ],
        [
          0.3844,
          0.6815
        ],
        [
          0.3944,
          0.6761
        ],
        [
          0.4045,
          0.6702
        ],
        [
          0.4149,
          0.662
        ],
        [
          0.4249,
          0.6574
        ],
        [
          0.4354,
          0.6515
        ],
        [
          0.4454,
          0.6456
        ],
        [
          0.4554,
          0.6397
        ],
        [
          0.4654,
          0.6328
        ],
        [
          0.4759,
          0.6269
        ],
        [
          0.4864,
          0.6219
        ],
        [
          0.4973,
          0.6146
        ],
        [
          0.5077,
          0.606
        ],
        [
          0.5177,
          0.601
        ],
        [
          0.5278,
          0.5933
        ],
        [
          0.5387,
          0.5855
        ],
        [
          0.5491,
          0.5778
        ],
        [
          0.5591,
          0.5696
        ],
        [
          0.5692,
          0.5619
        ],
        [
          0.5796,
          0.5555
        ],
        [
          0.5896,
          0.5478
        ],
        [
          0.5996,
          0.5391
        ],
        [
          0.6096,
          0.5332
        ],
        [
          0.6197,
          0.5278
        ],
        [
          0.6306,
          0.5223
        ],
        [
          0.641,
          0.5168
        ],
        [
          0.6515,
          0.5096
        ],
        [
          0.6624,
          0.5036
        ],
        [
          0.6724,
          0.4982
        ],
        [
          0.6824,
          0.4904
        ],
        [
          0.6924,
          0.4827
        ],
        [
          0.7029,
          0.4759
        ],
        [
          0.7129,
          0.4718
        ],
        [
          0.7229,
          0.4659
        ],
        [
          0.7334,
          0.4591
        ],
        [
          0.7434,
          0.4509
        ],
        [
          0.7539,
          0.444
        ],
        [
          0.7643,
          0.4368
        ],
        [
          0.7743,
          0.4286
        ],
        [
          0.7848,
          0.4204
        ],
        [
          0.7957,
          0.4117
        ],
        [
          0.8071,
          0.4054
        ],
        [
          0.8171,
          0.3999
        ],
        [
          0.8276,
          0.3926
        ],
        [
          0.838,
          0.3863
        ],
        [
          0.848,
          0.3785
        ],
        [
          0.8581,
          0.3712
        ],
        [
          0.8685,
          0.364
        ],
        [
          0.8794,
          0.3567
        ],
        [
          0.8894,
          0.3485
        ],
        [
          0.8995,
          0.3417
        ],
        [
          0.9104,
          0.3344
        ],
        [
          0.9208,
          0.3258
        ],
        [
          0.9308,
          0.3189
        ],
        [
          0.9418,
          0.3103
        ],
        [
          0.9522,
          0.3035
        ],
        [
          0.9622,
          0.2966
        ],
        [
          0.9722,
          0.2889
        ],
        [
          0.9823,
          0.2807
        ],
        [
          0.9927,
          0.2762
        ],
        [
          1.0,
          0.2712
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.0017280412206164926,
        "difference": -0.00047906128561831424,
        "worse": 0.17666666666666667,
        "kept_right": 0.0012489799349981788,
        "kept_wrong": 0.00047906128561831424,
        "handed_off": 0.9982719587793846
      },
      "usd_per_1000": 0.4402720741205182
    },
    {
      "decision": 243,
      "frontier": 47,
      "frontier_accuracy": 0.9044585987261147,
      "in_sample": {
        "threshold": 0.9256,
        "answered": 0.0018198362147406734,
        "accuracy": 0.9044585987261147
      },
      "curve": [
        [
          0.0,
          0.9045
        ],
        [
          0.01,
          0.9013
        ],
        [
          0.02,
          0.8981
        ],
        [
          0.03,
          0.8949
        ],
        [
          0.04,
          0.8885
        ],
        [
          0.05,
          0.8835
        ],
        [
          0.0601,
          0.8785
        ],
        [
          0.0705,
          0.8708
        ],
        [
          0.0805,
          0.8658
        ],
        [
          0.091,
          0.8599
        ],
        [
          0.101,
          0.8562
        ],
        [
          0.111,
          0.8499
        ],
        [
          0.121,
          0.8444
        ],
        [
          0.131,
          0.8403
        ],
        [
          0.141,
          0.8335
        ],
        [
          0.151,
          0.8271
        ],
        [
          0.1611,
          0.8203
        ],
        [
          0.1715,
          0.8139
        ],
        [
          0.1815,
          0.8062
        ],
        [
          0.1915,
          0.8003
        ],
        [
          0.2015,
          0.7948
        ],
        [
          0.2116,
          0.7907
        ],
        [
          0.222,
          0.7834
        ],
        [
          0.232,
          0.7757
        ],
        [
          0.242,
          0.7698
        ],
        [
          0.252,
          0.7639
        ],
        [
          0.2621,
          0.758
        ],
        [
          0.2725,
          0.7516
        ],
        [
          0.2825,
          0.7461
        ],
        [
          0.2925,
          0.7398
        ],
        [
          0.3025,
          0.7343
        ],
        [
          0.3126,
          0.727
        ],
        [
          0.3235,
          0.7179
        ],
        [
          0.3339,
          0.712
        ],
        [
          0.3444,
          0.7043
        ],
        [
          0.3544,
          0.6965
        ],
        [
          0.3644,
          0.6884
        ],
        [
          0.3744,
          0.6829
        ],
        [
          0.3844,
          0.6761
        ],
        [
          0.3944,
          0.6697
        ],
        [
          0.4045,
          0.6624
        ],
        [
          0.4149,
          0.6538
        ],
        [
          0.4249,
          0.6492
        ],
        [
          0.4354,
          0.6429
        ],
        [
          0.4454,
          0.6369
        ],
        [
          0.4554,
          0.6315
        ],
        [
          0.4654,
          0.6256
        ],
        [
          0.4759,
          0.6192
        ],
        [
          0.4864,
          0.6142
        ],
        [
          0.4973,
          0.6078
        ],
        [
          0.5077,
          0.5987
        ],
        [
          0.5177,
          0.5928
        ],
        [
          0.5278,
          0.5846
        ],
        [
          0.5387,
          0.5778
        ],
        [
          0.5491,
          0.5714
        ],
        [
          0.5591,
          0.5641
        ],
        [
          0.5692,
          0.556
        ],
        [
          0.5796,
          0.5505
        ],
        [
          0.5896,
          0.5441
        ],
        [
          0.5996,
          0.535
        ],
        [
          0.6096,
          0.5282
        ],
        [
          0.6197,
          0.5227
        ],
        [
          0.6306,
          0.5177
        ],
        [
          0.641,
          0.5114
        ],
        [
          0.6515,
          0.5036
        ],
        [
          0.6624,
          0.4977
        ],
        [
          0.6724,
          0.4927
        ],
        [
          0.6824,
          0.4836
        ],
        [
          0.6924,
          0.4759
        ],
        [
          0.7029,
          0.4686
        ],
        [
          0.7129,
          0.4641
        ],
        [
          0.7229,
          0.4581
        ],
        [
          0.7334,
          0.4522
        ],
        [
          0.7434,
          0.4445
        ],
        [
          0.7539,
          0.4377
        ],
        [
          0.7643,
          0.4299
        ],
        [
          0.7743,
          0.4217
        ],
        [
          0.7848,
          0.4149
        ],
        [
          0.7957,
          0.4058
        ],
        [
          0.8071,
          0.3995
        ],
        [
          0.8171,
          0.394
        ],
        [
          0.8276,
          0.3867
        ],
        [
          0.838,
          0.3803
        ],
        [
          0.848,
          0.3735
        ],
        [
          0.8581,
          0.3676
        ],
        [
          0.8685,
          0.3599
        ],
        [
          0.8794,
          0.3526
        ],
        [
          0.8894,
          0.3453
        ],
        [
          0.8995,
          0.3376
        ],
        [
          0.9104,
          0.3321
        ],
        [
          0.9208,
          0.3244
        ],
        [
          0.9308,
          0.318
        ],
        [
          0.9418,
          0.3098
        ],
        [
          0.9522,
          0.3039
        ],
        [
          0.9622,
          0.2971
        ],
        [
          0.9722,
          0.2889
        ],
        [
          0.9823,
          0.2821
        ],
        [
          0.9927,
          0.2762
        ],
        [
          1.0,
          0.2712
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.0017765701863429102,
        "difference": -0.0005063588288394243,
        "worse": 0.17666666666666667,
        "kept_right": 0.0012702113575034867,
        "kept_wrong": 0.0005063588288394243,
        "handed_off": 0.9982234298136581
      },
      "usd_per_1000": 4.936589407143141
    },
    {
      "decision": 243,
      "frontier": 193,
      "frontier_accuracy": 0.89171974522293,
      "in_sample": {
        "threshold": 0.9256,
        "answered": 0.0018198362147406734,
        "accuracy": 0.89171974522293
      },
      "curve": [
        [
          0.0,
          0.8917
        ],
        [
          0.01,
          0.8885
        ],
        [
          0.02,
          0.8858
        ],
        [
          0.03,
          0.8826
        ],
        [
          0.04,
          0.8767
        ],
        [
          0.05,
          0.8717
        ],
        [
          0.0601,
          0.8672
        ],
        [
          0.0705,
          0.8594
        ],
        [
          0.0805,
          0.8549
        ],
        [
          0.091,
          0.849
        ],
        [
          0.101,
          0.8444
        ],
        [
          0.111,
          0.838
        ],
        [
          0.121,
          0.833
        ],
        [
          0.131,
          0.8285
        ],
        [
          0.141,
          0.8207
        ],
        [
          0.151,
          0.8148
        ],
        [
          0.1611,
          0.808
        ],
        [
          0.1715,
          0.8025
        ],
        [
          0.1815,
          0.7948
        ],
        [
          0.1915,
          0.7898
        ],
        [
          0.2015,
          0.7843
        ],
        [
          0.2116,
          0.7798
        ],
        [
          0.222,
          0.7734
        ],
        [
          0.232,
          0.7671
        ],
        [
          0.242,
          0.7625
        ],
        [
          0.252,
          0.7557
        ],
        [
          0.2621,
          0.7484
        ],
        [
          0.2725,
          0.742
        ],
        [
          0.2825,
          0.737
        ],
        [
          0.2925,
          0.7307
        ],
        [
          0.3025,
          0.7247
        ],
        [
          0.3126,
          0.7179
        ],
        [
          0.3235,
          0.7084
        ],
        [
          0.3339,
          0.7025
        ],
        [
          0.3444,
          0.6956
        ],
        [
          0.3544,
          0.6874
        ],
        [
          0.3644,
          0.6811
        ],
        [
          0.3744,
          0.6756
        ],
        [
          0.3844,
          0.6688
        ],
        [
          0.3944,
          0.6629
        ],
        [
          0.4045,
          0.657
        ],
        [
          0.4149,
          0.6497
        ],
        [
          0.4249,
          0.646
        ],
        [
          0.4354,
          0.6392
        ],
        [
          0.4454,
          0.6324
        ],
        [
          0.4554,
          0.6265
        ],
        [
          0.4654,
          0.6215
        ],
        [
          0.4759,
          0.6142
        ],
        [
          0.4864,
          0.6083
        ],
        [
          0.4973,
          0.601
        ],
        [
          0.5077,
          0.5937
        ],
        [
          0.5177,
          0.5896
        ],
        [
          0.5278,
          0.5814
        ],
        [
          0.5387,
          0.5746
        ],
        [
          0.5491,
          0.566
        ],
        [
          0.5591,
          0.5591
        ],
        [
          0.5692,
          0.551
        ],
        [
          0.5796,
          0.5441
        ],
        [
          0.5896,
          0.5382
        ],
        [
          0.5996,
          0.53
        ],
        [
          0.6096,
          0.5237
        ],
        [
          0.6197,
          0.5196
        ],
        [
          0.6306,
          0.5146
        ],
        [
          0.641,
          0.5091
        ],
        [
          0.6515,
          0.5005
        ],
        [
          0.6624,
          0.4945
        ],
        [
          0.6724,
          0.4882
        ],
        [
          0.6824,
          0.4795
        ],
        [
          0.6924,
          0.4713
        ],
        [
          0.7029,
          0.4645
        ],
        [
          0.7129,
          0.4591
        ],
        [
          0.7229,
          0.4527
        ],
        [
          0.7334,
          0.4463
        ],
        [
          0.7434,
          0.4386
        ],
        [
          0.7539,
          0.4327
        ],
        [
          0.7643,
          0.4268
        ],
        [
          0.7743,
          0.419
        ],
        [
          0.7848,
          0.4126
        ],
        [
          0.7957,
          0.4045
        ],
        [
          0.8071,
          0.399
        ],
        [
          0.8171,
          0.3935
        ],
        [
          0.8276,
          0.3858
        ],
        [
          0.838,
          0.3803
        ],
        [
          0.848,
          0.3735
        ],
        [
          0.8581,
          0.3681
        ],
        [
          0.8685,
          0.3612
        ],
        [
          0.8794,
          0.354
        ],
        [
          0.8894,
          0.3462
        ],
        [
          0.8995,
          0.3389
        ],
        [
          0.9104,
          0.3326
        ],
        [
          0.9208,
          0.3258
        ],
        [
          0.9308,
          0.3194
        ],
        [
          0.9418,
          0.3107
        ],
        [
          0.9522,
          0.3053
        ],
        [
          0.9622,
          0.2985
        ],
        [
          0.9722,
          0.2907
        ],
        [
          0.9823,
          0.283
        ],
        [
          0.9927,
          0.2766
        ],
        [
          1.0,
          0.2712
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.0017765701863429102,
        "difference": -0.0005033257684815232,
        "worse": 0.17666666666666667,
        "kept_right": 0.0012702113575034867,
        "kept_wrong": 0.0005063588288394243,
        "handed_off": 0.9982234298136581
      },
      "usd_per_1000": 7.7485888198400055
    },
    {
      "decision": 243,
      "frontier": 55,
      "frontier_accuracy": 0.8503184713375797,
      "in_sample": {
        "threshold": 0.9219,
        "answered": 0.0031847133757961785,
        "accuracy": 0.8503184713375797
      },
      "curve": [
        [
          0.0,
          0.8503
        ],
        [
          0.01,
          0.848
        ],
        [
          0.02,
          0.8444
        ],
        [
          0.03,
          0.8412
        ],
        [
          0.04,
          0.8353
        ],
        [
          0.05,
          0.8303
        ],
        [
          0.0601,
          0.8258
        ],
        [
          0.0705,
          0.8189
        ],
        [
          0.0805,
          0.8148
        ],
        [
          0.091,
          0.8098
        ],
        [
          0.101,
          0.8048
        ],
        [
          0.111,
          0.7985
        ],
        [
          0.121,
          0.7934
        ],
        [
          0.131,
          0.7889
        ],
        [
          0.141,
          0.7839
        ],
        [
          0.151,
          0.7775
        ],
        [
          0.1611,
          0.7712
        ],
        [
          0.1715,
          0.7662
        ],
        [
          0.1815,
          0.7584
        ],
        [
          0.1915,
          0.7534
        ],
        [
          0.2015,
          0.7475
        ],
        [
          0.2116,
          0.7429
        ],
        [
          0.222,
          0.7384
        ],
        [
          0.232,
          0.7329
        ],
        [
          0.242,
          0.7288
        ],
        [
          0.252,
          0.7234
        ],
        [
          0.2621,
          0.7179
        ],
        [
          0.2725,
          0.7125
        ],
        [
          0.2825,
          0.7079
        ],
        [
          0.2925,
          0.702
        ],
        [
          0.3025,
          0.6979
        ],
        [
          0.3126,
          0.6911
        ],
        [
          0.3235,
          0.6833
        ],
        [
          0.3339,
          0.6774
        ],
        [
          0.3444,
          0.6702
        ],
        [
          0.3544,
          0.6624
        ],
        [
          0.3644,
          0.657
        ],
        [
          0.3744,
          0.6515
        ],
        [
          0.3844,
          0.6447
        ],
        [
          0.3944,
          0.6388
        ],
        [
          0.4045,
          0.6319
        ],
        [
          0.4149,
          0.6256
        ],
        [
          0.4249,
          0.621
        ],
        [
          0.4354,
          0.6146
        ],
        [
          0.4454,
          0.6087
        ],
        [
          0.4554,
          0.6037
        ],
        [
          0.4654,
          0.5987
        ],
        [
          0.4759,
          0.5919
        ],
        [
          0.4864,
          0.586
        ],
        [
          0.4973,
          0.5792
        ],
        [
          0.5077,
          0.5719
        ],
        [
          0.5177,
          0.5682
        ],
        [
          0.5278,
          0.5601
        ],
        [
          0.5387,
          0.5541
        ],
        [
          0.5491,
          0.5469
        ],
        [
          0.5591,
          0.5391
        ],
        [
          0.5692,
          0.5318
        ],
        [
          0.5796,
          0.5264
        ],
        [
          0.5896,
          0.5205
        ],
        [
          0.5996,
          0.5123
        ],
        [
          0.6096,
          0.5064
        ],
        [
          0.6197,
          0.5023
        ],
        [
          0.6306,
          0.4982
        ],
        [
          0.641,
          0.4936
        ],
        [
          0.6515,
          0.4864
        ],
        [
          0.6624,
          0.4813
        ],
        [
          0.6724,
          0.4763
        ],
        [
          0.6824,
          0.4677
        ],
        [
          0.6924,
          0.46
        ],
        [
          0.7029,
          0.4536
        ],
        [
          0.7129,
          0.4486
        ],
        [
          0.7229,
          0.4436
        ],
        [
          0.7334,
          0.4363
        ],
        [
          0.7434,
          0.4295
        ],
        [
          0.7539,
          0.4236
        ],
        [
          0.7643,
          0.4167
        ],
        [
          0.7743,
          0.4099
        ],
        [
          0.7848,
          0.4031
        ],
        [
          0.7957,
          0.3958
        ],
        [
          0.8071,
          0.3913
        ],
        [
          0.8171,
          0.3867
        ],
        [
          0.8276,
          0.3817
        ],
        [
          0.838,
          0.3758
        ],
        [
          0.848,
          0.3694
        ],
        [
          0.8581,
          0.3649
        ],
        [
          0.8685,
          0.3585
        ],
        [
          0.8794,
          0.3512
        ],
        [
          0.8894,
          0.3453
        ],
        [
          0.8995,
          0.3385
        ],
        [
          0.9104,
          0.3321
        ],
        [
          0.9208,
          0.3244
        ],
        [
          0.9308,
          0.3176
        ],
        [
          0.9418,
          0.308
        ],
        [
          0.9522,
          0.303
        ],
        [
          0.9622,
          0.2975
        ],
        [
          0.9722,
          0.2903
        ],
        [
          0.9823,
          0.2825
        ],
        [
          0.9927,
          0.2757
        ],
        [
          1.0,
          0.2712
        ]
      ],
      "held_out": {
        "splits": 300,
        "answered": 0.0038595655796828192,
        "difference": -0.0010915666648459773,
        "worse": 0.38666666666666666,
        "kept_right": 0.002367824841963769,
        "kept_wrong": 0.0014917407377190552,
        "handed_off": 0.9961404344203185
      },
      "usd_per_1000": 3.6549924459864522
    }
  ],
  "doubts": [
    {
      "decision": 199,
      "frontier": [
        48,
        47,
        55
      ],
      "questions": 2198,
      "rho": 0.3015448463087679,
      "p": 0.004975124378109453,
      "auroc": 0.7171352118757728,
      "top_tenth": {
        "questions": 219,
        "with_mistake": 74,
        "all_with_mistake": 433
      }
    },
    {
      "decision": 248,
      "frontier": [
        48,
        47,
        55
      ],
      "questions": 2198,
      "rho": 0.2645297074128647,
      "p": 0.004975124378109453,
      "auroc": 0.6889747397758572,
      "top_tenth": {
        "questions": 219,
        "with_mistake": 83,
        "all_with_mistake": 433
      }
    },
    {
      "decision": 129,
      "frontier": [
        48,
        47,
        55
      ],
      "questions": 2198,
      "rho": 0.23675409695076555,
      "p": 0.004975124378109453,
      "auroc": 0.6697067040019888,
      "top_tenth": {
        "questions": 219,
        "with_mistake": 68,
        "all_with_mistake": 433
      }
    },
    {
      "decision": 259,
      "frontier": [
        48,
        47,
        55
      ],
      "questions": 2198,
      "rho": 0.2737043445116408,
      "p": 0.004975124378109453,
      "auroc": 0.6954929374742392,
      "top_tenth": {
        "questions": 219,
        "with_mistake": 84,
        "all_with_mistake": 433
      }
    },
    {
      "decision": 269,
      "frontier": [
        48,
        47,
        55
      ],
      "questions": 2198,
      "rho": 0.2569423087710018,
      "p": 0.004975124378109453,
      "auroc": 0.6828988086281231,
      "top_tenth": {
        "questions": 219,
        "with_mistake": 78,
        "all_with_mistake": 433
      }
    },
    {
      "decision": 322,
      "frontier": [
        48,
        47,
        55
      ],
      "questions": 2198,
      "rho": 0.27650047767708213,
      "p": 0.004975124378109453,
      "auroc": 0.6978102571819247,
      "top_tenth": {
        "questions": 219,
        "with_mistake": 80,
        "all_with_mistake": 433
      }
    },
    {
      "decision": 326,
      "frontier": [
        48,
        47,
        55
      ],
      "questions": 2198,
      "rho": 0.2620013746165349,
      "p": 0.004975124378109453,
      "auroc": 0.6872913790734647,
      "top_tenth": {
        "questions": 219,
        "with_mistake": 82,
        "all_with_mistake": 433
      }
    },
    {
      "decision": 374,
      "frontier": [
        48,
        47,
        55
      ],
      "questions": 2198,
      "rho": 0.24964260853326162,
      "p": 0.004975124378109453,
      "auroc": 0.679736210246714,
      "top_tenth": {
        "questions": 219,
        "with_mistake": 74,
        "all_with_mistake": 433
      }
    },
    {
      "decision": 217,
      "frontier": [
        48,
        47,
        55
      ],
      "questions": 2198,
      "rho": 0.2679028075331068,
      "p": 0.004975124378109453,
      "auroc": 0.6919260184888354,
      "top_tenth": {
        "questions": 219,
        "with_mistake": 81,
        "all_with_mistake": 433
      }
    },
    {
      "decision": 145,
      "frontier": [
        48,
        47,
        55
      ],
      "questions": 2198,
      "rho": 0.28021944302267887,
      "p": 0.004975124378109453,
      "auroc": 0.7006470438144836,
      "top_tenth": {
        "questions": 219,
        "with_mistake": 71,
        "all_with_mistake": 433
      }
    },
    {
      "decision": 177,
      "frontier": [
        48,
        47,
        55
      ],
      "questions": 2198,
      "rho": 0.2533481390910471,
      "p": 0.004975124378109453,
      "auroc": 0.6812893770976585,
      "top_tenth": {
        "questions": 219,
        "with_mistake": 67,
        "all_with_mistake": 433
      }
    },
    {
      "decision": 346,
      "frontier": [
        48,
        47,
        55
      ],
      "questions": 2198,
      "rho": 0.2613270452103907,
      "p": 0.004975124378109453,
      "auroc": 0.6858271889250175,
      "top_tenth": {
        "questions": 219,
        "with_mistake": 72,
        "all_with_mistake": 433
      }
    },
    {
      "decision": 143,
      "frontier": [
        48,
        47,
        55
      ],
      "questions": 2194,
      "rho": 0.20893004529506504,
      "p": 0.004975124378109453,
      "auroc": 0.6512604048429814,
      "top_tenth": {
        "questions": 219,
        "with_mistake": 64,
        "all_with_mistake": 432
      }
    },
    {
      "decision": 254,
      "frontier": [
        48,
        47,
        55
      ],
      "questions": 2198,
      "rho": 0.24015206281823911,
      "p": 0.004975124378109453,
      "auroc": 0.6739245922446336,
      "top_tenth": {
        "questions": 219,
        "with_mistake": 71,
        "all_with_mistake": 433
      }
    },
    {
      "decision": 150,
      "frontier": [
        48,
        47,
        55
      ],
      "questions": 2198,
      "rho": 0.20426168955342888,
      "p": 0.004975124378109453,
      "auroc": 0.6490680344653874,
      "top_tenth": {
        "questions": 219,
        "with_mistake": 51,
        "all_with_mistake": 433
      }
    },
    {
      "decision": 44,
      "frontier": [
        48,
        47,
        55
      ],
      "questions": 2198,
      "rho": 0.22817878874345082,
      "p": 0.004975124378109453,
      "auroc": 0.6639232183396686,
      "top_tenth": {
        "questions": 219,
        "with_mistake": 63,
        "all_with_mistake": 433
      }
    },
    {
      "decision": 264,
      "frontier": [
        48,
        47,
        55
      ],
      "questions": 2198,
      "rho": 0.24093254444285928,
      "p": 0.004975124378109453,
      "auroc": 0.6740861896381396,
      "top_tenth": {
        "questions": 219,
        "with_mistake": 65,
        "all_with_mistake": 433
      }
    },
    {
      "decision": 194,
      "frontier": [
        48,
        47,
        55
      ],
      "questions": 2198,
      "rho": 0.20516341205319433,
      "p": 0.004975124378109453,
      "auroc": 0.6480153615659899,
      "top_tenth": {
        "questions": 219,
        "with_mistake": 56,
        "all_with_mistake": 433
      }
    },
    {
      "decision": 204,
      "frontier": [
        48,
        47,
        55
      ],
      "questions": 2198,
      "rho": 0.22509582510310908,
      "p": 0.004975124378109453,
      "auroc": 0.6612323273295867,
      "top_tenth": {
        "questions": 219,
        "with_mistake": 62,
        "all_with_mistake": 433
      }
    },
    {
      "decision": 49,
      "frontier": [
        48,
        47,
        55
      ],
      "questions": 2198,
      "rho": 0.20387571235084012,
      "p": 0.004975124378109453,
      "auroc": 0.6472786868085496,
      "top_tenth": {
        "questions": 219,
        "with_mistake": 61,
        "all_with_mistake": 433
      }
    },
    {
      "decision": 125,
      "frontier": [
        48,
        47,
        55
      ],
      "questions": 2198,
      "rho": 0.1812948326361934,
      "p": 0.004975124378109453,
      "auroc": 0.630554337941367,
      "top_tenth": {
        "questions": 219,
        "with_mistake": 62,
        "all_with_mistake": 433
      }
    },
    {
      "decision": 52,
      "frontier": [
        48,
        47,
        55
      ],
      "questions": 2198,
      "rho": 0.2030499388311332,
      "p": 0.004975124378109453,
      "auroc": 0.647787031645611,
      "top_tenth": {
        "questions": 219,
        "with_mistake": 64,
        "all_with_mistake": 433
      }
    },
    {
      "decision": 334,
      "frontier": [
        48,
        47,
        55
      ],
      "questions": 2198,
      "rho": 0.22407728119749107,
      "p": 0.004975124378109453,
      "auroc": 0.6621397588469666,
      "top_tenth": {
        "questions": 219,
        "with_mistake": 70,
        "all_with_mistake": 433
      }
    },
    {
      "decision": 45,
      "frontier": [
        48,
        47,
        55
      ],
      "questions": 2198,
      "rho": 0.2284969151183873,
      "p": 0.004975124378109453,
      "auroc": 0.6644610039974093,
      "top_tenth": {
        "questions": 219,
        "with_mistake": 76,
        "all_with_mistake": 433
      }
    },
    {
      "decision": 127,
      "frontier": [
        48,
        47,
        55
      ],
      "questions": 2198,
      "rho": 0.2025070031176271,
      "p": 0.004975124378109453,
      "auroc": 0.6465649104671931,
      "top_tenth": {
        "questions": 219,
        "with_mistake": 62,
        "all_with_mistake": 433
      }
    },
    {
      "decision": 128,
      "frontier": [
        48,
        47,
        55
      ],
      "questions": 2198,
      "rho": 0.18811807119240398,
      "p": 0.004975124378109453,
      "auroc": 0.6354532905023913,
      "top_tenth": {
        "questions": 219,
        "with_mistake": 61,
        "all_with_mistake": 433
      }
    },
    {
      "decision": 373,
      "frontier": [
        48,
        47,
        55
      ],
      "questions": 2198,
      "rho": 0.18953617571544865,
      "p": 0.004975124378109453,
      "auroc": 0.6375416260492381,
      "top_tenth": {
        "questions": 219,
        "with_mistake": 65,
        "all_with_mistake": 433
      }
    },
    {
      "decision": 223,
      "frontier": [
        48,
        47,
        55
      ],
      "questions": 2198,
      "rho": 0.18592964522183145,
      "p": 0.004975124378109453,
      "auroc": 0.6347022224548411,
      "top_tenth": {
        "questions": 219,
        "with_mistake": 59,
        "all_with_mistake": 433
      }
    },
    {
      "decision": 205,
      "frontier": [
        48,
        47,
        55
      ],
      "questions": 2198,
      "rho": 0.16939013163757866,
      "p": 0.004975124378109453,
      "auroc": 0.6231339426492813,
      "top_tenth": {
        "questions": 219,
        "with_mistake": 54,
        "all_with_mistake": 433
      }
    },
    {
      "decision": 139,
      "frontier": [
        48,
        47,
        55
      ],
      "questions": 2198,
      "rho": 0.12766387292232953,
      "p": 0.004975124378109453,
      "auroc": 0.5943303521776394,
      "top_tenth": {
        "questions": 219,
        "with_mistake": 55,
        "all_with_mistake": 433
      }
    },
    {
      "decision": 126,
      "frontier": [
        48,
        47,
        55
      ],
      "questions": 2198,
      "rho": 0.1758017514629497,
      "p": 0.004975124378109453,
      "auroc": 0.6264862707639566,
      "top_tenth": {
        "questions": 219,
        "with_mistake": 65,
        "all_with_mistake": 433
      }
    },
    {
      "decision": 340,
      "frontier": [
        48,
        47,
        55
      ],
      "questions": 2198,
      "rho": 0.14041333931441907,
      "p": 0.004975124378109453,
      "auroc": 0.603466165954635,
      "top_tenth": {
        "questions": 219,
        "with_mistake": 59,
        "all_with_mistake": 433
      }
    },
    {
      "decision": 141,
      "frontier": [
        48,
        47,
        55
      ],
      "questions": 2198,
      "rho": 0.12020230565086733,
      "p": 0.004975124378109453,
      "auroc": 0.5873535319171208,
      "top_tenth": {
        "questions": 219,
        "with_mistake": 53,
        "all_with_mistake": 433
      }
    },
    {
      "decision": 243,
      "frontier": [
        48,
        47,
        55
      ],
      "questions": 2198,
      "rho": 0.10405108588743903,
      "p": 0.004975124378109453,
      "auroc": 0.5755536509888844,
      "top_tenth": {
        "questions": 219,
        "with_mistake": 46,
        "all_with_mistake": 433
      }
    }
  ],
  "sources": [
    {
      "answers": 53,
      "questions": 2198,
      "correct": 1191,
      "sources": [
        {
          "run": 53,
          "kind": "stated",
          "questions": 2198,
          "auroc": 0.5147923394342041,
          "auroc_interval": [
            0.509732090367011,
            0.5202663316582915
          ],
          "risk_coverage": [
            [
              0.02,
              0.1174
            ],
            [
              0.04,
              0.291
            ],
            [
              0.06,
              0.3489
            ],
            [
              0.08,
              0.3778
            ],
            [
              0.1,
              0.3951
            ],
            [
              0.12,
              0.4067
            ],
            [
              0.14,
              0.415
            ],
            [
              0.16,
              0.4212
            ],
            [
              0.18,
              0.426
            ],
            [
              0.2,
              0.4299
            ],
            [
              0.22,
              0.433
            ],
            [
              0.24,
              0.4356
            ],
            [
              0.26,
              0.4379
            ],
            [
              0.28,
              0.4398
            ],
            [
              0.3,
              0.4414
            ],
            [
              0.32,
              0.4429
            ],
            [
              0.34,
              0.4441
            ],
            [
              0.36,
              0.4453
            ],
            [
              0.38,
              0.4463
            ],
            [
              0.4,
              0.4472
            ],
            [
              0.42,
              0.448
            ],
            [
              0.44,
              0.4488
            ],
            [
              0.46,
              0.4495
            ],
            [
              0.48,
              0.4501
            ],
            [
              0.5,
              0.4507
            ],
            [
              0.52,
              0.4512
            ],
            [
              0.54,
              0.4517
            ],
            [
              0.56,
              0.4522
            ],
            [
              0.58,
              0.4526
            ],
            [
              0.6,
              0.453
            ],
            [
              0.62,
              0.4534
            ],
            [
              0.64,
              0.4537
            ],
            [
              0.66,
              0.454
            ],
            [
              0.68,
              0.4544
            ],
            [
              0.7,
              0.4546
            ],
            [
              0.72,
              0.4549
            ],
            [
              0.74,
              0.4552
            ],
            [
              0.76,
              0.4554
            ],
            [
              0.78,
              0.4557
            ],
            [
              0.8,
              0.4559
            ],
            [
              0.82,
              0.4561
            ],
            [
              0.84,
              0.4563
            ],
            [
              0.86,
              0.4565
            ],
            [
              0.88,
              0.4567
            ],
            [
              0.9,
              0.4569
            ],
            [
              0.92,
              0.457
            ],
            [
              0.94,
              0.4572
            ],
            [
              0.96,
              0.4573
            ],
            [
              0.98,
              0.4575
            ],
            [
              1.0,
              0.4581
            ]
          ]
        },
        {
          "run": 45,
          "kind": "probability",
          "questions": 2198,
          "auroc": 0.8185047238599326,
          "auroc_interval": [
            0.8005678063625311,
            0.8366006167343264
          ],
          "risk_coverage": [
            [
              0.02,
              0.0
            ],
            [
              0.04,
              0.0
            ],
            [
              0.06,
              0.0
            ],
            [
              0.08,
              0.0
            ],
            [
              0.1,
              0.0
            ],
            [
              0.12,
              0.0038
            ],
            [
              0.14,
              0.0065
            ],
            [
              0.16,
              0.0085
            ],
            [
              0.18,
              0.0202
            ],
            [
              0.2,
              0.025
            ],
            [
              0.22,
              0.0331
            ],
            [
              0.24,
              0.0398
            ],
            [
              0.26,
              0.0463
            ],
            [
              0.28,
              0.0601
            ],
            [
              0.3,
              0.0637
            ],
            [
              0.32,
              0.0754
            ],
            [
              0.34,
              0.095
            ],
            [
              0.36,
              0.1049
            ],
            [
              0.38,
              0.1233
            ],
            [
              0.4,
              0.1354
            ],
            [
              0.42,
              0.1497
            ],
            [
              0.44,
              0.1718
            ],
            [
              0.46,
              0.1859
            ],
            [
              0.48,
              0.2038
            ],
            [
              0.5,
              0.2184
            ],
            [
              0.52,
              0.2336
            ],
            [
              0.54,
              0.2502
            ],
            [
              0.56,
              0.2566
            ],
            [
              0.58,
              0.2697
            ],
            [
              0.6,
              0.2783
            ],
            [
              0.62,
              0.2882
            ],
            [
              0.64,
              0.3
            ],
            [
              0.66,
              0.315
            ],
            [
              0.68,
              0.3249
            ],
            [
              0.7,
              0.3332
            ],
            [
              0.72,
              0.3456
            ],
            [
              0.74,
              0.3547
            ],
            [
              0.76,
              0.3655
            ],
            [
              0.78,
              0.3753
            ],
            [
              0.8,
              0.3833
            ],
            [
              0.82,
              0.393
            ],
            [
              0.84,
              0.4021
            ],
            [
              0.86,
              0.4095
            ],
            [
              0.88,
              0.4142
            ],
            [
              0.9,
              0.4216
            ],
            [
              0.92,
              0.4308
            ],
            [
              0.94,
              0.4361
            ],
            [
              0.96,
              0.4427
            ],
            [
              0.98,
              0.4499
            ],
            [
              1.0,
              0.4581
            ]
          ]
        },
        {
          "run": 83,
          "kind": "votes",
          "questions": 2198,
          "auroc": 0.7788444782408948,
          "auroc_interval": [
            0.7584856386009123,
            0.7971547529520745
          ],
          "risk_coverage": [
            [
              0.02,
              0.1474
            ],
            [
              0.04,
              0.1474
            ],
            [
              0.06,
              0.1474
            ],
            [
              0.08,
              0.1474
            ],
            [
              0.1,
              0.1474
            ],
            [
              0.12,
              0.1474
            ],
            [
              0.14,
              0.1474
            ],
            [
              0.16,
              0.1474
            ],
            [
              0.18,
              0.1474
            ],
            [
              0.2,
              0.1474
            ],
            [
              0.22,
              0.1474
            ],
            [
              0.24,
              0.1474
            ],
            [
              0.26,
              0.1474
            ],
            [
              0.28,
              0.1474
            ],
            [
              0.3,
              0.1474
            ],
            [
              0.32,
              0.1474
            ],
            [
              0.34,
              0.1474
            ],
            [
              0.36,
              0.1474
            ],
            [
              0.38,
              0.1474
            ],
            [
              0.4,
              0.1494
            ],
            [
              0.42,
              0.1694
            ],
            [
              0.44,
              0.1876
            ],
            [
              0.46,
              0.2042
            ],
            [
              0.48,
              0.218
            ],
            [
              0.5,
              0.2306
            ],
            [
              0.52,
              0.2422
            ],
            [
              0.54,
              0.2541
            ],
            [
              0.56,
              0.2658
            ],
            [
              0.58,
              0.2768
            ],
            [
              0.6,
              0.287
            ],
            [
              0.62,
              0.298
            ],
            [
              0.64,
              0.3107
            ],
            [
              0.66,
              0.3226
            ],
            [
              0.68,
              0.3338
            ],
            [
              0.7,
              0.3443
            ],
            [
              0.72,
              0.3543
            ],
            [
              0.74,
              0.3635
            ],
            [
              0.76,
              0.3721
            ],
            [
              0.78,
              0.3804
            ],
            [
              0.8,
              0.3882
            ],
            [
              0.82,
              0.3956
            ],
            [
              0.84,
              0.4036
            ],
            [
              0.86,
              0.4113
            ],
            [
              0.88,
              0.4187
            ],
            [
              0.9,
              0.4258
            ],
            [
              0.92,
              0.4329
            ],
            [
              0.94,
              0.4402
            ],
            [
              0.96,
              0.4472
            ],
            [
              0.98,
              0.4524
            ],
            [
              1.0,
              0.4581
            ]
          ]
        },
        {
          "run": 50,
          "kind": "probability",
          "questions": 2198,
          "auroc": 0.7820262361621463,
          "auroc_interval": [
            0.7615251753722931,
            0.8006139572157996
          ],
          "risk_coverage": [
            [
              0.02,
              0.0
            ],
            [
              0.04,
              0.0
            ],
            [
              0.06,
              0.0
            ],
            [
              0.08,
              0.0
            ],
            [
              0.1,
              0.0
            ],
            [
              0.12,
              0.0076
            ],
            [
              0.14,
              0.0065
            ],
            [
              0.16,
              0.0114
            ],
            [
              0.18,
              0.0278
            ],
            [
              0.2,
              0.0409
            ],
            [
              0.22,
              0.0476
            ],
            [
              0.24,
              0.0579
            ],
            [
              0.26,
              0.0665
            ],
            [
              0.28,
              0.0812
            ],
            [
              0.3,
              0.091
            ],
            [
              0.32,
              0.11
            ],
            [
              0.34,
              0.1204
            ],
            [
              0.36,
              0.1318
            ],
            [
              0.38,
              0.1497
            ],
            [
              0.4,
              0.1649
            ],
            [
              0.42,
              0.1798
            ],
            [
              0.44,
              0.1976
            ],
            [
              0.46,
              0.2167
            ],
            [
              0.48,
              0.2332
            ],
            [
              0.5,
              0.2484
            ],
            [
              0.52,
              0.2624
            ],
            [
              0.54,
              0.278
            ],
            [
              0.56,
              0.2917
            ],
            [
              0.58,
              0.3028
            ],
            [
              0.6,
              0.3147
            ],
            [
              0.62,
              0.3273
            ],
            [
              0.64,
              0.3368
            ],
            [
              0.66,
              0.3451
            ],
            [
              0.68,
              0.353
            ],
            [
              0.7,
              0.3611
            ],
            [
              0.72,
              0.369
            ],
            [
              0.74,
              0.3766
            ],
            [
              0.76,
              0.3825
            ],
            [
              0.78,
              0.3885
            ],
            [
              0.8,
              0.3943
            ],
            [
              0.82,
              0.4002
            ],
            [
              0.84,
              0.4058
            ],
            [
              0.86,
              0.4138
            ],
            [
              0.88,
              0.4224
            ],
            [
              0.9,
              0.4288
            ],
            [
              0.92,
              0.4347
            ],
            [
              0.94,
              0.4405
            ],
            [
              0.96,
              0.4455
            ],
            [
              0.98,
              0.4536
            ],
            [
              1.0,
              0.4581
            ]
          ]
        }
      ]
    }
  ],
  "repeats": [
    {
      "model": "gpt-6.1-sol",
      "name": "GPT-6.1 Sol",
      "variant": null,
      "reasoning": "high",
      "provider": "openai",
      "questions_per_request": 16,
      "whole_bank": false,
      "runs": [
        281,
        292,
        295,
        299,
        302
      ],
      "questions": 200,
      "accuracy": [
        0.985,
        0.985,
        0.98,
        0.98,
        0.98
      ],
      "changed_answer": 1,
      "wrong_any": 4,
      "wrong_every_run_same": 3,
      "scored": 200,
      "score_spread_median": 0.010000000000000009,
      "score_identical": 88,
      "sure_wrong": [
        1,
        0,
        1,
        1,
        0
      ],
      "sure_wrong_every_run": 0,
      "latency": {
        "runs": [
          281,
          292,
          295,
          299,
          302
        ],
        "concurrency": 1,
        "requests": 65,
        "median_request_seconds": 25.495603712974116,
        "p90_request_seconds": 60.84645333419903
      }
    },
    {
      "model": "gpt-6-astra",
      "name": "GPT-6 Astra",
      "variant": null,
      "reasoning": "high",
      "provider": "openai",
      "questions_per_request": 16,
      "whole_bank": false,
      "runs": [
        282,
        297,
        300,
        301,
        303
      ],
      "questions": 200,
      "accuracy": [
        0.985,
        0.98,
        0.975,
        0.98,
        0.985
      ],
      "changed_answer": 3,
      "wrong_any": 5,
      "wrong_every_run_same": 2,
      "scored": 200,
      "score_spread_median": 0.0,
      "score_identical": 105,
      "sure_wrong": [
        2,
        0,
        1,
        0,
        0
      ],
      "sure_wrong_every_run": 0,
      "latency": {
        "runs": [
          282,
          297,
          300,
          301,
          303
        ],
        "concurrency": 1,
        "requests": 65,
        "median_request_seconds": 23.566122614982305,
        "p90_request_seconds": 42.08493875039858
      }
    },
    {
      "model": "gpt-6-astra",
      "name": "GPT-6 Astra",
      "variant": null,
      "reasoning": "low",
      "provider": "openai",
      "questions_per_request": 16,
      "whole_bank": false,
      "runs": [
        69,
        78,
        82,
        87,
        91
      ],
      "questions": 200,
      "accuracy": [
        0.975,
        0.98,
        0.98,
        0.98,
        0.98
      ],
      "changed_answer": 1,
      "wrong_any": 5,
      "wrong_every_run_same": 4,
      "scored": 200,
      "score_spread_median": 0.0,
      "score_identical": 105,
      "sure_wrong": [
        1,
        0,
        1,
        1,
        1
      ],
      "sure_wrong_every_run": 0,
      "latency": {
        "runs": [
          69,
          78,
          82,
          87,
          91
        ],
        "concurrency": 1,
        "requests": 65,
        "median_request_seconds": 13.043534992961213,
        "p90_request_seconds": 16.78220434060786
      }
    },
    {
      "model": "claude-opus-5-5",
      "name": "Claude Opus 5.5",
      "variant": null,
      "reasoning": "high",
      "provider": "claude",
      "questions_per_request": 16,
      "whole_bank": false,
      "runs": [
        283,
        289,
        294,
        296,
        298
      ],
      "questions": 200,
      "accuracy": [
        0.985,
        0.975,
        0.975,
        0.97,
        0.98
      ],
      "changed_answer": 5,
      "wrong_any": 8,
      "wrong_every_run_same": 3,
      "scored": 200,
      "score_spread_median": 0.050000000000000044,
      "score_identical": 15,
      "sure_wrong": [
        0,
        0,
        0,
        0,
        0
      ],
      "sure_wrong_every_run": 0,
      "latency": {
        "runs": [
          283,
          289,
          294,
          296,
          298
        ],
        "concurrency": 1,
        "requests": 65,
        "median_request_seconds": 19.52282273999299,
        "p90_request_seconds": 35.27809036319377
      }
    },
    {
      "model": "gpt-6.1-terra",
      "name": "GPT-6.1 Terra",
      "variant": null,
      "reasoning": "high",
      "provider": "openai",
      "questions_per_request": 16,
      "whole_bank": false,
      "runs": [
        280,
        285,
        288,
        291,
        293
      ],
      "questions": 200,
      "accuracy": [
        0.965,
        0.97,
        0.95,
        0.965,
        0.955
      ],
      "changed_answer": 11,
      "wrong_any": 16,
      "wrong_every_run_same": 5,
      "scored": 200,
      "score_spread_median": 0.010000000000000009,
      "score_identical": 83,
      "sure_wrong": [
        1,
        0,
        2,
        1,
        1
      ],
      "sure_wrong_every_run": 0,
      "latency": {
        "runs": [
          280,
          285,
          288,
          291,
          293
        ],
        "concurrency": 1,
        "requests": 65,
        "median_request_seconds": 8.852608644025167,
        "p90_request_seconds": 14.96516182362102
      }
    },
    {
      "model": "claude-opus-5-5",
      "name": "Claude Opus 5.5",
      "variant": null,
      "reasoning": "low",
      "provider": "claude",
      "questions_per_request": 16,
      "whole_bank": false,
      "runs": [
        67,
        77,
        81,
        86,
        90
      ],
      "questions": 200,
      "accuracy": [
        0.94,
        0.95,
        0.95,
        0.96,
        0.975
      ],
      "changed_answer": 20,
      "wrong_any": 22,
      "wrong_every_run_same": 2,
      "scored": 200,
      "score_spread_median": 0.050000000000000044,
      "score_identical": 23,
      "sure_wrong": [
        0,
        0,
        0,
        0,
        0
      ],
      "sure_wrong_every_run": 0,
      "latency": {
        "runs": [
          67,
          77,
          81,
          86,
          90
        ],
        "concurrency": 1,
        "requests": 65,
        "median_request_seconds": 13.402853598003276,
        "p90_request_seconds": 19.79555194299901
      }
    },
    {
      "model": "gpt-6.1-terra",
      "name": "GPT-6.1 Terra",
      "variant": null,
      "reasoning": "low",
      "provider": "openai",
      "questions_per_request": 16,
      "whole_bank": false,
      "runs": [
        279,
        284,
        286,
        287,
        290
      ],
      "questions": 200,
      "accuracy": [
        0.93,
        0.96,
        0.96,
        0.955,
        0.96
      ],
      "changed_answer": 14,
      "wrong_any": 18,
      "wrong_every_run_same": 4,
      "scored": 200,
      "score_spread_median": 0.010000000000000009,
      "score_identical": 87,
      "sure_wrong": [
        2,
        0,
        1,
        1,
        1
      ],
      "sure_wrong_every_run": 0,
      "latency": {
        "runs": [
          279,
          284,
          286,
          287,
          290
        ],
        "concurrency": 1,
        "requests": 65,
        "median_request_seconds": 5.419412001996534,
        "p90_request_seconds": 6.897614485403755
      }
    },
    {
      "model": "deepseek-v4.1-flash",
      "name": "DeepSeek-V4.1-Flash, first release",
      "variant": null,
      "reasoning": "on",
      "provider": "vllm",
      "questions_per_request": 16,
      "whole_bank": false,
      "runs": [
        114,
        115,
        116,
        117,
        118
      ],
      "questions": 200,
      "accuracy": [
        0.95,
        0.97,
        0.94,
        0.935,
        0.95
      ],
      "changed_answer": 15,
      "wrong_any": 20,
      "wrong_every_run_same": 5,
      "scored": 0,
      "score_spread_median": null,
      "score_identical": 0,
      "sure_wrong": [
        0,
        0,
        0,
        0,
        0
      ],
      "sure_wrong_every_run": 0,
      "latency": {
        "runs": [
          114,
          115,
          116,
          117,
          118
        ],
        "concurrency": 8,
        "requests": 65,
        "median_request_seconds": 274.6319791179849,
        "p90_request_seconds": 443.7561121556093
      }
    },
    {
      "model": "claude-sonnet-5-5",
      "name": "Claude Sonnet 5.5",
      "variant": null,
      "reasoning": "high",
      "provider": "claude",
      "questions_per_request": 16,
      "whole_bank": false,
      "runs": [
        63,
        80,
        88,
        93,
        98
      ],
      "questions": 200,
      "accuracy": [
        0.895,
        0.91,
        0.895,
        0.87,
        0.87
      ],
      "changed_answer": 31,
      "wrong_any": 40,
      "wrong_every_run_same": 9,
      "scored": 200,
      "score_spread_median": 0.09999999999999998,
      "score_identical": 14,
      "sure_wrong": [
        0,
        0,
        0,
        0,
        0
      ],
      "sure_wrong_every_run": 0,
      "latency": {
        "runs": [
          63,
          80,
          88,
          93,
          98
        ],
        "concurrency": 1,
        "requests": 65,
        "median_request_seconds": 20.488734460028354,
        "p90_request_seconds": 37.048832100233994
      }
    },
    {
      "model": "gemma-4-31B-it",
      "name": "Gemma 4 31B",
      "variant": null,
      "reasoning": "off",
      "provider": "vllm",
      "questions_per_request": 16,
      "whole_bank": false,
      "runs": [
        61,
        72,
        73,
        76,
        79
      ],
      "questions": 200,
      "accuracy": [
        0.705,
        0.705,
        0.705,
        0.705,
        0.705
      ],
      "changed_answer": 0,
      "wrong_any": 59,
      "wrong_every_run_same": 59,
      "scored": 0,
      "score_spread_median": null,
      "score_identical": 0,
      "sure_wrong": [
        0,
        0,
        0,
        0,
        0
      ],
      "sure_wrong_every_run": 0,
      "latency": {
        "runs": [
          61,
          72,
          73,
          76,
          79
        ],
        "concurrency": 1,
        "requests": 65,
        "median_request_seconds": 3.5819392279954627,
        "p90_request_seconds": 3.9002420175937003
      }
    },
    {
      "model": "claude-sonnet-5-5",
      "name": "Claude Sonnet 5.5",
      "variant": null,
      "reasoning": "low",
      "provider": "claude",
      "questions_per_request": 16,
      "whole_bank": false,
      "runs": [
        229,
        230,
        231,
        232,
        233
      ],
      "questions": 200,
      "accuracy": [
        0.7,
        0.655,
        0.68,
        0.68,
        0.685
      ],
      "changed_answer": 72,
      "wrong_any": 92,
      "wrong_every_run_same": 20,
      "scored": 200,
      "score_spread_median": 0.09999999999999998,
      "score_identical": 40,
      "sure_wrong": [
        0,
        0,
        0,
        0,
        0
      ],
      "sure_wrong_every_run": 0,
      "latency": {
        "runs": [
          229,
          230,
          231,
          232,
          233
        ],
        "concurrency": 1,
        "requests": 65,
        "median_request_seconds": 3.873064635001356,
        "p90_request_seconds": 5.84579606440384
      }
    },
    {
      "model": "deepseek-v4.1-flash",
      "name": "DeepSeek-V4.1-Flash, first release",
      "variant": null,
      "reasoning": "off",
      "provider": "vllm",
      "questions_per_request": 16,
      "whole_bank": false,
      "runs": [
        109,
        110,
        111,
        112,
        113
      ],
      "questions": 200,
      "accuracy": [
        0.645,
        0.64,
        0.645,
        0.64,
        0.645
      ],
      "changed_answer": 12,
      "wrong_any": 75,
      "wrong_every_run_same": 63,
      "scored": 0,
      "score_spread_median": null,
      "score_identical": 0,
      "sure_wrong": [
        0,
        0,
        0,
        0,
        0
      ],
      "sure_wrong_every_run": 0,
      "latency": {
        "runs": [
          109,
          110,
          111,
          112,
          113
        ],
        "concurrency": 1,
        "requests": 65,
        "median_request_seconds": 0.5750969999935478,
        "p90_request_seconds": 5.717052210203837
      }
    },
    {
      "model": "d1-ultra",
      "name": "Liquid d1-ultra",
      "variant": null,
      "reasoning": null,
      "provider": "liquid",
      "questions_per_request": 1,
      "whole_bank": true,
      "runs": [
        199,
        200,
        201,
        202,
        203
      ],
      "questions": 2198,
      "accuracy": [
        0.7707006369426752,
        0.7670609645131938,
        0.759326660600546,
        0.7666060054595086,
        0.7597816196542311
      ],
      "changed_answer": 375,
      "wrong_any": 669,
      "wrong_every_run_same": 294,
      "scored": 2198,
      "score_spread_median": 0.07732904799822926,
      "score_identical": 0,
      "sure_wrong": [
        0,
        0,
        0,
        0,
        0
      ],
      "sure_wrong_every_run": 0,
      "latency": {
        "runs": [
          199,
          200,
          201,
          202,
          203
        ],
        "concurrency": 1,
        "requests": 10990,
        "median_request_seconds": 0.22276320450328058,
        "p90_request_seconds": 0.3530015264128451
      }
    },
    {
      "model": "glide-v1.2",
      "name": "Fastino GLiDE v1.2",
      "variant": "local",
      "reasoning": null,
      "provider": "glide",
      "questions_per_request": 1,
      "whole_bank": true,
      "runs": [
        248,
        250,
        251,
        252,
        253
      ],
      "questions": 2198,
      "accuracy": [
        0.6847133757961783,
        0.6847133757961783,
        0.6847133757961783,
        0.6847133757961783,
        0.6847133757961783
      ],
      "changed_answer": 0,
      "wrong_any": 693,
      "wrong_every_run_same": 693,
      "scored": 2198,
      "score_spread_median": 0.0,
      "score_identical": 2198,
      "sure_wrong": [
        0,
        0,
        0,
        0,
        0
      ],
      "sure_wrong_every_run": 0,
      "latency": {
        "runs": [
          248,
          250,
          251,
          252,
          253
        ],
        "concurrency": 1,
        "requests": 10990,
        "median_request_seconds": 0.4119799835025333,
        "p90_request_seconds": 0.5165845516894478
      }
    },
    {
      "model": "vllm-sr/d3",
      "name": "vLLM-SR d3",
      "variant": null,
      "reasoning": null,
      "provider": "vllm-sr",
      "questions_per_request": 1,
      "whole_bank": true,
      "runs": [
        259,
        260,
        261,
        262,
        263
      ],
      "questions": 2198,
      "accuracy": [
        0.6778889899909009,
        0.6778889899909009,
        0.6778889899909009,
        0.6778889899909009,
        0.6778889899909009
      ],
      "changed_answer": 0,
      "wrong_any": 708,
      "wrong_every_run_same": 708,
      "scored": 2198,
      "score_spread_median": 0.0,
      "score_identical": 2198,
      "sure_wrong": [
        0,
        0,
        0,
        0,
        0
      ],
      "sure_wrong_every_run": 0,
      "latency": {
        "runs": [
          259,
          260,
          261,
          262,
          263
        ],
        "concurrency": 1,
        "requests": 10990,
        "median_request_seconds": 0.08950294999522157,
        "p90_request_seconds": 0.13542871531099082
      }
    },
    {
      "model": "vllm-sr/d3-ultra",
      "name": "vLLM-SR d3-ultra",
      "variant": "transformers, rev 52ba9e7",
      "reasoning": null,
      "provider": "d3-server",
      "questions_per_request": 1,
      "whole_bank": true,
      "runs": [
        269,
        270,
        271,
        272,
        273
      ],
      "questions": 2198,
      "accuracy": [
        0.6778889899909009,
        0.6778889899909009,
        0.6778889899909009,
        0.6778889899909009,
        0.6778889899909009
      ],
      "changed_answer": 0,
      "wrong_any": 708,
      "wrong_every_run_same": 708,
      "scored": 2198,
      "score_spread_median": 0.0,
      "score_identical": 2198,
      "sure_wrong": [
        1,
        1,
        1,
        1,
        1
      ],
      "sure_wrong_every_run": 1,
      "latency": {
        "runs": [
          269,
          270,
          271,
          272,
          273
        ],
        "concurrency": 1,
        "requests": 10990,
        "median_request_seconds": 0.27599614550126716,
        "p90_request_seconds": 0.43350873780436816
      }
    },
    {
      "model": "drex-max-v1",
      "name": "Drex Max v1",
      "variant": null,
      "reasoning": null,
      "provider": "drex-max",
      "questions_per_request": 1,
      "whole_bank": true,
      "runs": [
        322,
        323,
        324,
        327,
        328
      ],
      "questions": 2198,
      "accuracy": [
        0.6774340309372157,
        0.6774340309372157,
        0.6774340309372157,
        0.6774340309372157,
        0.6774340309372157
      ],
      "changed_answer": 0,
      "wrong_any": 709,
      "wrong_every_run_same": 709,
      "scored": 2198,
      "score_spread_median": 0.0,
      "score_identical": 2198,
      "sure_wrong": [
        1,
        1,
        1,
        1,
        1
      ],
      "sure_wrong_every_run": 1,
      "latency": {
        "runs": [
          322,
          323,
          324,
          327,
          328
        ],
        "concurrency": 1,
        "requests": 10990,
        "median_request_seconds": 0.0814295545060304,
        "p90_request_seconds": 0.10906726221001009
      }
    },
    {
      "model": "drex-max-v1.1",
      "name": "Drex Max 1.1",
      "variant": null,
      "reasoning": null,
      "provider": "nace",
      "questions_per_request": 1,
      "whole_bank": true,
      "runs": [
        326,
        329,
        330,
        331,
        332
      ],
      "questions": 2198,
      "accuracy": [
        0.6737943585077343,
        0.6737943585077343,
        0.6737943585077343,
        0.6737943585077343,
        0.6737943585077343
      ],
      "changed_answer": 0,
      "wrong_any": 717,
      "wrong_every_run_same": 717,
      "scored": 2198,
      "score_spread_median": 0.0,
      "score_identical": 2194,
      "sure_wrong": [
        0,
        0,
        0,
        0,
        0
      ],
      "sure_wrong_every_run": 0,
      "latency": {
        "runs": [
          326,
          329,
          330,
          331,
          332
        ],
        "concurrency": 1,
        "requests": 10990,
        "median_request_seconds": 0.3142536500017741,
        "p90_request_seconds": 0.4515718497015769
      }
    },
    {
      "model": "baselabs/mammoth-1",
      "name": "Mammoth-1",
      "variant": null,
      "reasoning": null,
      "provider": "baselabs",
      "questions_per_request": 1,
      "whole_bank": true,
      "runs": [
        374,
        377,
        380,
        383,
        384
      ],
      "questions": 2198,
      "accuracy": [
        0.6710646041856233,
        0.6710646041856233,
        0.6710646041856233,
        0.6710646041856233,
        0.6710646041856233
      ],
      "changed_answer": 0,
      "wrong_any": 723,
      "wrong_every_run_same": 723,
      "scored": 2198,
      "score_spread_median": 0.0,
      "score_identical": 2198,
      "sure_wrong": [
        0,
        0,
        0,
        0,
        0
      ],
      "sure_wrong_every_run": 0,
      "latency": {
        "runs": [
          374,
          377,
          380,
          383,
          384
        ],
        "concurrency": 1,
        "requests": 10990,
        "median_request_seconds": 0.16727188749064226,
        "p90_request_seconds": 0.1937068673025351
      }
    },
    {
      "model": "pplx-decider-v1.2",
      "name": "Perplexity Decider 27B v1.2",
      "variant": "local",
      "reasoning": null,
      "provider": "pplx-decider",
      "questions_per_request": 1,
      "whole_bank": true,
      "runs": [
        217,
        218,
        219,
        220,
        221
      ],
      "questions": 2198,
      "accuracy": [
        0.6706096451319381,
        0.6706096451319381,
        0.6706096451319381,
        0.6706096451319381,
        0.6706096451319381
      ],
      "changed_answer": 0,
      "wrong_any": 724,
      "wrong_every_run_same": 724,
      "scored": 2198,
      "score_spread_median": 0.0,
      "score_identical": 2198,
      "sure_wrong": [
        0,
        0,
        0,
        0,
        0
      ],
      "sure_wrong_every_run": 0,
      "latency": {
        "runs": [
          217,
          218,
          219,
          220,
          221
        ],
        "concurrency": 1,
        "requests": 10990,
        "median_request_seconds": 0.5787490529983188,
        "p90_request_seconds": 0.7847130408059456
      }
    },
    {
      "model": "pplx-decider-v1.1-27b",
      "name": "Perplexity Decider 27B v1.1",
      "variant": null,
      "reasoning": null,
      "provider": "perplexity",
      "questions_per_request": 1,
      "whole_bank": true,
      "runs": [
        145,
        146,
        147,
        148,
        149
      ],
      "questions": 2198,
      "accuracy": [
        0.6687898089171974,
        0.6687898089171974,
        0.6687898089171974,
        0.6687898089171974,
        0.6687898089171974
      ],
      "changed_answer": 0,
      "wrong_any": 728,
      "wrong_every_run_same": 728,
      "scored": 2198,
      "score_spread_median": 0.0,
      "score_identical": 2198,
      "sure_wrong": [
        0,
        0,
        0,
        0,
        0
      ],
      "sure_wrong_every_run": 0,
      "latency": {
        "runs": [
          145,
          146,
          147,
          148,
          149
        ],
        "concurrency": 1,
        "requests": 10990,
        "median_request_seconds": 0.230697883060202,
        "p90_request_seconds": 0.28727554152719675
      }
    },
    {
      "model": "subq-snap-preview",
      "name": "SubQ Snap Preview",
      "variant": null,
      "reasoning": null,
      "provider": "subq",
      "questions_per_request": 1,
      "whole_bank": true,
      "runs": [
        177,
        178,
        179,
        180,
        181
      ],
      "questions": 2198,
      "accuracy": [
        0.6524112829845314,
        0.6519563239308462,
        0.6524112829845314,
        0.651501364877161,
        0.6528662420382165
      ],
      "changed_answer": 9,
      "wrong_any": 768,
      "wrong_every_run_same": 759,
      "scored": 2198,
      "score_spread_median": 0.0,
      "score_identical": 2069,
      "sure_wrong": [
        0,
        0,
        0,
        0,
        0
      ],
      "sure_wrong_every_run": 0,
      "latency": {
        "runs": [
          177,
          178,
          179,
          180,
          181
        ],
        "concurrency": 1,
        "requests": 10990,
        "median_request_seconds": 0.3907852455158718,
        "p90_request_seconds": 0.44049788439588156
      }
    },
    {
      "model": "subq-snap-1.1",
      "name": "SubQ Snap 1.1",
      "variant": null,
      "reasoning": null,
      "provider": "subq",
      "questions_per_request": 1,
      "whole_bank": true,
      "runs": [
        346,
        387,
        388,
        389,
        390
      ],
      "questions": 2198,
      "accuracy": [
        0.6387625113739763,
        0.6387625113739763,
        0.6387625113739763,
        0.6387625113739763,
        0.6387625113739763
      ],
      "changed_answer": 0,
      "wrong_any": 794,
      "wrong_every_run_same": 794,
      "scored": 2198,
      "score_spread_median": 0.0,
      "score_identical": 2198,
      "sure_wrong": [
        1,
        1,
        1,
        1,
        1
      ],
      "sure_wrong_every_run": 1,
      "latency": {
        "runs": [
          346,
          387,
          388,
          389,
          390
        ],
        "concurrency": 1,
        "requests": 10990,
        "median_request_seconds": 0.37816790348733775,
        "p90_request_seconds": 0.3870654027094133
      }
    },
    {
      "model": "d1.1",
      "name": "Liquid d1.1",
      "variant": null,
      "reasoning": null,
      "provider": "liquid",
      "questions_per_request": 1,
      "whole_bank": true,
      "runs": [
        254,
        255,
        256,
        257,
        258
      ],
      "questions": 2198,
      "accuracy": [
        0.6151046405823476,
        0.6141947224749773,
        0.6205641492265697,
        0.6128298453139217,
        0.6223839854413102
      ],
      "changed_answer": 659,
      "wrong_any": 1071,
      "wrong_every_run_same": 412,
      "scored": 2198,
      "score_spread_median": 0.10694811785745692,
      "score_identical": 0,
      "sure_wrong": [
        0,
        0,
        0,
        0,
        0
      ],
      "sure_wrong_every_run": 0,
      "latency": {
        "runs": [
          254,
          255,
          256,
          257,
          258
        ],
        "concurrency": 1,
        "requests": 10990,
        "median_request_seconds": 0.19625577199963118,
        "p90_request_seconds": 0.2373385658997904
      }
    },
    {
      "model": "pplx-decider-v1-27b",
      "name": "Perplexity Decider 27B v1",
      "variant": null,
      "reasoning": null,
      "provider": "perplexity",
      "questions_per_request": 1,
      "whole_bank": true,
      "runs": [
        44,
        64,
        84,
        96,
        100
      ],
      "questions": 2198,
      "accuracy": [
        0.6037306642402184,
        0.6037306642402184,
        0.6037306642402184,
        0.6037306642402184,
        0.6041856232939036
      ],
      "changed_answer": 2,
      "wrong_any": 871,
      "wrong_every_run_same": 869,
      "scored": 2198,
      "score_spread_median": 0.0,
      "score_identical": 2169,
      "sure_wrong": [
        0,
        0,
        0,
        0,
        0
      ],
      "sure_wrong_every_run": 0,
      "latency": {
        "runs": [
          64,
          84,
          96,
          100
        ],
        "concurrency": 1,
        "requests": 8792,
        "median_request_seconds": 0.21641260248725303,
        "p90_request_seconds": 0.2719852286914829
      }
    },
    {
      "model": "celeris-1-decision",
      "name": "Celeris-1 Decision",
      "variant": null,
      "reasoning": null,
      "provider": "celeris",
      "questions_per_request": 1,
      "whole_bank": true,
      "runs": [
        150,
        151,
        156,
        157,
        158
      ],
      "questions": 2198,
      "accuracy": [
        0.6046405823475887,
        0.6028207461328481,
        0.6010009099181074,
        0.600090991810737,
        0.5982711555959963
      ],
      "changed_answer": 206,
      "wrong_any": 944,
      "wrong_every_run_same": 738,
      "scored": 2198,
      "score_spread_median": 0.030000000000000027,
      "score_identical": 192,
      "sure_wrong": [
        0,
        0,
        0,
        0,
        0
      ],
      "sure_wrong_every_run": 0,
      "latency": {
        "runs": [
          150,
          151,
          156,
          157,
          158
        ],
        "concurrency": 1,
        "requests": 10990,
        "median_request_seconds": 0.40173603700168314,
        "p90_request_seconds": 0.41774079340102616
      }
    },
    {
      "model": "OmniJev/OneJev-27B",
      "name": "OneJev 27B",
      "variant": null,
      "reasoning": null,
      "provider": "onejev",
      "questions_per_request": 1,
      "whole_bank": true,
      "runs": [
        264,
        265,
        266,
        267,
        268
      ],
      "questions": 2198,
      "accuracy": [
        0.5959963603275705,
        0.5959963603275705,
        0.5959963603275705,
        0.5959963603275705,
        0.5959963603275705
      ],
      "changed_answer": 0,
      "wrong_any": 888,
      "wrong_every_run_same": 888,
      "scored": 2198,
      "score_spread_median": 0.0,
      "score_identical": 2198,
      "sure_wrong": [
        0,
        0,
        0,
        0,
        0
      ],
      "sure_wrong_every_run": 0,
      "latency": {
        "runs": [
          264,
          265,
          266,
          267,
          268
        ],
        "concurrency": 1,
        "requests": 10990,
        "median_request_seconds": 0.1524845355161233,
        "p90_request_seconds": 0.2812980904011056
      }
    },
    {
      "model": "inception/mercury-decide",
      "name": "Inception Mercury Decide",
      "variant": "via OpenRouter",
      "reasoning": null,
      "provider": "openrouter",
      "questions_per_request": 1,
      "whole_bank": true,
      "runs": [
        204,
        210,
        212,
        214,
        216
      ],
      "questions": 2198,
      "accuracy": [
        0.5828025477707006,
        0.5905368516833485,
        0.5859872611464968,
        0.5887170154686078,
        0.5878070973612375
      ],
      "changed_answer": 191,
      "wrong_any": 969,
      "wrong_every_run_same": 778,
      "scored": 2198,
      "score_spread_median": 0.029266423876022768,
      "score_identical": 7,
      "sure_wrong": [
        13,
        14,
        11,
        13,
        13
      ],
      "sure_wrong_every_run": 11,
      "latency": {
        "runs": [
          204,
          210,
          212,
          214,
          216
        ],
        "concurrency": 1,
        "requests": 10990,
        "median_request_seconds": 0.30808473999786656,
        "p90_request_seconds": 0.39868138850142715
      }
    },
    {
      "model": "jev-1.14.0",
      "name": "Jev 1.14.0",
      "variant": null,
      "reasoning": null,
      "provider": "jev",
      "questions_per_request": 1,
      "whole_bank": true,
      "runs": [
        194,
        195,
        196,
        197,
        198
      ],
      "questions": 2198,
      "accuracy": [
        0.591901728844404,
        0.5818926296633303,
        0.5882620564149227,
        0.578252957233849,
        0.5828025477707006
      ],
      "changed_answer": 291,
      "wrong_any": 1010,
      "wrong_every_run_same": 719,
      "scored": 2198,
      "score_spread_median": 0.04999999999999999,
      "score_identical": 227,
      "sure_wrong": [
        1,
        1,
        1,
        1,
        0
      ],
      "sure_wrong_every_run": 0,
      "latency": {
        "runs": [
          194,
          195,
          196,
          197,
          198
        ],
        "concurrency": 1,
        "requests": 10990,
        "median_request_seconds": 0.2106848010007525,
        "p90_request_seconds": 0.2690009778045351
      }
    },
    {
      "model": "jev-1.13.0",
      "name": "Jev 1.13.0",
      "variant": null,
      "reasoning": null,
      "provider": "jev",
      "questions_per_request": 1,
      "whole_bank": true,
      "runs": [
        49,
        65,
        85,
        97,
        101
      ],
      "questions": 2198,
      "accuracy": [
        0.5750682438580528,
        0.5714285714285714,
        0.5759781619654231,
        0.5727934485896269,
        0.5800727934485896
      ],
      "changed_answer": 314,
      "wrong_any": 1035,
      "wrong_every_run_same": 721,
      "scored": 2198,
      "score_spread_median": 0.04999999999999999,
      "score_identical": 271,
      "sure_wrong": [
        1,
        0,
        0,
        1,
        0
      ],
      "sure_wrong_every_run": 0,
      "latency": {
        "runs": [
          65,
          85,
          97,
          101
        ],
        "concurrency": 1,
        "requests": 8792,
        "median_request_seconds": 0.24339387452346273,
        "p90_request_seconds": 0.2807294782833196
      }
    },
    {
      "model": "d1:free",
      "name": "Liquid d1",
      "variant": null,
      "reasoning": null,
      "provider": "liquid",
      "questions_per_request": 1,
      "whole_bank": true,
      "runs": [
        52,
        66,
        94,
        105,
        108
      ],
      "questions": 2198,
      "accuracy": [
        0.5700636942675159,
        0.5691537761601456,
        0.5677888989990901,
        0.5686988171064604,
        0.5673339399454049
      ],
      "changed_answer": 217,
      "wrong_any": 1016,
      "wrong_every_run_same": 799,
      "scored": 2198,
      "score_spread_median": 0.008535373288807185,
      "score_identical": 824,
      "sure_wrong": [
        1,
        1,
        1,
        2,
        1
      ],
      "sure_wrong_every_run": 1,
      "latency": {
        "runs": [
          66,
          94,
          105,
          108
        ],
        "concurrency": 1,
        "requests": 8792,
        "median_request_seconds": 0.3054448555049021,
        "p90_request_seconds": 0.78636241707718
      }
    },
    {
      "model": "drex-v2.0",
      "name": "Drex 2.0",
      "variant": null,
      "reasoning": null,
      "provider": "nace",
      "questions_per_request": 1,
      "whole_bank": true,
      "runs": [
        334,
        335,
        336,
        337,
        338
      ],
      "questions": 2198,
      "accuracy": [
        0.5573248407643312,
        0.5582347588717016,
        0.5586897179253867,
        0.5577797998180164,
        0.5582347588717016
      ],
      "changed_answer": 19,
      "wrong_any": 978,
      "wrong_every_run_same": 959,
      "scored": 2198,
      "score_spread_median": 0.0,
      "score_identical": 1887,
      "sure_wrong": [
        0,
        0,
        0,
        0,
        0
      ],
      "sure_wrong_every_run": 0,
      "latency": {
        "runs": [
          334,
          335,
          336,
          337,
          338
        ],
        "concurrency": 1,
        "requests": 10990,
        "median_request_seconds": 0.3239434089991846,
        "p90_request_seconds": 0.4030103706943919
      }
    },
    {
      "model": "open-jev-27b-v1.1",
      "name": "Open-Jev 27B v1.1",
      "variant": null,
      "reasoning": null,
      "provider": "open-jev",
      "questions_per_request": 1,
      "whole_bank": true,
      "runs": [
        45,
        60,
        92,
        102,
        106
      ],
      "questions": 2198,
      "accuracy": [
        0.556869881710646,
        0.556869881710646,
        0.556869881710646,
        0.556869881710646,
        0.556869881710646
      ],
      "changed_answer": 0,
      "wrong_any": 974,
      "wrong_every_run_same": 974,
      "scored": 2198,
      "score_spread_median": 0.0,
      "score_identical": 2198,
      "sure_wrong": [
        0,
        0,
        0,
        0,
        0
      ],
      "sure_wrong_every_run": 0,
      "latency": {
        "runs": [
          60,
          92,
          102,
          106
        ],
        "concurrency": 1,
        "requests": 8792,
        "median_request_seconds": 0.386393806518754,
        "p90_request_seconds": 0.5727336546988226
      }
    },
    {
      "model": "qwen38-27b-bf16",
      "name": "Qwen3.8-27B",
      "variant": "stated confidence",
      "reasoning": "off",
      "provider": "vllm-verbal",
      "questions_per_request": 1,
      "whole_bank": true,
      "runs": [
        53,
        74,
        95,
        103,
        107
      ],
      "questions": 2198,
      "accuracy": [
        0.5418562329390355,
        0.5427661510464058,
        0.5427661510464058,
        0.5427661510464058,
        0.5427661510464058
      ],
      "changed_answer": 44,
      "wrong_any": 1019,
      "wrong_every_run_same": 975,
      "scored": 2198,
      "score_spread_median": 0.0,
      "score_identical": 2193,
      "sure_wrong": [
        1,
        2,
        2,
        2,
        2
      ],
      "sure_wrong_every_run": 1,
      "latency": {
        "runs": [
          74,
          95,
          103,
          107
        ],
        "concurrency": 1,
        "requests": 8792,
        "median_request_seconds": 0.43298505601705983,
        "p90_request_seconds": 0.4745744730113074
      }
    },
    {
      "model": "baselabs/elephant-1",
      "name": "Elephant-1",
      "variant": null,
      "reasoning": null,
      "provider": "baselabs",
      "questions_per_request": 1,
      "whole_bank": true,
      "runs": [
        373,
        375,
        376,
        378,
        379
      ],
      "questions": 2198,
      "accuracy": [
        0.5236578707916287,
        0.5236578707916287,
        0.5236578707916287,
        0.5236578707916287,
        0.5236578707916287
      ],
      "changed_answer": 0,
      "wrong_any": 1047,
      "wrong_every_run_same": 1047,
      "scored": 2198,
      "score_spread_median": 0.0,
      "score_identical": 2198,
      "sure_wrong": [
        0,
        0,
        0,
        0,
        0
      ],
      "sure_wrong_every_run": 0,
      "latency": {
        "runs": [
          373,
          375,
          376,
          378,
          379
        ],
        "concurrency": 1,
        "requests": 10990,
        "median_request_seconds": 0.08010628948977683,
        "p90_request_seconds": 0.09728465078223962
      }
    },
    {
      "model": "qwen38-27b-bf16",
      "name": "Qwen3.8-27B",
      "variant": "yes/no scoring",
      "reasoning": null,
      "provider": "vllm-yes-no",
      "questions_per_request": 1,
      "whole_bank": true,
      "runs": [
        50,
        75,
        89,
        99,
        104
      ],
      "questions": 2198,
      "accuracy": [
        0.5191082802547771,
        0.5168334849863512,
        0.5163785259326661,
        0.5141037306642402,
        0.5159235668789809
      ],
      "changed_answer": 160,
      "wrong_any": 1109,
      "wrong_every_run_same": 949,
      "scored": 2198,
      "score_spread_median": 0.016477080928385346,
      "score_identical": 0,
      "sure_wrong": [
        2,
        2,
        2,
        2,
        2
      ],
      "sure_wrong_every_run": 2,
      "latency": {
        "runs": [
          75,
          89,
          99,
          104
        ],
        "concurrency": 1,
        "requests": 8792,
        "median_request_seconds": 0.2798689884948544,
        "p90_request_seconds": 0.40955076351528985
      }
    },
    {
      "model": "pplx-decider-flash-v1.2",
      "name": "Perplexity Decider Flash v1.2",
      "variant": "local",
      "reasoning": null,
      "provider": "pplx-decider",
      "questions_per_request": 1,
      "whole_bank": true,
      "runs": [
        223,
        224,
        225,
        226,
        227
      ],
      "questions": 2198,
      "accuracy": [
        0.47770700636942676,
        0.47770700636942676,
        0.47770700636942676,
        0.47770700636942676,
        0.47770700636942676
      ],
      "changed_answer": 0,
      "wrong_any": 1148,
      "wrong_every_run_same": 1148,
      "scored": 2198,
      "score_spread_median": 0.0,
      "score_identical": 2198,
      "sure_wrong": [
        0,
        0,
        0,
        0,
        0
      ],
      "sure_wrong_every_run": 0,
      "latency": {
        "runs": [
          223,
          224,
          225,
          226,
          227
        ],
        "concurrency": 1,
        "requests": 10990,
        "median_request_seconds": 0.13157959599629976,
        "p90_request_seconds": 0.1874813156988239
      }
    },
    {
      "model": "microsoft/microsoft-decision-1",
      "name": "Microsoft-Decision-1",
      "variant": "via OpenRouter",
      "reasoning": null,
      "provider": "openrouter",
      "questions_per_request": 1,
      "whole_bank": true,
      "runs": [
        205,
        209,
        211,
        213,
        215
      ],
      "questions": 2198,
      "accuracy": [
        0.46451319381255685,
        0.4672429481346679,
        0.4640582347588717,
        0.46360327570518656,
        0.4626933575978162
      ],
      "changed_answer": 279,
      "wrong_any": 1256,
      "wrong_every_run_same": 977,
      "scored": 2198,
      "score_spread_median": 0.0537542191919925,
      "score_identical": 2,
      "sure_wrong": [
        3,
        3,
        3,
        2,
        2
      ],
      "sure_wrong_every_run": 2,
      "latency": {
        "runs": [
          205,
          209,
          211,
          213,
          215
        ],
        "concurrency": 1,
        "requests": 10990,
        "median_request_seconds": 0.28970416450465564,
        "p90_request_seconds": 0.33072930260095745
      }
    },
    {
      "model": "LiquidAI/d1-2B",
      "name": "Liquid d1-2B (open weights)",
      "variant": null,
      "reasoning": null,
      "provider": "d1-server",
      "questions_per_request": 1,
      "whole_bank": true,
      "runs": [
        340,
        341,
        342,
        343,
        344
      ],
      "questions": 2198,
      "accuracy": [
        0.29390354868061874,
        0.29390354868061874,
        0.29390354868061874,
        0.29390354868061874,
        0.29390354868061874
      ],
      "changed_answer": 0,
      "wrong_any": 1552,
      "wrong_every_run_same": 1552,
      "scored": 2198,
      "score_spread_median": 0.0,
      "score_identical": 2198,
      "sure_wrong": [
        0,
        0,
        0,
        0,
        0
      ],
      "sure_wrong_every_run": 0,
      "latency": {
        "runs": [
          340,
          341,
          342,
          343,
          344
        ],
        "concurrency": 1,
        "requests": 10990,
        "median_request_seconds": 0.015482019996852614,
        "p90_request_seconds": 0.02136149551370181
      }
    },
    {
      "model": "decider-1",
      "name": "meraGPT Decider 1",
      "variant": null,
      "reasoning": null,
      "provider": "meragpt",
      "questions_per_request": 1,
      "whole_bank": true,
      "runs": [
        243,
        244,
        245,
        246,
        247
      ],
      "questions": 2198,
      "accuracy": [
        0.27115559599636035,
        0.27115559599636035,
        0.27115559599636035,
        0.27115559599636035,
        0.27115559599636035
      ],
      "changed_answer": 0,
      "wrong_any": 1602,
      "wrong_every_run_same": 1602,
      "scored": 2198,
      "score_spread_median": 0.0,
      "score_identical": 2198,
      "sure_wrong": [
        0,
        0,
        0,
        0,
        0
      ],
      "sure_wrong_every_run": 0,
      "latency": {
        "runs": [
          243,
          244,
          245,
          246,
          247
        ],
        "concurrency": 1,
        "requests": 10990,
        "median_request_seconds": 0.3855947514966829,
        "p90_request_seconds": 0.4280789560987614
      }
    },
    {
      "model": "laya",
      "name": "Laya",
      "variant": null,
      "reasoning": null,
      "provider": "laya",
      "questions_per_request": 1,
      "whole_bank": true,
      "runs": [
        51,
        234,
        235,
        236,
        237
      ],
      "questions": 2198,
      "accuracy": [
        0.19017288444040037,
        0.19017288444040037,
        0.19017288444040037,
        0.19017288444040037,
        0.19017288444040037
      ],
      "changed_answer": 0,
      "wrong_any": 1780,
      "wrong_every_run_same": 1780,
      "scored": 2198,
      "score_spread_median": 0.0,
      "score_identical": 2198,
      "sure_wrong": [
        0,
        0,
        0,
        0,
        0
      ],
      "sure_wrong_every_run": 0,
      "latency": {
        "runs": [
          234,
          235,
          236,
          237
        ],
        "concurrency": 1,
        "requests": 8792,
        "median_request_seconds": 0.011380975003703497,
        "p90_request_seconds": 0.012440272996900602
      }
    },
    {
      "model": "laya-multilingual",
      "name": "Laya multilingual",
      "variant": null,
      "reasoning": null,
      "provider": "laya",
      "questions_per_request": 1,
      "whole_bank": true,
      "runs": [
        46,
        238,
        239,
        240,
        241
      ],
      "questions": 2198,
      "accuracy": [
        0.17879890809827115,
        0.17879890809827115,
        0.17879890809827115,
        0.17879890809827115,
        0.17879890809827115
      ],
      "changed_answer": 0,
      "wrong_any": 1805,
      "wrong_every_run_same": 1805,
      "scored": 2198,
      "score_spread_median": 0.0,
      "score_identical": 2198,
      "sure_wrong": [
        1,
        1,
        1,
        1,
        1
      ],
      "sure_wrong_every_run": 1,
      "latency": {
        "runs": [
          238,
          239,
          240,
          241
        ],
        "concurrency": 1,
        "requests": 8792,
        "median_request_seconds": 0.009499495485215448,
        "p90_request_seconds": 0.010441777898813598
      }
    }
  ],
  "pipelines": [
    {
      "decision": 44,
      "frontier": 48,
      "seed": 1,
      "tolerance_points": 0.5,
      "rule": "half",
      "batch_size": 16,
      "threshold": 0.9156852342507505,
      "held_out": 1099,
      "routed": 194,
      "simulated": {
        "accuracy": 0.9736123748862603,
        "frontier_accuracy": 0.9736123748862603
      },
      "measured": {
        "pipeline_run": 70,
        "alone_run": 71,
        "accuracy": 0.9708826205641492,
        "frontier_accuracy": 0.9745222929936306,
        "difference": -0.003639672429481347,
        "low": -0.008002885959281333,
        "high": 0.0007235411003186395,
        "p": 0.21875,
        "within_tolerance": true,
        "decision_cost_usd": 0.01707551607239819,
        "cost_usd": 4.714345516072398,
        "frontier_cost_usd": 5.67298,
        "request_seconds": 1127.6062055643858,
        "frontier_request_seconds": 992.3243447878631,
        "pipeline_tokens": {
          "input_tokens": 466347,
          "cached_input_tokens": 112000,
          "cache_write_input_tokens": 0,
          "cache_write_1h_input_tokens": 0,
          "output_tokens": 20836
        },
        "alone_tokens": {
          "input_tokens": 582668,
          "cached_input_tokens": 144000,
          "cache_write_input_tokens": 0,
          "cache_write_1h_input_tokens": 0,
          "output_tokens": 22846
        },
        "alternatives": [
          {
            "run": 137,
            "accuracy": 0.9681528662420382,
            "cost_usd": 1.3225,
            "request_seconds": 1362.2349697217578,
            "versus_pipeline": {
              "difference": -0.00272975432211101,
              "low": -0.01008129238170055,
              "high": 0.0046217837374785305,
              "p": 0.629058837890625
            },
            "versus_frontier": {
              "difference": -0.006369426751592357,
              "low": -0.012788672867968776,
              "high": 4.981936478406189e-05,
              "p": 0.09228515625
            },
            "model": "gpt-6.1-sol",
            "name": "GPT-6.1 Sol",
            "variant": null,
            "reasoning": "low",
            "provider": "openai",
            "tokens": {
              "input_tokens": 582670,
              "cached_input_tokens": 57600,
              "cache_write_input_tokens": 0,
              "cache_write_1h_input_tokens": 0,
              "output_tokens": 26660
            }
          },
          {
            "run": 138,
            "accuracy": 0.910828025477707,
            "cost_usd": 0.29709675,
            "request_seconds": 300.7470318475971,
            "versus_pipeline": {
              "difference": -0.06005459508644222,
              "low": -0.07580918063887844,
              "high": -0.044300009534005994,
              "p": 1.647020623262653e-14
            },
            "versus_frontier": {
              "difference": -0.06369426751592357,
              "low": -0.07939880684293842,
              "high": -0.04798972818890872,
              "p": 1.5686991122453906e-16
            },
            "model": "gemini-3.8-flash",
            "name": "Gemini 3.8 Flash",
            "variant": null,
            "reasoning": "low",
            "provider": "gemini",
            "tokens": {
              "input_tokens": 244424,
              "cached_input_tokens": 0,
              "cache_write_input_tokens": 0,
              "cache_write_1h_input_tokens": 0,
              "output_tokens": 30341
            }
          },
          {
            "run": 165,
            "accuracy": 0.8935395814376706,
            "cost_usd": 5.387050599999999,
            "request_seconds": 929.1633372180659,
            "versus_pipeline": {
              "difference": -0.07734303912647862,
              "low": -0.09591358971040771,
              "high": -0.05877248854254953,
              "p": 1.3522957587316645e-16
            },
            "versus_frontier": {
              "difference": -0.08098271155595996,
              "low": -0.09915214754244522,
              "high": -0.06281327556947469,
              "p": 4.0853725805661456e-19
            },
            "model": "claude-opus-5-5",
            "name": "Claude Opus 5.5",
            "variant": null,
            "reasoning": "low",
            "provider": "claude",
            "tokens": {
              "input_tokens": 628800,
              "cached_input_tokens": 171673,
              "cache_write_input_tokens": 0,
              "cache_write_1h_input_tokens": 456987,
              "output_tokens": 84813
            }
          },
          {
            "run": 68,
            "accuracy": 0.8489535941765242,
            "cost_usd": 4.1268408,
            "request_seconds": 1799.8465952617116,
            "versus_pipeline": {
              "difference": -0.12192902638762511,
              "low": -0.1425477397529321,
              "high": -0.1013103130223181,
              "p": 7.799950339037261e-33
            },
            "versus_frontier": {
              "difference": -0.12556869881710647,
              "low": -0.14611091193863107,
              "high": -0.10502648569558187,
              "p": 2.08925524731209e-35
            },
            "model": "claude-sonnet-5-5",
            "name": "Claude Sonnet 5.5",
            "variant": null,
            "reasoning": "high",
            "provider": "claude",
            "tokens": {
              "input_tokens": 609112,
              "cached_input_tokens": 148308,
              "cache_write_input_tokens": 0,
              "cache_write_1h_input_tokens": 460666,
              "output_tokens": 226907
            }
          }
        ]
      }
    },
    {
      "decision": 44,
      "frontier": 55,
      "seed": 1,
      "tolerance_points": 0.5,
      "rule": "half",
      "batch_size": 16,
      "threshold": 0.7190257589471052,
      "held_out": 1099,
      "routed": 343,
      "simulated": {
        "accuracy": 0.8525932666060054,
        "frontier_accuracy": 0.8462238398544131
      },
      "measured": {
        "pipeline_run": 62,
        "alone_run": 68,
        "accuracy": 0.8480436760691538,
        "frontier_accuracy": 0.8489535941765242,
        "difference": -0.0009099181073703367,
        "low": -0.01952949531030512,
        "high": 0.017709659095564444,
        "p": 1.0,
        "within_tolerance": true,
        "decision_cost_usd": 0.01707551607239819,
        "cost_usd": 3.2248042160723984,
        "frontier_cost_usd": 4.1268408,
        "request_seconds": 1845.901194359744,
        "frontier_request_seconds": 1799.8465952617116,
        "pipeline_tokens": {
          "input_tokens": 383982,
          "cached_input_tokens": 102507,
          "cache_write_input_tokens": 0,
          "cache_write_1h_input_tokens": 281379,
          "output_tokens": 207177
        },
        "alone_tokens": {
          "input_tokens": 609112,
          "cached_input_tokens": 148308,
          "cache_write_input_tokens": 0,
          "cache_write_1h_input_tokens": 460666,
          "output_tokens": 226907
        },
        "alternatives": [
          {
            "run": 71,
            "accuracy": 0.9745222929936306,
            "cost_usd": 5.67298,
            "request_seconds": 992.3243447878631,
            "versus_pipeline": {
              "difference": 0.1264786169244768,
              "low": 0.10618973142600935,
              "high": 0.14676750242294423,
              "p": 2.1525608033566037e-37
            },
            "versus_frontier": {
              "difference": 0.12556869881710647,
              "low": 0.10502648569558187,
              "high": 0.14611091193863107,
              "p": 2.08925524731209e-35
            },
            "model": "gpt-6-astra",
            "name": "GPT-6 Astra",
            "variant": null,
            "reasoning": "low",
            "provider": "openai",
            "tokens": {
              "input_tokens": 582668,
              "cached_input_tokens": 144000,
              "cache_write_input_tokens": 0,
              "cache_write_1h_input_tokens": 0,
              "output_tokens": 22846
            }
          },
          {
            "run": 137,
            "accuracy": 0.9681528662420382,
            "cost_usd": 1.3225,
            "request_seconds": 1362.2349697217578,
            "versus_pipeline": {
              "difference": 0.12010919017288443,
              "low": 0.0996077339118915,
              "high": 0.14061064643387738,
              "p": 2.797312157195774e-32
            },
            "versus_frontier": {
              "difference": 0.1191992720655141,
              "low": 0.09875690893398828,
              "high": 0.13964163519703993,
              "p": 5.294448923596039e-32
            },
            "model": "gpt-6.1-sol",
            "name": "GPT-6.1 Sol",
            "variant": null,
            "reasoning": "low",
            "provider": "openai",
            "tokens": {
              "input_tokens": 582670,
              "cached_input_tokens": 57600,
              "cache_write_input_tokens": 0,
              "cache_write_1h_input_tokens": 0,
              "output_tokens": 26660
            }
          },
          {
            "run": 138,
            "accuracy": 0.910828025477707,
            "cost_usd": 0.29709675,
            "request_seconds": 300.7470318475971,
            "versus_pipeline": {
              "difference": 0.06278434940855324,
              "low": 0.04046156514295063,
              "high": 0.08510713367415584,
              "p": 5.283208926292809e-08
            },
            "versus_frontier": {
              "difference": 0.06187443130118289,
              "low": 0.03877269078891368,
              "high": 0.0849761718134521,
              "p": 2.2849587651185474e-07
            },
            "model": "gemini-3.8-flash",
            "name": "Gemini 3.8 Flash",
            "variant": null,
            "reasoning": "low",
            "provider": "gemini",
            "tokens": {
              "input_tokens": 244424,
              "cached_input_tokens": 0,
              "cache_write_input_tokens": 0,
              "cache_write_1h_input_tokens": 0,
              "output_tokens": 30341
            }
          },
          {
            "run": 165,
            "accuracy": 0.8935395814376706,
            "cost_usd": 5.387050599999999,
            "request_seconds": 929.1633372180659,
            "versus_pipeline": {
              "difference": 0.04549590536851683,
              "low": 0.02324038085043073,
              "high": 0.06775142988660293,
              "p": 8.548342922070617e-05
            },
            "versus_frontier": {
              "difference": 0.044585987261146494,
              "low": 0.022829831897380495,
              "high": 0.06634214262491249,
              "p": 8.212852870228109e-05
            },
            "model": "claude-opus-5-5",
            "name": "Claude Opus 5.5",
            "variant": null,
            "reasoning": "low",
            "provider": "claude",
            "tokens": {
              "input_tokens": 628800,
              "cached_input_tokens": 171673,
              "cache_write_input_tokens": 0,
              "cache_write_1h_input_tokens": 456987,
              "output_tokens": 84813
            }
          }
        ]
      }
    },
    {
      "decision": 145,
      "frontier": 48,
      "seed": 1,
      "tolerance_points": 0.5,
      "rule": "cutoff",
      "batch_size": 16,
      "threshold": 0.99,
      "held_out": 1099,
      "routed": 176,
      "simulated": {
        "accuracy": 0.9736123748862603,
        "frontier_accuracy": 0.9736123748862603
      },
      "measured": {
        "pipeline_run": 176,
        "alone_run": 71,
        "accuracy": 0.9754322111010009,
        "frontier_accuracy": 0.9745222929936306,
        "difference": 0.0009099181073703367,
        "low": -0.003808312575386305,
        "high": 0.005628148790126978,
        "p": 1.0,
        "within_tolerance": true,
        "decision_cost_usd": 0.008533460000000001,
        "cost_usd": 4.83464346,
        "frontier_cost_usd": 5.67298,
        "request_seconds": 1244.687020608093,
        "frontier_request_seconds": 992.3243447878631,
        "pipeline_tokens": {
          "input_tokens": 479281,
          "cached_input_tokens": 115200,
          "cache_write_input_tokens": 0,
          "cache_write_1h_input_tokens": 0,
          "output_tokens": 21402
        },
        "alone_tokens": {
          "input_tokens": 582668,
          "cached_input_tokens": 144000,
          "cache_write_input_tokens": 0,
          "cache_write_1h_input_tokens": 0,
          "output_tokens": 22846
        },
        "alternatives": [
          {
            "run": 137,
            "accuracy": 0.9681528662420382,
            "cost_usd": 1.3225,
            "request_seconds": 1362.2349697217578,
            "versus_pipeline": {
              "difference": -0.007279344858962694,
              "low": -0.014400108715310153,
              "high": -0.0001585810026152332,
              "p": 0.076812744140625
            },
            "versus_frontier": {
              "difference": -0.006369426751592357,
              "low": -0.012788672867968776,
              "high": 4.981936478406189e-05,
              "p": 0.09228515625
            },
            "model": "gpt-6.1-sol",
            "name": "GPT-6.1 Sol",
            "variant": null,
            "reasoning": "low",
            "provider": "openai",
            "tokens": {
              "input_tokens": 582670,
              "cached_input_tokens": 57600,
              "cache_write_input_tokens": 0,
              "cache_write_1h_input_tokens": 0,
              "output_tokens": 26660
            }
          },
          {
            "run": 138,
            "accuracy": 0.910828025477707,
            "cost_usd": 0.29709675,
            "request_seconds": 300.7470318475971,
            "versus_pipeline": {
              "difference": -0.0646041856232939,
              "low": -0.08059688556495011,
              "high": -0.04861148568163769,
              "p": 2.7959374761947534e-16
            },
            "versus_frontier": {
              "difference": -0.06369426751592357,
              "low": -0.07939880684293842,
              "high": -0.04798972818890872,
              "p": 1.5686991122453906e-16
            },
            "model": "gemini-3.8-flash",
            "name": "Gemini 3.8 Flash",
            "variant": null,
            "reasoning": "low",
            "provider": "gemini",
            "tokens": {
              "input_tokens": 244424,
              "cached_input_tokens": 0,
              "cache_write_input_tokens": 0,
              "cache_write_1h_input_tokens": 0,
              "output_tokens": 30341
            }
          },
          {
            "run": 165,
            "accuracy": 0.8935395814376706,
            "cost_usd": 5.387050599999999,
            "request_seconds": 929.1633372180659,
            "versus_pipeline": {
              "difference": -0.0818926296633303,
              "low": -0.10013519006432656,
              "high": -0.06365006926233405,
              "p": 2.262519224355612e-19
            },
            "versus_frontier": {
              "difference": -0.08098271155595996,
              "low": -0.09915214754244522,
              "high": -0.06281327556947469,
              "p": 4.0853725805661456e-19
            },
            "model": "claude-opus-5-5",
            "name": "Claude Opus 5.5",
            "variant": null,
            "reasoning": "low",
            "provider": "claude",
            "tokens": {
              "input_tokens": 628800,
              "cached_input_tokens": 171673,
              "cache_write_input_tokens": 0,
              "cache_write_1h_input_tokens": 456987,
              "output_tokens": 84813
            }
          },
          {
            "run": 68,
            "accuracy": 0.8489535941765242,
            "cost_usd": 4.1268408,
            "request_seconds": 1799.8465952617116,
            "versus_pipeline": {
              "difference": -0.1264786169244768,
              "low": -0.14707865334275164,
              "high": -0.10587858050620194,
              "p": 1.0875369892924252e-35
            },
            "versus_frontier": {
              "difference": -0.12556869881710647,
              "low": -0.14611091193863107,
              "high": -0.10502648569558187,
              "p": 2.08925524731209e-35
            },
            "model": "claude-sonnet-5-5",
            "name": "Claude Sonnet 5.5",
            "variant": null,
            "reasoning": "high",
            "provider": "claude",
            "tokens": {
              "input_tokens": 609112,
              "cached_input_tokens": 148308,
              "cache_write_input_tokens": 0,
              "cache_write_1h_input_tokens": 460666,
              "output_tokens": 226907
            }
          }
        ]
      }
    },
    {
      "decision": 145,
      "frontier": 48,
      "seed": 1,
      "tolerance_points": 0.5,
      "rule": "half",
      "batch_size": 1,
      "threshold": 0.9825243141806727,
      "held_out": 1099,
      "routed": 219,
      "simulated": {
        "accuracy": 0.9727024567788899,
        "frontier_accuracy": 0.9736123748862603
      },
      "measured": {
        "pipeline_run": 169,
        "alone_run": 163,
        "accuracy": 0.9727024567788899,
        "frontier_accuracy": 0.9754322111010009,
        "difference": -0.00272975432211101,
        "low": -0.007445530788481704,
        "high": 0.001986022144259684,
        "p": 0.453125,
        "within_tolerance": true,
        "decision_cost_usd": 0.008533460000000001,
        "cost_usd": 23.68529946,
        "frontier_cost_usd": 31.648706,
        "request_seconds": 11274.370967360068,
        "frontier_request_seconds": 7964.2585057840915,
        "waits": {
          "pipeline": {
            "questions": 1099,
            "median_seconds": 6.631,
            "p90_seconds": 10.136600000000001,
            "mean_seconds": 10.271715195632392,
            "max_seconds": 394.978,
            "total_seconds": 11288.615
          },
          "kept": {
            "questions": 219,
            "median_seconds": 0.231,
            "p90_seconds": 0.2834,
            "mean_seconds": 0.2428812785388128,
            "max_seconds": 0.546,
            "total_seconds": 53.191
          },
          "passed": {
            "questions": 880,
            "median_seconds": 7.1995,
            "p90_seconds": 10.9358,
            "mean_seconds": 12.767527272727273,
            "max_seconds": 394.978,
            "total_seconds": 11235.424
          },
          "alone": {
            "questions": 1099,
            "median_seconds": 6.665,
            "p90_seconds": 9.7262,
            "mean_seconds": 7.246808917197453,
            "max_seconds": 28.965,
            "total_seconds": 7964.243
          }
        },
        "pipeline_tokens": {
          "input_tokens": 4074897,
          "cached_input_tokens": 2221056,
          "cache_write_input_tokens": 0,
          "cache_write_1h_input_tokens": 0,
          "output_tokens": 58346
        },
        "alone_tokens": {
          "input_tokens": 5103512,
          "cached_input_tokens": 2509696,
          "cache_write_input_tokens": 0,
          "cache_write_1h_input_tokens": 0,
          "output_tokens": 64017
        },
        "alternatives": [
          {
            "run": 166,
            "accuracy": 0.9272065514103731,
            "cost_usd": 12.8286324,
            "request_seconds": 4694.714676301097,
            "versus_pipeline": {
              "difference": -0.04549590536851683,
              "low": -0.059507567242236284,
              "high": -0.03148424349479738,
              "p": 7.638141345540223e-11
            },
            "versus_frontier": {
              "difference": -0.048225659690627844,
              "low": -0.06209114647456982,
              "high": -0.03436017290668587,
              "p": 1.6623508125590547e-12
            },
            "model": "claude-opus-5-5",
            "name": "Claude Opus 5.5",
            "variant": null,
            "reasoning": "low",
            "provider": "claude",
            "tokens": {
              "input_tokens": 1746017,
              "cached_input_tokens": 701622,
              "cache_write_input_tokens": 0,
              "cache_write_1h_input_tokens": 1042197,
              "output_tokens": 217097
            }
          },
          {
            "run": 167,
            "accuracy": 0.8935395814376706,
            "cost_usd": 8.6273988,
            "request_seconds": 6064.148409693618,
            "versus_pipeline": {
              "difference": -0.0791628753412193,
              "low": -0.09664720801122059,
              "high": -0.061678542671217994,
              "p": 5.109797372820077e-20
            },
            "versus_frontier": {
              "difference": -0.0818926296633303,
              "low": -0.09942390909910859,
              "high": -0.06436135022755202,
              "p": 2.253920747846914e-21
            },
            "model": "claude-sonnet-5-5",
            "name": "Claude Sonnet 5.5",
            "variant": null,
            "reasoning": "high",
            "provider": "claude",
            "tokens": {
              "input_tokens": 1648174,
              "cached_input_tokens": 599508,
              "cache_write_input_tokens": 0,
              "cache_write_1h_input_tokens": 1046468,
              "output_tokens": 437718
            }
          }
        ]
      }
    },
    {
      "decision": 145,
      "frontier": 48,
      "seed": 1,
      "tolerance_points": 0.5,
      "rule": "share",
      "batch_size": 16,
      "threshold": 0.9642039538871846,
      "held_out": 1099,
      "routed": 275,
      "simulated": {
        "accuracy": 0.9727024567788899,
        "frontier_accuracy": 0.9736123748862603
      },
      "share": 0.25,
      "measured": {
        "pipeline_run": 175,
        "alone_run": 71,
        "accuracy": 0.9708826205641492,
        "frontier_accuracy": 0.9745222929936306,
        "difference": -0.003639672429481347,
        "low": -0.00981393925260459,
        "high": 0.0025345943936418975,
        "p": 0.3876953125,
        "within_tolerance": true,
        "decision_cost_usd": 0.008533460000000001,
        "cost_usd": 4.67707346,
        "frontier_cost_usd": 5.67298,
        "request_seconds": 1113.8878569669614,
        "frontier_request_seconds": 992.3243447878631,
        "pipeline_tokens": {
          "input_tokens": 424904,
          "cached_input_tokens": 60800,
          "cache_write_input_tokens": 0,
          "cache_write_1h_input_tokens": 0,
          "output_tokens": 19334
        },
        "alone_tokens": {
          "input_tokens": 582668,
          "cached_input_tokens": 144000,
          "cache_write_input_tokens": 0,
          "cache_write_1h_input_tokens": 0,
          "output_tokens": 22846
        },
        "alternatives": [
          {
            "run": 137,
            "accuracy": 0.9681528662420382,
            "cost_usd": 1.3225,
            "request_seconds": 1362.2349697217578,
            "versus_pipeline": {
              "difference": -0.00272975432211101,
              "low": -0.010900907086924596,
              "high": 0.005441398442702574,
              "p": 0.6636238098144531
            },
            "versus_frontier": {
              "difference": -0.006369426751592357,
              "low": -0.012788672867968776,
              "high": 4.981936478406189e-05,
              "p": 0.09228515625
            },
            "model": "gpt-6.1-sol",
            "name": "GPT-6.1 Sol",
            "variant": null,
            "reasoning": "low",
            "provider": "openai",
            "tokens": {
              "input_tokens": 582670,
              "cached_input_tokens": 57600,
              "cache_write_input_tokens": 0,
              "cache_write_1h_input_tokens": 0,
              "output_tokens": 26660
            }
          },
          {
            "run": 138,
            "accuracy": 0.910828025477707,
            "cost_usd": 0.29709675,
            "request_seconds": 300.7470318475971,
            "versus_pipeline": {
              "difference": -0.06005459508644222,
              "low": -0.07600979106647317,
              "high": -0.044099399106411266,
              "p": 4.308584029113881e-14
            },
            "versus_frontier": {
              "difference": -0.06369426751592357,
              "low": -0.07939880684293842,
              "high": -0.04798972818890872,
              "p": 1.5686991122453906e-16
            },
            "model": "gemini-3.8-flash",
            "name": "Gemini 3.8 Flash",
            "variant": null,
            "reasoning": "low",
            "provider": "gemini",
            "tokens": {
              "input_tokens": 244424,
              "cached_input_tokens": 0,
              "cache_write_input_tokens": 0,
              "cache_write_1h_input_tokens": 0,
              "output_tokens": 30341
            }
          },
          {
            "run": 165,
            "accuracy": 0.8935395814376706,
            "cost_usd": 5.387050599999999,
            "request_seconds": 929.1633372180659,
            "versus_pipeline": {
              "difference": -0.07734303912647862,
              "low": -0.09608408130194028,
              "high": -0.05860199695101696,
              "p": 2.8669598238307665e-16
            },
            "versus_frontier": {
              "difference": -0.08098271155595996,
              "low": -0.09915214754244522,
              "high": -0.06281327556947469,
              "p": 4.0853725805661456e-19
            },
            "model": "claude-opus-5-5",
            "name": "Claude Opus 5.5",
            "variant": null,
            "reasoning": "low",
            "provider": "claude",
            "tokens": {
              "input_tokens": 628800,
              "cached_input_tokens": 171673,
              "cache_write_input_tokens": 0,
              "cache_write_1h_input_tokens": 456987,
              "output_tokens": 84813
            }
          },
          {
            "run": 68,
            "accuracy": 0.8489535941765242,
            "cost_usd": 4.1268408,
            "request_seconds": 1799.8465952617116,
            "versus_pipeline": {
              "difference": -0.12192902638762511,
              "low": -0.1425477397529321,
              "high": -0.1013103130223181,
              "p": 7.799950339037261e-33
            },
            "versus_frontier": {
              "difference": -0.12556869881710647,
              "low": -0.14611091193863107,
              "high": -0.10502648569558187,
              "p": 2.08925524731209e-35
            },
            "model": "claude-sonnet-5-5",
            "name": "Claude Sonnet 5.5",
            "variant": null,
            "reasoning": "high",
            "provider": "claude",
            "tokens": {
              "input_tokens": 609112,
              "cached_input_tokens": 148308,
              "cache_write_input_tokens": 0,
              "cache_write_1h_input_tokens": 460666,
              "output_tokens": 226907
            }
          }
        ]
      }
    },
    {
      "decision": 145,
      "frontier": 48,
      "seed": 1,
      "tolerance_points": 0.5,
      "rule": "half",
      "batch_size": 16,
      "threshold": 0.9825243141806727,
      "held_out": 1099,
      "routed": 219,
      "simulated": {
        "accuracy": 0.9727024567788899,
        "frontier_accuracy": 0.9736123748862603
      },
      "measured": {
        "pipeline_run": 161,
        "alone_run": 71,
        "accuracy": 0.9745222929936306,
        "frontier_accuracy": 0.9745222929936306,
        "difference": 0.0,
        "low": -0.0050443286301205936,
        "high": 0.0050443286301205936,
        "p": 1.0,
        "within_tolerance": true,
        "decision_cost_usd": 0.008533460000000001,
        "cost_usd": 4.9963634599999995,
        "frontier_cost_usd": 5.67298,
        "request_seconds": 1189.094736152052,
        "frontier_request_seconds": 992.3243447878631,
        "pipeline_tokens": {
          "input_tokens": 453928,
          "cached_input_tokens": 64000,
          "cache_write_input_tokens": 0,
          "cache_write_1h_input_tokens": 0,
          "output_tokens": 20491
        },
        "alone_tokens": {
          "input_tokens": 582668,
          "cached_input_tokens": 144000,
          "cache_write_input_tokens": 0,
          "cache_write_1h_input_tokens": 0,
          "output_tokens": 22846
        },
        "alternatives": [
          {
            "run": 137,
            "accuracy": 0.9681528662420382,
            "cost_usd": 1.3225,
            "request_seconds": 1362.2349697217578,
            "versus_pipeline": {
              "difference": -0.006369426751592357,
              "low": -0.014533492652310458,
              "high": 0.0017946391491257437,
              "p": 0.18924713134765625
            },
            "versus_frontier": {
              "difference": -0.006369426751592357,
              "low": -0.012788672867968776,
              "high": 4.981936478406189e-05,
              "p": 0.09228515625
            },
            "model": "gpt-6.1-sol",
            "name": "GPT-6.1 Sol",
            "variant": null,
            "reasoning": "low",
            "provider": "openai",
            "tokens": {
              "input_tokens": 582670,
              "cached_input_tokens": 57600,
              "cache_write_input_tokens": 0,
              "cache_write_1h_input_tokens": 0,
              "output_tokens": 26660
            }
          },
          {
            "run": 138,
            "accuracy": 0.910828025477707,
            "cost_usd": 0.29709675,
            "request_seconds": 300.7470318475971,
            "versus_pipeline": {
              "difference": -0.06369426751592357,
              "low": -0.07960004848719057,
              "high": -0.04778848654465656,
              "p": 5.137564699362503e-16
            },
            "versus_frontier": {
              "difference": -0.06369426751592357,
              "low": -0.07939880684293842,
              "high": -0.04798972818890872,
              "p": 1.5686991122453906e-16
            },
            "model": "gemini-3.8-flash",
            "name": "Gemini 3.8 Flash",
            "variant": null,
            "reasoning": "low",
            "provider": "gemini",
            "tokens": {
              "input_tokens": 244424,
              "cached_input_tokens": 0,
              "cache_write_input_tokens": 0,
              "cache_write_1h_input_tokens": 0,
              "output_tokens": 30341
            }
          },
          {
            "run": 165,
            "accuracy": 0.8935395814376706,
            "cost_usd": 5.387050599999999,
            "request_seconds": 929.1633372180659,
            "versus_pipeline": {
              "difference": -0.08098271155595996,
              "low": -0.09949894847741117,
              "high": -0.062466474634508745,
              "p": 2.687543585380812e-18
            },
            "versus_frontier": {
              "difference": -0.08098271155595996,
              "low": -0.09915214754244522,
              "high": -0.06281327556947469,
              "p": 4.0853725805661456e-19
            },
            "model": "claude-opus-5-5",
            "name": "Claude Opus 5.5",
            "variant": null,
            "reasoning": "low",
            "provider": "claude",
            "tokens": {
              "input_tokens": 628800,
              "cached_input_tokens": 171673,
              "cache_write_input_tokens": 0,
              "cache_write_1h_input_tokens": 456987,
              "output_tokens": 84813
            }
          },
          {
            "run": 68,
            "accuracy": 0.8489535941765242,
            "cost_usd": 4.1268408,
            "request_seconds": 1799.8465952617116,
            "versus_pipeline": {
              "difference": -0.12556869881710647,
              "low": -0.14611091193863107,
              "high": -0.10502648569558187,
              "p": 2.08925524731209e-35
            },
            "versus_frontier": {
              "difference": -0.12556869881710647,
              "low": -0.14611091193863107,
              "high": -0.10502648569558187,
              "p": 2.08925524731209e-35
            },
            "model": "claude-sonnet-5-5",
            "name": "Claude Sonnet 5.5",
            "variant": null,
            "reasoning": "high",
            "provider": "claude",
            "tokens": {
              "input_tokens": 609112,
              "cached_input_tokens": 148308,
              "cache_write_input_tokens": 0,
              "cache_write_1h_input_tokens": 460666,
              "output_tokens": 226907
            }
          }
        ]
      }
    },
    {
      "decision": 145,
      "frontier": 47,
      "seed": 1,
      "tolerance_points": 0.5,
      "rule": "cutoff",
      "batch_size": 16,
      "threshold": 0.99,
      "held_out": 1099,
      "routed": 176,
      "simulated": {
        "accuracy": 0.905368516833485,
        "frontier_accuracy": 0.905368516833485
      },
      "measured": {
        "pipeline_run": 172,
        "alone_run": 165,
        "accuracy": 0.8844404003639672,
        "frontier_accuracy": 0.8935395814376706,
        "difference": -0.009099181073703366,
        "low": -0.027796314332103306,
        "high": 0.009597952184696572,
        "p": 0.39092745669950485,
        "within_tolerance": false,
        "decision_cost_usd": 0.008533460000000001,
        "cost_usd": 4.4161930599999994,
        "frontier_cost_usd": 5.387050599999999,
        "request_seconds": 1078.980863467048,
        "frontier_request_seconds": 929.1633372180659,
        "pipeline_tokens": {
          "input_tokens": 495388,
          "cached_input_tokens": 129618,
          "cache_write_input_tokens": 0,
          "cache_write_1h_input_tokens": 365654,
          "output_tokens": 72802
        },
        "alone_tokens": {
          "input_tokens": 628800,
          "cached_input_tokens": 171673,
          "cache_write_input_tokens": 0,
          "cache_write_1h_input_tokens": 456987,
          "output_tokens": 84813
        },
        "alternatives": [
          {
            "run": 71,
            "accuracy": 0.9745222929936306,
            "cost_usd": 5.67298,
            "request_seconds": 992.3243447878631,
            "versus_pipeline": {
              "difference": 0.09008189262966333,
              "low": 0.07206280755970734,
              "high": 0.10810097769961932,
              "p": 1.8478334088755076e-24
            },
            "versus_frontier": {
              "difference": 0.08098271155595996,
              "low": 0.06281327556947469,
              "high": 0.09915214754244522,
              "p": 4.0853725805661456e-19
            },
            "model": "gpt-6-astra",
            "name": "GPT-6 Astra",
            "variant": null,
            "reasoning": "low",
            "provider": "openai",
            "tokens": {
              "input_tokens": 582668,
              "cached_input_tokens": 144000,
              "cache_write_input_tokens": 0,
              "cache_write_1h_input_tokens": 0,
              "output_tokens": 22846
            }
          },
          {
            "run": 137,
            "accuracy": 0.9681528662420382,
            "cost_usd": 1.3225,
            "request_seconds": 1362.2349697217578,
            "versus_pipeline": {
              "difference": 0.08371246587807098,
              "low": 0.06498223676411861,
              "high": 0.10244269499202334,
              "p": 4.764406071312224e-19
            },
            "versus_frontier": {
              "difference": 0.0746132848043676,
              "low": 0.05591847354625709,
              "high": 0.09330809606247811,
              "p": 2.9566626383126704e-15
            },
            "model": "gpt-6.1-sol",
            "name": "GPT-6.1 Sol",
            "variant": null,
            "reasoning": "low",
            "provider": "openai",
            "tokens": {
              "input_tokens": 582670,
              "cached_input_tokens": 57600,
              "cache_write_input_tokens": 0,
              "cache_write_1h_input_tokens": 0,
              "output_tokens": 26660
            }
          },
          {
            "run": 138,
            "accuracy": 0.910828025477707,
            "cost_usd": 0.29709675,
            "request_seconds": 300.7470318475971,
            "versus_pipeline": {
              "difference": 0.026387625113739762,
              "low": 0.0061918159303973284,
              "high": 0.0465834342970822,
              "p": 0.013380437039567964
            },
            "versus_frontier": {
              "difference": 0.017288444040036398,
              "low": -0.0034080250492140966,
              "high": 0.037984913129286896,
              "p": 0.12101372565511773
            },
            "model": "gemini-3.8-flash",
            "name": "Gemini 3.8 Flash",
            "variant": null,
            "reasoning": "low",
            "provider": "gemini",
            "tokens": {
              "input_tokens": 244424,
              "cached_input_tokens": 0,
              "cache_write_input_tokens": 0,
              "cache_write_1h_input_tokens": 0,
              "output_tokens": 30341
            }
          },
          {
            "run": 68,
            "accuracy": 0.8489535941765242,
            "cost_usd": 4.1268408,
            "request_seconds": 1799.8465952617116,
            "versus_pipeline": {
              "difference": -0.03548680618744313,
              "low": -0.05843822530069169,
              "high": -0.012535387074194576,
              "p": 0.00315764881359094
            },
            "versus_frontier": {
              "difference": -0.044585987261146494,
              "low": -0.06634214262491249,
              "high": -0.022829831897380495,
              "p": 8.212852870228109e-05
            },
            "model": "claude-sonnet-5-5",
            "name": "Claude Sonnet 5.5",
            "variant": null,
            "reasoning": "high",
            "provider": "claude",
            "tokens": {
              "input_tokens": 609112,
              "cached_input_tokens": 148308,
              "cache_write_input_tokens": 0,
              "cache_write_1h_input_tokens": 460666,
              "output_tokens": 226907
            }
          }
        ]
      }
    },
    {
      "decision": 145,
      "frontier": 47,
      "seed": 1,
      "tolerance_points": 0.5,
      "rule": "half",
      "batch_size": 1,
      "threshold": 0.9825243141806727,
      "held_out": 1099,
      "routed": 219,
      "simulated": {
        "accuracy": 0.9044585987261147,
        "frontier_accuracy": 0.905368516833485
      },
      "measured": {
        "pipeline_run": 168,
        "alone_run": 166,
        "accuracy": 0.9335759781619655,
        "frontier_accuracy": 0.9272065514103731,
        "difference": 0.006369426751592357,
        "low": -0.002889942045888322,
        "high": 0.015628795549073038,
        "p": 0.24778856337070465,
        "within_tolerance": true,
        "decision_cost_usd": 0.008533460000000001,
        "cost_usd": 10.56694966,
        "frontier_cost_usd": 12.8286324,
        "request_seconds": 4184.495180073995,
        "frontier_request_seconds": 4694.714676301097,
        "waits": {
          "pipeline": {
            "questions": 1099,
            "median_seconds": 3.924,
            "p90_seconds": 6.1256,
            "mean_seconds": 3.82051228389445,
            "max_seconds": 34.648,
            "total_seconds": 4198.743
          },
          "kept": {
            "questions": 219,
            "median_seconds": 0.231,
            "p90_seconds": 0.2834,
            "mean_seconds": 0.2428812785388128,
            "max_seconds": 0.546,
            "total_seconds": 53.191
          },
          "passed": {
            "questions": 880,
            "median_seconds": 4.334,
            "p90_seconds": 6.609100000000001,
            "mean_seconds": 4.710854545454545,
            "max_seconds": 34.648,
            "total_seconds": 4145.552
          },
          "alone": {
            "questions": 1099,
            "median_seconds": 3.77,
            "p90_seconds": 6.1594,
            "mean_seconds": 4.271797088262057,
            "max_seconds": 22.185,
            "total_seconds": 4694.705
          }
        },
        "pipeline_tokens": {
          "input_tokens": 1373985,
          "cached_input_tokens": 568721,
          "cache_write_input_tokens": 0,
          "cache_write_1h_input_tokens": 803504,
          "output_tokens": 200480
        },
        "alone_tokens": {
          "input_tokens": 1746017,
          "cached_input_tokens": 701622,
          "cache_write_input_tokens": 0,
          "cache_write_1h_input_tokens": 1042197,
          "output_tokens": 217097
        },
        "alternatives": [
          {
            "run": 163,
            "accuracy": 0.9754322111010009,
            "cost_usd": 31.648706,
            "request_seconds": 7964.2585057840915,
            "versus_pipeline": {
              "difference": 0.04185623293903549,
              "low": 0.02874163169875927,
              "high": 0.05497083417931171,
              "p": 1.170292474039769e-10
            },
            "versus_frontier": {
              "difference": 0.048225659690627844,
              "low": 0.03436017290668587,
              "high": 0.06209114647456982,
              "p": 1.6623508125590547e-12
            },
            "model": "gpt-6-astra",
            "name": "GPT-6 Astra",
            "variant": null,
            "reasoning": "low",
            "provider": "openai",
            "tokens": {
              "input_tokens": 5103512,
              "cached_input_tokens": 2509696,
              "cache_write_input_tokens": 0,
              "cache_write_1h_input_tokens": 0,
              "output_tokens": 64017
            }
          },
          {
            "run": 167,
            "accuracy": 0.8935395814376706,
            "cost_usd": 8.6273988,
            "request_seconds": 6064.148409693618,
            "versus_pipeline": {
              "difference": -0.040036396724294813,
              "low": -0.05771300818223522,
              "high": -0.0223597852663544,
              "p": 1.2579150016678854e-05
            },
            "versus_frontier": {
              "difference": -0.03366696997270246,
              "low": -0.051657121097316086,
              "high": -0.015676818848088834,
              "p": 0.00034099780128697693
            },
            "model": "claude-sonnet-5-5",
            "name": "Claude Sonnet 5.5",
            "variant": null,
            "reasoning": "high",
            "provider": "claude",
            "tokens": {
              "input_tokens": 1648174,
              "cached_input_tokens": 599508,
              "cache_write_input_tokens": 0,
              "cache_write_1h_input_tokens": 1046468,
              "output_tokens": 437718
            }
          }
        ]
      }
    },
    {
      "decision": 145,
      "frontier": 47,
      "seed": 1,
      "tolerance_points": 0.5,
      "rule": "share",
      "batch_size": 16,
      "threshold": 0.9642039538871846,
      "held_out": 1099,
      "routed": 275,
      "simulated": {
        "accuracy": 0.9044585987261147,
        "frontier_accuracy": 0.905368516833485
      },
      "share": 0.25,
      "measured": {
        "pipeline_run": 171,
        "alone_run": 165,
        "accuracy": 0.8871701546860783,
        "frontier_accuracy": 0.8935395814376706,
        "difference": -0.006369426751592357,
        "low": -0.0242888152823936,
        "high": 0.011549961779208884,
        "p": 0.5507089135982718,
        "within_tolerance": false,
        "decision_cost_usd": 0.008533460000000001,
        "cost_usd": 4.13014146,
        "frontier_cost_usd": 5.387050599999999,
        "request_seconds": 1111.0155438639922,
        "frontier_request_seconds": 929.1633372180659,
        "pipeline_tokens": {
          "input_tokens": 434443,
          "cached_input_tokens": 113700,
          "cache_write_input_tokens": 0,
          "cache_write_1h_input_tokens": 320639,
          "output_tokens": 76667
        },
        "alone_tokens": {
          "input_tokens": 628800,
          "cached_input_tokens": 171673,
          "cache_write_input_tokens": 0,
          "cache_write_1h_input_tokens": 456987,
          "output_tokens": 84813
        },
        "alternatives": [
          {
            "run": 71,
            "accuracy": 0.9745222929936306,
            "cost_usd": 5.67298,
            "request_seconds": 992.3243447878631,
            "versus_pipeline": {
              "difference": 0.08735213830755233,
              "low": 0.06973177466928716,
              "high": 0.10497250194581749,
              "p": 2.6256041585124165e-24
            },
            "versus_frontier": {
              "difference": 0.08098271155595996,
              "low": 0.06281327556947469,
              "high": 0.09915214754244522,
              "p": 4.0853725805661456e-19
            },
            "model": "gpt-6-astra",
            "name": "GPT-6 Astra",
            "variant": null,
            "reasoning": "low",
            "provider": "openai",
            "tokens": {
              "input_tokens": 582668,
              "cached_input_tokens": 144000,
              "cache_write_input_tokens": 0,
              "cache_write_1h_input_tokens": 0,
              "output_tokens": 22846
            }
          },
          {
            "run": 137,
            "accuracy": 0.9681528662420382,
            "cost_usd": 1.3225,
            "request_seconds": 1362.2349697217578,
            "versus_pipeline": {
              "difference": 0.08098271155595996,
              "low": 0.06298918240699025,
              "high": 0.09897624070492966,
              "p": 1.457836211253665e-19
            },
            "versus_frontier": {
              "difference": 0.0746132848043676,
              "low": 0.05591847354625709,
              "high": 0.09330809606247811,
              "p": 2.9566626383126704e-15
            },
            "model": "gpt-6.1-sol",
            "name": "GPT-6.1 Sol",
            "variant": null,
            "reasoning": "low",
            "provider": "openai",
            "tokens": {
              "input_tokens": 582670,
              "cached_input_tokens": 57600,
              "cache_write_input_tokens": 0,
              "cache_write_1h_input_tokens": 0,
              "output_tokens": 26660
            }
          },
          {
            "run": 138,
            "accuracy": 0.910828025477707,
            "cost_usd": 0.29709675,
            "request_seconds": 300.7470318475971,
            "versus_pipeline": {
              "difference": 0.023657870791628753,
              "low": 0.0036877362927157445,
              "high": 0.04362800529054176,
              "p": 0.02554336402978297
            },
            "versus_frontier": {
              "difference": 0.017288444040036398,
              "low": -0.0034080250492140966,
              "high": 0.037984913129286896,
              "p": 0.12101372565511773
            },
            "model": "gemini-3.8-flash",
            "name": "Gemini 3.8 Flash",
            "variant": null,
            "reasoning": "low",
            "provider": "gemini",
            "tokens": {
              "input_tokens": 244424,
              "cached_input_tokens": 0,
              "cache_write_input_tokens": 0,
              "cache_write_1h_input_tokens": 0,
              "output_tokens": 30341
            }
          },
          {
            "run": 68,
            "accuracy": 0.8489535941765242,
            "cost_usd": 4.1268408,
            "request_seconds": 1799.8465952617116,
            "versus_pipeline": {
              "difference": -0.03821656050955414,
              "low": -0.059197185877242015,
              "high": -0.01723593514186626,
              "p": 0.00048561896725776954
            },
            "versus_frontier": {
              "difference": -0.044585987261146494,
              "low": -0.06634214262491249,
              "high": -0.022829831897380495,
              "p": 8.212852870228109e-05
            },
            "model": "claude-sonnet-5-5",
            "name": "Claude Sonnet 5.5",
            "variant": null,
            "reasoning": "high",
            "provider": "claude",
            "tokens": {
              "input_tokens": 609112,
              "cached_input_tokens": 148308,
              "cache_write_input_tokens": 0,
              "cache_write_1h_input_tokens": 460666,
              "output_tokens": 226907
            }
          }
        ]
      }
    },
    {
      "decision": 145,
      "frontier": 47,
      "seed": 1,
      "tolerance_points": 0.5,
      "rule": "half",
      "batch_size": 16,
      "threshold": 0.9825243141806727,
      "held_out": 1099,
      "routed": 219,
      "simulated": {
        "accuracy": 0.9044585987261147,
        "frontier_accuracy": 0.905368516833485
      },
      "measured": {
        "pipeline_run": 164,
        "alone_run": 165,
        "accuracy": 0.8926296633303002,
        "frontier_accuracy": 0.8935395814376706,
        "difference": -0.0009099181073703367,
        "low": -0.01847467766231635,
        "high": 0.016654841447575675,
        "p": 1.0,
        "within_tolerance": true,
        "decision_cost_usd": 0.008533460000000001,
        "cost_usd": 4.38446586,
        "frontier_cost_usd": 5.387050599999999,
        "request_seconds": 1131.248071975031,
        "frontier_request_seconds": 929.1633372180659,
        "pipeline_tokens": {
          "input_tokens": 478223,
          "cached_input_tokens": 130962,
          "cache_write_input_tokens": 0,
          "cache_write_1h_input_tokens": 347149,
          "output_tokens": 78605
        },
        "alone_tokens": {
          "input_tokens": 628800,
          "cached_input_tokens": 171673,
          "cache_write_input_tokens": 0,
          "cache_write_1h_input_tokens": 456987,
          "output_tokens": 84813
        },
        "alternatives": [
          {
            "run": 71,
            "accuracy": 0.9745222929936306,
            "cost_usd": 5.67298,
            "request_seconds": 992.3243447878631,
            "versus_pipeline": {
              "difference": 0.0818926296633303,
              "low": 0.06454372636014774,
              "high": 0.09924153296651286,
              "p": 5.657185283317709e-22
            },
            "versus_frontier": {
              "difference": 0.08098271155595996,
              "low": 0.06281327556947469,
              "high": 0.09915214754244522,
              "p": 4.0853725805661456e-19
            },
            "model": "gpt-6-astra",
            "name": "GPT-6 Astra",
            "variant": null,
            "reasoning": "low",
            "provider": "openai",
            "tokens": {
              "input_tokens": 582668,
              "cached_input_tokens": 144000,
              "cache_write_input_tokens": 0,
              "cache_write_1h_input_tokens": 0,
              "output_tokens": 22846
            }
          },
          {
            "run": 137,
            "accuracy": 0.9681528662420382,
            "cost_usd": 1.3225,
            "request_seconds": 1362.2349697217578,
            "versus_pipeline": {
              "difference": 0.07552320291173795,
              "low": 0.05780227700995354,
              "high": 0.09324412881352236,
              "p": 1.3869287779064768e-17
            },
            "versus_frontier": {
              "difference": 0.0746132848043676,
              "low": 0.05591847354625709,
              "high": 0.09330809606247811,
              "p": 2.9566626383126704e-15
            },
            "model": "gpt-6.1-sol",
            "name": "GPT-6.1 Sol",
            "variant": null,
            "reasoning": "low",
            "provider": "openai",
            "tokens": {
              "input_tokens": 582670,
              "cached_input_tokens": 57600,
              "cache_write_input_tokens": 0,
              "cache_write_1h_input_tokens": 0,
              "output_tokens": 26660
            }
          },
          {
            "run": 138,
            "accuracy": 0.910828025477707,
            "cost_usd": 0.29709675,
            "request_seconds": 300.7470318475971,
            "versus_pipeline": {
              "difference": 0.018198362147406732,
              "low": -0.001308588285279768,
              "high": 0.03770531258009323,
              "p": 0.08240747932752655
            },
            "versus_frontier": {
              "difference": 0.017288444040036398,
              "low": -0.0034080250492140966,
              "high": 0.037984913129286896,
              "p": 0.12101372565511773
            },
            "model": "gemini-3.8-flash",
            "name": "Gemini 3.8 Flash",
            "variant": null,
            "reasoning": "low",
            "provider": "gemini",
            "tokens": {
              "input_tokens": 244424,
              "cached_input_tokens": 0,
              "cache_write_input_tokens": 0,
              "cache_write_1h_input_tokens": 0,
              "output_tokens": 30341
            }
          },
          {
            "run": 68,
            "accuracy": 0.8489535941765242,
            "cost_usd": 4.1268408,
            "request_seconds": 1799.8465952617116,
            "versus_pipeline": {
              "difference": -0.04367606915377616,
              "low": -0.0649209846060858,
              "high": -0.022431153701466536,
              "p": 7.799113896848694e-05
            },
            "versus_frontier": {
              "difference": -0.044585987261146494,
              "low": -0.06634214262491249,
              "high": -0.022829831897380495,
              "p": 8.212852870228109e-05
            },
            "model": "claude-sonnet-5-5",
            "name": "Claude Sonnet 5.5",
            "variant": null,
            "reasoning": "high",
            "provider": "claude",
            "tokens": {
              "input_tokens": 609112,
              "cached_input_tokens": 148308,
              "cache_write_input_tokens": 0,
              "cache_write_1h_input_tokens": 460666,
              "output_tokens": 226907
            }
          }
        ]
      }
    },
    {
      "decision": 145,
      "frontier": 55,
      "seed": 1,
      "tolerance_points": 0.5,
      "rule": "cutoff",
      "batch_size": 16,
      "threshold": 0.99,
      "held_out": 1099,
      "routed": 176,
      "simulated": {
        "accuracy": 0.8498635122838945,
        "frontier_accuracy": 0.8462238398544131
      },
      "measured": {
        "pipeline_run": 174,
        "alone_run": 68,
        "accuracy": 0.8571428571428571,
        "frontier_accuracy": 0.8489535941765242,
        "difference": 0.00818926296633303,
        "low": -0.009186810653492545,
        "high": 0.025565336586158605,
        "p": 0.41191007368097654,
        "within_tolerance": true,
        "decision_cost_usd": 0.008533460000000001,
        "cost_usd": 3.68969116,
        "frontier_cost_usd": 4.1268408,
        "request_seconds": 2001.7496087580075,
        "frontier_request_seconds": 1799.8465952617116,
        "pipeline_tokens": {
          "input_tokens": 490249,
          "cached_input_tokens": 124317,
          "cache_write_input_tokens": 0,
          "cache_write_1h_input_tokens": 365816,
          "output_tokens": 220523
        },
        "alone_tokens": {
          "input_tokens": 609112,
          "cached_input_tokens": 148308,
          "cache_write_input_tokens": 0,
          "cache_write_1h_input_tokens": 460666,
          "output_tokens": 226907
        },
        "alternatives": [
          {
            "run": 71,
            "accuracy": 0.9745222929936306,
            "cost_usd": 5.67298,
            "request_seconds": 992.3243447878631,
            "versus_pipeline": {
              "difference": 0.11737943585077343,
              "low": 0.09737165678674037,
              "high": 0.1373872149148065,
              "p": 7.352026653297796e-33
            },
            "versus_frontier": {
              "difference": 0.12556869881710647,
              "low": 0.10502648569558187,
              "high": 0.14611091193863107,
              "p": 2.08925524731209e-35
            },
            "model": "gpt-6-astra",
            "name": "GPT-6 Astra",
            "variant": null,
            "reasoning": "low",
            "provider": "openai",
            "tokens": {
              "input_tokens": 582668,
              "cached_input_tokens": 144000,
              "cache_write_input_tokens": 0,
              "cache_write_1h_input_tokens": 0,
              "output_tokens": 22846
            }
          },
          {
            "run": 137,
            "accuracy": 0.9681528662420382,
            "cost_usd": 1.3225,
            "request_seconds": 1362.2349697217578,
            "versus_pipeline": {
              "difference": 0.11101000909918107,
              "low": 0.09079672440902348,
              "high": 0.13122329378933867,
              "p": 2.5747366898777346e-28
            },
            "versus_frontier": {
              "difference": 0.1191992720655141,
              "low": 0.09875690893398828,
              "high": 0.13964163519703993,
              "p": 5.294448923596039e-32
            },
            "model": "gpt-6.1-sol",
            "name": "GPT-6.1 Sol",
            "variant": null,
            "reasoning": "low",
            "provider": "openai",
            "tokens": {
              "input_tokens": 582670,
              "cached_input_tokens": 57600,
              "cache_write_input_tokens": 0,
              "cache_write_1h_input_tokens": 0,
              "output_tokens": 26660
            }
          },
          {
            "run": 138,
            "accuracy": 0.910828025477707,
            "cost_usd": 0.29709675,
            "request_seconds": 300.7470318475971,
            "versus_pipeline": {
              "difference": 0.05368516833484986,
              "low": 0.03099742234231355,
              "high": 0.07637291432738617,
              "p": 5.076173568329694e-06
            },
            "versus_frontier": {
              "difference": 0.06187443130118289,
              "low": 0.03877269078891368,
              "high": 0.0849761718134521,
              "p": 2.2849587651185474e-07
            },
            "model": "gemini-3.8-flash",
            "name": "Gemini 3.8 Flash",
            "variant": null,
            "reasoning": "low",
            "provider": "gemini",
            "tokens": {
              "input_tokens": 244424,
              "cached_input_tokens": 0,
              "cache_write_input_tokens": 0,
              "cache_write_1h_input_tokens": 0,
              "output_tokens": 30341
            }
          },
          {
            "run": 165,
            "accuracy": 0.8935395814376706,
            "cost_usd": 5.387050599999999,
            "request_seconds": 929.1633372180659,
            "versus_pipeline": {
              "difference": 0.036396724294813464,
              "low": 0.01436968479581385,
              "high": 0.058423763793813074,
              "p": 0.0015876769917063935
            },
            "versus_frontier": {
              "difference": 0.044585987261146494,
              "low": 0.022829831897380495,
              "high": 0.06634214262491249,
              "p": 8.212852870228109e-05
            },
            "model": "claude-opus-5-5",
            "name": "Claude Opus 5.5",
            "variant": null,
            "reasoning": "low",
            "provider": "claude",
            "tokens": {
              "input_tokens": 628800,
              "cached_input_tokens": 171673,
              "cache_write_input_tokens": 0,
              "cache_write_1h_input_tokens": 456987,
              "output_tokens": 84813
            }
          }
        ]
      }
    },
    {
      "decision": 145,
      "frontier": 55,
      "seed": 1,
      "tolerance_points": 0.5,
      "rule": "half",
      "batch_size": 1,
      "threshold": 0.7171164205094459,
      "held_out": 1099,
      "routed": 558,
      "simulated": {
        "accuracy": 0.8343949044585988,
        "frontier_accuracy": 0.8462238398544131
      },
      "measured": {
        "pipeline_run": 170,
        "alone_run": 167,
        "accuracy": 0.8462238398544131,
        "frontier_accuracy": 0.8935395814376706,
        "difference": -0.047315741583257506,
        "low": -0.0647478551295116,
        "high": -0.029883628037003413,
        "p": 1.3172242120654852e-07,
        "within_tolerance": false,
        "decision_cost_usd": 0.008533460000000001,
        "cost_usd": 5.21685236,
        "frontier_cost_usd": 8.6273988,
        "request_seconds": 3980.5925824250007,
        "frontier_request_seconds": 6064.148409693618,
        "waits": {
          "pipeline": {
            "questions": 1099,
            "median_seconds": 0.4,
            "p90_seconds": 8.8948,
            "mean_seconds": 3.634992720655141,
            "max_seconds": 32.635999999999996,
            "total_seconds": 3994.857
          },
          "kept": {
            "questions": 558,
            "median_seconds": 0.232,
            "p90_seconds": 0.28300000000000003,
            "mean_seconds": 0.29999283154121864,
            "max_seconds": 31.381,
            "total_seconds": 167.396
          },
          "passed": {
            "questions": 541,
            "median_seconds": 5.985,
            "p90_seconds": 11.602,
            "mean_seconds": 7.074789279112753,
            "max_seconds": 32.635999999999996,
            "total_seconds": 3827.461
          },
          "alone": {
            "questions": 1099,
            "median_seconds": 4.774,
            "p90_seconds": 8.731800000000002,
            "mean_seconds": 5.517877161055505,
            "max_seconds": 21.935,
            "total_seconds": 6064.147
          }
        },
        "pipeline_tokens": {
          "input_tokens": 785837,
          "cached_input_tokens": 298909,
          "cache_write_input_tokens": 0,
          "cache_write_1h_input_tokens": 485846,
          "output_tokens": 323288
        },
        "alone_tokens": {
          "input_tokens": 1648174,
          "cached_input_tokens": 599508,
          "cache_write_input_tokens": 0,
          "cache_write_1h_input_tokens": 1046468,
          "output_tokens": 437718
        },
        "alternatives": [
          {
            "run": 163,
            "accuracy": 0.9754322111010009,
            "cost_usd": 31.648706,
            "request_seconds": 7964.2585057840915,
            "versus_pipeline": {
              "difference": 0.1292083712465878,
              "low": 0.10828410243890053,
              "high": 0.15013264005427504,
              "p": 8.935182693751113e-36
            },
            "versus_frontier": {
              "difference": 0.0818926296633303,
              "low": 0.06436135022755202,
              "high": 0.09942390909910859,
              "p": 2.253920747846914e-21
            },
            "model": "gpt-6-astra",
            "name": "GPT-6 Astra",
            "variant": null,
            "reasoning": "low",
            "provider": "openai",
            "tokens": {
              "input_tokens": 5103512,
              "cached_input_tokens": 2509696,
              "cache_write_input_tokens": 0,
              "cache_write_1h_input_tokens": 0,
              "output_tokens": 64017
            }
          },
          {
            "run": 166,
            "accuracy": 0.9272065514103731,
            "cost_usd": 12.8286324,
            "request_seconds": 4694.714676301097,
            "versus_pipeline": {
              "difference": 0.08098271155595996,
              "low": 0.05989641185577714,
              "high": 0.10206901125614277,
              "p": 6.155096365873754e-14
            },
            "versus_frontier": {
              "difference": 0.03366696997270246,
              "low": 0.015676818848088834,
              "high": 0.051657121097316086,
              "p": 0.00034099780128697693
            },
            "model": "claude-opus-5-5",
            "name": "Claude Opus 5.5",
            "variant": null,
            "reasoning": "low",
            "provider": "claude",
            "tokens": {
              "input_tokens": 1746017,
              "cached_input_tokens": 701622,
              "cache_write_input_tokens": 0,
              "cache_write_1h_input_tokens": 1042197,
              "output_tokens": 217097
            }
          }
        ]
      }
    },
    {
      "decision": 145,
      "frontier": 55,
      "seed": 1,
      "tolerance_points": 0.5,
      "rule": "share",
      "batch_size": 16,
      "threshold": 0.9642039538871846,
      "held_out": 1099,
      "routed": 275,
      "simulated": {
        "accuracy": 0.8553230209281165,
        "frontier_accuracy": 0.8462238398544131
      },
      "share": 0.25,
      "measured": {
        "pipeline_run": 173,
        "alone_run": 68,
        "accuracy": 0.8607825295723385,
        "frontier_accuracy": 0.8489535941765242,
        "difference": 0.011828935395814377,
        "low": -0.006777580673092972,
        "high": 0.030435451464721725,
        "p": 0.25028200469192174,
        "within_tolerance": true,
        "decision_cost_usd": 0.008533460000000001,
        "cost_usd": 3.37300646,
        "frontier_cost_usd": 4.1268408,
        "request_seconds": 1886.570894726974,
        "frontier_request_seconds": 1799.8465952617116,
        "pipeline_tokens": {
          "input_tokens": 429829,
          "cached_input_tokens": 109050,
          "cache_write_input_tokens": 0,
          "cache_write_1h_input_tokens": 320675,
          "output_tokens": 207066
        },
        "alone_tokens": {
          "input_tokens": 609112,
          "cached_input_tokens": 148308,
          "cache_write_input_tokens": 0,
          "cache_write_1h_input_tokens": 460666,
          "output_tokens": 226907
        },
        "alternatives": [
          {
            "run": 71,
            "accuracy": 0.9745222929936306,
            "cost_usd": 5.67298,
            "request_seconds": 992.3243447878631,
            "versus_pipeline": {
              "difference": 0.11373976342129208,
              "low": 0.09381765627121447,
              "high": 0.1336618705713697,
              "p": 5.166045778807018e-31
            },
            "versus_frontier": {
              "difference": 0.12556869881710647,
              "low": 0.10502648569558187,
              "high": 0.14611091193863107,
              "p": 2.08925524731209e-35
            },
            "model": "gpt-6-astra",
            "name": "GPT-6 Astra",
            "variant": null,
            "reasoning": "low",
            "provider": "openai",
            "tokens": {
              "input_tokens": 582668,
              "cached_input_tokens": 144000,
              "cache_write_input_tokens": 0,
              "cache_write_1h_input_tokens": 0,
              "output_tokens": 22846
            }
          },
          {
            "run": 137,
            "accuracy": 0.9681528662420382,
            "cost_usd": 1.3225,
            "request_seconds": 1362.2349697217578,
            "versus_pipeline": {
              "difference": 0.10737033666969972,
              "low": 0.08724587552101651,
              "high": 0.12749479781838294,
              "p": 1.0618810807448801e-26
            },
            "versus_frontier": {
              "difference": 0.1191992720655141,
              "low": 0.09875690893398828,
              "high": 0.13964163519703993,
              "p": 5.294448923596039e-32
            },
            "model": "gpt-6.1-sol",
            "name": "GPT-6.1 Sol",
            "variant": null,
            "reasoning": "low",
            "provider": "openai",
            "tokens": {
              "input_tokens": 582670,
              "cached_input_tokens": 57600,
              "cache_write_input_tokens": 0,
              "cache_write_1h_input_tokens": 0,
              "output_tokens": 26660
            }
          },
          {
            "run": 138,
            "accuracy": 0.910828025477707,
            "cost_usd": 0.29709675,
            "request_seconds": 300.7470318475971,
            "versus_pipeline": {
              "difference": 0.05004549590536852,
              "low": 0.02718909948193704,
              "high": 0.0729018923288,
              "p": 2.505115608706493e-05
            },
            "versus_frontier": {
              "difference": 0.06187443130118289,
              "low": 0.03877269078891368,
              "high": 0.0849761718134521,
              "p": 2.2849587651185474e-07
            },
            "model": "gemini-3.8-flash",
            "name": "Gemini 3.8 Flash",
            "variant": null,
            "reasoning": "low",
            "provider": "gemini",
            "tokens": {
              "input_tokens": 244424,
              "cached_input_tokens": 0,
              "cache_write_input_tokens": 0,
              "cache_write_1h_input_tokens": 0,
              "output_tokens": 30341
            }
          },
          {
            "run": 165,
            "accuracy": 0.8935395814376706,
            "cost_usd": 5.387050599999999,
            "request_seconds": 929.1633372180659,
            "versus_pipeline": {
              "difference": 0.03275705186533212,
              "low": 0.011443588455153298,
              "high": 0.05407051527551095,
              "p": 0.003393884063573226
            },
            "versus_frontier": {
              "difference": 0.044585987261146494,
              "low": 0.022829831897380495,
              "high": 0.06634214262491249,
              "p": 8.212852870228109e-05
            },
            "model": "claude-opus-5-5",
            "name": "Claude Opus 5.5",
            "variant": null,
            "reasoning": "low",
            "provider": "claude",
            "tokens": {
              "input_tokens": 628800,
              "cached_input_tokens": 171673,
              "cache_write_input_tokens": 0,
              "cache_write_1h_input_tokens": 456987,
              "output_tokens": 84813
            }
          }
        ]
      }
    },
    {
      "decision": 145,
      "frontier": 55,
      "seed": 1,
      "tolerance_points": 0.5,
      "rule": "half",
      "batch_size": 16,
      "threshold": 0.7171164205094459,
      "held_out": 1099,
      "routed": 558,
      "simulated": {
        "accuracy": 0.8343949044585988,
        "frontier_accuracy": 0.8462238398544131
      },
      "measured": {
        "pipeline_run": 162,
        "alone_run": 68,
        "accuracy": 0.8425841674249318,
        "frontier_accuracy": 0.8489535941765242,
        "difference": -0.006369426751592357,
        "low": -0.028281462396915237,
        "high": 0.015542608893730525,
        "p": 0.6255036238897121,
        "within_tolerance": false,
        "decision_cost_usd": 0.008533460000000001,
        "cost_usd": 2.5804246600000003,
        "frontier_cost_usd": 4.1268408,
        "request_seconds": 1677.1019263799535,
        "frontier_request_seconds": 1799.8465952617116,
        "pipeline_tokens": {
          "input_tokens": 274389,
          "cached_input_tokens": 69792,
          "cache_write_input_tokens": 0,
          "cache_write_1h_input_tokens": 204529,
          "output_tokens": 174666
        },
        "alone_tokens": {
          "input_tokens": 609112,
          "cached_input_tokens": 148308,
          "cache_write_input_tokens": 0,
          "cache_write_1h_input_tokens": 460666,
          "output_tokens": 226907
        },
        "alternatives": [
          {
            "run": 71,
            "accuracy": 0.9745222929936306,
            "cost_usd": 5.67298,
            "request_seconds": 992.3243447878631,
            "versus_pipeline": {
              "difference": 0.13193812556869883,
              "low": 0.11069579859223826,
              "high": 0.15318045254515938,
              "p": 6.769507367528149e-36
            },
            "versus_frontier": {
              "difference": 0.12556869881710647,
              "low": 0.10502648569558187,
              "high": 0.14611091193863107,
              "p": 2.08925524731209e-35
            },
            "model": "gpt-6-astra",
            "name": "GPT-6 Astra",
            "variant": null,
            "reasoning": "low",
            "provider": "openai",
            "tokens": {
              "input_tokens": 582668,
              "cached_input_tokens": 144000,
              "cache_write_input_tokens": 0,
              "cache_write_1h_input_tokens": 0,
              "output_tokens": 22846
            }
          },
          {
            "run": 137,
            "accuracy": 0.9681528662420382,
            "cost_usd": 1.3225,
            "request_seconds": 1362.2349697217578,
            "versus_pipeline": {
              "difference": 0.12556869881710647,
              "low": 0.10426637302645733,
              "high": 0.1468710246077556,
              "p": 4.576636824126176e-32
            },
            "versus_frontier": {
              "difference": 0.1191992720655141,
              "low": 0.09875690893398828,
              "high": 0.13964163519703993,
              "p": 5.294448923596039e-32
            },
            "model": "gpt-6.1-sol",
            "name": "GPT-6.1 Sol",
            "variant": null,
            "reasoning": "low",
            "provider": "openai",
            "tokens": {
              "input_tokens": 582670,
              "cached_input_tokens": 57600,
              "cache_write_input_tokens": 0,
              "cache_write_1h_input_tokens": 0,
              "output_tokens": 26660
            }
          },
          {
            "run": 138,
            "accuracy": 0.910828025477707,
            "cost_usd": 0.29709675,
            "request_seconds": 300.7470318475971,
            "versus_pipeline": {
              "difference": 0.06824385805277525,
              "low": 0.0448623126424854,
              "high": 0.0916254034630651,
              "p": 1.6420148695755048e-08
            },
            "versus_frontier": {
              "difference": 0.06187443130118289,
              "low": 0.03877269078891368,
              "high": 0.0849761718134521,
              "p": 2.2849587651185474e-07
            },
            "model": "gemini-3.8-flash",
            "name": "Gemini 3.8 Flash",
            "variant": null,
            "reasoning": "low",
            "provider": "gemini",
            "tokens": {
              "input_tokens": 244424,
              "cached_input_tokens": 0,
              "cache_write_input_tokens": 0,
              "cache_write_1h_input_tokens": 0,
              "output_tokens": 30341
            }
          },
          {
            "run": 165,
            "accuracy": 0.8935395814376706,
            "cost_usd": 5.387050599999999,
            "request_seconds": 929.1633372180659,
            "versus_pipeline": {
              "difference": 0.050955414012738856,
              "low": 0.027760654036543823,
              "high": 0.07415017398893389,
              "p": 2.350182078373937e-05
            },
            "versus_frontier": {
              "difference": 0.044585987261146494,
              "low": 0.022829831897380495,
              "high": 0.06634214262491249,
              "p": 8.212852870228109e-05
            },
            "model": "claude-opus-5-5",
            "name": "Claude Opus 5.5",
            "variant": null,
            "reasoning": "low",
            "provider": "claude",
            "tokens": {
              "input_tokens": 628800,
              "cached_input_tokens": 171673,
              "cache_write_input_tokens": 0,
              "cache_write_1h_input_tokens": 456987,
              "output_tokens": 84813
            }
          }
        ]
      }
    }
  ],
  "batching": [
    {
      "run": 136,
      "model": "gemma-4-31B-it",
      "name": "Gemma 4 31B",
      "variant": null,
      "reasoning": "off",
      "provider": "vllm",
      "batch_size": 1,
      "compared_batch_size": 16,
      "questions": 200,
      "correct": 137,
      "median_request_seconds": 0.29488859651610255,
      "p90_request_seconds": 0.30137023626593873,
      "others": [
        {
          "run": 61,
          "correct": 141,
          "first_only": 15,
          "second_only": 19,
          "difference": -0.02,
          "low": -0.07707606153195926,
          "high": 0.03707606153195926,
          "p": 0.6075913612730801,
          "repeat": null
        },
        {
          "run": 72,
          "correct": 141,
          "first_only": 15,
          "second_only": 19,
          "difference": -0.02,
          "low": -0.07707606153195926,
          "high": 0.03707606153195926,
          "p": 0.6075913612730801,
          "repeat": 2
        },
        {
          "run": 73,
          "correct": 141,
          "first_only": 15,
          "second_only": 19,
          "difference": -0.02,
          "low": -0.07707606153195926,
          "high": 0.03707606153195926,
          "p": 0.6075913612730801,
          "repeat": 3
        },
        {
          "run": 76,
          "correct": 141,
          "first_only": 15,
          "second_only": 19,
          "difference": -0.02,
          "low": -0.07707606153195926,
          "high": 0.03707606153195926,
          "p": 0.6075913612730801,
          "repeat": 4
        },
        {
          "run": 79,
          "correct": 141,
          "first_only": 15,
          "second_only": 19,
          "difference": -0.02,
          "low": -0.07707606153195926,
          "high": 0.03707606153195926,
          "p": 0.6075913612730801,
          "repeat": 5
        },
        {
          "run": 119,
          "correct": 145,
          "first_only": 12,
          "second_only": 20,
          "difference": -0.04,
          "low": -0.09515928933552353,
          "high": 0.015159289335523532,
          "p": 0.21532714972272515,
          "repeat": null
        }
      ]
    },
    {
      "run": 142,
      "model": "gpt-6-astra",
      "name": "GPT-6 Astra",
      "variant": null,
      "reasoning": "low",
      "provider": "openai",
      "batch_size": 1,
      "compared_batch_size": 16,
      "questions": 200,
      "correct": 197,
      "median_request_seconds": 6.135667477443349,
      "p90_request_seconds": 8.567763022321742,
      "others": [
        {
          "run": 48,
          "correct": 199,
          "first_only": 0,
          "second_only": 2,
          "difference": -0.01,
          "low": -0.023789822333880883,
          "high": 0.003789822333880883,
          "p": 0.5,
          "repeat": null
        },
        {
          "run": 69,
          "correct": 195,
          "first_only": 2,
          "second_only": 0,
          "difference": 0.01,
          "low": -0.003789822333880883,
          "high": 0.023789822333880883,
          "p": 0.5,
          "repeat": null
        },
        {
          "run": 78,
          "correct": 196,
          "first_only": 1,
          "second_only": 0,
          "difference": 0.005,
          "low": -0.004775469298197401,
          "high": 0.0147754692981974,
          "p": 1.0,
          "repeat": 2
        },
        {
          "run": 82,
          "correct": 196,
          "first_only": 1,
          "second_only": 0,
          "difference": 0.005,
          "low": -0.004775469298197401,
          "high": 0.0147754692981974,
          "p": 1.0,
          "repeat": 3
        },
        {
          "run": 87,
          "correct": 196,
          "first_only": 1,
          "second_only": 0,
          "difference": 0.005,
          "low": -0.004775469298197401,
          "high": 0.0147754692981974,
          "p": 1.0,
          "repeat": 4
        },
        {
          "run": 91,
          "correct": 196,
          "first_only": 1,
          "second_only": 0,
          "difference": 0.005,
          "low": -0.004775469298197401,
          "high": 0.0147754692981974,
          "p": 1.0,
          "repeat": 5
        }
      ]
    }
  ],
  "dont_know": {
    "option": "Bilmiyorum",
    "option_en": "I don't know",
    "sentence": "If you do not know the answer, choose Bilmiyorum.",
    "instructions": "Which option is the correct answer to this Turkish multiple-choice question? If you do not know the answer, choose Bilmiyorum.",
    "prompt_version": "1cdd542b597f",
    "vs_confidence_share": 0.01,
    "questions": 2198,
    "models": [
      {
        "name": "Drex Max v1",
        "model": "drex-max-v1",
        "provider": "drex-max",
        "run": 367,
        "runs": [
          367
        ],
        "right": 1485,
        "wrong": 711,
        "dont_know": 2,
        "invalid": 0,
        "accuracy": 0.675614194722475,
        "attempt_rate": 0.9990900818926297,
        "non_hallucination": 0.002805049088359046,
        "mean_dont_know_probability": 0.015052588128069306,
        "spread": null,
        "answers_changed": null,
        "original_run": 322,
        "original_accuracy": 0.6774340309372157,
        "derived": {
          "cut_off": 0.235659473972748,
          "attempt_rate": 0.9990900818926297,
          "non_hallucination": 0.0028208744710860366,
          "right": 1489,
          "wrong": 707,
          "abstained": 2
        },
        "vs_confidence": {
          "option_only_wrong": 45,
          "confidence_only_wrong": 41,
          "p": 0.7465343822890573,
          "p_holm": null
        }
      },
      {
        "name": "vLLM-SR d3-ultra",
        "model": "vllm-sr/d3-ultra",
        "provider": "d3-server",
        "run": 363,
        "runs": [
          363
        ],
        "right": 1484,
        "wrong": 704,
        "dont_know": 10,
        "invalid": 0,
        "accuracy": 0.6751592356687898,
        "attempt_rate": 0.9954504094631483,
        "non_hallucination": 0.014005602240896359,
        "mean_dont_know_probability": 0.02463415869057922,
        "spread": null,
        "answers_changed": null,
        "original_run": 269,
        "original_accuracy": 0.6778889899909009,
        "derived": {
          "cut_off": 0.2875844645345384,
          "attempt_rate": 0.9954504094631483,
          "non_hallucination": 0.014044943820224719,
          "right": 1486,
          "wrong": 702,
          "abstained": 10
        },
        "vs_confidence": {
          "option_only_wrong": 54,
          "confidence_only_wrong": 52,
          "p": 0.9226851594412759,
          "p_holm": null
        }
      },
      {
        "name": "vLLM-SR d3",
        "model": "vllm-sr/d3",
        "provider": "vllm-sr",
        "run": 358,
        "runs": [
          358
        ],
        "right": 1483,
        "wrong": 715,
        "dont_know": 0,
        "invalid": 0,
        "accuracy": 0.6747042766151047,
        "attempt_rate": 1.0,
        "non_hallucination": 0.0,
        "mean_dont_know_probability": 0.029138429914467894,
        "spread": null,
        "answers_changed": null,
        "original_run": 259,
        "original_accuracy": 0.6778889899909009,
        "derived": {
          "cut_off": 0.22253464583592555,
          "attempt_rate": 1.0,
          "non_hallucination": 0.0,
          "right": 1490,
          "wrong": 708,
          "abstained": 0
        },
        "vs_confidence": {
          "option_only_wrong": 40,
          "confidence_only_wrong": 33,
          "p": 0.4827885568440213,
          "p_holm": null
        }
      },
      {
        "name": "Perplexity Decider 27B v1.2",
        "model": "pplx-decider-v1.2",
        "provider": "pplx-decider",
        "run": 372,
        "runs": [
          372
        ],
        "right": 1479,
        "wrong": 717,
        "dont_know": 2,
        "invalid": 0,
        "accuracy": 0.672884440400364,
        "attempt_rate": 0.9990900818926297,
        "non_hallucination": 0.0027816411682892906,
        "mean_dont_know_probability": 0.023578840422335552,
        "spread": null,
        "answers_changed": null,
        "original_run": 217,
        "original_accuracy": 0.6706096451319381,
        "derived": {
          "cut_off": 0.22050537826136882,
          "attempt_rate": 0.9990900818926297,
          "non_hallucination": 0.0027624309392265192,
          "right": 1474,
          "wrong": 722,
          "abstained": 2
        },
        "vs_confidence": {
          "option_only_wrong": 45,
          "confidence_only_wrong": 50,
          "p": 0.6817425059181855,
          "p_holm": null
        }
      },
      {
        "name": "Fastino GLiDE v1.2",
        "model": "glide-v1.2",
        "provider": "glide",
        "run": 369,
        "runs": [
          369
        ],
        "right": 1386,
        "wrong": 491,
        "dont_know": 321,
        "invalid": 0,
        "accuracy": 0.6305732484076433,
        "attempt_rate": 0.8539581437670609,
        "non_hallucination": 0.39532019704433496,
        "mean_dont_know_probability": 0.18583695105631345,
        "spread": null,
        "answers_changed": null,
        "original_run": 248,
        "original_accuracy": 0.6847133757961783,
        "derived": {
          "cut_off": 0.4134294302863545,
          "attempt_rate": 0.8539581437670609,
          "non_hallucination": 0.40428211586901763,
          "right": 1404,
          "wrong": 473,
          "abstained": 321
        },
        "vs_confidence": {
          "option_only_wrong": 77,
          "confidence_only_wrong": 59,
          "p": 0.14463646515752107,
          "p_holm": 0.7203551558140382
        }
      },
      {
        "name": "Liquid d1-ultra",
        "model": "d1-ultra",
        "provider": "liquid",
        "run": 350,
        "runs": [
          350,
          354,
          360,
          365,
          370
        ],
        "right": 1376,
        "wrong": 205,
        "dont_know": 617,
        "invalid": 0,
        "accuracy": 0.6260236578707916,
        "attempt_rate": 0.7192902638762512,
        "non_hallucination": 0.7506082725060828,
        "mean_dont_know_probability": 0.30688419671830985,
        "spread": {
          "accuracy": [
            0.6255686988171064,
            0.6346678798908099
          ],
          "attempt_rate": [
            0.7192902638762512,
            0.7397634212920837
          ],
          "non_hallucination": [
            0.7123287671232876,
            0.7506082725060828
          ],
          "mean_dont_know_probability": [
            0.3060557578090563,
            0.3076800813405489
          ]
        },
        "answers_changed": 518,
        "original_run": 199,
        "original_accuracy": 0.7707006369426752,
        "derived": {
          "cut_off": 0.5632390296623447,
          "attempt_rate": 0.7192902638762512,
          "non_hallucination": 0.7869897959183674,
          "right": 1414,
          "wrong": 167,
          "abstained": 617
        },
        "vs_confidence": {
          "option_only_wrong": 93,
          "confidence_only_wrong": 55,
          "p": 0.002244365752749707,
          "p_holm": 0.01571056026924795
        }
      },
      {
        "name": "Drex Max 1.1",
        "model": "drex-max-v1.1",
        "provider": "nace",
        "run": 352,
        "runs": [
          352
        ],
        "right": 1361,
        "wrong": 517,
        "dont_know": 320,
        "invalid": 0,
        "accuracy": 0.6191992720655141,
        "attempt_rate": 0.8544131028207461,
        "non_hallucination": 0.3823178016726404,
        "mean_dont_know_probability": 0.16972642800384946,
        "spread": null,
        "answers_changed": null,
        "original_run": 326,
        "original_accuracy": 0.6737943585077343,
        "derived": {
          "cut_off": 0.38063265738985963,
          "attempt_rate": 0.8544131028207461,
          "non_hallucination": 0.39263803680981596,
          "right": 1383,
          "wrong": 495,
          "abstained": 320
        },
        "vs_confidence": {
          "option_only_wrong": 125,
          "confidence_only_wrong": 103,
          "p": 0.16415489186828677,
          "p_holm": 0.7203551558140382
        }
      },
      {
        "name": "Fastino GLiDE",
        "model": "fastino/GLiDE",
        "provider": "fastino",
        "run": 351,
        "runs": [
          351,
          353,
          357,
          361,
          364
        ],
        "right": 1321,
        "wrong": 356,
        "dont_know": 521,
        "invalid": 0,
        "accuracy": 0.6010009099181074,
        "attempt_rate": 0.7629663330300273,
        "non_hallucination": 0.5940706955530216,
        "mean_dont_know_probability": 0.2201435400931123,
        "spread": {
          "accuracy": [
            0.5905368516833485,
            0.6010009099181074
          ],
          "attempt_rate": [
            0.7552320291173794,
            0.7629663330300273
          ],
          "non_hallucination": [
            0.5940706955530216,
            0.5991091314031181
          ],
          "mean_dont_know_probability": [
            0.21628588574582433,
            0.22014875465701816
          ]
        },
        "answers_changed": 189,
        "original_run": 129,
        "original_accuracy": 0.6787989080982711,
        "derived": {
          "cut_off": 0.5206454205284788,
          "attempt_rate": 0.7629663330300273,
          "non_hallucination": 0.5801781737193764,
          "right": 1300,
          "wrong": 377,
          "abstained": 521
        },
        "vs_confidence": {
          "option_only_wrong": 192,
          "confidence_only_wrong": 213,
          "p": 0.32031790636017266,
          "p_holm": 0.7203551558140382
        }
      },
      {
        "name": "SubQ Snap 1.1",
        "model": "subq-snap-1.1",
        "provider": "subq",
        "run": 392,
        "runs": [
          392
        ],
        "right": 1271,
        "wrong": 490,
        "dont_know": 437,
        "invalid": 0,
        "accuracy": 0.578252957233849,
        "attempt_rate": 0.8011828935395814,
        "non_hallucination": 0.4714131607335491,
        "mean_dont_know_probability": 0.20102079041315762,
        "spread": null,
        "answers_changed": null,
        "original_run": 346,
        "original_accuracy": 0.6387625113739763,
        "derived": {
          "cut_off": 0.512372612953186,
          "attempt_rate": 0.8011828935395814,
          "non_hallucination": 0.4760348583877996,
          "right": 1280,
          "wrong": 481,
          "abstained": 437
        },
        "vs_confidence": {
          "option_only_wrong": 148,
          "confidence_only_wrong": 139,
          "p": 0.6368426841662184,
          "p_holm": 0.7203551558140382
        }
      },
      {
        "name": "Mammoth-1",
        "model": "baselabs/mammoth-1",
        "provider": "baselabs",
        "run": 386,
        "runs": [
          386
        ],
        "right": 1194,
        "wrong": 314,
        "dont_know": 690,
        "invalid": 0,
        "accuracy": 0.543221110100091,
        "attempt_rate": 0.6860782529572339,
        "non_hallucination": 0.6872509960159362,
        "mean_dont_know_probability": 0.2445804940855323,
        "spread": null,
        "answers_changed": null,
        "original_run": 374,
        "original_accuracy": 0.6710646041856233,
        "derived": {
          "cut_off": 0.569673,
          "attempt_rate": 0.6860782529572339,
          "non_hallucination": 0.7091469681397738,
          "right": 1225,
          "wrong": 283,
          "abstained": 690
        },
        "vs_confidence": {
          "option_only_wrong": 66,
          "confidence_only_wrong": 35,
          "p": 0.0026537860572285053,
          "p_holm": 0.015922716343371033
        }
      },
      {
        "name": "Elephant-1",
        "model": "baselabs/elephant-1",
        "provider": "baselabs",
        "run": 382,
        "runs": [
          382
        ],
        "right": 1035,
        "wrong": 758,
        "dont_know": 405,
        "invalid": 0,
        "accuracy": 0.4708826205641492,
        "attempt_rate": 0.8157415832575068,
        "non_hallucination": 0.3482373172828891,
        "mean_dont_know_probability": 0.16036186169244768,
        "spread": null,
        "answers_changed": null,
        "original_run": 373,
        "original_accuracy": 0.5236578707916287,
        "derived": {
          "cut_off": 0.388276,
          "attempt_rate": 0.8157415832575068,
          "non_hallucination": 0.35526315789473684,
          "right": 1058,
          "wrong": 735,
          "abstained": 405
        },
        "vs_confidence": {
          "option_only_wrong": 125,
          "confidence_only_wrong": 102,
          "p": 0.14407103116280764,
          "p_holm": 0.7203551558140382
        }
      }
    ]
  },
  "dont_know_option_word": {
    "option": "I don't know",
    "sentence": "If you do not know the answer, choose I don't know.",
    "instructions": "Which option is the correct answer to this Turkish multiple-choice question? If you do not know the answer, choose I don't know.",
    "prompt_version": "fb17d583a3af",
    "questions": 2198,
    "models": [
      {
        "name": "Drex Max v1",
        "model": "drex-max-v1",
        "provider": "drex-max",
        "bilmiyorum": {
          "run": 367,
          "date": "2026-11-01",
          "prompt_version": "1cdd542b597f",
          "right": 1485,
          "wrong": 711,
          "dont_know": 2,
          "invalid": 0,
          "accuracy": 0.675614194722475,
          "attempt_rate": 0.9990900818926297,
          "non_hallucination": 0.002805049088359046,
          "mean_dont_know_probability": 0.015052588128069306
        },
        "i_dont_know": {
          "run": 542,
          "date": "2026-11-02",
          "prompt_version": "fb17d583a3af",
          "right": 1484,
          "wrong": 712,
          "dont_know": 2,
          "invalid": 0,
          "accuracy": 0.6751592356687898,
          "attempt_rate": 0.9990900818926297,
          "non_hallucination": 0.0028011204481792717,
          "mean_dont_know_probability": 0.022460676766798113
        },
        "paired": {
          "bilmiyorum_only_wrong": 28,
          "i_dont_know_only_wrong": 29,
          "p": 1.0,
          "p_holm": 1.0
        }
      },
      {
        "name": "vLLM-SR d3-ultra",
        "model": "vllm-sr/d3-ultra",
        "provider": "d3-server",
        "bilmiyorum": {
          "run": 363,
          "date": "2026-11-01",
          "prompt_version": "1cdd542b597f",
          "right": 1484,
          "wrong": 704,
          "dont_know": 10,
          "invalid": 0,
          "accuracy": 0.6751592356687898,
          "attempt_rate": 0.9954504094631483,
          "non_hallucination": 0.014005602240896359,
          "mean_dont_know_probability": 0.02463415869057922
        },
        "i_dont_know": {
          "run": 540,
          "date": "2026-11-02",
          "prompt_version": "fb17d583a3af",
          "right": 1485,
          "wrong": 704,
          "dont_know": 9,
          "invalid": 0,
          "accuracy": 0.675614194722475,
          "attempt_rate": 0.9959053685168335,
          "non_hallucination": 0.012622720897615708,
          "mean_dont_know_probability": 0.029017655024615365
        },
        "paired": {
          "bilmiyorum_only_wrong": 39,
          "i_dont_know_only_wrong": 39,
          "p": 1.0,
          "p_holm": 1.0
        }
      },
      {
        "name": "vLLM-SR d3",
        "model": "vllm-sr/d3",
        "provider": "vllm-sr",
        "bilmiyorum": {
          "run": 358,
          "date": "2026-11-01",
          "prompt_version": "1cdd542b597f",
          "right": 1483,
          "wrong": 715,
          "dont_know": 0,
          "invalid": 0,
          "accuracy": 0.6747042766151047,
          "attempt_rate": 1.0,
          "non_hallucination": 0.0,
          "mean_dont_know_probability": 0.029138429914467894
        },
        "i_dont_know": {
          "run": 544,
          "date": "2026-11-02",
          "prompt_version": "fb17d583a3af",
          "right": 1482,
          "wrong": 714,
          "dont_know": 2,
          "invalid": 0,
          "accuracy": 0.6742493175614195,
          "attempt_rate": 0.9990900818926297,
          "non_hallucination": 0.002793296089385475,
          "mean_dont_know_probability": 0.033758577840294166
        },
        "paired": {
          "bilmiyorum_only_wrong": 22,
          "i_dont_know_only_wrong": 21,
          "p": 1.0,
          "p_holm": 1.0
        }
      },
      {
        "name": "Perplexity Decider 27B v1.2",
        "model": "pplx-decider-v1.2",
        "provider": "pplx-decider",
        "bilmiyorum": {
          "run": 372,
          "date": "2026-11-01",
          "prompt_version": "1cdd542b597f",
          "right": 1479,
          "wrong": 717,
          "dont_know": 2,
          "invalid": 0,
          "accuracy": 0.672884440400364,
          "attempt_rate": 0.9990900818926297,
          "non_hallucination": 0.0027816411682892906,
          "mean_dont_know_probability": 0.023578840422335552
        },
        "i_dont_know": {
          "run": 531,
          "date": "2026-11-02",
          "prompt_version": "fb17d583a3af",
          "right": 1466,
          "wrong": 728,
          "dont_know": 4,
          "invalid": 0,
          "accuracy": 0.6669699727024567,
          "attempt_rate": 0.9981801637852593,
          "non_hallucination": 0.00546448087431694,
          "mean_dont_know_probability": 0.03379872297063549
        },
        "paired": {
          "bilmiyorum_only_wrong": 29,
          "i_dont_know_only_wrong": 40,
          "p": 0.22840022535329696,
          "p_holm": 1.0
        }
      },
      {
        "name": "Fastino GLiDE v1.2",
        "model": "glide-v1.2",
        "provider": "glide",
        "bilmiyorum": {
          "run": 369,
          "date": "2026-11-01",
          "prompt_version": "1cdd542b597f",
          "right": 1386,
          "wrong": 491,
          "dont_know": 321,
          "invalid": 0,
          "accuracy": 0.6305732484076433,
          "attempt_rate": 0.8539581437670609,
          "non_hallucination": 0.39532019704433496,
          "mean_dont_know_probability": 0.18583695105631345
        },
        "i_dont_know": {
          "run": 525,
          "date": "2026-11-02",
          "prompt_version": "fb17d583a3af",
          "right": 1345,
          "wrong": 427,
          "dont_know": 426,
          "invalid": 0,
          "accuracy": 0.6119199272065514,
          "attempt_rate": 0.8061874431301183,
          "non_hallucination": 0.49941383352872215,
          "mean_dont_know_probability": 0.2091311634897189
        },
        "paired": {
          "bilmiyorum_only_wrong": 83,
          "i_dont_know_only_wrong": 19,
          "p": 1.0158259578897715e-10,
          "p_holm": 9.142433621007943e-10
        }
      },
      {
        "name": "Liquid d1-ultra",
        "model": "d1-ultra",
        "provider": "liquid",
        "bilmiyorum": {
          "run": 350,
          "date": "2026-11-01",
          "prompt_version": "1cdd542b597f",
          "right": 1376,
          "wrong": 205,
          "dont_know": 617,
          "invalid": 0,
          "accuracy": 0.6260236578707916,
          "attempt_rate": 0.7192902638762512,
          "non_hallucination": 0.7506082725060828,
          "mean_dont_know_probability": 0.30688419671830985
        },
        "i_dont_know": {
          "run": 511,
          "date": "2026-11-02",
          "prompt_version": "fb17d583a3af",
          "right": 1366,
          "wrong": 216,
          "dont_know": 616,
          "invalid": 0,
          "accuracy": 0.62147406733394,
          "attempt_rate": 0.7197452229299363,
          "non_hallucination": 0.7403846153846154,
          "mean_dont_know_probability": 0.31407871994345526
        },
        "paired": {
          "bilmiyorum_only_wrong": 55,
          "i_dont_know_only_wrong": 66,
          "p": 0.3633632605480977,
          "p_holm": 1.0
        }
      },
      {
        "name": "Drex Max 1.1",
        "model": "drex-max-v1.1",
        "provider": "nace",
        "bilmiyorum": {
          "run": 352,
          "date": "2026-11-01",
          "prompt_version": "1cdd542b597f",
          "right": 1361,
          "wrong": 517,
          "dont_know": 320,
          "invalid": 0,
          "accuracy": 0.6191992720655141,
          "attempt_rate": 0.8544131028207461,
          "non_hallucination": 0.3823178016726404,
          "mean_dont_know_probability": 0.16972642800384946
        },
        "i_dont_know": {
          "run": 513,
          "date": "2026-11-02",
          "prompt_version": "fb17d583a3af",
          "right": 1299,
          "wrong": 413,
          "dont_know": 486,
          "invalid": 0,
          "accuracy": 0.5909918107370337,
          "attempt_rate": 0.7788898999090081,
          "non_hallucination": 0.5406006674082313,
          "mean_dont_know_probability": 0.20933764122653578
        },
        "paired": {
          "bilmiyorum_only_wrong": 144,
          "i_dont_know_only_wrong": 40,
          "p": 5.518207697056318e-15,
          "p_holm": 5.518207697056318e-14
        }
      },
      {
        "name": "Fastino GLiDE",
        "model": "fastino/GLiDE",
        "provider": "fastino",
        "bilmiyorum": {
          "run": 351,
          "date": "2026-11-01",
          "prompt_version": "1cdd542b597f",
          "right": 1321,
          "wrong": 356,
          "dont_know": 521,
          "invalid": 0,
          "accuracy": 0.6010009099181074,
          "attempt_rate": 0.7629663330300273,
          "non_hallucination": 0.5940706955530216,
          "mean_dont_know_probability": 0.2201435400931123
        },
        "i_dont_know": {
          "run": 510,
          "date": "2026-11-02",
          "prompt_version": "fb17d583a3af",
          "right": 1282,
          "wrong": 335,
          "dont_know": 581,
          "invalid": 0,
          "accuracy": 0.5832575068243858,
          "attempt_rate": 0.7356687898089171,
          "non_hallucination": 0.6342794759825328,
          "mean_dont_know_probability": 0.22846753147822998
        },
        "paired": {
          "bilmiyorum_only_wrong": 73,
          "i_dont_know_only_wrong": 52,
          "p": 0.07321432008545255,
          "p_holm": 0.4392859205127153
        }
      },
      {
        "name": "SubQ Snap 1.1",
        "model": "subq-snap-1.1",
        "provider": "subq",
        "bilmiyorum": {
          "run": 392,
          "date": "2026-11-01",
          "prompt_version": "1cdd542b597f",
          "right": 1271,
          "wrong": 490,
          "dont_know": 437,
          "invalid": 0,
          "accuracy": 0.578252957233849,
          "attempt_rate": 0.8011828935395814,
          "non_hallucination": 0.4714131607335491,
          "mean_dont_know_probability": 0.20102079041315762
        },
        "i_dont_know": {
          "run": 512,
          "date": "2026-11-02",
          "prompt_version": "fb17d583a3af",
          "right": 1246,
          "wrong": 454,
          "dont_know": 498,
          "invalid": 0,
          "accuracy": 0.5668789808917197,
          "attempt_rate": 0.7734303912647862,
          "non_hallucination": 0.523109243697479,
          "mean_dont_know_probability": 0.21841116926581552
        },
        "paired": {
          "bilmiyorum_only_wrong": 144,
          "i_dont_know_only_wrong": 108,
          "p": 0.02727041690722833,
          "p_holm": 0.1908929183505983
        }
      },
      {
        "name": "Mammoth-1",
        "model": "baselabs/mammoth-1",
        "provider": "baselabs",
        "bilmiyorum": {
          "run": 386,
          "date": "2026-11-01",
          "prompt_version": "1cdd542b597f",
          "right": 1194,
          "wrong": 314,
          "dont_know": 690,
          "invalid": 0,
          "accuracy": 0.543221110100091,
          "attempt_rate": 0.6860782529572339,
          "non_hallucination": 0.6872509960159362,
          "mean_dont_know_probability": 0.2445804940855323
        },
        "i_dont_know": {
          "run": 519,
          "date": "2026-11-02",
          "prompt_version": "fb17d583a3af",
          "right": 1169,
          "wrong": 276,
          "dont_know": 753,
          "invalid": 0,
          "accuracy": 0.5318471337579618,
          "attempt_rate": 0.6574158325750682,
          "non_hallucination": 0.7317784256559767,
          "mean_dont_know_probability": 0.2590707324840764
        },
        "paired": {
          "bilmiyorum_only_wrong": 47,
          "i_dont_know_only_wrong": 9,
          "p": 2.571572048082338e-07,
          "p_holm": 2.0572576384658703e-06
        }
      },
      {
        "name": "Elephant-1",
        "model": "baselabs/elephant-1",
        "provider": "baselabs",
        "bilmiyorum": {
          "run": 382,
          "date": "2026-11-01",
          "prompt_version": "1cdd542b597f",
          "right": 1035,
          "wrong": 758,
          "dont_know": 405,
          "invalid": 0,
          "accuracy": 0.4708826205641492,
          "attempt_rate": 0.8157415832575068,
          "non_hallucination": 0.3482373172828891,
          "mean_dont_know_probability": 0.16036186169244768
        },
        "i_dont_know": {
          "run": 515,
          "date": "2026-11-02",
          "prompt_version": "fb17d583a3af",
          "right": 972,
          "wrong": 622,
          "dont_know": 604,
          "invalid": 0,
          "accuracy": 0.4422202001819836,
          "attempt_rate": 0.7252047315741583,
          "non_hallucination": 0.4926590538336052,
          "mean_dont_know_probability": 0.2117174194722475
        },
        "paired": {
          "bilmiyorum_only_wrong": 154,
          "i_dont_know_only_wrong": 18,
          "p": 4.074445781426416e-28,
          "p_holm": 4.481890359569058e-27
        }
      }
    ],
    "not_run": [],
    "vs_confidence": {
      "share": 0.01,
      "family": [
        "Fastino GLiDE v1.2",
        "Liquid d1-ultra",
        "Drex Max 1.1",
        "Fastino GLiDE",
        "SubQ Snap 1.1",
        "Mammoth-1",
        "Elephant-1"
      ],
      "models": [
        {
          "name": "Drex Max v1",
          "option_run": 542,
          "five_option_run": 322,
          "replaces_run": null,
          "option": {
            "answered": 2196,
            "attempt_rate": 0.9990900818926297,
            "right": 1484,
            "wrong": 712,
            "dont_know": 2
          },
          "derived": {
            "cut_off": 0.235659473972748,
            "answered": 2196,
            "attempt_rate": 0.9990900818926297,
            "right": 1489,
            "wrong": 707,
            "abstained": 2,
            "below": {
              "cut_off": 0.23480254670996567,
              "answered": 2197
            },
            "above": {
              "cut_off": 0.23753862082958221,
              "answered": 2195
            }
          },
          "option_only_wrong": 49,
          "confidence_only_wrong": 44,
          "p": 0.6785320136533252,
          "p_holm": null
        },
        {
          "name": "vLLM-SR d3-ultra",
          "option_run": 540,
          "five_option_run": 269,
          "replaces_run": null,
          "option": {
            "answered": 2189,
            "attempt_rate": 0.9959053685168335,
            "right": 1485,
            "wrong": 704,
            "dont_know": 9
          },
          "derived": {
            "cut_off": 0.28390823189469805,
            "answered": 2189,
            "attempt_rate": 0.9959053685168335,
            "right": 1486,
            "wrong": 703,
            "abstained": 9,
            "below": {
              "cut_off": 0.2830032727346068,
              "answered": 2190
            },
            "above": {
              "cut_off": 0.2875844645345384,
              "answered": 2188
            }
          },
          "option_only_wrong": 54,
          "confidence_only_wrong": 53,
          "p": 1.0,
          "p_holm": null
        },
        {
          "name": "vLLM-SR d3",
          "option_run": 544,
          "five_option_run": 259,
          "replaces_run": null,
          "option": {
            "answered": 2196,
            "attempt_rate": 0.9990900818926297,
            "right": 1482,
            "wrong": 714,
            "dont_know": 2
          },
          "derived": {
            "cut_off": 0.23305362956033399,
            "answered": 2196,
            "attempt_rate": 0.9990900818926297,
            "right": 1490,
            "wrong": 706,
            "abstained": 2,
            "below": {
              "cut_off": 0.22942051369818334,
              "answered": 2197
            },
            "above": {
              "cut_off": 0.23517204742681724,
              "answered": 2195
            }
          },
          "option_only_wrong": 39,
          "confidence_only_wrong": 31,
          "p": 0.40296307504607976,
          "p_holm": null
        },
        {
          "name": "Perplexity Decider 27B v1.2",
          "option_run": 531,
          "five_option_run": 217,
          "replaces_run": null,
          "option": {
            "answered": 2194,
            "attempt_rate": 0.9981801637852593,
            "right": 1466,
            "wrong": 728,
            "dont_know": 4
          },
          "derived": {
            "cut_off": 0.22165176602679143,
            "answered": 2194,
            "attempt_rate": 0.9981801637852593,
            "right": 1474,
            "wrong": 720,
            "abstained": 4,
            "below": {
              "cut_off": 0.2205054759979248,
              "answered": 2195
            },
            "above": {
              "cut_off": 0.22231751017816334,
              "answered": 2193
            }
          },
          "option_only_wrong": 52,
          "confidence_only_wrong": 44,
          "p": 0.4751520499865724,
          "p_holm": null
        },
        {
          "name": "Fastino GLiDE v1.2",
          "option_run": 525,
          "five_option_run": 248,
          "replaces_run": null,
          "option": {
            "answered": 1772,
            "attempt_rate": 0.8061874431301183,
            "right": 1345,
            "wrong": 427,
            "dont_know": 426
          },
          "derived": {
            "cut_off": 0.4535287127682595,
            "answered": 1772,
            "attempt_rate": 0.8061874431301183,
            "right": 1369,
            "wrong": 403,
            "abstained": 426,
            "below": {
              "cut_off": 0.45340738860669905,
              "answered": 1773
            },
            "above": {
              "cut_off": 0.4535807144797621,
              "answered": 1771
            }
          },
          "option_only_wrong": 77,
          "confidence_only_wrong": 53,
          "p": 0.04325513078582877,
          "p_holm": 0.21627565392914386
        },
        {
          "name": "Liquid d1-ultra",
          "option_run": 511,
          "five_option_run": 199,
          "replaces_run": null,
          "option": {
            "answered": 1582,
            "attempt_rate": 0.7197452229299363,
            "right": 1366,
            "wrong": 216,
            "dont_know": 616
          },
          "derived": {
            "cut_off": 0.5630411363694499,
            "answered": 1582,
            "attempt_rate": 0.7197452229299363,
            "right": 1415,
            "wrong": 167,
            "abstained": 616,
            "below": {
              "cut_off": 0.5621712429171062,
              "answered": 1583
            },
            "above": {
              "cut_off": 0.5632390296623447,
              "answered": 1581
            }
          },
          "option_only_wrong": 98,
          "confidence_only_wrong": 49,
          "p": 6.51752575962746e-05,
          "p_holm": 0.00045622680317392223
        },
        {
          "name": "Drex Max 1.1",
          "option_run": 513,
          "five_option_run": 326,
          "replaces_run": null,
          "option": {
            "answered": 1712,
            "attempt_rate": 0.7788898999090081,
            "right": 1299,
            "wrong": 413,
            "dont_know": 486
          },
          "derived": {
            "cut_off": 0.447757591729814,
            "answered": 1712,
            "attempt_rate": 0.7788898999090081,
            "right": 1316,
            "wrong": 396,
            "abstained": 486,
            "below": {
              "cut_off": 0.44701865315437317,
              "answered": 1713
            },
            "above": {
              "cut_off": 0.44790264940914626,
              "answered": 1711
            }
          },
          "option_only_wrong": 106,
          "confidence_only_wrong": 89,
          "p": 0.2518223958737491,
          "p_holm": 0.6238752645558847
        },
        {
          "name": "Fastino GLiDE",
          "option_run": 510,
          "five_option_run": 517,
          "replaces_run": 129,
          "option": {
            "answered": 1617,
            "attempt_rate": 0.7356687898089171,
            "right": 1282,
            "wrong": 335,
            "dont_know": 581
          },
          "derived": {
            "cut_off": 0.5222132078753563,
            "answered": 1617,
            "attempt_rate": 0.7356687898089171,
            "right": 1298,
            "wrong": 319,
            "abstained": 581,
            "below": {
              "cut_off": 0.5220279305992286,
              "answered": 1618
            },
            "above": {
              "cut_off": 0.5231063044519938,
              "answered": 1616
            }
          },
          "option_only_wrong": 79,
          "confidence_only_wrong": 63,
          "p": 0.20795842151862826,
          "p_holm": 0.6238752645558847
        },
        {
          "name": "SubQ Snap 1.1",
          "option_run": 512,
          "five_option_run": 346,
          "replaces_run": null,
          "option": {
            "answered": 1700,
            "attempt_rate": 0.7734303912647862,
            "right": 1246,
            "wrong": 454,
            "dont_know": 498
          },
          "derived": {
            "cut_off": 0.537179708480835,
            "answered": 1700,
            "attempt_rate": 0.7734303912647862,
            "right": 1255,
            "wrong": 445,
            "abstained": 498,
            "below": {
              "cut_off": 0.537083089351654,
              "answered": 1701
            },
            "above": {
              "cut_off": 0.5380315780639648,
              "answered": 1699
            }
          },
          "option_only_wrong": 141,
          "confidence_only_wrong": 132,
          "p": 0.6283375482150046,
          "p_holm": 0.6283375482150046
        },
        {
          "name": "Mammoth-1",
          "option_run": 519,
          "five_option_run": 374,
          "replaces_run": null,
          "option": {
            "answered": 1445,
            "attempt_rate": 0.6574158325750682,
            "right": 1169,
            "wrong": 276,
            "dont_know": 753
          },
          "derived": {
            "cut_off": 0.595068,
            "answered": 1445,
            "attempt_rate": 0.6574158325750682,
            "right": 1195,
            "wrong": 250,
            "abstained": 753,
            "below": {
              "cut_off": 0.594775,
              "answered": 1446
            },
            "above": {
              "cut_off": 0.595632,
              "answered": 1444
            }
          },
          "option_only_wrong": 64,
          "confidence_only_wrong": 38,
          "p": 0.0129204519111262,
          "p_holm": 0.0775227114667572
        },
        {
          "name": "Elephant-1",
          "option_run": 515,
          "five_option_run": 373,
          "replaces_run": null,
          "option": {
            "answered": 1594,
            "attempt_rate": 0.7252047315741583,
            "right": 972,
            "wrong": 622,
            "dont_know": 604
          },
          "derived": {
            "cut_off": 0.437598,
            "answered": 1594,
            "attempt_rate": 0.7252047315741583,
            "right": 1000,
            "wrong": 594,
            "abstained": 604,
            "below": {
              "cut_off": 0.437402,
              "answered": 1595
            },
            "above": {
              "cut_off": 0.437758,
              "answered": 1593
            }
          },
          "option_only_wrong": 120,
          "confidence_only_wrong": 92,
          "p": 0.06343187528222884,
          "p_holm": 0.25372750112891534
        }
      ]
    }
  },
  "hosted_glide_rerun": {
    "run": 517,
    "rerun": "2026-11-02",
    "date": "2026-11-02",
    "prompt_version": "254597b8be16",
    "replaces_run": 129,
    "accuracy": 0.681073703366697,
    "right": 1497,
    "wrong": 701,
    "invalid": 0,
    "dont_know_run": 351,
    "derived": {
      "cut_off": 0.4955913130694101,
      "attempt_rate": 0.7629663330300273,
      "non_hallucination": 0.5974770642201835,
      "right": 1326,
      "wrong": 351,
      "abstained": 521
    },
    "vs_confidence": {
      "option_only_wrong": 90,
      "confidence_only_wrong": 85,
      "p": 0.7624694402612475
    }
  },
  "english_reading": {
    "instructions": "Which option is the correct answer to this English multiple-choice question?",
    "options_heading": "Options:",
    "prompt_version": "47f5776e71db",
    "translations_sha256": "bf0d47cd385a93050c65906977e199ef7e45d5848dbbdef2bc8d124f4d5abf68",
    "reading_questions": 656,
    "questions": 547,
    "flagged": 109,
    "models": [
      {
        "name": "Liquid d1-ultra",
        "model": "d1-ultra",
        "provider": "liquid",
        "run": 411,
        "runs": [
          411,
          418,
          422,
          426,
          428
        ],
        "turkish_run": 199,
        "turkish_runs": [
          199,
          200,
          201,
          202,
          203
        ],
        "english": {
          "right": 496,
          "wrong": 51,
          "invalid": 0,
          "accuracy": 0.906764168190128
        },
        "turkish": {
          "right": 512,
          "wrong": 35,
          "invalid": 0,
          "accuracy": 0.9360146252285192
        },
        "difference": -0.029250457038391242,
        "spread": {
          "english": {
            "low": 0.9012797074954296,
            "high": 0.9104204753199269
          },
          "turkish": {
            "low": 0.926873857404022,
            "high": 0.9360146252285192
          }
        },
        "answers_changed": {
          "english": 28,
          "turkish": 18
        },
        "paired": {
          "english_only_right": 8,
          "turkish_only_right": 24,
          "p": 0.0070003666914999485,
          "p_holm": 0.10500550037249923
        }
      },
      {
        "name": "Fastino GLiDE v1.2",
        "model": "glide-v1.2",
        "provider": "glide",
        "run": 521,
        "runs": [
          521
        ],
        "turkish_run": 248,
        "turkish_runs": [
          248,
          250,
          251,
          252,
          253
        ],
        "english": {
          "right": 494,
          "wrong": 53,
          "invalid": 0,
          "accuracy": 0.903107861060329
        },
        "turkish": {
          "right": 495,
          "wrong": 52,
          "invalid": 0,
          "accuracy": 0.9049360146252285
        },
        "difference": -0.001828153564899515,
        "spread": {
          "english": null,
          "turkish": {
            "low": 0.9049360146252285,
            "high": 0.9049360146252285
          }
        },
        "answers_changed": {
          "english": null,
          "turkish": 0
        },
        "paired": {
          "english_only_right": 21,
          "turkish_only_right": 22,
          "p": 1.0,
          "p_holm": 1.0
        }
      },
      {
        "name": "Mammoth-1",
        "model": "baselabs/mammoth-1",
        "provider": "baselabs",
        "run": 503,
        "runs": [
          503
        ],
        "turkish_run": 374,
        "turkish_runs": [
          374,
          377,
          380,
          383,
          384
        ],
        "english": {
          "right": 493,
          "wrong": 54,
          "invalid": 0,
          "accuracy": 0.9012797074954296
        },
        "turkish": {
          "right": 508,
          "wrong": 39,
          "invalid": 0,
          "accuracy": 0.9287020109689214
        },
        "difference": -0.027422303473491727,
        "spread": {
          "english": null,
          "turkish": {
            "low": 0.9287020109689214,
            "high": 0.9287020109689214
          }
        },
        "answers_changed": {
          "english": null,
          "turkish": 0
        },
        "paired": {
          "english_only_right": 15,
          "turkish_only_right": 30,
          "p": 0.035697803555194696,
          "p_holm": 0.462575417879012
        }
      },
      {
        "name": "vLLM-SR d3",
        "model": "vllm-sr/d3",
        "provider": "vllm-sr",
        "run": 554,
        "runs": [
          554
        ],
        "turkish_run": 259,
        "turkish_runs": [
          259,
          260,
          261,
          262,
          263
        ],
        "english": {
          "right": 491,
          "wrong": 56,
          "invalid": 0,
          "accuracy": 0.8976234003656307
        },
        "turkish": {
          "right": 498,
          "wrong": 49,
          "invalid": 0,
          "accuracy": 0.9104204753199269
        },
        "difference": -0.012797074954296161,
        "spread": {
          "english": null,
          "turkish": {
            "low": 0.9104204753199269,
            "high": 0.9104204753199269
          }
        },
        "answers_changed": {
          "english": null,
          "turkish": 0
        },
        "paired": {
          "english_only_right": 15,
          "turkish_only_right": 22,
          "p": 0.3240086000878364,
          "p_holm": 1.0
        }
      },
      {
        "name": "Drex Max v1",
        "model": "drex-max-v1",
        "provider": "drex-max",
        "run": 552,
        "runs": [
          552
        ],
        "turkish_run": 322,
        "turkish_runs": [
          322,
          323,
          324,
          327,
          328
        ],
        "english": {
          "right": 490,
          "wrong": 57,
          "invalid": 0,
          "accuracy": 0.8957952468007313
        },
        "turkish": {
          "right": 500,
          "wrong": 47,
          "invalid": 0,
          "accuracy": 0.9140767824497258
        },
        "difference": -0.018281535648994485,
        "spread": {
          "english": null,
          "turkish": {
            "low": 0.9140767824497258,
            "high": 0.9140767824497258
          }
        },
        "answers_changed": {
          "english": null,
          "turkish": 0
        },
        "paired": {
          "english_only_right": 19,
          "turkish_only_right": 29,
          "p": 0.19341265286193732,
          "p_holm": 1.0
        }
      },
      {
        "name": "vLLM-SR d3-ultra",
        "model": "vllm-sr/d3-ultra",
        "provider": "d3-server",
        "run": 550,
        "runs": [
          550
        ],
        "turkish_run": 269,
        "turkish_runs": [
          269,
          270,
          271,
          272,
          273
        ],
        "english": {
          "right": 489,
          "wrong": 58,
          "invalid": 0,
          "accuracy": 0.8939670932358318
        },
        "turkish": {
          "right": 500,
          "wrong": 47,
          "invalid": 0,
          "accuracy": 0.9140767824497258
        },
        "difference": -0.020109689213894,
        "spread": {
          "english": null,
          "turkish": {
            "low": 0.9140767824497258,
            "high": 0.9140767824497258
          }
        },
        "answers_changed": {
          "english": null,
          "turkish": 0
        },
        "paired": {
          "english_only_right": 22,
          "turkish_only_right": 33,
          "p": 0.17700136477035816,
          "p_holm": 1.0
        }
      },
      {
        "name": "Perplexity Decider 27B v1.2",
        "model": "pplx-decider-v1.2",
        "provider": "pplx-decider",
        "run": 527,
        "runs": [
          527
        ],
        "turkish_run": 217,
        "turkish_runs": [
          217,
          218,
          219,
          220,
          221
        ],
        "english": {
          "right": 485,
          "wrong": 62,
          "invalid": 0,
          "accuracy": 0.886654478976234
        },
        "turkish": {
          "right": 500,
          "wrong": 47,
          "invalid": 0,
          "accuracy": 0.9140767824497258
        },
        "difference": -0.027422303473491838,
        "spread": {
          "english": null,
          "turkish": {
            "low": 0.9140767824497258,
            "high": 0.9140767824497258
          }
        },
        "answers_changed": {
          "english": null,
          "turkish": 0
        },
        "paired": {
          "english_only_right": 15,
          "turkish_only_right": 30,
          "p": 0.035697803555194696,
          "p_holm": 0.462575417879012
        }
      },
      {
        "name": "Drex Max 1.1",
        "model": "drex-max-v1.1",
        "provider": "nace",
        "run": 399,
        "runs": [
          399
        ],
        "turkish_run": 326,
        "turkish_runs": [
          326,
          329,
          330,
          331,
          332
        ],
        "english": {
          "right": 484,
          "wrong": 63,
          "invalid": 0,
          "accuracy": 0.8848263254113345
        },
        "turkish": {
          "right": 493,
          "wrong": 54,
          "invalid": 0,
          "accuracy": 0.9012797074954296
        },
        "difference": -0.01645338208409508,
        "spread": {
          "english": null,
          "turkish": {
            "low": 0.9012797074954296,
            "high": 0.9012797074954296
          }
        },
        "answers_changed": {
          "english": null,
          "turkish": 0
        },
        "paired": {
          "english_only_right": 18,
          "turkish_only_right": 27,
          "p": 0.23269319190831084,
          "p_holm": 1.0
        }
      },
      {
        "name": "Liquid d1",
        "model": "d1",
        "provider": "liquid",
        "run": 430,
        "runs": [
          430,
          432,
          433,
          434,
          435
        ],
        "turkish_run": 52,
        "turkish_runs": [
          52,
          66,
          94,
          105,
          108
        ],
        "english": {
          "right": 475,
          "wrong": 72,
          "invalid": 0,
          "accuracy": 0.8683729433272395
        },
        "turkish": {
          "right": 465,
          "wrong": 82,
          "invalid": 0,
          "accuracy": 0.850091407678245
        },
        "difference": 0.018281535648994485,
        "spread": {
          "english": {
            "low": 0.8574040219378428,
            "high": 0.8793418647166362
          },
          "turkish": {
            "low": 0.850091407678245,
            "high": 0.8574040219378428
          }
        },
        "answers_changed": {
          "english": 50,
          "turkish": 19
        },
        "paired": {
          "english_only_right": 36,
          "turkish_only_right": 26,
          "p": 0.2528543941881108,
          "p_holm": 1.0
        }
      },
      {
        "name": "Jev 1.14.0",
        "model": "jev-1.14.0",
        "provider": "jev",
        "run": 410,
        "runs": [
          410,
          416,
          421,
          423,
          425
        ],
        "turkish_run": 194,
        "turkish_runs": [
          194,
          195,
          196,
          197,
          198
        ],
        "english": {
          "right": 471,
          "wrong": 76,
          "invalid": 0,
          "accuracy": 0.8610603290676416
        },
        "turkish": {
          "right": 477,
          "wrong": 70,
          "invalid": 0,
          "accuracy": 0.8720292504570384
        },
        "difference": -0.010968921389396757,
        "spread": {
          "english": {
            "low": 0.8610603290676416,
            "high": 0.8647166361974405
          },
          "turkish": {
            "low": 0.8647166361974405,
            "high": 0.8720292504570384
          }
        },
        "answers_changed": {
          "english": 14,
          "turkish": 17
        },
        "paired": {
          "english_only_right": 17,
          "turkish_only_right": 23,
          "p": 0.42959050784338615,
          "p_holm": 1.0
        }
      },
      {
        "name": "SubQ Snap 1.1",
        "model": "subq-snap-1.1",
        "provider": "subq",
        "run": 413,
        "runs": [
          413
        ],
        "turkish_run": 346,
        "turkish_runs": [
          346,
          387,
          388,
          389,
          390
        ],
        "english": {
          "right": 470,
          "wrong": 77,
          "invalid": 0,
          "accuracy": 0.8592321755027422
        },
        "turkish": {
          "right": 488,
          "wrong": 59,
          "invalid": 0,
          "accuracy": 0.8921389396709324
        },
        "difference": -0.03290676416819016,
        "spread": {
          "english": null,
          "turkish": {
            "low": 0.8921389396709324,
            "high": 0.8921389396709324
          }
        },
        "answers_changed": {
          "english": null,
          "turkish": 0
        },
        "paired": {
          "english_only_right": 23,
          "turkish_only_right": 41,
          "p": 0.0327657590988232,
          "p_holm": 0.45872062738352476
        }
      },
      {
        "name": "Inception Mercury Decide",
        "model": "inception/mercury-decide",
        "provider": "openrouter",
        "run": 398,
        "runs": [
          398,
          400,
          401,
          402,
          403
        ],
        "turkish_run": 204,
        "turkish_runs": [
          204,
          210,
          212,
          214,
          216
        ],
        "english": {
          "right": 469,
          "wrong": 78,
          "invalid": 0,
          "accuracy": 0.8574040219378428
        },
        "turkish": {
          "right": 476,
          "wrong": 71,
          "invalid": 0,
          "accuracy": 0.870201096892139
        },
        "difference": -0.012797074954296161,
        "spread": {
          "english": {
            "low": 0.850091407678245,
            "high": 0.8592321755027422
          },
          "turkish": {
            "low": 0.8665447897623401,
            "high": 0.8738574040219378
          }
        },
        "answers_changed": {
          "english": 20,
          "turkish": 19
        },
        "paired": {
          "english_only_right": 28,
          "turkish_only_right": 35,
          "p": 0.4499625362902225,
          "p_holm": 1.0
        }
      },
      {
        "name": "Microsoft-Decision-1",
        "model": "microsoft/microsoft-decision-1",
        "provider": "openrouter",
        "run": 412,
        "runs": [
          412,
          420,
          424,
          427,
          431
        ],
        "turkish_run": 205,
        "turkish_runs": [
          205,
          209,
          211,
          213,
          215
        ],
        "english": {
          "right": 469,
          "wrong": 78,
          "invalid": 0,
          "accuracy": 0.8574040219378428
        },
        "turkish": {
          "right": 418,
          "wrong": 129,
          "invalid": 0,
          "accuracy": 0.7641681901279708
        },
        "difference": 0.09323583180987205,
        "spread": {
          "english": {
            "low": 0.850091407678245,
            "high": 0.8610603290676416
          },
          "turkish": {
            "low": 0.7623400365630713,
            "high": 0.7714808043875686
          }
        },
        "answers_changed": {
          "english": 19,
          "turkish": 36
        },
        "paired": {
          "english_only_right": 74,
          "turkish_only_right": 23,
          "p": 2.027109075312698e-07,
          "p_holm": 3.4460854280315867e-06
        }
      },
      {
        "name": "Elephant-1",
        "model": "baselabs/elephant-1",
        "provider": "baselabs",
        "run": 499,
        "runs": [
          499
        ],
        "turkish_run": 373,
        "turkish_runs": [
          373,
          375,
          376,
          378,
          379
        ],
        "english": {
          "right": 467,
          "wrong": 80,
          "invalid": 0,
          "accuracy": 0.8537477148080439
        },
        "turkish": {
          "right": 457,
          "wrong": 90,
          "invalid": 0,
          "accuracy": 0.8354661791590493
        },
        "difference": 0.018281535648994596,
        "spread": {
          "english": null,
          "turkish": {
            "low": 0.8354661791590493,
            "high": 0.8354661791590493
          }
        },
        "answers_changed": {
          "english": null,
          "turkish": 0
        },
        "paired": {
          "english_only_right": 39,
          "turkish_only_right": 29,
          "p": 0.2749903961710005,
          "p_holm": 1.0
        }
      },
      {
        "name": "Celeris-1 Decision",
        "model": "celeris-1-decision",
        "provider": "celeris",
        "run": 546,
        "runs": [
          546
        ],
        "turkish_run": 150,
        "turkish_runs": [
          150,
          151,
          156,
          157,
          158
        ],
        "english": {
          "right": 460,
          "wrong": 87,
          "invalid": 0,
          "accuracy": 0.8409506398537477
        },
        "turkish": {
          "right": 478,
          "wrong": 69,
          "invalid": 0,
          "accuracy": 0.8738574040219378
        },
        "difference": -0.03290676416819005,
        "spread": {
          "english": null,
          "turkish": {
            "low": 0.8665447897623401,
            "high": 0.8738574040219378
          }
        },
        "answers_changed": {
          "english": null,
          "turkish": 19
        },
        "paired": {
          "english_only_right": 24,
          "turkish_only_right": 42,
          "p": 0.03558272445223169,
          "p_holm": 0.462575417879012
        }
      },
      {
        "name": "GPT-6 Luna",
        "model": "gpt-6-luna",
        "provider": "openai-decisions",
        "run": 397,
        "runs": [
          397
        ],
        "turkish_run": 143,
        "turkish_runs": [
          143
        ],
        "english": {
          "right": 460,
          "wrong": 85,
          "invalid": 2,
          "accuracy": 0.8409506398537477
        },
        "turkish": {
          "right": 474,
          "wrong": 72,
          "invalid": 1,
          "accuracy": 0.8665447897623401
        },
        "difference": -0.025594149908592323,
        "spread": null,
        "answers_changed": null,
        "paired": {
          "english_only_right": 28,
          "turkish_only_right": 42,
          "p": 0.11960933562988493,
          "p_holm": 1.0
        }
      },
      {
        "name": "Perplexity Decider Flash v1.2",
        "model": "pplx-decider-flash-v1.2",
        "provider": "pplx-decider",
        "run": 533,
        "runs": [
          533
        ],
        "turkish_run": 223,
        "turkish_runs": [
          223,
          224,
          225,
          226,
          227
        ],
        "english": {
          "right": 448,
          "wrong": 99,
          "invalid": 0,
          "accuracy": 0.8190127970749543
        },
        "turkish": {
          "right": 419,
          "wrong": 128,
          "invalid": 0,
          "accuracy": 0.7659963436928702
        },
        "difference": 0.05301645338208416,
        "spread": {
          "english": null,
          "turkish": {
            "low": 0.7659963436928702,
            "high": 0.7659963436928702
          }
        },
        "answers_changed": {
          "english": null,
          "turkish": 0
        },
        "paired": {
          "english_only_right": 63,
          "turkish_only_right": 34,
          "p": 0.00422580392168276,
          "p_holm": 0.06761286274692416
        }
      }
    ],
    "dont_know": {
      "option": "I don't know",
      "sentence": "If you do not know the answer, choose I don't know.",
      "instructions": "Which option is the correct answer to this English multiple-choice question? If you do not know the answer, choose I don't know.",
      "prompt_version": "413d6775fc58",
      "questions": 547,
      "models": [
        {
          "name": "Liquid d1-ultra",
          "model": "d1-ultra",
          "provider": "liquid",
          "text_run": 199,
          "english": {
            "run": 449,
            "runs": [
              449,
              456,
              462,
              466,
              474
            ],
            "right": 457,
            "wrong": 39,
            "dont_know": 51,
            "invalid": 0,
            "accuracy": 0.8354661791590493,
            "attempt_rate": 0.906764168190128,
            "non_hallucination": 0.5666666666666667,
            "mean_dont_know_probability": 0.2527647853971477
          },
          "turkish": {
            "run": 350,
            "runs": [
              350,
              354,
              360,
              365,
              370
            ],
            "subset": false,
            "right": 491,
            "wrong": 26,
            "dont_know": 30,
            "invalid": 0,
            "accuracy": 0.8976234003656307,
            "attempt_rate": 0.9451553930530164,
            "non_hallucination": 0.5357142857142857,
            "mean_dont_know_probability": 0.2275111857445987
          },
          "spread": {
            "english": {
              "accuracy": [
                0.8263254113345521,
                0.8446069469835467
              ],
              "attempt_rate": [
                0.8957952468007313,
                0.9140767824497258
              ],
              "non_hallucination": [
                0.5529411764705883,
                0.6129032258064516
              ],
              "mean_dont_know_probability": [
                0.2527647853971477,
                0.2564300726801178
              ]
            },
            "turkish": {
              "accuracy": [
                0.8921389396709324,
                0.9012797074954296
              ],
              "attempt_rate": [
                0.9414990859232175,
                0.9561243144424132
              ],
              "non_hallucination": [
                0.4444444444444444,
                0.5925925925925926
              ],
              "mean_dont_know_probability": [
                0.22441245809913582,
                0.23042049881081036
              ]
            }
          },
          "paired": {
            "english_only_wrong": 20,
            "turkish_only_wrong": 7,
            "p": 0.0191572904586792,
            "p_holm": 0.2107301950454712
          }
        },
        {
          "name": "Drex Max 1.1",
          "model": "drex-max-v1.1",
          "provider": "nace",
          "text_run": 326,
          "english": {
            "run": 447,
            "runs": [
              447
            ],
            "right": 463,
            "wrong": 41,
            "dont_know": 43,
            "invalid": 0,
            "accuracy": 0.8464351005484461,
            "attempt_rate": 0.9213893967093236,
            "non_hallucination": 0.5119047619047619,
            "mean_dont_know_probability": 0.1425561642833011
          },
          "turkish": {
            "run": 352,
            "runs": [
              352
            ],
            "subset": false,
            "right": 487,
            "wrong": 51,
            "dont_know": 9,
            "invalid": 0,
            "accuracy": 0.8903107861060329,
            "attempt_rate": 0.9835466179159049,
            "non_hallucination": 0.15,
            "mean_dont_know_probability": 0.08407015445670117
          },
          "spread": null,
          "paired": {
            "english_only_wrong": 20,
            "turkish_only_wrong": 30,
            "p": 0.20263875106454066,
            "p_holm": 1.0
          }
        },
        {
          "name": "Mammoth-1",
          "model": "baselabs/mammoth-1",
          "provider": "baselabs",
          "text_run": 374,
          "english": {
            "run": 505,
            "runs": [
              505
            ],
            "right": 474,
            "wrong": 36,
            "dont_know": 37,
            "invalid": 0,
            "accuracy": 0.8665447897623401,
            "attempt_rate": 0.9323583180987203,
            "non_hallucination": 0.5068493150684932,
            "mean_dont_know_probability": 0.09803614442413162
          },
          "turkish": {
            "run": 386,
            "runs": [
              386
            ],
            "subset": false,
            "right": 490,
            "wrong": 33,
            "dont_know": 24,
            "invalid": 0,
            "accuracy": 0.8957952468007313,
            "attempt_rate": 0.9561243144424132,
            "non_hallucination": 0.42105263157894735,
            "mean_dont_know_probability": 0.07447928702010968
          },
          "spread": null,
          "paired": {
            "english_only_wrong": 18,
            "turkish_only_wrong": 15,
            "p": 0.7283324808813632,
            "p_holm": 1.0
          }
        },
        {
          "name": "Fastino GLiDE v1.2",
          "model": "glide-v1.2",
          "provider": "glide",
          "text_run": 248,
          "english": {
            "run": 523,
            "runs": [
              523
            ],
            "right": 481,
            "wrong": 36,
            "dont_know": 30,
            "invalid": 0,
            "accuracy": 0.8793418647166362,
            "attempt_rate": 0.9451553930530164,
            "non_hallucination": 0.45454545454545453,
            "mean_dont_know_probability": 0.12485239376322405
          },
          "turkish": {
            "run": 369,
            "runs": [
              369
            ],
            "subset": false,
            "right": 494,
            "wrong": 47,
            "dont_know": 6,
            "invalid": 0,
            "accuracy": 0.903107861060329,
            "attempt_rate": 0.9890310786106032,
            "non_hallucination": 0.11320754716981132,
            "mean_dont_know_probability": 0.08693889552230191
          },
          "spread": null,
          "paired": {
            "english_only_wrong": 16,
            "turkish_only_wrong": 27,
            "p": 0.12628947438543037,
            "p_holm": 0.8840263206980126
          }
        },
        {
          "name": "Elephant-1",
          "model": "baselabs/elephant-1",
          "provider": "baselabs",
          "text_run": 373,
          "english": {
            "run": 501,
            "runs": [
              501
            ],
            "right": 457,
            "wrong": 67,
            "dont_know": 23,
            "invalid": 0,
            "accuracy": 0.8354661791590493,
            "attempt_rate": 0.9579524680073126,
            "non_hallucination": 0.25555555555555554,
            "mean_dont_know_probability": 0.06769455393053016
          },
          "turkish": {
            "run": 382,
            "runs": [
              382
            ],
            "subset": false,
            "right": 450,
            "wrong": 82,
            "dont_know": 15,
            "invalid": 0,
            "accuracy": 0.8226691042047533,
            "attempt_rate": 0.9725776965265083,
            "non_hallucination": 0.15463917525773196,
            "mean_dont_know_probability": 0.057047244972577696
          },
          "spread": null,
          "paired": {
            "english_only_wrong": 31,
            "turkish_only_wrong": 46,
            "p": 0.11001559898513336,
            "p_holm": 0.8801247918810668
          }
        },
        {
          "name": "SubQ Snap 1.1",
          "model": "subq-snap-1.1",
          "provider": "subq",
          "text_run": 346,
          "english": {
            "run": 448,
            "runs": [
              448
            ],
            "right": 463,
            "wrong": 63,
            "dont_know": 21,
            "invalid": 0,
            "accuracy": 0.8464351005484461,
            "attempt_rate": 0.9616087751371115,
            "non_hallucination": 0.25,
            "mean_dont_know_probability": 0.11709256155179708
          },
          "turkish": {
            "run": 392,
            "runs": [
              392
            ],
            "subset": false,
            "right": 477,
            "wrong": 51,
            "dont_know": 19,
            "invalid": 0,
            "accuracy": 0.8720292504570384,
            "attempt_rate": 0.9652650822669104,
            "non_hallucination": 0.2714285714285714,
            "mean_dont_know_probability": 0.08417631924225587
          },
          "spread": null,
          "paired": {
            "english_only_wrong": 39,
            "turkish_only_wrong": 27,
            "p": 0.1752857944430353,
            "p_holm": 1.0
          }
        },
        {
          "name": "Inception Mercury Decide",
          "model": "inception/mercury-decide",
          "provider": "openrouter",
          "text_run": 204,
          "english": {
            "run": 450,
            "runs": [
              450,
              460,
              469,
              478,
              484
            ],
            "right": 465,
            "wrong": 71,
            "dont_know": 11,
            "invalid": 0,
            "accuracy": 0.850091407678245,
            "attempt_rate": 0.979890310786106,
            "non_hallucination": 0.13414634146341464,
            "mean_dont_know_probability": 0.031531572744804136
          },
          "turkish": {
            "run": 489,
            "runs": [
              489,
              491,
              493,
              495,
              497
            ],
            "subset": true,
            "right": 468,
            "wrong": 71,
            "dont_know": 8,
            "invalid": 0,
            "accuracy": 0.8555758683729433,
            "attempt_rate": 0.9853747714808044,
            "non_hallucination": 0.10126582278481013,
            "mean_dont_know_probability": 0.019188870343915032
          },
          "spread": {
            "english": {
              "accuracy": [
                0.850091407678245,
                0.8537477148080439
              ],
              "attempt_rate": [
                0.979890310786106,
                0.9835466179159049
              ],
              "non_hallucination": [
                0.1111111111111111,
                0.13414634146341464
              ],
              "mean_dont_know_probability": [
                0.030897669793960476,
                0.03153736749203556
              ]
            },
            "turkish": {
              "accuracy": [
                0.8555758683729433,
                0.8610603290676416
              ],
              "attempt_rate": [
                0.9853747714808044,
                0.9853747714808044
              ],
              "non_hallucination": [
                0.10126582278481013,
                0.10526315789473684
              ],
              "mean_dont_know_probability": [
                0.018941783254423132,
                0.019928634779118805
              ]
            }
          },
          "paired": {
            "english_only_wrong": 29,
            "turkish_only_wrong": 29,
            "p": 1.0,
            "p_holm": 1.0
          }
        },
        {
          "name": "GPT-6 Luna",
          "model": "gpt-6-luna",
          "provider": "openai-decisions",
          "text_run": 143,
          "english": {
            "run": 444,
            "runs": [
              444
            ],
            "right": 441,
            "wrong": 96,
            "dont_know": 10,
            "invalid": 0,
            "accuracy": 0.8062157221206582,
            "attempt_rate": 0.9817184643510055,
            "non_hallucination": 0.09433962264150944,
            "mean_dont_know_probability": 0.030073126142595975
          },
          "turkish": {
            "run": 453,
            "runs": [
              453
            ],
            "subset": true,
            "right": 463,
            "wrong": 78,
            "dont_know": 6,
            "invalid": 0,
            "accuracy": 0.8464351005484461,
            "attempt_rate": 0.9890310786106032,
            "non_hallucination": 0.07142857142857142,
            "mean_dont_know_probability": 0.02712979890310786
          },
          "spread": null,
          "paired": {
            "english_only_wrong": 47,
            "turkish_only_wrong": 29,
            "p": 0.05045241024721763,
            "p_holm": 0.48766765904474596
          }
        },
        {
          "name": "Perplexity Decider Flash v1.2",
          "model": "pplx-decider-flash-v1.2",
          "provider": "pplx-decider",
          "text_run": 223,
          "english": {
            "run": 535,
            "runs": [
              535
            ],
            "right": 442,
            "wrong": 100,
            "dont_know": 5,
            "invalid": 0,
            "accuracy": 0.8080438756855576,
            "attempt_rate": 0.9908592321755028,
            "non_hallucination": 0.047619047619047616,
            "mean_dont_know_probability": 0.05526566245284184
          },
          "turkish": {
            "run": 537,
            "runs": [
              537
            ],
            "subset": true,
            "right": 408,
            "wrong": 133,
            "dont_know": 6,
            "invalid": 0,
            "accuracy": 0.7458866544789763,
            "attempt_rate": 0.9890310786106032,
            "non_hallucination": 0.04316546762589928,
            "mean_dont_know_probability": 0.06262391641941556
          },
          "spread": null,
          "paired": {
            "english_only_wrong": 36,
            "turkish_only_wrong": 69,
            "p": 0.00165902688989955,
            "p_holm": 0.02156734956869415
          }
        },
        {
          "name": "Jev 1.14.0",
          "model": "jev-1.14.0",
          "provider": "jev",
          "text_run": 194,
          "english": {
            "run": 446,
            "runs": [
              446,
              454,
              457,
              461,
              464
            ],
            "right": 472,
            "wrong": 72,
            "dont_know": 3,
            "invalid": 0,
            "accuracy": 0.8628884826325411,
            "attempt_rate": 0.9945155393053017,
            "non_hallucination": 0.04,
            "mean_dont_know_probability": 0.015521023765996345
          },
          "turkish": {
            "run": 470,
            "runs": [
              470,
              473,
              476,
              480,
              482
            ],
            "subset": true,
            "right": 475,
            "wrong": 69,
            "dont_know": 3,
            "invalid": 0,
            "accuracy": 0.8683729433272395,
            "attempt_rate": 0.9945155393053017,
            "non_hallucination": 0.041666666666666664,
            "mean_dont_know_probability": 0.016691042047531995
          },
          "spread": {
            "english": {
              "accuracy": [
                0.8592321755027422,
                0.8665447897623401
              ],
              "attempt_rate": [
                0.9926873857404022,
                0.9945155393053017
              ],
              "non_hallucination": [
                0.04,
                0.05194805194805195
              ],
              "mean_dont_know_probability": [
                0.015521023765996345,
                0.015630712979890313
              ]
            },
            "turkish": {
              "accuracy": [
                0.8628884826325411,
                0.8775137111517367
              ],
              "attempt_rate": [
                0.9945155393053017,
                0.9945155393053017
              ],
              "non_hallucination": [
                0.04,
                0.04477611940298507
              ],
              "mean_dont_know_probability": [
                0.01627056672760512,
                0.016691042047531995
              ]
            }
          },
          "paired": {
            "english_only_wrong": 20,
            "turkish_only_wrong": 17,
            "p": 0.7428293587290682,
            "p_holm": 1.0
          }
        },
        {
          "name": "Microsoft-Decision-1",
          "model": "microsoft/microsoft-decision-1",
          "provider": "openrouter",
          "text_run": 205,
          "english": {
            "run": 451,
            "runs": [
              451,
              459,
              467,
              477,
              483
            ],
            "right": 467,
            "wrong": 77,
            "dont_know": 3,
            "invalid": 0,
            "accuracy": 0.8537477148080439,
            "attempt_rate": 0.9945155393053017,
            "non_hallucination": 0.0375,
            "mean_dont_know_probability": 0.03592171404356558
          },
          "turkish": {
            "run": 487,
            "runs": [
              487,
              490,
              492,
              494,
              496
            ],
            "subset": true,
            "right": 457,
            "wrong": 88,
            "dont_know": 2,
            "invalid": 0,
            "accuracy": 0.8354661791590493,
            "attempt_rate": 0.9963436928702011,
            "non_hallucination": 0.022222222222222223,
            "mean_dont_know_probability": 0.023238344633348
          },
          "spread": {
            "english": {
              "accuracy": [
                0.8537477148080439,
                0.8628884826325411
              ],
              "attempt_rate": [
                0.9945155393053017,
                0.9945155393053017
              ],
              "non_hallucination": [
                0.0375,
                0.04
              ],
              "mean_dont_know_probability": [
                0.03576906694841837,
                0.03594965361105969
              ]
            },
            "turkish": {
              "accuracy": [
                0.8281535648994516,
                0.8354661791590493
              ],
              "attempt_rate": [
                0.9926873857404022,
                0.9963436928702011
              ],
              "non_hallucination": [
                0.022222222222222223,
                0.043478260869565216
              ],
              "mean_dont_know_probability": [
                0.023238344633348,
                0.023497202805421846
              ]
            }
          },
          "paired": {
            "english_only_wrong": 24,
            "turkish_only_wrong": 35,
            "p": 0.19252645776146274,
            "p_holm": 1.0
          }
        },
        {
          "name": "Perplexity Decider 27B v1.2",
          "model": "pplx-decider-v1.2",
          "provider": "pplx-decider",
          "text_run": 217,
          "english": {
            "run": 529,
            "runs": [
              529
            ],
            "right": 481,
            "wrong": 64,
            "dont_know": 2,
            "invalid": 0,
            "accuracy": 0.8793418647166362,
            "attempt_rate": 0.9963436928702011,
            "non_hallucination": 0.030303030303030304,
            "mean_dont_know_probability": 0.027666686351686155
          },
          "turkish": {
            "run": 372,
            "runs": [
              372
            ],
            "subset": false,
            "right": 497,
            "wrong": 50,
            "dont_know": 0,
            "invalid": 0,
            "accuracy": 0.9085923217550275,
            "attempt_rate": 1.0,
            "non_hallucination": 0.0,
            "mean_dont_know_probability": 0.009278084188072258
          },
          "spread": null,
          "paired": {
            "english_only_wrong": 29,
            "turkish_only_wrong": 15,
            "p": 0.048766765904474596,
            "p_holm": 0.48766765904474596
          }
        },
        {
          "name": "Liquid d1",
          "model": "d1",
          "provider": "liquid",
          "text_run": 52,
          "english": {
            "run": 445,
            "runs": [
              445,
              455,
              458,
              463,
              465
            ],
            "right": 478,
            "wrong": 67,
            "dont_know": 2,
            "invalid": 0,
            "accuracy": 0.8738574040219378,
            "attempt_rate": 0.9963436928702011,
            "non_hallucination": 0.028985507246376812,
            "mean_dont_know_probability": 0.009200322658382633
          },
          "turkish": {
            "run": 472,
            "runs": [
              472,
              475,
              479,
              481,
              485
            ],
            "subset": true,
            "right": 457,
            "wrong": 89,
            "dont_know": 1,
            "invalid": 0,
            "accuracy": 0.8354661791590493,
            "attempt_rate": 0.9981718464351006,
            "non_hallucination": 0.011111111111111112,
            "mean_dont_know_probability": 0.012543721557333359
          },
          "spread": {
            "english": {
              "accuracy": [
                0.8592321755027422,
                0.8738574040219378
              ],
              "attempt_rate": [
                0.9963436928702011,
                0.9963436928702011
              ],
              "non_hallucination": [
                0.025974025974025976,
                0.028985507246376812
              ],
              "mean_dont_know_probability": [
                0.009001428730236674,
                0.010092346863690762
              ]
            },
            "turkish": {
              "accuracy": [
                0.829981718464351,
                0.850091407678245
              ],
              "attempt_rate": [
                0.9981718464351006,
                1.0
              ],
              "non_hallucination": [
                0.0,
                0.011111111111111112
              ],
              "mean_dont_know_probability": [
                0.011311550737785406,
                0.012813937215386527
              ]
            }
          },
          "paired": {
            "english_only_wrong": 21,
            "turkish_only_wrong": 43,
            "p": 0.008146902228698568,
            "p_holm": 0.09776282674438282
          }
        }
      ],
      "english_only": [
        {
          "name": "Celeris-1 Decision",
          "model": "celeris-1-decision",
          "provider": "celeris",
          "text_run": 150,
          "reason": "no Turkish run with “Bilmiyorum” on these questions; new Turkish runs were outside the request of 3 November 2026",
          "english": {
            "run": 548,
            "runs": [
              548
            ],
            "right": 418,
            "wrong": 37,
            "dont_know": 92,
            "invalid": 0,
            "accuracy": 0.7641681901279708,
            "attempt_rate": 0.8318098720292505,
            "non_hallucination": 0.7131782945736435,
            "mean_dont_know_probability": 0.12687385740402193
          }
        }
      ]
    },
    "coverage": {
      "questions": 547,
      "definitions": "All series are counted on the same kept reading questions; a full-bank run is restricted to them; first runs only. Answered = right + wrong (an invalid answer or one without a score is never answered). Coverage = answered / kept questions. Risk = error rate among answered = wrong / answered. Abstained: for a native point, the times the model chose the option; for a curve point, kept questions - answered - invalid. A native point is the run with its abstention option (Turkish \"Bilmiyorum\", Turkish \"I don't know\" as the wording control, English \"I don't know\"); Turkish \"Bilmiyorum\" is the Turkish side of the English \"I don't know\" comparison or, for a model outside it, its Hallucination tab run; a missing one is Not measured. A curve is one five-option run held back by its recorded score (the probability of the chosen option): one point per distinct score s, answering when the score is at least s; answers with equal scores enter together, so coverage moves in jumps and every point is an achieved one; all distinct points kept, ordered by coverage.",
      "models": [
        {
          "name": "Liquid d1-ultra",
          "text_run": 199,
          "native": [
            {
              "language": "Turkish",
              "wording": "Bilmiyorum",
              "run": 350,
              "full_bank_restricted": true,
              "right": 491,
              "wrong": 26,
              "abstained": 30,
              "invalid": 0,
              "coverage": 0.9451553930530164,
              "risk": 0.05029013539651837
            },
            {
              "language": "Turkish",
              "wording": "I don't know",
              "run": 511,
              "full_bank_restricted": true,
              "right": 492,
              "wrong": 28,
              "abstained": 27,
              "invalid": 0,
              "coverage": 0.9506398537477148,
              "risk": 0.05384615384615385
            },
            {
              "language": "English",
              "wording": "I don't know",
              "run": 449,
              "full_bank_restricted": false,
              "right": 457,
              "wrong": 39,
              "abstained": 51,
              "invalid": 0,
              "coverage": 0.906764168190128,
              "risk": 0.07862903225806452
            }
          ],
          "curves": [
            {
              "language": "Turkish",
              "run": 199,
              "points": [
                [
                  0.0018281535648994515,
                  0.0,
                  1,
                  0
                ],
                [
                  0.003656307129798903,
                  0.0,
                  2,
                  0
                ],
                [
                  0.005484460694698354,
                  0.0,
                  3,
                  0
                ],
                [
                  0.007312614259597806,
                  0.0,
                  4,
                  0
                ],
                [
                  0.009140767824497258,
                  0.0,
                  5,
                  0
                ],
                [
                  0.010968921389396709,
                  0.0,
                  6,
                  0
                ],
                [
                  0.012797074954296161,
                  0.0,
                  7,
                  0
                ],
                [
                  0.014625228519195612,
                  0.0,
                  8,
                  0
                ],
                [
                  0.016453382084095063,
                  0.0,
                  9,
                  0
                ],
                [
                  0.018281535648994516,
                  0.0,
                  10,
                  0
                ],
                [
                  0.02010968921389397,
                  0.0,
                  11,
                  0
                ],
                [
                  0.021937842778793418,
                  0.0,
                  12,
                  0
                ],
                [
                  0.02376599634369287,
                  0.0,
                  13,
                  0
                ],
                [
                  0.025594149908592323,
                  0.0,
                  14,
                  0
                ],
                [
                  0.027422303473491772,
                  0.0,
                  15,
                  0
                ],
                [
                  0.029250457038391225,
                  0.0,
                  16,
                  0
                ],
                [
                  0.031078610603290677,
                  0.0,
                  17,
                  0
                ],
                [
                  0.03290676416819013,
                  0.0,
                  18,
                  0
                ],
                [
                  0.03473491773308958,
                  0.0,
                  19,
                  0
                ],
                [
                  0.03656307129798903,
                  0.0,
                  20,
                  0
                ],
                [
                  0.038391224862888484,
                  0.0,
                  21,
                  0
                ],
                [
                  0.04021937842778794,
                  0.0,
                  22,
                  0
                ],
                [
                  0.04204753199268738,
                  0.0,
                  23,
                  0
                ],
                [
                  0.043875685557586835,
                  0.0,
                  24,
                  0
                ],
                [
                  0.04570383912248629,
                  0.0,
                  25,
                  0
                ],
                [
                  0.04753199268738574,
                  0.0,
                  26,
                  0
                ],
                [
                  0.04936014625228519,
                  0.0,
                  27,
                  0
                ],
                [
                  0.051188299817184646,
                  0.0,
                  28,
                  0
                ],
                [
                  0.05301645338208409,
                  0.0,
                  29,
                  0
                ],
                [
                  0.054844606946983544,
                  0.0,
                  30,
                  0
                ],
                [
                  0.056672760511883,
                  0.0,
                  31,
                  0
                ],
                [
                  0.05850091407678245,
                  0.0,
                  32,
                  0
                ],
                [
                  0.0603290676416819,
                  0.0,
                  33,
                  0
                ],
                [
                  0.062157221206581355,
                  0.0,
                  34,
                  0
                ],
                [
                  0.06398537477148081,
                  0.0,
                  35,
                  0
                ],
                [
                  0.06581352833638025,
                  0.0,
                  36,
                  0
                ],
                [
                  0.06764168190127971,
                  0.0,
                  37,
                  0
                ],
                [
                  0.06946983546617916,
                  0.0,
                  38,
                  0
                ],
                [
                  0.0712979890310786,
                  0.0,
                  39,
                  0
                ],
                [
                  0.07312614259597806,
                  0.0,
                  40,
                  0
                ],
                [
                  0.07495429616087751,
                  0.0,
                  41,
                  0
                ],
                [
                  0.07678244972577697,
                  0.0,
                  42,
                  0
                ],
                [
                  0.07861060329067641,
                  0.0,
                  43,
                  0
                ],
                [
                  0.08043875685557587,
                  0.0,
                  44,
                  0
                ],
                [
                  0.08226691042047532,
                  0.0,
                  45,
                  0
                ],
                [
                  0.08409506398537477,
                  0.0,
                  46,
                  0
                ],
                [
                  0.08592321755027423,
                  0.0,
                  47,
                  0
                ],
                [
                  0.08775137111517367,
                  0.0,
                  48,
                  0
                ],
                [
                  0.08957952468007313,
                  0.0,
                  49,
                  0
                ],
                [
                  0.09140767824497258,
                  0.0,
                  50,
                  0
                ],
                [
                  0.09323583180987204,
                  0.0,
                  51,
                  0
                ],
                [
                  0.09506398537477148,
                  0.0,
                  52,
                  0
                ],
                [
                  0.09689213893967093,
                  0.0,
                  53,
                  0
                ],
                [
                  0.09872029250457039,
                  0.0,
                  54,
                  0
                ],
                [
                  0.10054844606946983,
                  0.0,
                  55,
                  0
                ],
                [
                  0.10237659963436929,
                  0.0,
                  56,
                  0
                ],
                [
                  0.10420475319926874,
                  0.0,
                  57,
                  0
                ],
                [
                  0.10603290676416818,
                  0.0,
                  58,
                  0
                ],
                [
                  0.10786106032906764,
                  0.0,
                  59,
                  0
                ],
                [
                  0.10968921389396709,
                  0.0,
                  60,
                  0
                ],
                [
                  0.11151736745886655,
                  0.0,
                  61,
                  0
                ],
                [
                  0.113345521023766,
                  0.0,
                  62,
                  0
                ],
                [
                  0.11517367458866545,
                  0.0,
                  63,
                  0
                ],
                [
                  0.1170018281535649,
                  0.0,
                  64,
                  0
                ],
                [
                  0.11882998171846434,
                  0.0,
                  65,
                  0
                ],
                [
                  0.1206581352833638,
                  0.0,
                  66,
                  0
                ],
                [
                  0.12248628884826325,
                  0.0,
                  67,
                  0
                ],
                [
                  0.12431444241316271,
                  0.0,
                  68,
                  0
                ],
                [
                  0.12614259597806216,
                  0.0,
                  69,
                  0
                ],
                [
                  0.12797074954296161,
                  0.0,
                  70,
                  0
                ],
                [
                  0.12979890310786105,
                  0.0,
                  71,
                  0
                ],
                [
                  0.1316270566727605,
                  0.0,
                  72,
                  0
                ],
                [
                  0.13345521023765997,
                  0.0,
                  73,
                  0
                ],
                [
                  0.13528336380255943,
                  0.0,
                  74,
                  0
                ],
                [
                  0.13711151736745886,
                  0.0,
                  75,
                  0
                ],
                [
                  0.13893967093235832,
                  0.0,
                  76,
                  0
                ],
                [
                  0.14076782449725778,
                  0.0,
                  77,
                  0
                ],
                [
                  0.1425959780621572,
                  0.0,
                  78,
                  0
                ],
                [
                  0.14442413162705667,
                  0.0,
                  79,
                  0
                ],
                [
                  0.14625228519195613,
                  0.0,
                  80,
                  0
                ],
                [
                  0.1480804387568556,
                  0.0,
                  81,
                  0
                ],
                [
                  0.14990859232175502,
                  0.0,
                  82,
                  0
                ],
                [
                  0.15173674588665448,
                  0.0,
                  83,
                  0
                ],
                [
                  0.15356489945155394,
                  0.0,
                  84,
                  0
                ],
                [
                  0.15539305301645337,
                  0.0,
                  85,
                  0
                ],
                [
                  0.15722120658135283,
                  0.0,
                  86,
                  0
                ],
                [
                  0.1590493601462523,
                  0.0,
                  87,
                  0
                ],
                [
                  0.16087751371115175,
                  0.0,
                  88,
                  0
                ],
                [
                  0.16270566727605118,
                  0.0,
                  89,
                  0
                ],
                [
                  0.16453382084095064,
                  0.0,
                  90,
                  0
                ],
                [
                  0.1663619744058501,
                  0.0,
                  91,
                  0
                ],
                [
                  0.16819012797074953,
                  0.0,
                  92,
                  0
                ],
                [
                  0.170018281535649,
                  0.0,
                  93,
                  0
                ],
                [
                  0.17184643510054845,
                  0.0,
                  94,
                  0
                ],
                [
                  0.1736745886654479,
                  0.0,
                  95,
                  0
                ],
                [
                  0.17550274223034734,
                  0.0,
                  96,
                  0
                ],
                [
                  0.1773308957952468,
                  0.0,
                  97,
                  0
                ],
                [
                  0.17915904936014626,
                  0.0,
                  98,
                  0
                ],
                [
                  0.1809872029250457,
                  0.0,
                  99,
                  0
                ],
                [
                  0.18281535648994515,
                  0.0,
                  100,
                  0
                ],
                [
                  0.1846435100548446,
                  0.0,
                  101,
                  0
                ],
                [
                  0.18647166361974407,
                  0.0,
                  102,
                  0
                ],
                [
                  0.1882998171846435,
                  0.0,
                  103,
                  0
                ],
                [
                  0.19012797074954296,
                  0.0,
                  104,
                  0
                ],
                [
                  0.19195612431444242,
                  0.0,
                  105,
                  0
                ],
                [
                  0.19378427787934185,
                  0.0,
                  106,
                  0
                ],
                [
                  0.1956124314442413,
                  0.0,
                  107,
                  0
                ],
                [
                  0.19744058500914077,
                  0.0,
                  108,
                  0
                ],
                [
                  0.19926873857404023,
                  0.0,
                  109,
                  0
                ],
                [
                  0.20109689213893966,
                  0.0,
                  110,
                  0
                ],
                [
                  0.20292504570383912,
                  0.0,
                  111,
                  0
                ],
                [
                  0.20475319926873858,
                  0.0,
                  112,
                  0
                ],
                [
                  0.20658135283363802,
                  0.0,
                  113,
                  0
                ],
                [
                  0.20840950639853748,
                  0.0,
                  114,
                  0
                ],
                [
                  0.21023765996343693,
                  0.0,
                  115,
                  0
                ],
                [
                  0.21206581352833637,
                  0.0,
                  116,
                  0
                ],
                [
                  0.21389396709323583,
                  0.0,
                  117,
                  0
                ],
                [
                  0.21572212065813529,
                  0.0,
                  118,
                  0
                ],
                [
                  0.21755027422303475,
                  0.0,
                  119,
                  0
                ],
                [
                  0.21937842778793418,
                  0.0,
                  120,
                  0
                ],
                [
                  0.22120658135283364,
                  0.0,
                  121,
                  0
                ],
                [
                  0.2230347349177331,
                  0.0,
                  122,
                  0
                ],
                [
                  0.22486288848263253,
                  0.0,
                  123,
                  0
                ],
                [
                  0.226691042047532,
                  0.0,
                  124,
                  0
                ],
                [
                  0.22851919561243145,
                  0.0,
                  125,
                  0
                ],
                [
                  0.2303473491773309,
                  0.0,
                  126,
                  0
                ],
                [
                  0.23217550274223034,
                  0.0,
                  127,
                  0
                ],
                [
                  0.2340036563071298,
                  0.0,
                  128,
                  0
                ],
                [
                  0.23583180987202926,
                  0.0,
                  129,
                  0
                ],
                [
                  0.2376599634369287,
                  0.0,
                  130,
                  0
                ],
                [
                  0.23948811700182815,
                  0.0,
                  131,
                  0
                ],
                [
                  0.2413162705667276,
                  0.0,
                  132,
                  0
                ],
                [
                  0.24314442413162707,
                  0.0,
                  133,
                  0
                ],
                [
                  0.2449725776965265,
                  0.0,
                  134,
                  0
                ],
                [
                  0.24680073126142596,
                  0.0,
                  135,
                  0
                ],
                [
                  0.24862888482632542,
                  0.0,
                  136,
                  0
                ],
                [
                  0.25045703839122485,
                  0.0,
                  137,
                  0
                ],
                [
                  0.2522851919561243,
                  0.0,
                  138,
                  0
                ],
                [
                  0.25411334552102377,
                  0.0,
                  139,
                  0
                ],
                [
                  0.25594149908592323,
                  0.0,
                  140,
                  0
                ],
                [
                  0.2577696526508227,
                  0.0,
                  141,
                  0
                ],
                [
                  0.2595978062157221,
                  0.0,
                  142,
                  0
                ],
                [
                  0.26142595978062155,
                  0.0,
                  143,
                  0
                ],
                [
                  0.263254113345521,
                  0.0,
                  144,
                  0
                ],
                [
                  0.26508226691042047,
                  0.0,
                  145,
                  0
                ],
                [
                  0.26691042047531993,
                  0.0,
                  146,
                  0
                ],
                [
                  0.2687385740402194,
                  0.0,
                  147,
                  0
                ],
                [
                  0.27056672760511885,
                  0.0,
                  148,
                  0
                ],
                [
                  0.27239488117001825,
                  0.0,
                  149,
                  0
                ],
                [
                  0.2742230347349177,
                  0.0,
                  150,
                  0
                ],
                [
                  0.2760511882998172,
                  0.0,
                  151,
                  0
                ],
                [
                  0.27787934186471663,
                  0.0,
                  152,
                  0
                ],
                [
                  0.2797074954296161,
                  0.0,
                  153,
                  0
                ],
                [
                  0.28153564899451555,
                  0.0,
                  154,
                  0
                ],
                [
                  0.283363802559415,
                  0.0,
                  155,
                  0
                ],
                [
                  0.2851919561243144,
                  0.0,
                  156,
                  0
                ],
                [
                  0.2870201096892139,
                  0.0,
                  157,
                  0
                ],
                [
                  0.28884826325411334,
                  0.0,
                  158,
                  0
                ],
                [
                  0.2906764168190128,
                  0.0,
                  159,
                  0
                ],
                [
                  0.29250457038391225,
                  0.0,
                  160,
                  0
                ],
                [
                  0.2943327239488117,
                  0.0,
                  161,
                  0
                ],
                [
                  0.2961608775137112,
                  0.0,
                  162,
                  0
                ],
                [
                  0.2979890310786106,
                  0.0,
                  163,
                  0
                ],
                [
                  0.29981718464351004,
                  0.0,
                  164,
                  0
                ],
                [
                  0.3016453382084095,
                  0.0,
                  165,
                  0
                ],
                [
                  0.30347349177330896,
                  0.0,
                  166,
                  0
                ],
                [
                  0.3053016453382084,
                  0.0,
                  167,
                  0
                ],
                [
                  0.3071297989031079,
                  0.0,
                  168,
                  0
                ],
                [
                  0.30895795246800734,
                  0.0,
                  169,
                  0
                ],
                [
                  0.31078610603290674,
                  0.0,
                  170,
                  0
                ],
                [
                  0.3126142595978062,
                  0.0,
                  171,
                  0
                ],
                [
                  0.31444241316270566,
                  0.0,
                  172,
                  0
                ],
                [
                  0.3162705667276051,
                  0.0,
                  173,
                  0
                ],
                [
                  0.3180987202925046,
                  0.0,
                  174,
                  0
                ],
                [
                  0.31992687385740404,
                  0.0,
                  175,
                  0
                ],
                [
                  0.3217550274223035,
                  0.0,
                  176,
                  0
                ],
                [
                  0.3235831809872029,
                  0.0,
                  177,
                  0
                ],
                [
                  0.32541133455210236,
                  0.0,
                  178,
                  0
                ],
                [
                  0.3272394881170018,
                  0.0,
                  179,
                  0
                ],
                [
                  0.3290676416819013,
                  0.0,
                  180,
                  0
                ],
                [
                  0.33089579524680074,
                  0.0,
                  181,
                  0
                ],
                [
                  0.3327239488117002,
                  0.0,
                  182,
                  0
                ],
                [
                  0.33455210237659966,
                  0.0,
                  183,
                  0
                ],
                [
                  0.33638025594149906,
                  0.0,
                  184,
                  0
                ],
                [
                  0.3382084095063985,
                  0.0,
                  185,
                  0
                ],
                [
                  0.340036563071298,
                  0.0,
                  186,
                  0
                ],
                [
                  0.34186471663619744,
                  0.0,
                  187,
                  0
                ],
                [
                  0.3436928702010969,
                  0.0,
                  188,
                  0
                ],
                [
                  0.34552102376599636,
                  0.0,
                  189,
                  0
                ],
                [
                  0.3473491773308958,
                  0.0,
                  190,
                  0
                ],
                [
                  0.3491773308957952,
                  0.0,
                  191,
                  0
                ],
                [
                  0.3510054844606947,
                  0.0,
                  192,
                  0
                ],
                [
                  0.35283363802559414,
                  0.0,
                  193,
                  0
                ],
                [
                  0.3546617915904936,
                  0.0,
                  194,
                  0
                ],
                [
                  0.35648994515539306,
                  0.0,
                  195,
                  0
                ],
                [
                  0.3583180987202925,
                  0.0,
                  196,
                  0
                ],
                [
                  0.360146252285192,
                  0.0,
                  197,
                  0
                ],
                [
                  0.3619744058500914,
                  0.0,
                  198,
                  0
                ],
                [
                  0.36380255941499084,
                  0.0,
                  199,
                  0
                ],
                [
                  0.3656307129798903,
                  0.0,
                  200,
                  0
                ],
                [
                  0.36745886654478976,
                  0.0,
                  201,
                  0
                ],
                [
                  0.3692870201096892,
                  0.0,
                  202,
                  0
                ],
                [
                  0.3711151736745887,
                  0.0,
                  203,
                  0
                ],
                [
                  0.37294332723948814,
                  0.0,
                  204,
                  0
                ],
                [
                  0.37477148080438755,
                  0.0,
                  205,
                  0
                ],
                [
                  0.376599634369287,
                  0.0,
                  206,
                  0
                ],
                [
                  0.37842778793418647,
                  0.0,
                  207,
                  0
                ],
                [
                  0.3802559414990859,
                  0.0,
                  208,
                  0
                ],
                [
                  0.3820840950639854,
                  0.0,
                  209,
                  0
                ],
                [
                  0.38391224862888484,
                  0.0,
                  210,
                  0
                ],
                [
                  0.3857404021937843,
                  0.0,
                  211,
                  0
                ],
                [
                  0.3875685557586837,
                  0.0,
                  212,
                  0
                ],
                [
                  0.38939670932358317,
                  0.0,
                  213,
                  0
                ],
                [
                  0.3912248628884826,
                  0.0,
                  214,
                  0
                ],
                [
                  0.3930530164533821,
                  0.0,
                  215,
                  0
                ],
                [
                  0.39488117001828155,
                  0.0,
                  216,
                  0
                ],
                [
                  0.396709323583181,
                  0.0,
                  217,
                  0
                ],
                [
                  0.39853747714808047,
                  0.0,
                  218,
                  0
                ],
                [
                  0.40036563071297987,
                  0.0,
                  219,
                  0
                ],
                [
                  0.40219378427787933,
                  0.0,
                  220,
                  0
                ],
                [
                  0.4040219378427788,
                  0.0,
                  221,
                  0
                ],
                [
                  0.40585009140767825,
                  0.0,
                  222,
                  0
                ],
                [
                  0.4076782449725777,
                  0.0,
                  223,
                  0
                ],
                [
                  0.40950639853747717,
                  0.0,
                  224,
                  0
                ],
                [
                  0.4113345521023766,
                  0.0,
                  225,
                  0
                ],
                [
                  0.41316270566727603,
                  0.0,
                  226,
                  0
                ],
                [
                  0.4149908592321755,
                  0.0,
                  227,
                  0
                ],
                [
                  0.41681901279707495,
                  0.0,
                  228,
                  0
                ],
                [
                  0.4186471663619744,
                  0.0,
                  229,
                  0
                ],
                [
                  0.42047531992687387,
                  0.0,
                  230,
                  0
                ],
                [
                  0.42230347349177333,
                  0.0,
                  231,
                  0
                ],
                [
                  0.42413162705667273,
                  0.0,
                  232,
                  0
                ],
                [
                  0.4259597806215722,
                  0.0,
                  233,
                  0
                ],
                [
                  0.42778793418647165,
                  0.0,
                  234,
                  0
                ],
                [
                  0.4296160877513711,
                  0.0,
                  235,
                  0
                ],
                [
                  0.43144424131627057,
                  0.0,
                  236,
                  0
                ],
                [
                  0.43327239488117003,
                  0.0,
                  237,
                  0
                ],
                [
                  0.4351005484460695,
                  0.0,
                  238,
                  0
                ],
                [
                  0.4369287020109689,
                  0.0,
                  239,
                  0
                ],
                [
                  0.43875685557586835,
                  0.0,
                  240,
                  0
                ],
                [
                  0.4405850091407678,
                  0.0,
                  241,
                  0
                ],
                [
                  0.4424131627056673,
                  0.0,
                  242,
                  0
                ],
                [
                  0.44424131627056673,
                  0.0,
                  243,
                  0
                ],
                [
                  0.4460694698354662,
                  0.0,
                  244,
                  0
                ],
                [
                  0.44789762340036565,
                  0.0,
                  245,
                  0
                ],
                [
                  0.44972577696526506,
                  0.0,
                  246,
                  0
                ],
                [
                  0.4515539305301645,
                  0.0,
                  247,
                  0
                ],
                [
                  0.453382084095064,
                  0.0,
                  248,
                  0
                ],
                [
                  0.45521023765996343,
                  0.0,
                  249,
                  0
                ],
                [
                  0.4570383912248629,
                  0.0,
                  250,
                  0
                ],
                [
                  0.45886654478976235,
                  0.0,
                  251,
                  0
                ],
                [
                  0.4606946983546618,
                  0.0,
                  252,
                  0
                ],
                [
                  0.4625228519195612,
                  0.0,
                  253,
                  0
                ],
                [
                  0.4643510054844607,
                  0.0,
                  254,
                  0
                ],
                [
                  0.46617915904936014,
                  0.0,
                  255,
                  0
                ],
                [
                  0.4680073126142596,
                  0.0,
                  256,
                  0
                ],
                [
                  0.46983546617915906,
                  0.0,
                  257,
                  0
                ],
                [
                  0.4716636197440585,
                  0.0,
                  258,
                  0
                ],
                [
                  0.473491773308958,
                  0.0,
                  259,
                  0
                ],
                [
                  0.4753199268738574,
                  0.0,
                  260,
                  0
                ],
                [
                  0.47714808043875684,
                  0.0,
                  261,
                  0
                ],
                [
                  0.4789762340036563,
                  0.0,
                  262,
                  0
                ],
                [
                  0.48080438756855576,
                  0.0,
                  263,
                  0
                ],
                [
                  0.4826325411334552,
                  0.0,
                  264,
                  0
                ],
                [
                  0.4844606946983547,
                  0.0,
                  265,
                  0
                ],
                [
                  0.48628884826325414,
                  0.0,
                  266,
                  0
                ],
                [
                  0.48811700182815354,
                  0.0,
                  267,
                  0
                ],
                [
                  0.489945155393053,
                  0.0,
                  268,
                  0
                ],
                [
                  0.49177330895795246,
                  0.0,
                  269,
                  0
                ],
                [
                  0.4936014625228519,
                  0.0,
                  270,
                  0
                ],
                [
                  0.4954296160877514,
                  0.0,
                  271,
                  0
                ],
                [
                  0.49725776965265084,
                  0.0,
                  272,
                  0
                ],
                [
                  0.4990859232175503,
                  0.0,
                  273,
                  0
                ],
                [
                  0.5009140767824497,
                  0.0,
                  274,
                  0
                ],
                [
                  0.5027422303473492,
                  0.0,
                  275,
                  0
                ],
                [
                  0.5045703839122486,
                  0.0,
                  276,
                  0
                ],
                [
                  0.506398537477148,
                  0.0,
                  277,
                  0
                ],
                [
                  0.5082266910420475,
                  0.0,
                  278,
                  0
                ],
                [
                  0.5100548446069469,
                  0.0,
                  279,
                  0
                ],
                [
                  0.5118829981718465,
                  0.0,
                  280,
                  0
                ],
                [
                  0.5137111517367459,
                  0.0,
                  281,
                  0
                ],
                [
                  0.5155393053016454,
                  0.0,
                  282,
                  0
                ],
                [
                  0.5173674588665448,
                  0.0,
                  283,
                  0
                ],
                [
                  0.5191956124314442,
                  0.0,
                  284,
                  0
                ],
                [
                  0.5210237659963437,
                  0.0,
                  285,
                  0
                ],
                [
                  0.5228519195612431,
                  0.0,
                  286,
                  0
                ],
                [
                  0.5246800731261426,
                  0.0,
                  287,
                  0
                ],
                [
                  0.526508226691042,
                  0.0,
                  288,
                  0
                ],
                [
                  0.5283363802559415,
                  0.0,
                  289,
                  0
                ],
                [
                  0.5301645338208409,
                  0.0,
                  290,
                  0
                ],
                [
                  0.5319926873857403,
                  0.0,
                  291,
                  0
                ],
                [
                  0.5338208409506399,
                  0.0,
                  292,
                  0
                ],
                [
                  0.5356489945155393,
                  0.0,
                  293,
                  0
                ],
                [
                  0.5374771480804388,
                  0.0,
                  294,
                  0
                ],
                [
                  0.5393053016453382,
                  0.0,
                  295,
                  0
                ],
                [
                  0.5411334552102377,
                  0.0,
                  296,
                  0
                ],
                [
                  0.5429616087751371,
                  0.0,
                  297,
                  0
                ],
                [
                  0.5447897623400365,
                  0.0,
                  298,
                  0
                ],
                [
                  0.546617915904936,
                  0.0,
                  299,
                  0
                ],
                [
                  0.5484460694698354,
                  0.0,
                  300,
                  0
                ],
                [
                  0.5502742230347349,
                  0.0,
                  301,
                  0
                ],
                [
                  0.5521023765996343,
                  0.0,
                  302,
                  0
                ],
                [
                  0.5539305301645339,
                  0.0,
                  303,
                  0
                ],
                [
                  0.5557586837294333,
                  0.0,
                  304,
                  0
                ],
                [
                  0.5575868372943327,
                  0.0,
                  305,
                  0
                ],
                [
                  0.5594149908592322,
                  0.0,
                  306,
                  0
                ],
                [
                  0.5612431444241316,
                  0.0,
                  307,
                  0
                ],
                [
                  0.5630712979890311,
                  0.0,
                  308,
                  0
                ],
                [
                  0.5648994515539305,
                  0.0,
                  309,
                  0
                ],
                [
                  0.56672760511883,
                  0.0,
                  310,
                  0
                ],
                [
                  0.5685557586837294,
                  0.0,
                  311,
                  0
                ],
                [
                  0.5703839122486288,
                  0.0,
                  312,
                  0
                ],
                [
                  0.5722120658135283,
                  0.0,
                  313,
                  0
                ],
                [
                  0.5740402193784278,
                  0.0,
                  314,
                  0
                ],
                [
                  0.5758683729433273,
                  0.0,
                  315,
                  0
                ],
                [
                  0.5776965265082267,
                  0.0,
                  316,
                  0
                ],
                [
                  0.5795246800731262,
                  0.0,
                  317,
                  0
                ],
                [
                  0.5813528336380256,
                  0.0,
                  318,
                  0
                ],
                [
                  0.583180987202925,
                  0.0,
                  319,
                  0
                ],
                [
                  0.5850091407678245,
                  0.0,
                  320,
                  0
                ],
                [
                  0.5868372943327239,
                  0.0,
                  321,
                  0
                ],
                [
                  0.5886654478976234,
                  0.0,
                  322,
                  0
                ],
                [
                  0.5904936014625228,
                  0.0,
                  323,
                  0
                ],
                [
                  0.5923217550274223,
                  0.0,
                  324,
                  0
                ],
                [
                  0.5941499085923218,
                  0.0,
                  325,
                  0
                ],
                [
                  0.5959780621572212,
                  0.0,
                  326,
                  0
                ],
                [
                  0.5978062157221207,
                  0.0,
                  327,
                  0
                ],
                [
                  0.5996343692870201,
                  0.0,
                  328,
                  0
                ],
                [
                  0.6014625228519196,
                  0.0,
                  329,
                  0
                ],
                [
                  0.603290676416819,
                  0.0,
                  330,
                  0
                ],
                [
                  0.6051188299817185,
                  0.0,
                  331,
                  0
                ],
                [
                  0.6069469835466179,
                  0.0,
                  332,
                  0
                ],
                [
                  0.6087751371115173,
                  0.0,
                  333,
                  0
                ],
                [
                  0.6106032906764168,
                  0.0,
                  334,
                  0
                ],
                [
                  0.6124314442413162,
                  0.0,
                  335,
                  0
                ],
                [
                  0.6142595978062158,
                  0.0,
                  336,
                  0
                ],
                [
                  0.6160877513711152,
                  0.0,
                  337,
                  0
                ],
                [
                  0.6179159049360147,
                  0.0,
                  338,
                  0
                ],
                [
                  0.6197440585009141,
                  0.0,
                  339,
                  0
                ],
                [
                  0.6215722120658135,
                  0.0,
                  340,
                  0
                ],
                [
                  0.623400365630713,
                  0.0,
                  341,
                  0
                ],
                [
                  0.6252285191956124,
                  0.0,
                  342,
                  0
                ],
                [
                  0.6270566727605119,
                  0.0,
                  343,
                  0
                ],
                [
                  0.6288848263254113,
                  0.0,
                  344,
                  0
                ],
                [
                  0.6307129798903108,
                  0.0,
                  345,
                  0
                ],
                [
                  0.6325411334552102,
                  0.0,
                  346,
                  0
                ],
                [
                  0.6343692870201096,
                  0.0,
                  347,
                  0
                ],
                [
                  0.6361974405850092,
                  0.0,
                  348,
                  0
                ],
                [
                  0.6380255941499086,
                  0.0,
                  349,
                  0
                ],
                [
                  0.6398537477148081,
                  0.0,
                  350,
                  0
                ],
                [
                  0.6416819012797075,
                  0.0,
                  351,
                  0
                ],
                [
                  0.643510054844607,
                  0.0,
                  352,
                  0
                ],
                [
                  0.6453382084095064,
                  0.0,
                  353,
                  0
                ],
                [
                  0.6471663619744058,
                  0.0,
                  354,
                  0
                ],
                [
                  0.6489945155393053,
                  0.0,
                  355,
                  0
                ],
                [
                  0.6508226691042047,
                  0.0,
                  356,
                  0
                ],
                [
                  0.6526508226691042,
                  0.0,
                  357,
                  0
                ],
                [
                  0.6544789762340036,
                  0.0,
                  358,
                  0
                ],
                [
                  0.6563071297989032,
                  0.0,
                  359,
                  0
                ],
                [
                  0.6581352833638026,
                  0.0,
                  360,
                  0
                ],
                [
                  0.659963436928702,
                  0.0,
                  361,
                  0
                ],
                [
                  0.6617915904936015,
                  0.0,
                  362,
                  0
                ],
                [
                  0.6636197440585009,
                  0.0,
                  363,
                  0
                ],
                [
                  0.6654478976234004,
                  0.0,
                  364,
                  0
                ],
                [
                  0.6672760511882998,
                  0.0,
                  365,
                  0
                ],
                [
                  0.6691042047531993,
                  0.0,
                  366,
                  0
                ],
                [
                  0.6709323583180987,
                  0.0027247956403269754,
                  367,
                  1
                ],
                [
                  0.6727605118829981,
                  0.002717391304347826,
                  368,
                  1
                ],
                [
                  0.6745886654478976,
                  0.0027100271002710027,
                  369,
                  1
                ],
                [
                  0.676416819012797,
                  0.002702702702702703,
                  370,
                  1
                ],
                [
                  0.6782449725776966,
                  0.0026954177897574125,
                  371,
                  1
                ],
                [
                  0.680073126142596,
                  0.002688172043010753,
                  372,
                  1
                ],
                [
                  0.6819012797074955,
                  0.002680965147453083,
                  373,
                  1
                ],
                [
                  0.6837294332723949,
                  0.00267379679144385,
                  374,
                  1
                ],
                [
                  0.6855575868372943,
                  0.0026666666666666666,
                  375,
                  1
                ],
                [
                  0.6873857404021938,
                  0.0026595744680851063,
                  376,
                  1
                ],
                [
                  0.6892138939670932,
                  0.002652519893899204,
                  377,
                  1
                ],
                [
                  0.6910420475319927,
                  0.0026455026455026454,
                  378,
                  1
                ],
                [
                  0.6928702010968921,
                  0.002638522427440633,
                  379,
                  1
                ],
                [
                  0.6946983546617916,
                  0.002631578947368421,
                  380,
                  1
                ],
                [
                  0.696526508226691,
                  0.0026246719160104987,
                  381,
                  1
                ],
                [
                  0.6983546617915904,
                  0.002617801047120419,
                  382,
                  1
                ],
                [
                  0.70018281535649,
                  0.0026109660574412533,
                  383,
                  1
                ],
                [
                  0.7020109689213894,
                  0.0026041666666666665,
                  384,
                  1
                ],
                [
                  0.7038391224862889,
                  0.0025974025974025974,
                  385,
                  1
                ],
                [
                  0.7056672760511883,
                  0.0025906735751295338,
                  386,
                  1
                ],
                [
                  0.7074954296160878,
                  0.002583979328165375,
                  387,
                  1
                ],
                [
                  0.7093235831809872,
                  0.002577319587628866,
                  388,
                  1
                ],
                [
                  0.7111517367458866,
                  0.002570694087403599,
                  389,
                  1
                ],
                [
                  0.7129798903107861,
                  0.002564102564102564,
                  390,
                  1
                ],
                [
                  0.7148080438756855,
                  0.005115089514066497,
                  391,
                  2
                ],
                [
                  0.716636197440585,
                  0.007653061224489796,
                  392,
                  3
                ],
                [
                  0.7184643510054844,
                  0.007633587786259542,
                  393,
                  3
                ],
                [
                  0.720292504570384,
                  0.007614213197969543,
                  394,
                  3
                ],
                [
                  0.7221206581352834,
                  0.007594936708860759,
                  395,
                  3
                ],
                [
                  0.7239488117001828,
                  0.007575757575757576,
                  396,
                  3
                ],
                [
                  0.7257769652650823,
                  0.007556675062972292,
                  397,
                  3
                ],
                [
                  0.7276051188299817,
                  0.007537688442211055,
                  398,
                  3
                ],
                [
                  0.7294332723948812,
                  0.007518796992481203,
                  399,
                  3
                ],
                [
                  0.7312614259597806,
                  0.0075,
                  400,
                  3
                ],
                [
                  0.7330895795246801,
                  0.007481296758104738,
                  401,
                  3
                ],
                [
                  0.7349177330895795,
                  0.007462686567164179,
                  402,
                  3
                ],
                [
                  0.7367458866544789,
                  0.007444168734491315,
                  403,
                  3
                ],
                [
                  0.7385740402193784,
                  0.007425742574257425,
                  404,
                  3
                ],
                [
                  0.7404021937842779,
                  0.007407407407407408,
                  405,
                  3
                ],
                [
                  0.7422303473491774,
                  0.009852216748768473,
                  406,
                  4
                ],
                [
                  0.7440585009140768,
                  0.009828009828009828,
                  407,
                  4
                ],
                [
                  0.7458866544789763,
                  0.00980392156862745,
                  408,
                  4
                ],
                [
                  0.7477148080438757,
                  0.009779951100244499,
                  409,
                  4
                ],
                [
                  0.7495429616087751,
                  0.00975609756097561,
                  410,
                  4
                ],
                [
                  0.7513711151736746,
                  0.009732360097323601,
                  411,
                  4
                ],
                [
                  0.753199268738574,
                  0.009708737864077669,
                  412,
                  4
                ],
                [
                  0.7550274223034735,
                  0.009685230024213076,
                  413,
                  4
                ],
                [
                  0.7568555758683729,
                  0.00966183574879227,
                  414,
                  4
                ],
                [
                  0.7586837294332724,
                  0.00963855421686747,
                  415,
                  4
                ],
                [
                  0.7605118829981719,
                  0.009615384615384616,
                  416,
                  4
                ],
                [
                  0.7623400365630713,
                  0.011990407673860911,
                  417,
                  5
                ],
                [
                  0.7641681901279708,
                  0.014354066985645933,
                  418,
                  6
                ],
                [
                  0.7659963436928702,
                  0.014319809069212411,
                  419,
                  6
                ],
                [
                  0.7678244972577697,
                  0.014285714285714285,
                  420,
                  6
                ],
                [
                  0.7696526508226691,
                  0.014251781472684086,
                  421,
                  6
                ],
                [
                  0.7714808043875686,
                  0.014218009478672985,
                  422,
                  6
                ],
                [
                  0.773308957952468,
                  0.014184397163120567,
                  423,
                  6
                ],
                [
                  0.7751371115173674,
                  0.01650943396226415,
                  424,
                  7
                ],
                [
                  0.7769652650822669,
                  0.01647058823529412,
                  425,
                  7
                ],
                [
                  0.7787934186471663,
                  0.01643192488262911,
                  426,
                  7
                ],
                [
                  0.7806215722120659,
                  0.01873536299765808,
                  427,
                  8
                ],
                [
                  0.7824497257769653,
                  0.018691588785046728,
                  428,
                  8
                ],
                [
                  0.7842778793418648,
                  0.018648018648018648,
                  429,
                  8
                ],
                [
                  0.7861060329067642,
                  0.018604651162790697,
                  430,
                  8
                ],
                [
                  0.7879341864716636,
                  0.018561484918793503,
                  431,
                  8
                ],
                [
                  0.7897623400365631,
                  0.018518518518518517,
                  432,
                  8
                ],
                [
                  0.7915904936014625,
                  0.018475750577367205,
                  433,
                  8
                ],
                [
                  0.793418647166362,
                  0.018433179723502304,
                  434,
                  8
                ],
                [
                  0.7952468007312614,
                  0.01839080459770115,
                  435,
                  8
                ],
                [
                  0.7970749542961609,
                  0.01834862385321101,
                  436,
                  8
                ],
                [
                  0.7989031078610603,
                  0.020594965675057208,
                  437,
                  9
                ],
                [
                  0.8007312614259597,
                  0.02054794520547945,
                  438,
                  9
                ],
                [
                  0.8025594149908593,
                  0.02050113895216401,
                  439,
                  9
                ],
                [
                  0.8043875685557587,
                  0.020454545454545454,
                  440,
                  9
                ],
                [
                  0.8062157221206582,
                  0.02040816326530612,
                  441,
                  9
                ],
                [
                  0.8080438756855576,
                  0.020361990950226245,
                  442,
                  9
                ],
                [
                  0.8098720292504571,
                  0.020316027088036117,
                  443,
                  9
                ],
                [
                  0.8117001828153565,
                  0.02027027027027027,
                  444,
                  9
                ],
                [
                  0.8135283363802559,
                  0.020224719101123594,
                  445,
                  9
                ],
                [
                  0.8153564899451554,
                  0.020179372197309416,
                  446,
                  9
                ],
                [
                  0.8171846435100548,
                  0.020134228187919462,
                  447,
                  9
                ],
                [
                  0.8190127970749543,
                  0.022321428571428572,
                  448,
                  10
                ],
                [
                  0.8208409506398537,
                  0.022271714922048998,
                  449,
                  10
                ],
                [
                  0.8226691042047533,
                  0.022222222222222223,
                  450,
                  10
                ],
                [
                  0.8244972577696527,
                  0.022172949002217297,
                  451,
                  10
                ],
                [
                  0.8263254113345521,
                  0.022123893805309734,
                  452,
                  10
                ],
                [
                  0.8281535648994516,
                  0.02207505518763797,
                  453,
                  10
                ],
                [
                  0.829981718464351,
                  0.022026431718061675,
                  454,
                  10
                ],
                [
                  0.8318098720292505,
                  0.02197802197802198,
                  455,
                  10
                ],
                [
                  0.8336380255941499,
                  0.021929824561403508,
                  456,
                  10
                ],
                [
                  0.8354661791590493,
                  0.02188183807439825,
                  457,
                  10
                ],
                [
                  0.8372943327239488,
                  0.021834061135371178,
                  458,
                  10
                ],
                [
                  0.8391224862888482,
                  0.02178649237472767,
                  459,
                  10
                ],
                [
                  0.8409506398537477,
                  0.021739130434782608,
                  460,
                  10
                ],
                [
                  0.8427787934186471,
                  0.021691973969631236,
                  461,
                  10
                ],
                [
                  0.8446069469835467,
                  0.021645021645021644,
                  462,
                  10
                ],
                [
                  0.8464351005484461,
                  0.02159827213822894,
                  463,
                  10
                ],
                [
                  0.8482632541133455,
                  0.021551724137931036,
                  464,
                  10
                ],
                [
                  0.850091407678245,
                  0.021505376344086023,
                  465,
                  10
                ],
                [
                  0.8519195612431444,
                  0.023605150214592276,
                  466,
                  11
                ],
                [
                  0.8537477148080439,
                  0.023554603854389723,
                  467,
                  11
                ],
                [
                  0.8555758683729433,
                  0.023504273504273504,
                  468,
                  11
                ],
                [
                  0.8574040219378428,
                  0.023454157782515993,
                  469,
                  11
                ],
                [
                  0.8592321755027422,
                  0.023404255319148935,
                  470,
                  11
                ],
                [
                  0.8610603290676416,
                  0.025477707006369428,
                  471,
                  12
                ],
                [
                  0.8628884826325411,
                  0.025423728813559324,
                  472,
                  12
                ],
                [
                  0.8647166361974405,
                  0.02536997885835095,
                  473,
                  12
                ],
                [
                  0.8665447897623401,
                  0.02531645569620253,
                  474,
                  12
                ],
                [
                  0.8683729433272395,
                  0.02526315789473684,
                  475,
                  12
                ],
                [
                  0.870201096892139,
                  0.025210084033613446,
                  476,
                  12
                ],
                [
                  0.8720292504570384,
                  0.025157232704402517,
                  477,
                  12
                ],
                [
                  0.8738574040219378,
                  0.02510460251046025,
                  478,
                  12
                ],
                [
                  0.8756855575868373,
                  0.025052192066805846,
                  479,
                  12
                ],
                [
                  0.8775137111517367,
                  0.025,
                  480,
                  12
                ],
                [
                  0.8793418647166362,
                  0.02494802494802495,
                  481,
                  12
                ],
                [
                  0.8811700182815356,
                  0.024896265560165973,
                  482,
                  12
                ],
                [
                  0.8829981718464351,
                  0.024844720496894408,
                  483,
                  12
                ],
                [
                  0.8848263254113345,
                  0.024793388429752067,
                  484,
                  12
                ],
                [
                  0.886654478976234,
                  0.026804123711340205,
                  485,
                  13
                ],
                [
                  0.8884826325411335,
                  0.026748971193415638,
                  486,
                  13
                ],
                [
                  0.8903107861060329,
                  0.026694045174537988,
                  487,
                  13
                ],
                [
                  0.8921389396709324,
                  0.02663934426229508,
                  488,
                  13
                ],
                [
                  0.8939670932358318,
                  0.026584867075664622,
                  489,
                  13
                ],
                [
                  0.8957952468007313,
                  0.026530612244897958,
                  490,
                  13
                ],
                [
                  0.8976234003656307,
                  0.026476578411405296,
                  491,
                  13
                ],
                [
                  0.8994515539305301,
                  0.026422764227642278,
                  492,
                  13
                ],
                [
                  0.9012797074954296,
                  0.028397565922920892,
                  493,
                  14
                ],
                [
                  0.903107861060329,
                  0.02834008097165992,
                  494,
                  14
                ],
                [
                  0.9049360146252285,
                  0.028282828282828285,
                  495,
                  14
                ],
                [
                  0.906764168190128,
                  0.03024193548387097,
                  496,
                  15
                ],
                [
                  0.9085923217550275,
                  0.030181086519114688,
                  497,
                  15
                ],
                [
                  0.9104204753199269,
                  0.030120481927710843,
                  498,
                  15
                ],
                [
                  0.9122486288848263,
                  0.03006012024048096,
                  499,
                  15
                ],
                [
                  0.9140767824497258,
                  0.032,
                  500,
                  16
                ],
                [
                  0.9159049360146252,
                  0.031936127744510975,
                  501,
                  16
                ],
                [
                  0.9177330895795247,
                  0.03187250996015936,
                  502,
                  16
                ],
                [
                  0.9195612431444241,
                  0.033797216699801194,
                  503,
                  17
                ],
                [
                  0.9213893967093236,
                  0.03373015873015873,
                  504,
                  17
                ],
                [
                  0.923217550274223,
                  0.03564356435643564,
                  505,
                  18
                ],
                [
                  0.9250457038391224,
                  0.03557312252964427,
                  506,
                  18
                ],
                [
                  0.926873857404022,
                  0.03550295857988166,
                  507,
                  18
                ],
                [
                  0.9287020109689214,
                  0.03543307086614173,
                  508,
                  18
                ],
                [
                  0.9305301645338209,
                  0.03732809430255403,
                  509,
                  19
                ],
                [
                  0.9323583180987203,
                  0.03725490196078431,
                  510,
                  19
                ],
                [
                  0.9341864716636198,
                  0.03718199608610567,
                  511,
                  19
                ],
                [
                  0.9360146252285192,
                  0.037109375,
                  512,
                  19
                ],
                [
                  0.9378427787934186,
                  0.03898635477582846,
                  513,
                  20
                ],
                [
                  0.9396709323583181,
                  0.038910505836575876,
                  514,
                  20
                ],
                [
                  0.9414990859232175,
                  0.040776699029126215,
                  515,
                  21
                ],
                [
                  0.943327239488117,
                  0.040697674418604654,
                  516,
                  21
                ],
                [
                  0.9451553930530164,
                  0.04061895551257253,
                  517,
                  21
                ],
                [
                  0.946983546617916,
                  0.04247104247104247,
                  518,
                  22
                ],
                [
                  0.9488117001828154,
                  0.04238921001926782,
                  519,
                  22
                ],
                [
                  0.9506398537477148,
                  0.04423076923076923,
                  520,
                  23
                ],
                [
                  0.9524680073126143,
                  0.044145873320537425,
                  521,
                  23
                ],
                [
                  0.9542961608775137,
                  0.044061302681992334,
                  522,
                  23
                ],
                [
                  0.9561243144424132,
                  0.045889101338432124,
                  523,
                  24
                ],
                [
                  0.9579524680073126,
                  0.04770992366412214,
                  524,
                  25
                ],
                [
                  0.9597806215722121,
                  0.049523809523809526,
                  525,
                  26
                ],
                [
                  0.9616087751371115,
                  0.049429657794676805,
                  526,
                  26
                ],
                [
                  0.9634369287020109,
                  0.051233396584440226,
                  527,
                  27
                ],
                [
                  0.9652650822669104,
                  0.05113636363636364,
                  528,
                  27
                ],
                [
                  0.9670932358318098,
                  0.05103969754253308,
                  529,
                  27
                ],
                [
                  0.9689213893967094,
                  0.0509433962264151,
                  530,
                  27
                ],
                [
                  0.9707495429616088,
                  0.05084745762711865,
                  531,
                  27
                ],
                [
                  0.9725776965265083,
                  0.05263157894736842,
                  532,
                  28
                ],
                [
                  0.9744058500914077,
                  0.0525328330206379,
                  533,
                  28
                ],
                [
                  0.9762340036563071,
                  0.054307116104868915,
                  534,
                  29
                ],
                [
                  0.9780621572212066,
                  0.056074766355140186,
                  535,
                  30
                ],
                [
                  0.979890310786106,
                  0.05783582089552239,
                  536,
                  31
                ],
                [
                  0.9817184643510055,
                  0.05772811918063315,
                  537,
                  31
                ],
                [
                  0.9835466179159049,
                  0.05947955390334572,
                  538,
                  32
                ],
                [
                  0.9853747714808044,
                  0.059369202226345084,
                  539,
                  32
                ],
                [
                  0.9872029250457038,
                  0.05925925925925926,
                  540,
                  32
                ],
                [
                  0.9890310786106032,
                  0.059149722735674676,
                  541,
                  32
                ],
                [
                  0.9908592321755028,
                  0.05904059040590406,
                  542,
                  32
                ],
                [
                  0.9926873857404022,
                  0.06077348066298342,
                  543,
                  33
                ],
                [
                  0.9945155393053017,
                  0.0625,
                  544,
                  34
                ],
                [
                  0.9963436928702011,
                  0.062385321100917435,
                  545,
                  34
                ],
                [
                  0.9981718464351006,
                  0.0641025641025641,
                  546,
                  35
                ],
                [
                  1.0,
                  0.06398537477148081,
                  547,
                  35
                ]
              ]
            },
            {
              "language": "English",
              "run": 411,
              "points": [
                [
                  0.0018281535648994515,
                  0.0,
                  1,
                  0
                ],
                [
                  0.003656307129798903,
                  0.0,
                  2,
                  0
                ],
                [
                  0.005484460694698354,
                  0.0,
                  3,
                  0
                ],
                [
                  0.007312614259597806,
                  0.0,
                  4,
                  0
                ],
                [
                  0.009140767824497258,
                  0.0,
                  5,
                  0
                ],
                [
                  0.010968921389396709,
                  0.0,
                  6,
                  0
                ],
                [
                  0.012797074954296161,
                  0.0,
                  7,
                  0
                ],
                [
                  0.014625228519195612,
                  0.0,
                  8,
                  0
                ],
                [
                  0.016453382084095063,
                  0.0,
                  9,
                  0
                ],
                [
                  0.018281535648994516,
                  0.0,
                  10,
                  0
                ],
                [
                  0.02010968921389397,
                  0.0,
                  11,
                  0
                ],
                [
                  0.021937842778793418,
                  0.0,
                  12,
                  0
                ],
                [
                  0.02376599634369287,
                  0.0,
                  13,
                  0
                ],
                [
                  0.025594149908592323,
                  0.0,
                  14,
                  0
                ],
                [
                  0.027422303473491772,
                  0.0,
                  15,
                  0
                ],
                [
                  0.029250457038391225,
                  0.0,
                  16,
                  0
                ],
                [
                  0.031078610603290677,
                  0.0,
                  17,
                  0
                ],
                [
                  0.03290676416819013,
                  0.0,
                  18,
                  0
                ],
                [
                  0.03473491773308958,
                  0.0,
                  19,
                  0
                ],
                [
                  0.03656307129798903,
                  0.0,
                  20,
                  0
                ],
                [
                  0.038391224862888484,
                  0.0,
                  21,
                  0
                ],
                [
                  0.04021937842778794,
                  0.0,
                  22,
                  0
                ],
                [
                  0.04204753199268738,
                  0.0,
                  23,
                  0
                ],
                [
                  0.043875685557586835,
                  0.0,
                  24,
                  0
                ],
                [
                  0.04570383912248629,
                  0.0,
                  25,
                  0
                ],
                [
                  0.04753199268738574,
                  0.0,
                  26,
                  0
                ],
                [
                  0.04936014625228519,
                  0.0,
                  27,
                  0
                ],
                [
                  0.051188299817184646,
                  0.0,
                  28,
                  0
                ],
                [
                  0.05301645338208409,
                  0.0,
                  29,
                  0
                ],
                [
                  0.054844606946983544,
                  0.03333333333333333,
                  30,
                  1
                ],
                [
                  0.056672760511883,
                  0.03225806451612903,
                  31,
                  1
                ],
                [
                  0.05850091407678245,
                  0.03125,
                  32,
                  1
                ],
                [
                  0.0603290676416819,
                  0.030303030303030304,
                  33,
                  1
                ],
                [
                  0.062157221206581355,
                  0.029411764705882353,
                  34,
                  1
                ],
                [
                  0.06398537477148081,
                  0.02857142857142857,
                  35,
                  1
                ],
                [
                  0.06581352833638025,
                  0.027777777777777776,
                  36,
                  1
                ],
                [
                  0.06764168190127971,
                  0.02702702702702703,
                  37,
                  1
                ],
                [
                  0.06946983546617916,
                  0.02631578947368421,
                  38,
                  1
                ],
                [
                  0.0712979890310786,
                  0.02564102564102564,
                  39,
                  1
                ],
                [
                  0.07312614259597806,
                  0.025,
                  40,
                  1
                ],
                [
                  0.07495429616087751,
                  0.024390243902439025,
                  41,
                  1
                ],
                [
                  0.07678244972577697,
                  0.023809523809523808,
                  42,
                  1
                ],
                [
                  0.07861060329067641,
                  0.023255813953488372,
                  43,
                  1
                ],
                [
                  0.08043875685557587,
                  0.022727272727272728,
                  44,
                  1
                ],
                [
                  0.08226691042047532,
                  0.022222222222222223,
                  45,
                  1
                ],
                [
                  0.08409506398537477,
                  0.021739130434782608,
                  46,
                  1
                ],
                [
                  0.08592321755027423,
                  0.02127659574468085,
                  47,
                  1
                ],
                [
                  0.08775137111517367,
                  0.020833333333333332,
                  48,
                  1
                ],
                [
                  0.08957952468007313,
                  0.02040816326530612,
                  49,
                  1
                ],
                [
                  0.09140767824497258,
                  0.02,
                  50,
                  1
                ],
                [
                  0.09323583180987204,
                  0.0196078431372549,
                  51,
                  1
                ],
                [
                  0.09506398537477148,
                  0.019230769230769232,
                  52,
                  1
                ],
                [
                  0.09689213893967093,
                  0.018867924528301886,
                  53,
                  1
                ],
                [
                  0.09872029250457039,
                  0.018518518518518517,
                  54,
                  1
                ],
                [
                  0.10054844606946983,
                  0.01818181818181818,
                  55,
                  1
                ],
                [
                  0.10237659963436929,
                  0.017857142857142856,
                  56,
                  1
                ],
                [
                  0.10420475319926874,
                  0.017543859649122806,
                  57,
                  1
                ],
                [
                  0.10603290676416818,
                  0.017241379310344827,
                  58,
                  1
                ],
                [
                  0.10786106032906764,
                  0.01694915254237288,
                  59,
                  1
                ],
                [
                  0.10968921389396709,
                  0.016666666666666666,
                  60,
                  1
                ],
                [
                  0.11151736745886655,
                  0.01639344262295082,
                  61,
                  1
                ],
                [
                  0.113345521023766,
                  0.016129032258064516,
                  62,
                  1
                ],
                [
                  0.11517367458866545,
                  0.015873015873015872,
                  63,
                  1
                ],
                [
                  0.1170018281535649,
                  0.015625,
                  64,
                  1
                ],
                [
                  0.11882998171846434,
                  0.015384615384615385,
                  65,
                  1
                ],
                [
                  0.1206581352833638,
                  0.015151515151515152,
                  66,
                  1
                ],
                [
                  0.12248628884826325,
                  0.014925373134328358,
                  67,
                  1
                ],
                [
                  0.12431444241316271,
                  0.014705882352941176,
                  68,
                  1
                ],
                [
                  0.12614259597806216,
                  0.014492753623188406,
                  69,
                  1
                ],
                [
                  0.12797074954296161,
                  0.014285714285714285,
                  70,
                  1
                ],
                [
                  0.12979890310786105,
                  0.014084507042253521,
                  71,
                  1
                ],
                [
                  0.1316270566727605,
                  0.013888888888888888,
                  72,
                  1
                ],
                [
                  0.13345521023765997,
                  0.0136986301369863,
                  73,
                  1
                ],
                [
                  0.13528336380255943,
                  0.013513513513513514,
                  74,
                  1
                ],
                [
                  0.13711151736745886,
                  0.013333333333333334,
                  75,
                  1
                ],
                [
                  0.13893967093235832,
                  0.013157894736842105,
                  76,
                  1
                ],
                [
                  0.14076782449725778,
                  0.012987012987012988,
                  77,
                  1
                ],
                [
                  0.1425959780621572,
                  0.01282051282051282,
                  78,
                  1
                ],
                [
                  0.14442413162705667,
                  0.012658227848101266,
                  79,
                  1
                ],
                [
                  0.14625228519195613,
                  0.0125,
                  80,
                  1
                ],
                [
                  0.1480804387568556,
                  0.012345679012345678,
                  81,
                  1
                ],
                [
                  0.14990859232175502,
                  0.012195121951219513,
                  82,
                  1
                ],
                [
                  0.15173674588665448,
                  0.012048192771084338,
                  83,
                  1
                ],
                [
                  0.15356489945155394,
                  0.011904761904761904,
                  84,
                  1
                ],
                [
                  0.15539305301645337,
                  0.011764705882352941,
                  85,
                  1
                ],
                [
                  0.15722120658135283,
                  0.011627906976744186,
                  86,
                  1
                ],
                [
                  0.1590493601462523,
                  0.011494252873563218,
                  87,
                  1
                ],
                [
                  0.16087751371115175,
                  0.011363636363636364,
                  88,
                  1
                ],
                [
                  0.16270566727605118,
                  0.011235955056179775,
                  89,
                  1
                ],
                [
                  0.16453382084095064,
                  0.011111111111111112,
                  90,
                  1
                ],
                [
                  0.1663619744058501,
                  0.01098901098901099,
                  91,
                  1
                ],
                [
                  0.16819012797074953,
                  0.010869565217391304,
                  92,
                  1
                ],
                [
                  0.170018281535649,
                  0.010752688172043012,
                  93,
                  1
                ],
                [
                  0.17184643510054845,
                  0.010638297872340425,
                  94,
                  1
                ],
                [
                  0.1736745886654479,
                  0.010526315789473684,
                  95,
                  1
                ],
                [
                  0.17550274223034734,
                  0.010416666666666666,
                  96,
                  1
                ],
                [
                  0.1773308957952468,
                  0.010309278350515464,
                  97,
                  1
                ],
                [
                  0.17915904936014626,
                  0.01020408163265306,
                  98,
                  1
                ],
                [
                  0.1809872029250457,
                  0.010101010101010102,
                  99,
                  1
                ],
                [
                  0.18281535648994515,
                  0.01,
                  100,
                  1
                ],
                [
                  0.1846435100548446,
                  0.009900990099009901,
                  101,
                  1
                ],
                [
                  0.18647166361974407,
                  0.00980392156862745,
                  102,
                  1
                ],
                [
                  0.1882998171846435,
                  0.009708737864077669,
                  103,
                  1
                ],
                [
                  0.19012797074954296,
                  0.009615384615384616,
                  104,
                  1
                ],
                [
                  0.19195612431444242,
                  0.009523809523809525,
                  105,
                  1
                ],
                [
                  0.19378427787934185,
                  0.009433962264150943,
                  106,
                  1
                ],
                [
                  0.1956124314442413,
                  0.009345794392523364,
                  107,
                  1
                ],
                [
                  0.19744058500914077,
                  0.009259259259259259,
                  108,
                  1
                ],
                [
                  0.19926873857404023,
                  0.009174311926605505,
                  109,
                  1
                ],
                [
                  0.20109689213893966,
                  0.00909090909090909,
                  110,
                  1
                ],
                [
                  0.20292504570383912,
                  0.009009009009009009,
                  111,
                  1
                ],
                [
                  0.20475319926873858,
                  0.008928571428571428,
                  112,
                  1
                ],
                [
                  0.20658135283363802,
                  0.008849557522123894,
                  113,
                  1
                ],
                [
                  0.20840950639853748,
                  0.008771929824561403,
                  114,
                  1
                ],
                [
                  0.21023765996343693,
                  0.008695652173913044,
                  115,
                  1
                ],
                [
                  0.21206581352833637,
                  0.008620689655172414,
                  116,
                  1
                ],
                [
                  0.21389396709323583,
                  0.008547008547008548,
                  117,
                  1
                ],
                [
                  0.21572212065813529,
                  0.00847457627118644,
                  118,
                  1
                ],
                [
                  0.21755027422303475,
                  0.008403361344537815,
                  119,
                  1
                ],
                [
                  0.21937842778793418,
                  0.008333333333333333,
                  120,
                  1
                ],
                [
                  0.22120658135283364,
                  0.008264462809917356,
                  121,
                  1
                ],
                [
                  0.2230347349177331,
                  0.00819672131147541,
                  122,
                  1
                ],
                [
                  0.22486288848263253,
                  0.008130081300813009,
                  123,
                  1
                ],
                [
                  0.226691042047532,
                  0.008064516129032258,
                  124,
                  1
                ],
                [
                  0.22851919561243145,
                  0.008,
                  125,
                  1
                ],
                [
                  0.2303473491773309,
                  0.007936507936507936,
                  126,
                  1
                ],
                [
                  0.23217550274223034,
                  0.007874015748031496,
                  127,
                  1
                ],
                [
                  0.2340036563071298,
                  0.0078125,
                  128,
                  1
                ],
                [
                  0.23583180987202926,
                  0.007751937984496124,
                  129,
                  1
                ],
                [
                  0.2376599634369287,
                  0.007692307692307693,
                  130,
                  1
                ],
                [
                  0.23948811700182815,
                  0.007633587786259542,
                  131,
                  1
                ],
                [
                  0.2413162705667276,
                  0.007575757575757576,
                  132,
                  1
                ],
                [
                  0.24314442413162707,
                  0.007518796992481203,
                  133,
                  1
                ],
                [
                  0.2449725776965265,
                  0.007462686567164179,
                  134,
                  1
                ],
                [
                  0.24680073126142596,
                  0.007407407407407408,
                  135,
                  1
                ],
                [
                  0.24862888482632542,
                  0.007352941176470588,
                  136,
                  1
                ],
                [
                  0.25045703839122485,
                  0.0072992700729927005,
                  137,
                  1
                ],
                [
                  0.2522851919561243,
                  0.007246376811594203,
                  138,
                  1
                ],
                [
                  0.25411334552102377,
                  0.007194244604316547,
                  139,
                  1
                ],
                [
                  0.25594149908592323,
                  0.007142857142857143,
                  140,
                  1
                ],
                [
                  0.2577696526508227,
                  0.0070921985815602835,
                  141,
                  1
                ],
                [
                  0.2595978062157221,
                  0.007042253521126761,
                  142,
                  1
                ],
                [
                  0.26142595978062155,
                  0.006993006993006993,
                  143,
                  1
                ],
                [
                  0.263254113345521,
                  0.006944444444444444,
                  144,
                  1
                ],
                [
                  0.26508226691042047,
                  0.006896551724137931,
                  145,
                  1
                ],
                [
                  0.26691042047531993,
                  0.00684931506849315,
                  146,
                  1
                ],
                [
                  0.2687385740402194,
                  0.006802721088435374,
                  147,
                  1
                ],
                [
                  0.27056672760511885,
                  0.006756756756756757,
                  148,
                  1
                ],
                [
                  0.27239488117001825,
                  0.006711409395973154,
                  149,
                  1
                ],
                [
                  0.2742230347349177,
                  0.006666666666666667,
                  150,
                  1
                ],
                [
                  0.2760511882998172,
                  0.006622516556291391,
                  151,
                  1
                ],
                [
                  0.27787934186471663,
                  0.006578947368421052,
                  152,
                  1
                ],
                [
                  0.2797074954296161,
                  0.006535947712418301,
                  153,
                  1
                ],
                [
                  0.28153564899451555,
                  0.006493506493506494,
                  154,
                  1
                ],
                [
                  0.283363802559415,
                  0.0064516129032258064,
                  155,
                  1
                ],
                [
                  0.2851919561243144,
                  0.00641025641025641,
                  156,
                  1
                ],
                [
                  0.2870201096892139,
                  0.006369426751592357,
                  157,
                  1
                ],
                [
                  0.28884826325411334,
                  0.006329113924050633,
                  158,
                  1
                ],
                [
                  0.2906764168190128,
                  0.006289308176100629,
                  159,
                  1
                ],
                [
                  0.29250457038391225,
                  0.00625,
                  160,
                  1
                ],
                [
                  0.2943327239488117,
                  0.006211180124223602,
                  161,
                  1
                ],
                [
                  0.2961608775137112,
                  0.006172839506172839,
                  162,
                  1
                ],
                [
                  0.2979890310786106,
                  0.006134969325153374,
                  163,
                  1
                ],
                [
                  0.29981718464351004,
                  0.006097560975609756,
                  164,
                  1
                ],
                [
                  0.3016453382084095,
                  0.006060606060606061,
                  165,
                  1
                ],
                [
                  0.30347349177330896,
                  0.006024096385542169,
                  166,
                  1
                ],
                [
                  0.3053016453382084,
                  0.005988023952095809,
                  167,
                  1
                ],
                [
                  0.3071297989031079,
                  0.005952380952380952,
                  168,
                  1
                ],
                [
                  0.30895795246800734,
                  0.005917159763313609,
                  169,
                  1
                ],
                [
                  0.31078610603290674,
                  0.0058823529411764705,
                  170,
                  1
                ],
                [
                  0.3126142595978062,
                  0.005847953216374269,
                  171,
                  1
                ],
                [
                  0.31444241316270566,
                  0.005813953488372093,
                  172,
                  1
                ],
                [
                  0.3162705667276051,
                  0.005780346820809248,
                  173,
                  1
                ],
                [
                  0.3180987202925046,
                  0.005747126436781609,
                  174,
                  1
                ],
                [
                  0.31992687385740404,
                  0.005714285714285714,
                  175,
                  1
                ],
                [
                  0.3217550274223035,
                  0.005681818181818182,
                  176,
                  1
                ],
                [
                  0.3235831809872029,
                  0.005649717514124294,
                  177,
                  1
                ],
                [
                  0.32541133455210236,
                  0.0056179775280898875,
                  178,
                  1
                ],
                [
                  0.3272394881170018,
                  0.00558659217877095,
                  179,
                  1
                ],
                [
                  0.3290676416819013,
                  0.005555555555555556,
                  180,
                  1
                ],
                [
                  0.33089579524680074,
                  0.0055248618784530384,
                  181,
                  1
                ],
                [
                  0.3327239488117002,
                  0.005494505494505495,
                  182,
                  1
                ],
                [
                  0.33455210237659966,
                  0.00546448087431694,
                  183,
                  1
                ],
                [
                  0.33638025594149906,
                  0.005434782608695652,
                  184,
                  1
                ],
                [
                  0.3382084095063985,
                  0.005405405405405406,
                  185,
                  1
                ],
                [
                  0.340036563071298,
                  0.005376344086021506,
                  186,
                  1
                ],
                [
                  0.34186471663619744,
                  0.0053475935828877,
                  187,
                  1
                ],
                [
                  0.3436928702010969,
                  0.005319148936170213,
                  188,
                  1
                ],
                [
                  0.34552102376599636,
                  0.005291005291005291,
                  189,
                  1
                ],
                [
                  0.3473491773308958,
                  0.005263157894736842,
                  190,
                  1
                ],
                [
                  0.3491773308957952,
                  0.005235602094240838,
                  191,
                  1
                ],
                [
                  0.3510054844606947,
                  0.005208333333333333,
                  192,
                  1
                ],
                [
                  0.35283363802559414,
                  0.0051813471502590676,
                  193,
                  1
                ],
                [
                  0.3546617915904936,
                  0.005154639175257732,
                  194,
                  1
                ],
                [
                  0.35648994515539306,
                  0.005128205128205128,
                  195,
                  1
                ],
                [
                  0.3583180987202925,
                  0.00510204081632653,
                  196,
                  1
                ],
                [
                  0.360146252285192,
                  0.005076142131979695,
                  197,
                  1
                ],
                [
                  0.3619744058500914,
                  0.005050505050505051,
                  198,
                  1
                ],
                [
                  0.36380255941499084,
                  0.005025125628140704,
                  199,
                  1
                ],
                [
                  0.3656307129798903,
                  0.005,
                  200,
                  1
                ],
                [
                  0.36745886654478976,
                  0.004975124378109453,
                  201,
                  1
                ],
                [
                  0.3692870201096892,
                  0.0049504950495049506,
                  202,
                  1
                ],
                [
                  0.3711151736745887,
                  0.0049261083743842365,
                  203,
                  1
                ],
                [
                  0.37294332723948814,
                  0.004901960784313725,
                  204,
                  1
                ],
                [
                  0.37477148080438755,
                  0.004878048780487805,
                  205,
                  1
                ],
                [
                  0.376599634369287,
                  0.0048543689320388345,
                  206,
                  1
                ],
                [
                  0.37842778793418647,
                  0.004830917874396135,
                  207,
                  1
                ],
                [
                  0.3802559414990859,
                  0.004807692307692308,
                  208,
                  1
                ],
                [
                  0.3820840950639854,
                  0.004784688995215311,
                  209,
                  1
                ],
                [
                  0.38391224862888484,
                  0.004761904761904762,
                  210,
                  1
                ],
                [
                  0.3857404021937843,
                  0.004739336492890996,
                  211,
                  1
                ],
                [
                  0.3875685557586837,
                  0.0047169811320754715,
                  212,
                  1
                ],
                [
                  0.38939670932358317,
                  0.004694835680751174,
                  213,
                  1
                ],
                [
                  0.3912248628884826,
                  0.004672897196261682,
                  214,
                  1
                ],
                [
                  0.3930530164533821,
                  0.004651162790697674,
                  215,
                  1
                ],
                [
                  0.39488117001828155,
                  0.004629629629629629,
                  216,
                  1
                ],
                [
                  0.396709323583181,
                  0.004608294930875576,
                  217,
                  1
                ],
                [
                  0.39853747714808047,
                  0.0045871559633027525,
                  218,
                  1
                ],
                [
                  0.40036563071297987,
                  0.0045662100456621,
                  219,
                  1
                ],
                [
                  0.40219378427787933,
                  0.004545454545454545,
                  220,
                  1
                ],
                [
                  0.4040219378427788,
                  0.004524886877828055,
                  221,
                  1
                ],
                [
                  0.40585009140767825,
                  0.0045045045045045045,
                  222,
                  1
                ],
                [
                  0.4076782449725777,
                  0.004484304932735426,
                  223,
                  1
                ],
                [
                  0.40950639853747717,
                  0.004464285714285714,
                  224,
                  1
                ],
                [
                  0.4113345521023766,
                  0.0044444444444444444,
                  225,
                  1
                ],
                [
                  0.41316270566727603,
                  0.004424778761061947,
                  226,
                  1
                ],
                [
                  0.4149908592321755,
                  0.004405286343612335,
                  227,
                  1
                ],
                [
                  0.41681901279707495,
                  0.0043859649122807015,
                  228,
                  1
                ],
                [
                  0.4186471663619744,
                  0.004366812227074236,
                  229,
                  1
                ],
                [
                  0.42047531992687387,
                  0.004347826086956522,
                  230,
                  1
                ],
                [
                  0.42230347349177333,
                  0.004329004329004329,
                  231,
                  1
                ],
                [
                  0.42413162705667273,
                  0.004310344827586207,
                  232,
                  1
                ],
                [
                  0.4259597806215722,
                  0.004291845493562232,
                  233,
                  1
                ],
                [
                  0.42778793418647165,
                  0.004273504273504274,
                  234,
                  1
                ],
                [
                  0.4296160877513711,
                  0.00425531914893617,
                  235,
                  1
                ],
                [
                  0.43144424131627057,
                  0.00423728813559322,
                  236,
                  1
                ],
                [
                  0.43327239488117003,
                  0.004219409282700422,
                  237,
                  1
                ],
                [
                  0.4351005484460695,
                  0.004201680672268907,
                  238,
                  1
                ],
                [
                  0.4369287020109689,
                  0.0041841004184100415,
                  239,
                  1
                ],
                [
                  0.43875685557586835,
                  0.004166666666666667,
                  240,
                  1
                ],
                [
                  0.4405850091407678,
                  0.004149377593360996,
                  241,
                  1
                ],
                [
                  0.4424131627056673,
                  0.004132231404958678,
                  242,
                  1
                ],
                [
                  0.44424131627056673,
                  0.00411522633744856,
                  243,
                  1
                ],
                [
                  0.4460694698354662,
                  0.004098360655737705,
                  244,
                  1
                ],
                [
                  0.44789762340036565,
                  0.004081632653061225,
                  245,
                  1
                ],
                [
                  0.44972577696526506,
                  0.0040650406504065045,
                  246,
                  1
                ],
                [
                  0.4515539305301645,
                  0.004048582995951417,
                  247,
                  1
                ],
                [
                  0.453382084095064,
                  0.004032258064516129,
                  248,
                  1
                ],
                [
                  0.45521023765996343,
                  0.004016064257028112,
                  249,
                  1
                ],
                [
                  0.4570383912248629,
                  0.004,
                  250,
                  1
                ],
                [
                  0.45886654478976235,
                  0.00398406374501992,
                  251,
                  1
                ],
                [
                  0.4606946983546618,
                  0.003968253968253968,
                  252,
                  1
                ],
                [
                  0.4625228519195612,
                  0.003952569169960474,
                  253,
                  1
                ],
                [
                  0.4643510054844607,
                  0.003937007874015748,
                  254,
                  1
                ],
                [
                  0.46617915904936014,
                  0.00392156862745098,
                  255,
                  1
                ],
                [
                  0.4680073126142596,
                  0.00390625,
                  256,
                  1
                ],
                [
                  0.46983546617915906,
                  0.0038910505836575876,
                  257,
                  1
                ],
                [
                  0.4716636197440585,
                  0.003875968992248062,
                  258,
                  1
                ],
                [
                  0.473491773308958,
                  0.003861003861003861,
                  259,
                  1
                ],
                [
                  0.4753199268738574,
                  0.0038461538461538464,
                  260,
                  1
                ],
                [
                  0.47714808043875684,
                  0.0038314176245210726,
                  261,
                  1
                ],
                [
                  0.4789762340036563,
                  0.003816793893129771,
                  262,
                  1
                ],
                [
                  0.48080438756855576,
                  0.0038022813688212928,
                  263,
                  1
                ],
                [
                  0.4826325411334552,
                  0.003787878787878788,
                  264,
                  1
                ],
                [
                  0.4844606946983547,
                  0.0037735849056603774,
                  265,
                  1
                ],
                [
                  0.48628884826325414,
                  0.0037593984962406013,
                  266,
                  1
                ],
                [
                  0.48811700182815354,
                  0.003745318352059925,
                  267,
                  1
                ],
                [
                  0.489945155393053,
                  0.0037313432835820895,
                  268,
                  1
                ],
                [
                  0.49177330895795246,
                  0.0037174721189591076,
                  269,
                  1
                ],
                [
                  0.4936014625228519,
                  0.003703703703703704,
                  270,
                  1
                ],
                [
                  0.4954296160877514,
                  0.0036900369003690036,
                  271,
                  1
                ],
                [
                  0.49725776965265084,
                  0.003676470588235294,
                  272,
                  1
                ],
                [
                  0.4990859232175503,
                  0.003663003663003663,
                  273,
                  1
                ],
                [
                  0.5009140767824497,
                  0.0036496350364963502,
                  274,
                  1
                ],
                [
                  0.5027422303473492,
                  0.0036363636363636364,
                  275,
                  1
                ],
                [
                  0.5045703839122486,
                  0.0036231884057971015,
                  276,
                  1
                ],
                [
                  0.506398537477148,
                  0.0036101083032490976,
                  277,
                  1
                ],
                [
                  0.5082266910420475,
                  0.0035971223021582736,
                  278,
                  1
                ],
                [
                  0.5100548446069469,
                  0.0035842293906810036,
                  279,
                  1
                ],
                [
                  0.5118829981718465,
                  0.0035714285714285713,
                  280,
                  1
                ],
                [
                  0.5137111517367459,
                  0.0035587188612099642,
                  281,
                  1
                ],
                [
                  0.5155393053016454,
                  0.0035460992907801418,
                  282,
                  1
                ],
                [
                  0.5173674588665448,
                  0.0035335689045936395,
                  283,
                  1
                ],
                [
                  0.5191956124314442,
                  0.0035211267605633804,
                  284,
                  1
                ],
                [
                  0.5210237659963437,
                  0.0035087719298245615,
                  285,
                  1
                ],
                [
                  0.5228519195612431,
                  0.0034965034965034965,
                  286,
                  1
                ],
                [
                  0.5246800731261426,
                  0.003484320557491289,
                  287,
                  1
                ],
                [
                  0.526508226691042,
                  0.003472222222222222,
                  288,
                  1
                ],
                [
                  0.5283363802559415,
                  0.0034602076124567475,
                  289,
                  1
                ],
                [
                  0.5301645338208409,
                  0.0034482758620689655,
                  290,
                  1
                ],
                [
                  0.5319926873857403,
                  0.003436426116838488,
                  291,
                  1
                ],
                [
                  0.5338208409506399,
                  0.003424657534246575,
                  292,
                  1
                ],
                [
                  0.5356489945155393,
                  0.0034129692832764505,
                  293,
                  1
                ],
                [
                  0.5374771480804388,
                  0.003401360544217687,
                  294,
                  1
                ],
                [
                  0.5393053016453382,
                  0.003389830508474576,
                  295,
                  1
                ],
                [
                  0.5411334552102377,
                  0.0033783783783783786,
                  296,
                  1
                ],
                [
                  0.5429616087751371,
                  0.003367003367003367,
                  297,
                  1
                ],
                [
                  0.5447897623400365,
                  0.003355704697986577,
                  298,
                  1
                ],
                [
                  0.546617915904936,
                  0.0033444816053511705,
                  299,
                  1
                ],
                [
                  0.5484460694698354,
                  0.0033333333333333335,
                  300,
                  1
                ],
                [
                  0.5502742230347349,
                  0.0033222591362126247,
                  301,
                  1
                ],
                [
                  0.5521023765996343,
                  0.0033112582781456954,
                  302,
                  1
                ],
                [
                  0.5539305301645339,
                  0.0033003300330033004,
                  303,
                  1
                ],
                [
                  0.5557586837294333,
                  0.003289473684210526,
                  304,
                  1
                ],
                [
                  0.5575868372943327,
                  0.003278688524590164,
                  305,
                  1
                ],
                [
                  0.5594149908592322,
                  0.0032679738562091504,
                  306,
                  1
                ],
                [
                  0.5612431444241316,
                  0.003257328990228013,
                  307,
                  1
                ],
                [
                  0.5630712979890311,
                  0.003246753246753247,
                  308,
                  1
                ],
                [
                  0.5648994515539305,
                  0.003236245954692557,
                  309,
                  1
                ],
                [
                  0.56672760511883,
                  0.0032258064516129032,
                  310,
                  1
                ],
                [
                  0.5685557586837294,
                  0.003215434083601286,
                  311,
                  1
                ],
                [
                  0.5703839122486288,
                  0.003205128205128205,
                  312,
                  1
                ],
                [
                  0.5722120658135283,
                  0.003194888178913738,
                  313,
                  1
                ],
                [
                  0.5740402193784278,
                  0.0031847133757961785,
                  314,
                  1
                ],
                [
                  0.5758683729433273,
                  0.0031746031746031746,
                  315,
                  1
                ],
                [
                  0.5776965265082267,
                  0.0031645569620253164,
                  316,
                  1
                ],
                [
                  0.5795246800731262,
                  0.0031545741324921135,
                  317,
                  1
                ],
                [
                  0.5813528336380256,
                  0.0031446540880503146,
                  318,
                  1
                ],
                [
                  0.583180987202925,
                  0.003134796238244514,
                  319,
                  1
                ],
                [
                  0.5850091407678245,
                  0.003125,
                  320,
                  1
                ],
                [
                  0.5868372943327239,
                  0.003115264797507788,
                  321,
                  1
                ],
                [
                  0.5886654478976234,
                  0.006211180124223602,
                  322,
                  2
                ],
                [
                  0.5904936014625228,
                  0.006191950464396285,
                  323,
                  2
                ],
                [
                  0.5923217550274223,
                  0.006172839506172839,
                  324,
                  2
                ],
                [
                  0.5941499085923218,
                  0.006153846153846154,
                  325,
                  2
                ],
                [
                  0.5959780621572212,
                  0.006134969325153374,
                  326,
                  2
                ],
                [
                  0.5978062157221207,
                  0.0061162079510703364,
                  327,
                  2
                ],
                [
                  0.5996343692870201,
                  0.006097560975609756,
                  328,
                  2
                ],
                [
                  0.6014625228519196,
                  0.0060790273556231,
                  329,
                  2
                ],
                [
                  0.603290676416819,
                  0.006060606060606061,
                  330,
                  2
                ],
                [
                  0.6051188299817185,
                  0.006042296072507553,
                  331,
                  2
                ],
                [
                  0.6069469835466179,
                  0.006024096385542169,
                  332,
                  2
                ],
                [
                  0.6087751371115173,
                  0.006006006006006006,
                  333,
                  2
                ],
                [
                  0.6106032906764168,
                  0.005988023952095809,
                  334,
                  2
                ],
                [
                  0.6124314442413162,
                  0.005970149253731343,
                  335,
                  2
                ],
                [
                  0.6142595978062158,
                  0.005952380952380952,
                  336,
                  2
                ],
                [
                  0.6160877513711152,
                  0.005934718100890208,
                  337,
                  2
                ],
                [
                  0.6179159049360147,
                  0.005917159763313609,
                  338,
                  2
                ],
                [
                  0.6197440585009141,
                  0.0058997050147492625,
                  339,
                  2
                ],
                [
                  0.6215722120658135,
                  0.0058823529411764705,
                  340,
                  2
                ],
                [
                  0.623400365630713,
                  0.005865102639296188,
                  341,
                  2
                ],
                [
                  0.6252285191956124,
                  0.005847953216374269,
                  342,
                  2
                ],
                [
                  0.6270566727605119,
                  0.0058309037900874635,
                  343,
                  2
                ],
                [
                  0.6288848263254113,
                  0.005813953488372093,
                  344,
                  2
                ],
                [
                  0.6307129798903108,
                  0.005797101449275362,
                  345,
                  2
                ],
                [
                  0.6325411334552102,
                  0.005780346820809248,
                  346,
                  2
                ],
                [
                  0.6343692870201096,
                  0.005763688760806916,
                  347,
                  2
                ],
                [
                  0.6361974405850092,
                  0.005747126436781609,
                  348,
                  2
                ],
                [
                  0.6380255941499086,
                  0.0057306590257879654,
                  349,
                  2
                ],
                [
                  0.6398537477148081,
                  0.005714285714285714,
                  350,
                  2
                ],
                [
                  0.6416819012797075,
                  0.005698005698005698,
                  351,
                  2
                ],
                [
                  0.643510054844607,
                  0.005681818181818182,
                  352,
                  2
                ],
                [
                  0.6453382084095064,
                  0.0056657223796034,
                  353,
                  2
                ],
                [
                  0.6471663619744058,
                  0.005649717514124294,
                  354,
                  2
                ],
                [
                  0.6489945155393053,
                  0.005633802816901409,
                  355,
                  2
                ],
                [
                  0.6508226691042047,
                  0.0056179775280898875,
                  356,
                  2
                ],
                [
                  0.6526508226691042,
                  0.0056022408963585435,
                  357,
                  2
                ],
                [
                  0.6544789762340036,
                  0.00558659217877095,
                  358,
                  2
                ],
                [
                  0.6563071297989032,
                  0.005571030640668524,
                  359,
                  2
                ],
                [
                  0.6581352833638026,
                  0.005555555555555556,
                  360,
                  2
                ],
                [
                  0.659963436928702,
                  0.00554016620498615,
                  361,
                  2
                ],
                [
                  0.6617915904936015,
                  0.0055248618784530384,
                  362,
                  2
                ],
                [
                  0.6636197440585009,
                  0.005509641873278237,
                  363,
                  2
                ],
                [
                  0.6654478976234004,
                  0.005494505494505495,
                  364,
                  2
                ],
                [
                  0.6672760511882998,
                  0.005479452054794521,
                  365,
                  2
                ],
                [
                  0.6691042047531993,
                  0.00546448087431694,
                  366,
                  2
                ],
                [
                  0.6709323583180987,
                  0.005449591280653951,
                  367,
                  2
                ],
                [
                  0.6727605118829981,
                  0.005434782608695652,
                  368,
                  2
                ],
                [
                  0.6745886654478976,
                  0.005420054200542005,
                  369,
                  2
                ],
                [
                  0.676416819012797,
                  0.005405405405405406,
                  370,
                  2
                ],
                [
                  0.6782449725776966,
                  0.005390835579514825,
                  371,
                  2
                ],
                [
                  0.680073126142596,
                  0.005376344086021506,
                  372,
                  2
                ],
                [
                  0.6819012797074955,
                  0.005361930294906166,
                  373,
                  2
                ],
                [
                  0.6837294332723949,
                  0.0053475935828877,
                  374,
                  2
                ],
                [
                  0.6855575868372943,
                  0.005333333333333333,
                  375,
                  2
                ],
                [
                  0.6873857404021938,
                  0.005319148936170213,
                  376,
                  2
                ],
                [
                  0.6892138939670932,
                  0.005305039787798408,
                  377,
                  2
                ],
                [
                  0.6910420475319927,
                  0.005291005291005291,
                  378,
                  2
                ],
                [
                  0.6928702010968921,
                  0.005277044854881266,
                  379,
                  2
                ],
                [
                  0.6946983546617916,
                  0.007894736842105263,
                  380,
                  3
                ],
                [
                  0.696526508226691,
                  0.007874015748031496,
                  381,
                  3
                ],
                [
                  0.6983546617915904,
                  0.007853403141361256,
                  382,
                  3
                ],
                [
                  0.70018281535649,
                  0.007832898172323759,
                  383,
                  3
                ],
                [
                  0.7020109689213894,
                  0.0078125,
                  384,
                  3
                ],
                [
                  0.7038391224862889,
                  0.007792207792207792,
                  385,
                  3
                ],
                [
                  0.7056672760511883,
                  0.007772020725388601,
                  386,
                  3
                ],
                [
                  0.7074954296160878,
                  0.0103359173126615,
                  387,
                  4
                ],
                [
                  0.7093235831809872,
                  0.010309278350515464,
                  388,
                  4
                ],
                [
                  0.7111517367458866,
                  0.010282776349614395,
                  389,
                  4
                ],
                [
                  0.7129798903107861,
                  0.010256410256410256,
                  390,
                  4
                ],
                [
                  0.7148080438756855,
                  0.01278772378516624,
                  391,
                  5
                ],
                [
                  0.716636197440585,
                  0.015306122448979591,
                  392,
                  6
                ],
                [
                  0.7184643510054844,
                  0.017811704834605598,
                  393,
                  7
                ],
                [
                  0.720292504570384,
                  0.02030456852791878,
                  394,
                  8
                ],
                [
                  0.7221206581352834,
                  0.020253164556962026,
                  395,
                  8
                ],
                [
                  0.7239488117001828,
                  0.020202020202020204,
                  396,
                  8
                ],
                [
                  0.7257769652650823,
                  0.020151133501259445,
                  397,
                  8
                ],
                [
                  0.7276051188299817,
                  0.020100502512562814,
                  398,
                  8
                ],
                [
                  0.7294332723948812,
                  0.020050125313283207,
                  399,
                  8
                ],
                [
                  0.7312614259597806,
                  0.02,
                  400,
                  8
                ],
                [
                  0.7330895795246801,
                  0.0199501246882793,
                  401,
                  8
                ],
                [
                  0.7349177330895795,
                  0.01990049751243781,
                  402,
                  8
                ],
                [
                  0.7367458866544789,
                  0.019851116625310174,
                  403,
                  8
                ],
                [
                  0.7385740402193784,
                  0.019801980198019802,
                  404,
                  8
                ],
                [
                  0.7404021937842779,
                  0.019753086419753086,
                  405,
                  8
                ],
                [
                  0.7422303473491774,
                  0.019704433497536946,
                  406,
                  8
                ],
                [
                  0.7440585009140768,
                  0.022113022113022112,
                  407,
                  9
                ],
                [
                  0.7458866544789763,
                  0.022058823529411766,
                  408,
                  9
                ],
                [
                  0.7477148080438757,
                  0.022004889975550123,
                  409,
                  9
                ],
                [
                  0.7495429616087751,
                  0.02195121951219512,
                  410,
                  9
                ],
                [
                  0.7513711151736746,
                  0.024330900243309004,
                  411,
                  10
                ],
                [
                  0.753199268738574,
                  0.024271844660194174,
                  412,
                  10
                ],
                [
                  0.7550274223034735,
                  0.024213075060532687,
                  413,
                  10
                ],
                [
                  0.7568555758683729,
                  0.024154589371980676,
                  414,
                  10
                ],
                [
                  0.7586837294332724,
                  0.024096385542168676,
                  415,
                  10
                ],
                [
                  0.7605118829981719,
                  0.02403846153846154,
                  416,
                  10
                ],
                [
                  0.7623400365630713,
                  0.026378896882494004,
                  417,
                  11
                ],
                [
                  0.7641681901279708,
                  0.02631578947368421,
                  418,
                  11
                ],
                [
                  0.7659963436928702,
                  0.028639618138424822,
                  419,
                  12
                ],
                [
                  0.7678244972577697,
                  0.02857142857142857,
                  420,
                  12
                ],
                [
                  0.7696526508226691,
                  0.028503562945368172,
                  421,
                  12
                ],
                [
                  0.7714808043875686,
                  0.02843601895734597,
                  422,
                  12
                ],
                [
                  0.773308957952468,
                  0.028368794326241134,
                  423,
                  12
                ],
                [
                  0.7751371115173674,
                  0.02830188679245283,
                  424,
                  12
                ],
                [
                  0.7769652650822669,
                  0.02823529411764706,
                  425,
                  12
                ],
                [
                  0.7787934186471663,
                  0.028169014084507043,
                  426,
                  12
                ],
                [
                  0.7806215722120659,
                  0.02810304449648712,
                  427,
                  12
                ],
                [
                  0.7824497257769653,
                  0.028037383177570093,
                  428,
                  12
                ],
                [
                  0.7842778793418648,
                  0.027972027972027972,
                  429,
                  12
                ],
                [
                  0.7861060329067642,
                  0.027906976744186046,
                  430,
                  12
                ],
                [
                  0.7879341864716636,
                  0.027842227378190254,
                  431,
                  12
                ],
                [
                  0.7897623400365631,
                  0.027777777777777776,
                  432,
                  12
                ],
                [
                  0.7915904936014625,
                  0.02771362586605081,
                  433,
                  12
                ],
                [
                  0.793418647166362,
                  0.027649769585253458,
                  434,
                  12
                ],
                [
                  0.7952468007312614,
                  0.027586206896551724,
                  435,
                  12
                ],
                [
                  0.7970749542961609,
                  0.027522935779816515,
                  436,
                  12
                ],
                [
                  0.7989031078610603,
                  0.02745995423340961,
                  437,
                  12
                ],
                [
                  0.8007312614259597,
                  0.0273972602739726,
                  438,
                  12
                ],
                [
                  0.8025594149908593,
                  0.02733485193621868,
                  439,
                  12
                ],
                [
                  0.8043875685557587,
                  0.02727272727272727,
                  440,
                  12
                ],
                [
                  0.8062157221206582,
                  0.027210884353741496,
                  441,
                  12
                ],
                [
                  0.8080438756855576,
                  0.029411764705882353,
                  442,
                  13
                ],
                [
                  0.8098720292504571,
                  0.029345372460496615,
                  443,
                  13
                ],
                [
                  0.8117001828153565,
                  0.02927927927927928,
                  444,
                  13
                ],
                [
                  0.8135283363802559,
                  0.03146067415730337,
                  445,
                  14
                ],
                [
                  0.8153564899451554,
                  0.03139013452914798,
                  446,
                  14
                ],
                [
                  0.8171846435100548,
                  0.03355704697986577,
                  447,
                  15
                ],
                [
                  0.8190127970749543,
                  0.033482142857142856,
                  448,
                  15
                ],
                [
                  0.8208409506398537,
                  0.0334075723830735,
                  449,
                  15
                ],
                [
                  0.8226691042047533,
                  0.03333333333333333,
                  450,
                  15
                ],
                [
                  0.8244972577696527,
                  0.03325942350332594,
                  451,
                  15
                ],
                [
                  0.8263254113345521,
                  0.035398230088495575,
                  452,
                  16
                ],
                [
                  0.8281535648994516,
                  0.037527593818984545,
                  453,
                  17
                ],
                [
                  0.829981718464351,
                  0.037444933920704845,
                  454,
                  17
                ],
                [
                  0.8318098720292505,
                  0.03736263736263736,
                  455,
                  17
                ],
                [
                  0.8336380255941499,
                  0.03728070175438596,
                  456,
                  17
                ],
                [
                  0.8354661791590493,
                  0.037199124726477024,
                  457,
                  17
                ],
                [
                  0.8372943327239488,
                  0.03711790393013101,
                  458,
                  17
                ],
                [
                  0.8391224862888482,
                  0.037037037037037035,
                  459,
                  17
                ],
                [
                  0.8409506398537477,
                  0.03695652173913044,
                  460,
                  17
                ],
                [
                  0.8427787934186471,
                  0.0368763557483731,
                  461,
                  17
                ],
                [
                  0.8446069469835467,
                  0.0367965367965368,
                  462,
                  17
                ],
                [
                  0.8464351005484461,
                  0.038876889848812095,
                  463,
                  18
                ],
                [
                  0.8482632541133455,
                  0.040948275862068964,
                  464,
                  19
                ],
                [
                  0.850091407678245,
                  0.04086021505376344,
                  465,
                  19
                ],
                [
                  0.8519195612431444,
                  0.04291845493562232,
                  466,
                  20
                ],
                [
                  0.8537477148080439,
                  0.042826552462526764,
                  467,
                  20
                ],
                [
                  0.8555758683729433,
                  0.042735042735042736,
                  468,
                  20
                ],
                [
                  0.8574040219378428,
                  0.04477611940298507,
                  469,
                  21
                ],
                [
                  0.8592321755027422,
                  0.04468085106382979,
                  470,
                  21
                ],
                [
                  0.8610603290676416,
                  0.044585987261146494,
                  471,
                  21
                ],
                [
                  0.8628884826325411,
                  0.04449152542372881,
                  472,
                  21
                ],
                [
                  0.8647166361974405,
                  0.046511627906976744,
                  473,
                  22
                ],
                [
                  0.8665447897623401,
                  0.046413502109704644,
                  474,
                  22
                ],
                [
                  0.8683729433272395,
                  0.04631578947368421,
                  475,
                  22
                ],
                [
                  0.870201096892139,
                  0.046218487394957986,
                  476,
                  22
                ],
                [
                  0.8720292504570384,
                  0.04821802935010482,
                  477,
                  23
                ],
                [
                  0.8738574040219378,
                  0.04811715481171548,
                  478,
                  23
                ],
                [
                  0.8756855575868373,
                  0.04801670146137787,
                  479,
                  23
                ],
                [
                  0.8775137111517367,
                  0.04791666666666667,
                  480,
                  23
                ],
                [
                  0.8793418647166362,
                  0.04781704781704782,
                  481,
                  23
                ],
                [
                  0.8811700182815356,
                  0.04771784232365145,
                  482,
                  23
                ],
                [
                  0.8829981718464351,
                  0.049689440993788817,
                  483,
                  24
                ],
                [
                  0.8848263254113345,
                  0.05165289256198347,
                  484,
                  25
                ],
                [
                  0.886654478976234,
                  0.05360824742268041,
                  485,
                  26
                ],
                [
                  0.8884826325411335,
                  0.053497942386831275,
                  486,
                  26
                ],
                [
                  0.8903107861060329,
                  0.055441478439425054,
                  487,
                  27
                ],
                [
                  0.8921389396709324,
                  0.055327868852459015,
                  488,
                  27
                ],
                [
                  0.8939670932358318,
                  0.05521472392638037,
                  489,
                  27
                ],
                [
                  0.8957952468007313,
                  0.05510204081632653,
                  490,
                  27
                ],
                [
                  0.8976234003656307,
                  0.054989816700611,
                  491,
                  27
                ],
                [
                  0.8994515539305301,
                  0.054878048780487805,
                  492,
                  27
                ],
                [
                  0.9012797074954296,
                  0.056795131845841784,
                  493,
                  28
                ],
                [
                  0.903107861060329,
                  0.05668016194331984,
                  494,
                  28
                ],
                [
                  0.9049360146252285,
                  0.05858585858585859,
                  495,
                  29
                ],
                [
                  0.906764168190128,
                  0.05846774193548387,
                  496,
                  29
                ],
                [
                  0.9085923217550275,
                  0.05835010060362173,
                  497,
                  29
                ],
                [
                  0.9104204753199269,
                  0.05823293172690763,
                  498,
                  29
                ],
                [
                  0.9122486288848263,
                  0.06012024048096192,
                  499,
                  30
                ],
                [
                  0.9140767824497258,
                  0.062,
                  500,
                  31
                ],
                [
                  0.9159049360146252,
                  0.06187624750499002,
                  501,
                  31
                ],
                [
                  0.9177330895795247,
                  0.061752988047808766,
                  502,
                  31
                ],
                [
                  0.9195612431444241,
                  0.061630218687872766,
                  503,
                  31
                ],
                [
                  0.9213893967093236,
                  0.06349206349206349,
                  504,
                  32
                ],
                [
                  0.923217550274223,
                  0.06336633663366337,
                  505,
                  32
                ],
                [
                  0.9250457038391224,
                  0.06324110671936758,
                  506,
                  32
                ],
                [
                  0.926873857404022,
                  0.0650887573964497,
                  507,
                  33
                ],
                [
                  0.9287020109689214,
                  0.06496062992125984,
                  508,
                  33
                ],
                [
                  0.9305301645338209,
                  0.06483300589390963,
                  509,
                  33
                ],
                [
                  0.9323583180987203,
                  0.06666666666666667,
                  510,
                  34
                ],
                [
                  0.9341864716636198,
                  0.06653620352250489,
                  511,
                  34
                ],
                [
                  0.9360146252285192,
                  0.068359375,
                  512,
                  35
                ],
                [
                  0.9378427787934186,
                  0.07017543859649122,
                  513,
                  36
                ],
                [
                  0.9396709323583181,
                  0.07003891050583658,
                  514,
                  36
                ],
                [
                  0.9414990859232175,
                  0.06990291262135923,
                  515,
                  36
                ],
                [
                  0.943327239488117,
                  0.07170542635658915,
                  516,
                  37
                ],
                [
                  0.9451553930530164,
                  0.0735009671179884,
                  517,
                  38
                ],
                [
                  0.946983546617916,
                  0.07528957528957529,
                  518,
                  39
                ],
                [
                  0.9488117001828154,
                  0.07707129094412331,
                  519,
                  40
                ],
                [
                  0.9506398537477148,
                  0.07692307692307693,
                  520,
                  40
                ],
                [
                  0.9524680073126143,
                  0.07677543186180422,
                  521,
                  40
                ],
                [
                  0.9542961608775137,
                  0.07854406130268199,
                  522,
                  41
                ],
                [
                  0.9561243144424132,
                  0.08030592734225621,
                  523,
                  42
                ],
                [
                  0.9579524680073126,
                  0.08206106870229007,
                  524,
                  43
                ],
                [
                  0.9597806215722121,
                  0.0838095238095238,
                  525,
                  44
                ],
                [
                  0.9616087751371115,
                  0.08365019011406843,
                  526,
                  44
                ],
                [
                  0.9634369287020109,
                  0.08538899430740038,
                  527,
                  45
                ],
                [
                  0.9652650822669104,
                  0.08522727272727272,
                  528,
                  45
                ],
                [
                  0.9670932358318098,
                  0.08506616257088846,
                  529,
                  45
                ],
                [
                  0.9689213893967094,
                  0.08679245283018867,
                  530,
                  46
                ],
                [
                  0.9707495429616088,
                  0.08662900188323917,
                  531,
                  46
                ],
                [
                  0.9725776965265083,
                  0.08834586466165413,
                  532,
                  47
                ],
                [
                  0.9744058500914077,
                  0.08818011257035648,
                  533,
                  47
                ],
                [
                  0.9762340036563071,
                  0.0898876404494382,
                  534,
                  48
                ],
                [
                  0.9780621572212066,
                  0.08971962616822429,
                  535,
                  48
                ],
                [
                  0.979890310786106,
                  0.08955223880597014,
                  536,
                  48
                ],
                [
                  0.9817184643510055,
                  0.0893854748603352,
                  537,
                  48
                ],
                [
                  0.9835466179159049,
                  0.09107806691449814,
                  538,
                  49
                ],
                [
                  0.9853747714808044,
                  0.09090909090909091,
                  539,
                  49
                ],
                [
                  0.9872029250457038,
                  0.09074074074074075,
                  540,
                  49
                ],
                [
                  0.9890310786106032,
                  0.09057301293900184,
                  541,
                  49
                ],
                [
                  0.9908592321755028,
                  0.09225092250922509,
                  542,
                  50
                ],
                [
                  0.9926873857404022,
                  0.09208103130755065,
                  543,
                  50
                ],
                [
                  0.9945155393053017,
                  0.09191176470588236,
                  544,
                  50
                ],
                [
                  0.9963436928702011,
                  0.09174311926605505,
                  545,
                  50
                ],
                [
                  0.9981718464351006,
                  0.09157509157509157,
                  546,
                  50
                ],
                [
                  1.0,
                  0.09323583180987204,
                  547,
                  51
                ]
              ]
            }
          ]
        },
        {
          "name": "Fastino GLiDE v1.2",
          "text_run": 248,
          "native": [
            {
              "language": "Turkish",
              "wording": "Bilmiyorum",
              "run": 369,
              "full_bank_restricted": true,
              "right": 494,
              "wrong": 47,
              "abstained": 6,
              "invalid": 0,
              "coverage": 0.9890310786106032,
              "risk": 0.08687615526802218
            },
            {
              "language": "Turkish",
              "wording": "I don't know",
              "run": 525,
              "full_bank_restricted": true,
              "right": 491,
              "wrong": 47,
              "abstained": 9,
              "invalid": 0,
              "coverage": 0.9835466179159049,
              "risk": 0.08736059479553904
            },
            {
              "language": "English",
              "wording": "I don't know",
              "run": 523,
              "full_bank_restricted": false,
              "right": 481,
              "wrong": 36,
              "abstained": 30,
              "invalid": 0,
              "coverage": 0.9451553930530164,
              "risk": 0.06963249516441006
            }
          ],
          "curves": [
            {
              "language": "Turkish",
              "run": 248,
              "points": [
                [
                  0.0018281535648994515,
                  0.0,
                  1,
                  0
                ],
                [
                  0.003656307129798903,
                  0.0,
                  2,
                  0
                ],
                [
                  0.005484460694698354,
                  0.0,
                  3,
                  0
                ],
                [
                  0.007312614259597806,
                  0.0,
                  4,
                  0
                ],
                [
                  0.009140767824497258,
                  0.0,
                  5,
                  0
                ],
                [
                  0.010968921389396709,
                  0.0,
                  6,
                  0
                ],
                [
                  0.012797074954296161,
                  0.0,
                  7,
                  0
                ],
                [
                  0.014625228519195612,
                  0.0,
                  8,
                  0
                ],
                [
                  0.016453382084095063,
                  0.0,
                  9,
                  0
                ],
                [
                  0.018281535648994516,
                  0.0,
                  10,
                  0
                ],
                [
                  0.02010968921389397,
                  0.0,
                  11,
                  0
                ],
                [
                  0.021937842778793418,
                  0.0,
                  12,
                  0
                ],
                [
                  0.02376599634369287,
                  0.0,
                  13,
                  0
                ],
                [
                  0.025594149908592323,
                  0.0,
                  14,
                  0
                ],
                [
                  0.027422303473491772,
                  0.0,
                  15,
                  0
                ],
                [
                  0.029250457038391225,
                  0.0,
                  16,
                  0
                ],
                [
                  0.031078610603290677,
                  0.0,
                  17,
                  0
                ],
                [
                  0.03290676416819013,
                  0.0,
                  18,
                  0
                ],
                [
                  0.03473491773308958,
                  0.0,
                  19,
                  0
                ],
                [
                  0.03656307129798903,
                  0.0,
                  20,
                  0
                ],
                [
                  0.038391224862888484,
                  0.0,
                  21,
                  0
                ],
                [
                  0.04021937842778794,
                  0.0,
                  22,
                  0
                ],
                [
                  0.04204753199268738,
                  0.0,
                  23,
                  0
                ],
                [
                  0.043875685557586835,
                  0.0,
                  24,
                  0
                ],
                [
                  0.04570383912248629,
                  0.0,
                  25,
                  0
                ],
                [
                  0.04753199268738574,
                  0.0,
                  26,
                  0
                ],
                [
                  0.04936014625228519,
                  0.0,
                  27,
                  0
                ],
                [
                  0.051188299817184646,
                  0.0,
                  28,
                  0
                ],
                [
                  0.05301645338208409,
                  0.0,
                  29,
                  0
                ],
                [
                  0.054844606946983544,
                  0.0,
                  30,
                  0
                ],
                [
                  0.056672760511883,
                  0.0,
                  31,
                  0
                ],
                [
                  0.05850091407678245,
                  0.0,
                  32,
                  0
                ],
                [
                  0.0603290676416819,
                  0.0,
                  33,
                  0
                ],
                [
                  0.062157221206581355,
                  0.0,
                  34,
                  0
                ],
                [
                  0.06398537477148081,
                  0.0,
                  35,
                  0
                ],
                [
                  0.06581352833638025,
                  0.0,
                  36,
                  0
                ],
                [
                  0.06764168190127971,
                  0.0,
                  37,
                  0
                ],
                [
                  0.06946983546617916,
                  0.0,
                  38,
                  0
                ],
                [
                  0.0712979890310786,
                  0.0,
                  39,
                  0
                ],
                [
                  0.07312614259597806,
                  0.0,
                  40,
                  0
                ],
                [
                  0.07495429616087751,
                  0.0,
                  41,
                  0
                ],
                [
                  0.07678244972577697,
                  0.0,
                  42,
                  0
                ],
                [
                  0.07861060329067641,
                  0.0,
                  43,
                  0
                ],
                [
                  0.08043875685557587,
                  0.0,
                  44,
                  0
                ],
                [
                  0.08226691042047532,
                  0.0,
                  45,
                  0
                ],
                [
                  0.08409506398537477,
                  0.0,
                  46,
                  0
                ],
                [
                  0.08592321755027423,
                  0.0,
                  47,
                  0
                ],
                [
                  0.08775137111517367,
                  0.0,
                  48,
                  0
                ],
                [
                  0.08957952468007313,
                  0.0,
                  49,
                  0
                ],
                [
                  0.09140767824497258,
                  0.0,
                  50,
                  0
                ],
                [
                  0.09323583180987204,
                  0.0,
                  51,
                  0
                ],
                [
                  0.09506398537477148,
                  0.0,
                  52,
                  0
                ],
                [
                  0.09689213893967093,
                  0.0,
                  53,
                  0
                ],
                [
                  0.09872029250457039,
                  0.0,
                  54,
                  0
                ],
                [
                  0.10054844606946983,
                  0.0,
                  55,
                  0
                ],
                [
                  0.10237659963436929,
                  0.0,
                  56,
                  0
                ],
                [
                  0.10420475319926874,
                  0.0,
                  57,
                  0
                ],
                [
                  0.10603290676416818,
                  0.0,
                  58,
                  0
                ],
                [
                  0.10786106032906764,
                  0.0,
                  59,
                  0
                ],
                [
                  0.10968921389396709,
                  0.0,
                  60,
                  0
                ],
                [
                  0.11151736745886655,
                  0.0,
                  61,
                  0
                ],
                [
                  0.113345521023766,
                  0.0,
                  62,
                  0
                ],
                [
                  0.11517367458866545,
                  0.0,
                  63,
                  0
                ],
                [
                  0.1170018281535649,
                  0.0,
                  64,
                  0
                ],
                [
                  0.11882998171846434,
                  0.0,
                  65,
                  0
                ],
                [
                  0.1206581352833638,
                  0.0,
                  66,
                  0
                ],
                [
                  0.12248628884826325,
                  0.0,
                  67,
                  0
                ],
                [
                  0.12431444241316271,
                  0.0,
                  68,
                  0
                ],
                [
                  0.12614259597806216,
                  0.0,
                  69,
                  0
                ],
                [
                  0.12797074954296161,
                  0.0,
                  70,
                  0
                ],
                [
                  0.12979890310786105,
                  0.0,
                  71,
                  0
                ],
                [
                  0.1316270566727605,
                  0.0,
                  72,
                  0
                ],
                [
                  0.13345521023765997,
                  0.0,
                  73,
                  0
                ],
                [
                  0.13528336380255943,
                  0.0,
                  74,
                  0
                ],
                [
                  0.13711151736745886,
                  0.0,
                  75,
                  0
                ],
                [
                  0.13893967093235832,
                  0.0,
                  76,
                  0
                ],
                [
                  0.14076782449725778,
                  0.0,
                  77,
                  0
                ],
                [
                  0.1425959780621572,
                  0.0,
                  78,
                  0
                ],
                [
                  0.14442413162705667,
                  0.0,
                  79,
                  0
                ],
                [
                  0.14625228519195613,
                  0.0,
                  80,
                  0
                ],
                [
                  0.1480804387568556,
                  0.0,
                  81,
                  0
                ],
                [
                  0.14990859232175502,
                  0.0,
                  82,
                  0
                ],
                [
                  0.15173674588665448,
                  0.0,
                  83,
                  0
                ],
                [
                  0.15356489945155394,
                  0.0,
                  84,
                  0
                ],
                [
                  0.15539305301645337,
                  0.0,
                  85,
                  0
                ],
                [
                  0.15722120658135283,
                  0.0,
                  86,
                  0
                ],
                [
                  0.1590493601462523,
                  0.0,
                  87,
                  0
                ],
                [
                  0.16087751371115175,
                  0.0,
                  88,
                  0
                ],
                [
                  0.16270566727605118,
                  0.0,
                  89,
                  0
                ],
                [
                  0.16453382084095064,
                  0.0,
                  90,
                  0
                ],
                [
                  0.1663619744058501,
                  0.0,
                  91,
                  0
                ],
                [
                  0.16819012797074953,
                  0.0,
                  92,
                  0
                ],
                [
                  0.170018281535649,
                  0.0,
                  93,
                  0
                ],
                [
                  0.17184643510054845,
                  0.0,
                  94,
                  0
                ],
                [
                  0.1736745886654479,
                  0.0,
                  95,
                  0
                ],
                [
                  0.17550274223034734,
                  0.0,
                  96,
                  0
                ],
                [
                  0.1773308957952468,
                  0.0,
                  97,
                  0
                ],
                [
                  0.17915904936014626,
                  0.0,
                  98,
                  0
                ],
                [
                  0.1809872029250457,
                  0.0,
                  99,
                  0
                ],
                [
                  0.18281535648994515,
                  0.0,
                  100,
                  0
                ],
                [
                  0.1846435100548446,
                  0.0,
                  101,
                  0
                ],
                [
                  0.18647166361974407,
                  0.0,
                  102,
                  0
                ],
                [
                  0.1882998171846435,
                  0.0,
                  103,
                  0
                ],
                [
                  0.19012797074954296,
                  0.0,
                  104,
                  0
                ],
                [
                  0.19195612431444242,
                  0.0,
                  105,
                  0
                ],
                [
                  0.19378427787934185,
                  0.0,
                  106,
                  0
                ],
                [
                  0.1956124314442413,
                  0.0,
                  107,
                  0
                ],
                [
                  0.19744058500914077,
                  0.0,
                  108,
                  0
                ],
                [
                  0.19926873857404023,
                  0.0,
                  109,
                  0
                ],
                [
                  0.20109689213893966,
                  0.0,
                  110,
                  0
                ],
                [
                  0.20292504570383912,
                  0.0,
                  111,
                  0
                ],
                [
                  0.20475319926873858,
                  0.0,
                  112,
                  0
                ],
                [
                  0.20658135283363802,
                  0.0,
                  113,
                  0
                ],
                [
                  0.20840950639853748,
                  0.0,
                  114,
                  0
                ],
                [
                  0.21023765996343693,
                  0.0,
                  115,
                  0
                ],
                [
                  0.21206581352833637,
                  0.0,
                  116,
                  0
                ],
                [
                  0.21389396709323583,
                  0.0,
                  117,
                  0
                ],
                [
                  0.21572212065813529,
                  0.0,
                  118,
                  0
                ],
                [
                  0.21755027422303475,
                  0.0,
                  119,
                  0
                ],
                [
                  0.21937842778793418,
                  0.0,
                  120,
                  0
                ],
                [
                  0.22120658135283364,
                  0.0,
                  121,
                  0
                ],
                [
                  0.2230347349177331,
                  0.0,
                  122,
                  0
                ],
                [
                  0.22486288848263253,
                  0.0,
                  123,
                  0
                ],
                [
                  0.226691042047532,
                  0.0,
                  124,
                  0
                ],
                [
                  0.22851919561243145,
                  0.0,
                  125,
                  0
                ],
                [
                  0.2303473491773309,
                  0.0,
                  126,
                  0
                ],
                [
                  0.23217550274223034,
                  0.0,
                  127,
                  0
                ],
                [
                  0.2340036563071298,
                  0.0,
                  128,
                  0
                ],
                [
                  0.23583180987202926,
                  0.0,
                  129,
                  0
                ],
                [
                  0.2376599634369287,
                  0.0,
                  130,
                  0
                ],
                [
                  0.23948811700182815,
                  0.0,
                  131,
                  0
                ],
                [
                  0.2413162705667276,
                  0.0,
                  132,
                  0
                ],
                [
                  0.24314442413162707,
                  0.0,
                  133,
                  0
                ],
                [
                  0.2449725776965265,
                  0.0,
                  134,
                  0
                ],
                [
                  0.24680073126142596,
                  0.0,
                  135,
                  0
                ],
                [
                  0.24862888482632542,
                  0.0,
                  136,
                  0
                ],
                [
                  0.25045703839122485,
                  0.0,
                  137,
                  0
                ],
                [
                  0.2522851919561243,
                  0.0,
                  138,
                  0
                ],
                [
                  0.25411334552102377,
                  0.0,
                  139,
                  0
                ],
                [
                  0.25594149908592323,
                  0.0,
                  140,
                  0
                ],
                [
                  0.2577696526508227,
                  0.0,
                  141,
                  0
                ],
                [
                  0.2595978062157221,
                  0.0,
                  142,
                  0
                ],
                [
                  0.26142595978062155,
                  0.0,
                  143,
                  0
                ],
                [
                  0.263254113345521,
                  0.0,
                  144,
                  0
                ],
                [
                  0.26508226691042047,
                  0.0,
                  145,
                  0
                ],
                [
                  0.26691042047531993,
                  0.0,
                  146,
                  0
                ],
                [
                  0.2687385740402194,
                  0.0,
                  147,
                  0
                ],
                [
                  0.27056672760511885,
                  0.0,
                  148,
                  0
                ],
                [
                  0.27239488117001825,
                  0.0,
                  149,
                  0
                ],
                [
                  0.2742230347349177,
                  0.0,
                  150,
                  0
                ],
                [
                  0.2760511882998172,
                  0.0,
                  151,
                  0
                ],
                [
                  0.27787934186471663,
                  0.0,
                  152,
                  0
                ],
                [
                  0.2797074954296161,
                  0.0,
                  153,
                  0
                ],
                [
                  0.28153564899451555,
                  0.0,
                  154,
                  0
                ],
                [
                  0.283363802559415,
                  0.0,
                  155,
                  0
                ],
                [
                  0.2851919561243144,
                  0.0,
                  156,
                  0
                ],
                [
                  0.2870201096892139,
                  0.0,
                  157,
                  0
                ],
                [
                  0.28884826325411334,
                  0.0,
                  158,
                  0
                ],
                [
                  0.2906764168190128,
                  0.0,
                  159,
                  0
                ],
                [
                  0.29250457038391225,
                  0.0,
                  160,
                  0
                ],
                [
                  0.2943327239488117,
                  0.0,
                  161,
                  0
                ],
                [
                  0.2961608775137112,
                  0.0,
                  162,
                  0
                ],
                [
                  0.2979890310786106,
                  0.0,
                  163,
                  0
                ],
                [
                  0.29981718464351004,
                  0.0,
                  164,
                  0
                ],
                [
                  0.3016453382084095,
                  0.0,
                  165,
                  0
                ],
                [
                  0.30347349177330896,
                  0.0,
                  166,
                  0
                ],
                [
                  0.3053016453382084,
                  0.0,
                  167,
                  0
                ],
                [
                  0.3071297989031079,
                  0.0,
                  168,
                  0
                ],
                [
                  0.30895795246800734,
                  0.0,
                  169,
                  0
                ],
                [
                  0.31078610603290674,
                  0.0,
                  170,
                  0
                ],
                [
                  0.3126142595978062,
                  0.0,
                  171,
                  0
                ],
                [
                  0.31444241316270566,
                  0.0,
                  172,
                  0
                ],
                [
                  0.3162705667276051,
                  0.0,
                  173,
                  0
                ],
                [
                  0.3180987202925046,
                  0.0,
                  174,
                  0
                ],
                [
                  0.31992687385740404,
                  0.0,
                  175,
                  0
                ],
                [
                  0.3217550274223035,
                  0.0,
                  176,
                  0
                ],
                [
                  0.3235831809872029,
                  0.0,
                  177,
                  0
                ],
                [
                  0.32541133455210236,
                  0.0,
                  178,
                  0
                ],
                [
                  0.3272394881170018,
                  0.0,
                  179,
                  0
                ],
                [
                  0.3290676416819013,
                  0.0,
                  180,
                  0
                ],
                [
                  0.33089579524680074,
                  0.0,
                  181,
                  0
                ],
                [
                  0.3327239488117002,
                  0.0,
                  182,
                  0
                ],
                [
                  0.33455210237659966,
                  0.0,
                  183,
                  0
                ],
                [
                  0.33638025594149906,
                  0.0,
                  184,
                  0
                ],
                [
                  0.3382084095063985,
                  0.0,
                  185,
                  0
                ],
                [
                  0.340036563071298,
                  0.0,
                  186,
                  0
                ],
                [
                  0.34186471663619744,
                  0.0,
                  187,
                  0
                ],
                [
                  0.3436928702010969,
                  0.0,
                  188,
                  0
                ],
                [
                  0.34552102376599636,
                  0.0,
                  189,
                  0
                ],
                [
                  0.3473491773308958,
                  0.0,
                  190,
                  0
                ],
                [
                  0.3491773308957952,
                  0.0,
                  191,
                  0
                ],
                [
                  0.3510054844606947,
                  0.0,
                  192,
                  0
                ],
                [
                  0.35283363802559414,
                  0.0,
                  193,
                  0
                ],
                [
                  0.3546617915904936,
                  0.0,
                  194,
                  0
                ],
                [
                  0.35648994515539306,
                  0.0,
                  195,
                  0
                ],
                [
                  0.3583180987202925,
                  0.0,
                  196,
                  0
                ],
                [
                  0.360146252285192,
                  0.0,
                  197,
                  0
                ],
                [
                  0.3619744058500914,
                  0.0,
                  198,
                  0
                ],
                [
                  0.36380255941499084,
                  0.0,
                  199,
                  0
                ],
                [
                  0.3656307129798903,
                  0.0,
                  200,
                  0
                ],
                [
                  0.36745886654478976,
                  0.0,
                  201,
                  0
                ],
                [
                  0.3692870201096892,
                  0.0,
                  202,
                  0
                ],
                [
                  0.3711151736745887,
                  0.0,
                  203,
                  0
                ],
                [
                  0.37294332723948814,
                  0.0,
                  204,
                  0
                ],
                [
                  0.37477148080438755,
                  0.0,
                  205,
                  0
                ],
                [
                  0.376599634369287,
                  0.0,
                  206,
                  0
                ],
                [
                  0.37842778793418647,
                  0.0,
                  207,
                  0
                ],
                [
                  0.3802559414990859,
                  0.0,
                  208,
                  0
                ],
                [
                  0.3820840950639854,
                  0.0,
                  209,
                  0
                ],
                [
                  0.38391224862888484,
                  0.0,
                  210,
                  0
                ],
                [
                  0.3857404021937843,
                  0.0,
                  211,
                  0
                ],
                [
                  0.3875685557586837,
                  0.0,
                  212,
                  0
                ],
                [
                  0.38939670932358317,
                  0.0,
                  213,
                  0
                ],
                [
                  0.3912248628884826,
                  0.0,
                  214,
                  0
                ],
                [
                  0.3930530164533821,
                  0.0,
                  215,
                  0
                ],
                [
                  0.39488117001828155,
                  0.0,
                  216,
                  0
                ],
                [
                  0.396709323583181,
                  0.0,
                  217,
                  0
                ],
                [
                  0.39853747714808047,
                  0.0,
                  218,
                  0
                ],
                [
                  0.40036563071297987,
                  0.0,
                  219,
                  0
                ],
                [
                  0.40219378427787933,
                  0.0,
                  220,
                  0
                ],
                [
                  0.4040219378427788,
                  0.0,
                  221,
                  0
                ],
                [
                  0.40585009140767825,
                  0.0,
                  222,
                  0
                ],
                [
                  0.4076782449725777,
                  0.0,
                  223,
                  0
                ],
                [
                  0.40950639853747717,
                  0.0,
                  224,
                  0
                ],
                [
                  0.4113345521023766,
                  0.0,
                  225,
                  0
                ],
                [
                  0.41316270566727603,
                  0.0,
                  226,
                  0
                ],
                [
                  0.4149908592321755,
                  0.0,
                  227,
                  0
                ],
                [
                  0.41681901279707495,
                  0.0,
                  228,
                  0
                ],
                [
                  0.4186471663619744,
                  0.0,
                  229,
                  0
                ],
                [
                  0.42047531992687387,
                  0.0,
                  230,
                  0
                ],
                [
                  0.42230347349177333,
                  0.0,
                  231,
                  0
                ],
                [
                  0.42413162705667273,
                  0.0,
                  232,
                  0
                ],
                [
                  0.4259597806215722,
                  0.0,
                  233,
                  0
                ],
                [
                  0.42778793418647165,
                  0.0,
                  234,
                  0
                ],
                [
                  0.4296160877513711,
                  0.0,
                  235,
                  0
                ],
                [
                  0.43144424131627057,
                  0.0,
                  236,
                  0
                ],
                [
                  0.43327239488117003,
                  0.0,
                  237,
                  0
                ],
                [
                  0.4351005484460695,
                  0.0,
                  238,
                  0
                ],
                [
                  0.4369287020109689,
                  0.0,
                  239,
                  0
                ],
                [
                  0.43875685557586835,
                  0.0,
                  240,
                  0
                ],
                [
                  0.4405850091407678,
                  0.0,
                  241,
                  0
                ],
                [
                  0.4424131627056673,
                  0.0,
                  242,
                  0
                ],
                [
                  0.44424131627056673,
                  0.0,
                  243,
                  0
                ],
                [
                  0.4460694698354662,
                  0.0,
                  244,
                  0
                ],
                [
                  0.44789762340036565,
                  0.0,
                  245,
                  0
                ],
                [
                  0.44972577696526506,
                  0.0,
                  246,
                  0
                ],
                [
                  0.4515539305301645,
                  0.0,
                  247,
                  0
                ],
                [
                  0.453382084095064,
                  0.0,
                  248,
                  0
                ],
                [
                  0.45521023765996343,
                  0.0,
                  249,
                  0
                ],
                [
                  0.4570383912248629,
                  0.0,
                  250,
                  0
                ],
                [
                  0.45886654478976235,
                  0.0,
                  251,
                  0
                ],
                [
                  0.4606946983546618,
                  0.0,
                  252,
                  0
                ],
                [
                  0.4625228519195612,
                  0.0,
                  253,
                  0
                ],
                [
                  0.4643510054844607,
                  0.0,
                  254,
                  0
                ],
                [
                  0.46617915904936014,
                  0.0,
                  255,
                  0
                ],
                [
                  0.4680073126142596,
                  0.0,
                  256,
                  0
                ],
                [
                  0.46983546617915906,
                  0.0,
                  257,
                  0
                ],
                [
                  0.4716636197440585,
                  0.0,
                  258,
                  0
                ],
                [
                  0.473491773308958,
                  0.0,
                  259,
                  0
                ],
                [
                  0.4753199268738574,
                  0.0,
                  260,
                  0
                ],
                [
                  0.47714808043875684,
                  0.0,
                  261,
                  0
                ],
                [
                  0.4789762340036563,
                  0.0,
                  262,
                  0
                ],
                [
                  0.48080438756855576,
                  0.0,
                  263,
                  0
                ],
                [
                  0.4826325411334552,
                  0.0,
                  264,
                  0
                ],
                [
                  0.4844606946983547,
                  0.0,
                  265,
                  0
                ],
                [
                  0.48628884826325414,
                  0.0,
                  266,
                  0
                ],
                [
                  0.48811700182815354,
                  0.0,
                  267,
                  0
                ],
                [
                  0.489945155393053,
                  0.0,
                  268,
                  0
                ],
                [
                  0.49177330895795246,
                  0.0,
                  269,
                  0
                ],
                [
                  0.4936014625228519,
                  0.0,
                  270,
                  0
                ],
                [
                  0.4954296160877514,
                  0.0,
                  271,
                  0
                ],
                [
                  0.49725776965265084,
                  0.0,
                  272,
                  0
                ],
                [
                  0.4990859232175503,
                  0.0,
                  273,
                  0
                ],
                [
                  0.5009140767824497,
                  0.0,
                  274,
                  0
                ],
                [
                  0.5027422303473492,
                  0.0,
                  275,
                  0
                ],
                [
                  0.5045703839122486,
                  0.0,
                  276,
                  0
                ],
                [
                  0.506398537477148,
                  0.0,
                  277,
                  0
                ],
                [
                  0.5082266910420475,
                  0.0,
                  278,
                  0
                ],
                [
                  0.5100548446069469,
                  0.0,
                  279,
                  0
                ],
                [
                  0.5118829981718465,
                  0.0,
                  280,
                  0
                ],
                [
                  0.5137111517367459,
                  0.0,
                  281,
                  0
                ],
                [
                  0.5155393053016454,
                  0.0,
                  282,
                  0
                ],
                [
                  0.5173674588665448,
                  0.0,
                  283,
                  0
                ],
                [
                  0.5191956124314442,
                  0.0,
                  284,
                  0
                ],
                [
                  0.5210237659963437,
                  0.0,
                  285,
                  0
                ],
                [
                  0.5228519195612431,
                  0.0,
                  286,
                  0
                ],
                [
                  0.5246800731261426,
                  0.0,
                  287,
                  0
                ],
                [
                  0.526508226691042,
                  0.0,
                  288,
                  0
                ],
                [
                  0.5283363802559415,
                  0.0,
                  289,
                  0
                ],
                [
                  0.5301645338208409,
                  0.0,
                  290,
                  0
                ],
                [
                  0.5319926873857403,
                  0.0,
                  291,
                  0
                ],
                [
                  0.5338208409506399,
                  0.0,
                  292,
                  0
                ],
                [
                  0.5356489945155393,
                  0.0,
                  293,
                  0
                ],
                [
                  0.5374771480804388,
                  0.0,
                  294,
                  0
                ],
                [
                  0.5393053016453382,
                  0.0,
                  295,
                  0
                ],
                [
                  0.5411334552102377,
                  0.0,
                  296,
                  0
                ],
                [
                  0.5429616087751371,
                  0.0,
                  297,
                  0
                ],
                [
                  0.5447897623400365,
                  0.0,
                  298,
                  0
                ],
                [
                  0.546617915904936,
                  0.0,
                  299,
                  0
                ],
                [
                  0.5484460694698354,
                  0.0,
                  300,
                  0
                ],
                [
                  0.5502742230347349,
                  0.0,
                  301,
                  0
                ],
                [
                  0.5521023765996343,
                  0.0,
                  302,
                  0
                ],
                [
                  0.5539305301645339,
                  0.0,
                  303,
                  0
                ],
                [
                  0.5557586837294333,
                  0.0,
                  304,
                  0
                ],
                [
                  0.5575868372943327,
                  0.0,
                  305,
                  0
                ],
                [
                  0.5594149908592322,
                  0.0,
                  306,
                  0
                ],
                [
                  0.5612431444241316,
                  0.0,
                  307,
                  0
                ],
                [
                  0.5630712979890311,
                  0.0,
                  308,
                  0
                ],
                [
                  0.5648994515539305,
                  0.0,
                  309,
                  0
                ],
                [
                  0.56672760511883,
                  0.0,
                  310,
                  0
                ],
                [
                  0.5685557586837294,
                  0.0,
                  311,
                  0
                ],
                [
                  0.5703839122486288,
                  0.0,
                  312,
                  0
                ],
                [
                  0.5722120658135283,
                  0.0,
                  313,
                  0
                ],
                [
                  0.5740402193784278,
                  0.0,
                  314,
                  0
                ],
                [
                  0.5758683729433273,
                  0.0,
                  315,
                  0
                ],
                [
                  0.5776965265082267,
                  0.0,
                  316,
                  0
                ],
                [
                  0.5795246800731262,
                  0.0,
                  317,
                  0
                ],
                [
                  0.5813528336380256,
                  0.0,
                  318,
                  0
                ],
                [
                  0.583180987202925,
                  0.0,
                  319,
                  0
                ],
                [
                  0.5850091407678245,
                  0.0,
                  320,
                  0
                ],
                [
                  0.5868372943327239,
                  0.0,
                  321,
                  0
                ],
                [
                  0.5886654478976234,
                  0.0,
                  322,
                  0
                ],
                [
                  0.5904936014625228,
                  0.0,
                  323,
                  0
                ],
                [
                  0.5923217550274223,
                  0.0,
                  324,
                  0
                ],
                [
                  0.5941499085923218,
                  0.0,
                  325,
                  0
                ],
                [
                  0.5959780621572212,
                  0.0,
                  326,
                  0
                ],
                [
                  0.5978062157221207,
                  0.0,
                  327,
                  0
                ],
                [
                  0.5996343692870201,
                  0.0,
                  328,
                  0
                ],
                [
                  0.6014625228519196,
                  0.0,
                  329,
                  0
                ],
                [
                  0.603290676416819,
                  0.0,
                  330,
                  0
                ],
                [
                  0.6051188299817185,
                  0.0,
                  331,
                  0
                ],
                [
                  0.6069469835466179,
                  0.0,
                  332,
                  0
                ],
                [
                  0.6087751371115173,
                  0.0,
                  333,
                  0
                ],
                [
                  0.6106032906764168,
                  0.0,
                  334,
                  0
                ],
                [
                  0.6124314442413162,
                  0.0,
                  335,
                  0
                ],
                [
                  0.6142595978062158,
                  0.0,
                  336,
                  0
                ],
                [
                  0.6160877513711152,
                  0.0,
                  337,
                  0
                ],
                [
                  0.6179159049360147,
                  0.0,
                  338,
                  0
                ],
                [
                  0.6197440585009141,
                  0.0,
                  339,
                  0
                ],
                [
                  0.6215722120658135,
                  0.0,
                  340,
                  0
                ],
                [
                  0.623400365630713,
                  0.0,
                  341,
                  0
                ],
                [
                  0.6252285191956124,
                  0.0,
                  342,
                  0
                ],
                [
                  0.6270566727605119,
                  0.0,
                  343,
                  0
                ],
                [
                  0.6288848263254113,
                  0.0,
                  344,
                  0
                ],
                [
                  0.6307129798903108,
                  0.0,
                  345,
                  0
                ],
                [
                  0.6325411334552102,
                  0.0,
                  346,
                  0
                ],
                [
                  0.6343692870201096,
                  0.0,
                  347,
                  0
                ],
                [
                  0.6361974405850092,
                  0.0,
                  348,
                  0
                ],
                [
                  0.6380255941499086,
                  0.0,
                  349,
                  0
                ],
                [
                  0.6398537477148081,
                  0.0,
                  350,
                  0
                ],
                [
                  0.6416819012797075,
                  0.0,
                  351,
                  0
                ],
                [
                  0.643510054844607,
                  0.0,
                  352,
                  0
                ],
                [
                  0.6453382084095064,
                  0.0,
                  353,
                  0
                ],
                [
                  0.6471663619744058,
                  0.0,
                  354,
                  0
                ],
                [
                  0.6489945155393053,
                  0.0,
                  355,
                  0
                ],
                [
                  0.6508226691042047,
                  0.0,
                  356,
                  0
                ],
                [
                  0.6526508226691042,
                  0.0,
                  357,
                  0
                ],
                [
                  0.6544789762340036,
                  0.0,
                  358,
                  0
                ],
                [
                  0.6563071297989032,
                  0.0,
                  359,
                  0
                ],
                [
                  0.6581352833638026,
                  0.0,
                  360,
                  0
                ],
                [
                  0.659963436928702,
                  0.0,
                  361,
                  0
                ],
                [
                  0.6617915904936015,
                  0.0,
                  362,
                  0
                ],
                [
                  0.6636197440585009,
                  0.0,
                  363,
                  0
                ],
                [
                  0.6654478976234004,
                  0.0,
                  364,
                  0
                ],
                [
                  0.6672760511882998,
                  0.0,
                  365,
                  0
                ],
                [
                  0.6691042047531993,
                  0.0,
                  366,
                  0
                ],
                [
                  0.6709323583180987,
                  0.0,
                  367,
                  0
                ],
                [
                  0.6727605118829981,
                  0.0,
                  368,
                  0
                ],
                [
                  0.6745886654478976,
                  0.0027100271002710027,
                  369,
                  1
                ],
                [
                  0.676416819012797,
                  0.002702702702702703,
                  370,
                  1
                ],
                [
                  0.6782449725776966,
                  0.0026954177897574125,
                  371,
                  1
                ],
                [
                  0.680073126142596,
                  0.002688172043010753,
                  372,
                  1
                ],
                [
                  0.6819012797074955,
                  0.002680965147453083,
                  373,
                  1
                ],
                [
                  0.6837294332723949,
                  0.00267379679144385,
                  374,
                  1
                ],
                [
                  0.6855575868372943,
                  0.0026666666666666666,
                  375,
                  1
                ],
                [
                  0.6873857404021938,
                  0.0026595744680851063,
                  376,
                  1
                ],
                [
                  0.6892138939670932,
                  0.002652519893899204,
                  377,
                  1
                ],
                [
                  0.6910420475319927,
                  0.0026455026455026454,
                  378,
                  1
                ],
                [
                  0.6928702010968921,
                  0.002638522427440633,
                  379,
                  1
                ],
                [
                  0.6946983546617916,
                  0.002631578947368421,
                  380,
                  1
                ],
                [
                  0.696526508226691,
                  0.0026246719160104987,
                  381,
                  1
                ],
                [
                  0.6983546617915904,
                  0.002617801047120419,
                  382,
                  1
                ],
                [
                  0.70018281535649,
                  0.0026109660574412533,
                  383,
                  1
                ],
                [
                  0.7020109689213894,
                  0.0026041666666666665,
                  384,
                  1
                ],
                [
                  0.7038391224862889,
                  0.0025974025974025974,
                  385,
                  1
                ],
                [
                  0.7056672760511883,
                  0.0025906735751295338,
                  386,
                  1
                ],
                [
                  0.7074954296160878,
                  0.002583979328165375,
                  387,
                  1
                ],
                [
                  0.7093235831809872,
                  0.002577319587628866,
                  388,
                  1
                ],
                [
                  0.7111517367458866,
                  0.002570694087403599,
                  389,
                  1
                ],
                [
                  0.7129798903107861,
                  0.002564102564102564,
                  390,
                  1
                ],
                [
                  0.7148080438756855,
                  0.0025575447570332483,
                  391,
                  1
                ],
                [
                  0.716636197440585,
                  0.002551020408163265,
                  392,
                  1
                ],
                [
                  0.7184643510054844,
                  0.002544529262086514,
                  393,
                  1
                ],
                [
                  0.720292504570384,
                  0.0025380710659898475,
                  394,
                  1
                ],
                [
                  0.7221206581352834,
                  0.002531645569620253,
                  395,
                  1
                ],
                [
                  0.7239488117001828,
                  0.0025252525252525255,
                  396,
                  1
                ],
                [
                  0.7257769652650823,
                  0.0025188916876574307,
                  397,
                  1
                ],
                [
                  0.7276051188299817,
                  0.002512562814070352,
                  398,
                  1
                ],
                [
                  0.7294332723948812,
                  0.002506265664160401,
                  399,
                  1
                ],
                [
                  0.7312614259597806,
                  0.0025,
                  400,
                  1
                ],
                [
                  0.7330895795246801,
                  0.0024937655860349127,
                  401,
                  1
                ],
                [
                  0.7349177330895795,
                  0.0024875621890547263,
                  402,
                  1
                ],
                [
                  0.7367458866544789,
                  0.0024813895781637717,
                  403,
                  1
                ],
                [
                  0.7385740402193784,
                  0.0024752475247524753,
                  404,
                  1
                ],
                [
                  0.7404021937842779,
                  0.0024691358024691358,
                  405,
                  1
                ],
                [
                  0.7422303473491774,
                  0.0024630541871921183,
                  406,
                  1
                ],
                [
                  0.7440585009140768,
                  0.002457002457002457,
                  407,
                  1
                ],
                [
                  0.7458866544789763,
                  0.0024509803921568627,
                  408,
                  1
                ],
                [
                  0.7477148080438757,
                  0.0024449877750611247,
                  409,
                  1
                ],
                [
                  0.7495429616087751,
                  0.0024390243902439024,
                  410,
                  1
                ],
                [
                  0.7513711151736746,
                  0.0024330900243309003,
                  411,
                  1
                ],
                [
                  0.753199268738574,
                  0.0024271844660194173,
                  412,
                  1
                ],
                [
                  0.7550274223034735,
                  0.002421307506053269,
                  413,
                  1
                ],
                [
                  0.7568555758683729,
                  0.0024154589371980675,
                  414,
                  1
                ],
                [
                  0.7586837294332724,
                  0.0024096385542168677,
                  415,
                  1
                ],
                [
                  0.7605118829981719,
                  0.002403846153846154,
                  416,
                  1
                ],
                [
                  0.7623400365630713,
                  0.004796163069544364,
                  417,
                  2
                ],
                [
                  0.7641681901279708,
                  0.004784688995215311,
                  418,
                  2
                ],
                [
                  0.7659963436928702,
                  0.00477326968973747,
                  419,
                  2
                ],
                [
                  0.7678244972577697,
                  0.004761904761904762,
                  420,
                  2
                ],
                [
                  0.7696526508226691,
                  0.004750593824228029,
                  421,
                  2
                ],
                [
                  0.7714808043875686,
                  0.004739336492890996,
                  422,
                  2
                ],
                [
                  0.773308957952468,
                  0.0070921985815602835,
                  423,
                  3
                ],
                [
                  0.7751371115173674,
                  0.007075471698113208,
                  424,
                  3
                ],
                [
                  0.7769652650822669,
                  0.007058823529411765,
                  425,
                  3
                ],
                [
                  0.7787934186471663,
                  0.007042253521126761,
                  426,
                  3
                ],
                [
                  0.7806215722120659,
                  0.00702576112412178,
                  427,
                  3
                ],
                [
                  0.7824497257769653,
                  0.007009345794392523,
                  428,
                  3
                ],
                [
                  0.7842778793418648,
                  0.006993006993006993,
                  429,
                  3
                ],
                [
                  0.7861060329067642,
                  0.0069767441860465115,
                  430,
                  3
                ],
                [
                  0.7879341864716636,
                  0.0069605568445475635,
                  431,
                  3
                ],
                [
                  0.7897623400365631,
                  0.006944444444444444,
                  432,
                  3
                ],
                [
                  0.7915904936014625,
                  0.006928406466512702,
                  433,
                  3
                ],
                [
                  0.793418647166362,
                  0.0069124423963133645,
                  434,
                  3
                ],
                [
                  0.7952468007312614,
                  0.006896551724137931,
                  435,
                  3
                ],
                [
                  0.7970749542961609,
                  0.006880733944954129,
                  436,
                  3
                ],
                [
                  0.7989031078610603,
                  0.006864988558352402,
                  437,
                  3
                ],
                [
                  0.8007312614259597,
                  0.0091324200913242,
                  438,
                  4
                ],
                [
                  0.8025594149908593,
                  0.009111617312072893,
                  439,
                  4
                ],
                [
                  0.8043875685557587,
                  0.00909090909090909,
                  440,
                  4
                ],
                [
                  0.8062157221206582,
                  0.009070294784580499,
                  441,
                  4
                ],
                [
                  0.8080438756855576,
                  0.00904977375565611,
                  442,
                  4
                ],
                [
                  0.8098720292504571,
                  0.009029345372460496,
                  443,
                  4
                ],
                [
                  0.8117001828153565,
                  0.009009009009009009,
                  444,
                  4
                ],
                [
                  0.8135283363802559,
                  0.008988764044943821,
                  445,
                  4
                ],
                [
                  0.8153564899451554,
                  0.008968609865470852,
                  446,
                  4
                ],
                [
                  0.8171846435100548,
                  0.011185682326621925,
                  447,
                  5
                ],
                [
                  0.8190127970749543,
                  0.011160714285714286,
                  448,
                  5
                ],
                [
                  0.8208409506398537,
                  0.011135857461024499,
                  449,
                  5
                ],
                [
                  0.8226691042047533,
                  0.013333333333333334,
                  450,
                  6
                ],
                [
                  0.8244972577696527,
                  0.015521064301552107,
                  451,
                  7
                ],
                [
                  0.8263254113345521,
                  0.015486725663716814,
                  452,
                  7
                ],
                [
                  0.8281535648994516,
                  0.017660044150110375,
                  453,
                  8
                ],
                [
                  0.829981718464351,
                  0.019823788546255508,
                  454,
                  9
                ],
                [
                  0.8318098720292505,
                  0.01978021978021978,
                  455,
                  9
                ],
                [
                  0.8336380255941499,
                  0.019736842105263157,
                  456,
                  9
                ],
                [
                  0.8354661791590493,
                  0.019693654266958426,
                  457,
                  9
                ],
                [
                  0.8372943327239488,
                  0.019650655021834062,
                  458,
                  9
                ],
                [
                  0.8391224862888482,
                  0.0196078431372549,
                  459,
                  9
                ],
                [
                  0.8409506398537477,
                  0.01956521739130435,
                  460,
                  9
                ],
                [
                  0.8427787934186471,
                  0.019522776572668113,
                  461,
                  9
                ],
                [
                  0.8446069469835467,
                  0.021645021645021644,
                  462,
                  10
                ],
                [
                  0.8464351005484461,
                  0.02159827213822894,
                  463,
                  10
                ],
                [
                  0.8482632541133455,
                  0.021551724137931036,
                  464,
                  10
                ],
                [
                  0.850091407678245,
                  0.023655913978494623,
                  465,
                  11
                ],
                [
                  0.8519195612431444,
                  0.023605150214592276,
                  466,
                  11
                ],
                [
                  0.8537477148080439,
                  0.023554603854389723,
                  467,
                  11
                ],
                [
                  0.8555758683729433,
                  0.023504273504273504,
                  468,
                  11
                ],
                [
                  0.8574040219378428,
                  0.0255863539445629,
                  469,
                  12
                ],
                [
                  0.8592321755027422,
                  0.027659574468085105,
                  470,
                  13
                ],
                [
                  0.8610603290676416,
                  0.029723991507430998,
                  471,
                  14
                ],
                [
                  0.8628884826325411,
                  0.029661016949152543,
                  472,
                  14
                ],
                [
                  0.8647166361974405,
                  0.02959830866807611,
                  473,
                  14
                ],
                [
                  0.8665447897623401,
                  0.029535864978902954,
                  474,
                  14
                ],
                [
                  0.8683729433272395,
                  0.029473684210526315,
                  475,
                  14
                ],
                [
                  0.870201096892139,
                  0.029411764705882353,
                  476,
                  14
                ],
                [
                  0.8720292504570384,
                  0.029350104821802937,
                  477,
                  14
                ],
                [
                  0.8738574040219378,
                  0.029288702928870293,
                  478,
                  14
                ],
                [
                  0.8756855575868373,
                  0.031315240083507306,
                  479,
                  15
                ],
                [
                  0.8775137111517367,
                  0.03125,
                  480,
                  15
                ],
                [
                  0.8793418647166362,
                  0.033264033264033266,
                  481,
                  16
                ],
                [
                  0.8811700182815356,
                  0.03319502074688797,
                  482,
                  16
                ],
                [
                  0.8829981718464351,
                  0.035196687370600416,
                  483,
                  17
                ],
                [
                  0.8848263254113345,
                  0.0371900826446281,
                  484,
                  18
                ],
                [
                  0.886654478976234,
                  0.03711340206185567,
                  485,
                  18
                ],
                [
                  0.8884826325411335,
                  0.03909465020576132,
                  486,
                  19
                ],
                [
                  0.8903107861060329,
                  0.039014373716632446,
                  487,
                  19
                ],
                [
                  0.8921389396709324,
                  0.0389344262295082,
                  488,
                  19
                ],
                [
                  0.8939670932358318,
                  0.03885480572597137,
                  489,
                  19
                ],
                [
                  0.8957952468007313,
                  0.03877551020408163,
                  490,
                  19
                ],
                [
                  0.8976234003656307,
                  0.04073319755600815,
                  491,
                  20
                ],
                [
                  0.8994515539305301,
                  0.04065040650406504,
                  492,
                  20
                ],
                [
                  0.9012797074954296,
                  0.04259634888438134,
                  493,
                  21
                ],
                [
                  0.903107861060329,
                  0.04251012145748988,
                  494,
                  21
                ],
                [
                  0.9049360146252285,
                  0.044444444444444446,
                  495,
                  22
                ],
                [
                  0.906764168190128,
                  0.04435483870967742,
                  496,
                  22
                ],
                [
                  0.9085923217550275,
                  0.04627766599597585,
                  497,
                  23
                ],
                [
                  0.9104204753199269,
                  0.04819277108433735,
                  498,
                  24
                ],
                [
                  0.9122486288848263,
                  0.050100200400801605,
                  499,
                  25
                ],
                [
                  0.9140767824497258,
                  0.05,
                  500,
                  25
                ],
                [
                  0.9159049360146252,
                  0.0499001996007984,
                  501,
                  25
                ],
                [
                  0.9177330895795247,
                  0.05179282868525897,
                  502,
                  26
                ],
                [
                  0.9195612431444241,
                  0.0536779324055666,
                  503,
                  27
                ],
                [
                  0.9213893967093236,
                  0.05357142857142857,
                  504,
                  27
                ],
                [
                  0.923217550274223,
                  0.053465346534653464,
                  505,
                  27
                ],
                [
                  0.9250457038391224,
                  0.0533596837944664,
                  506,
                  27
                ],
                [
                  0.926873857404022,
                  0.055226824457593686,
                  507,
                  28
                ],
                [
                  0.9287020109689214,
                  0.05511811023622047,
                  508,
                  28
                ],
                [
                  0.9305301645338209,
                  0.0550098231827112,
                  509,
                  28
                ],
                [
                  0.9323583180987203,
                  0.054901960784313725,
                  510,
                  28
                ],
                [
                  0.9341864716636198,
                  0.0547945205479452,
                  511,
                  28
                ],
                [
                  0.9360146252285192,
                  0.0546875,
                  512,
                  28
                ],
                [
                  0.9378427787934186,
                  0.056530214424951264,
                  513,
                  29
                ],
                [
                  0.9396709323583181,
                  0.058365758754863814,
                  514,
                  30
                ],
                [
                  0.9414990859232175,
                  0.06019417475728155,
                  515,
                  31
                ],
                [
                  0.943327239488117,
                  0.06201550387596899,
                  516,
                  32
                ],
                [
                  0.9451553930530164,
                  0.06382978723404255,
                  517,
                  33
                ],
                [
                  0.946983546617916,
                  0.0637065637065637,
                  518,
                  33
                ],
                [
                  0.9488117001828154,
                  0.06358381502890173,
                  519,
                  33
                ],
                [
                  0.9506398537477148,
                  0.06346153846153846,
                  520,
                  33
                ],
                [
                  0.9524680073126143,
                  0.06525911708253358,
                  521,
                  34
                ],
                [
                  0.9542961608775137,
                  0.06513409961685823,
                  522,
                  34
                ],
                [
                  0.9561243144424132,
                  0.06692160611854685,
                  523,
                  35
                ],
                [
                  0.9579524680073126,
                  0.06679389312977099,
                  524,
                  35
                ],
                [
                  0.9597806215722121,
                  0.06857142857142857,
                  525,
                  36
                ],
                [
                  0.9616087751371115,
                  0.07034220532319392,
                  526,
                  37
                ],
                [
                  0.9634369287020109,
                  0.07020872865275142,
                  527,
                  37
                ],
                [
                  0.9652650822669104,
                  0.07007575757575757,
                  528,
                  37
                ],
                [
                  0.9670932358318098,
                  0.07183364839319471,
                  529,
                  38
                ],
                [
                  0.9689213893967094,
                  0.07169811320754717,
                  530,
                  38
                ],
                [
                  0.9707495429616088,
                  0.07344632768361582,
                  531,
                  39
                ],
                [
                  0.9725776965265083,
                  0.07518796992481203,
                  532,
                  40
                ],
                [
                  0.9744058500914077,
                  0.075046904315197,
                  533,
                  40
                ],
                [
                  0.9762340036563071,
                  0.07677902621722846,
                  534,
                  41
                ],
                [
                  0.9780621572212066,
                  0.07850467289719626,
                  535,
                  42
                ],
                [
                  0.979890310786106,
                  0.08022388059701492,
                  536,
                  43
                ],
                [
                  0.9817184643510055,
                  0.08193668528864059,
                  537,
                  44
                ],
                [
                  0.9835466179159049,
                  0.08364312267657993,
                  538,
                  45
                ],
                [
                  0.9853747714808044,
                  0.08534322820037106,
                  539,
                  46
                ],
                [
                  0.9872029250457038,
                  0.08703703703703704,
                  540,
                  47
                ],
                [
                  0.9890310786106032,
                  0.08687615526802218,
                  541,
                  47
                ],
                [
                  0.9908592321755028,
                  0.08856088560885608,
                  542,
                  48
                ],
                [
                  0.9926873857404022,
                  0.08839779005524862,
                  543,
                  48
                ],
                [
                  0.9945155393053017,
                  0.0900735294117647,
                  544,
                  49
                ],
                [
                  0.9963436928702011,
                  0.09174311926605505,
                  545,
                  50
                ],
                [
                  0.9981718464351006,
                  0.09340659340659341,
                  546,
                  51
                ],
                [
                  1.0,
                  0.09506398537477148,
                  547,
                  52
                ]
              ]
            },
            {
              "language": "English",
              "run": 521,
              "points": [
                [
                  0.0018281535648994515,
                  0.0,
                  1,
                  0
                ],
                [
                  0.003656307129798903,
                  0.0,
                  2,
                  0
                ],
                [
                  0.005484460694698354,
                  0.0,
                  3,
                  0
                ],
                [
                  0.007312614259597806,
                  0.0,
                  4,
                  0
                ],
                [
                  0.009140767824497258,
                  0.0,
                  5,
                  0
                ],
                [
                  0.010968921389396709,
                  0.0,
                  6,
                  0
                ],
                [
                  0.012797074954296161,
                  0.0,
                  7,
                  0
                ],
                [
                  0.014625228519195612,
                  0.0,
                  8,
                  0
                ],
                [
                  0.016453382084095063,
                  0.0,
                  9,
                  0
                ],
                [
                  0.018281535648994516,
                  0.0,
                  10,
                  0
                ],
                [
                  0.02010968921389397,
                  0.0,
                  11,
                  0
                ],
                [
                  0.021937842778793418,
                  0.0,
                  12,
                  0
                ],
                [
                  0.02376599634369287,
                  0.0,
                  13,
                  0
                ],
                [
                  0.025594149908592323,
                  0.0,
                  14,
                  0
                ],
                [
                  0.027422303473491772,
                  0.0,
                  15,
                  0
                ],
                [
                  0.029250457038391225,
                  0.0,
                  16,
                  0
                ],
                [
                  0.031078610603290677,
                  0.0,
                  17,
                  0
                ],
                [
                  0.03290676416819013,
                  0.0,
                  18,
                  0
                ],
                [
                  0.03473491773308958,
                  0.0,
                  19,
                  0
                ],
                [
                  0.03656307129798903,
                  0.0,
                  20,
                  0
                ],
                [
                  0.038391224862888484,
                  0.0,
                  21,
                  0
                ],
                [
                  0.04021937842778794,
                  0.0,
                  22,
                  0
                ],
                [
                  0.04204753199268738,
                  0.0,
                  23,
                  0
                ],
                [
                  0.043875685557586835,
                  0.0,
                  24,
                  0
                ],
                [
                  0.04570383912248629,
                  0.0,
                  25,
                  0
                ],
                [
                  0.04753199268738574,
                  0.0,
                  26,
                  0
                ],
                [
                  0.04936014625228519,
                  0.0,
                  27,
                  0
                ],
                [
                  0.051188299817184646,
                  0.0,
                  28,
                  0
                ],
                [
                  0.05301645338208409,
                  0.0,
                  29,
                  0
                ],
                [
                  0.054844606946983544,
                  0.0,
                  30,
                  0
                ],
                [
                  0.056672760511883,
                  0.0,
                  31,
                  0
                ],
                [
                  0.05850091407678245,
                  0.0,
                  32,
                  0
                ],
                [
                  0.0603290676416819,
                  0.0,
                  33,
                  0
                ],
                [
                  0.062157221206581355,
                  0.0,
                  34,
                  0
                ],
                [
                  0.06398537477148081,
                  0.0,
                  35,
                  0
                ],
                [
                  0.06581352833638025,
                  0.0,
                  36,
                  0
                ],
                [
                  0.06764168190127971,
                  0.0,
                  37,
                  0
                ],
                [
                  0.06946983546617916,
                  0.0,
                  38,
                  0
                ],
                [
                  0.0712979890310786,
                  0.0,
                  39,
                  0
                ],
                [
                  0.07312614259597806,
                  0.0,
                  40,
                  0
                ],
                [
                  0.07495429616087751,
                  0.0,
                  41,
                  0
                ],
                [
                  0.07678244972577697,
                  0.0,
                  42,
                  0
                ],
                [
                  0.07861060329067641,
                  0.0,
                  43,
                  0
                ],
                [
                  0.08043875685557587,
                  0.0,
                  44,
                  0
                ],
                [
                  0.08226691042047532,
                  0.0,
                  45,
                  0
                ],
                [
                  0.08409506398537477,
                  0.0,
                  46,
                  0
                ],
                [
                  0.08592321755027423,
                  0.0,
                  47,
                  0
                ],
                [
                  0.08775137111517367,
                  0.0,
                  48,
                  0
                ],
                [
                  0.08957952468007313,
                  0.0,
                  49,
                  0
                ],
                [
                  0.09140767824497258,
                  0.0,
                  50,
                  0
                ],
                [
                  0.09323583180987204,
                  0.0,
                  51,
                  0
                ],
                [
                  0.09506398537477148,
                  0.0,
                  52,
                  0
                ],
                [
                  0.09689213893967093,
                  0.0,
                  53,
                  0
                ],
                [
                  0.09872029250457039,
                  0.0,
                  54,
                  0
                ],
                [
                  0.10054844606946983,
                  0.0,
                  55,
                  0
                ],
                [
                  0.10237659963436929,
                  0.0,
                  56,
                  0
                ],
                [
                  0.10420475319926874,
                  0.0,
                  57,
                  0
                ],
                [
                  0.10603290676416818,
                  0.0,
                  58,
                  0
                ],
                [
                  0.10786106032906764,
                  0.0,
                  59,
                  0
                ],
                [
                  0.10968921389396709,
                  0.0,
                  60,
                  0
                ],
                [
                  0.11151736745886655,
                  0.0,
                  61,
                  0
                ],
                [
                  0.113345521023766,
                  0.0,
                  62,
                  0
                ],
                [
                  0.11517367458866545,
                  0.0,
                  63,
                  0
                ],
                [
                  0.1170018281535649,
                  0.0,
                  64,
                  0
                ],
                [
                  0.11882998171846434,
                  0.0,
                  65,
                  0
                ],
                [
                  0.1206581352833638,
                  0.0,
                  66,
                  0
                ],
                [
                  0.12248628884826325,
                  0.0,
                  67,
                  0
                ],
                [
                  0.12431444241316271,
                  0.0,
                  68,
                  0
                ],
                [
                  0.12614259597806216,
                  0.0,
                  69,
                  0
                ],
                [
                  0.12797074954296161,
                  0.0,
                  70,
                  0
                ],
                [
                  0.12979890310786105,
                  0.0,
                  71,
                  0
                ],
                [
                  0.1316270566727605,
                  0.0,
                  72,
                  0
                ],
                [
                  0.13345521023765997,
                  0.0,
                  73,
                  0
                ],
                [
                  0.13528336380255943,
                  0.0,
                  74,
                  0
                ],
                [
                  0.13711151736745886,
                  0.0,
                  75,
                  0
                ],
                [
                  0.13893967093235832,
                  0.0,
                  76,
                  0
                ],
                [
                  0.14076782449725778,
                  0.0,
                  77,
                  0
                ],
                [
                  0.1425959780621572,
                  0.0,
                  78,
                  0
                ],
                [
                  0.14442413162705667,
                  0.0,
                  79,
                  0
                ],
                [
                  0.14625228519195613,
                  0.0,
                  80,
                  0
                ],
                [
                  0.1480804387568556,
                  0.0,
                  81,
                  0
                ],
                [
                  0.14990859232175502,
                  0.0,
                  82,
                  0
                ],
                [
                  0.15173674588665448,
                  0.0,
                  83,
                  0
                ],
                [
                  0.15356489945155394,
                  0.0,
                  84,
                  0
                ],
                [
                  0.15539305301645337,
                  0.0,
                  85,
                  0
                ],
                [
                  0.15722120658135283,
                  0.0,
                  86,
                  0
                ],
                [
                  0.1590493601462523,
                  0.0,
                  87,
                  0
                ],
                [
                  0.16087751371115175,
                  0.0,
                  88,
                  0
                ],
                [
                  0.16270566727605118,
                  0.0,
                  89,
                  0
                ],
                [
                  0.16453382084095064,
                  0.0,
                  90,
                  0
                ],
                [
                  0.1663619744058501,
                  0.0,
                  91,
                  0
                ],
                [
                  0.16819012797074953,
                  0.0,
                  92,
                  0
                ],
                [
                  0.170018281535649,
                  0.0,
                  93,
                  0
                ],
                [
                  0.17184643510054845,
                  0.0,
                  94,
                  0
                ],
                [
                  0.1736745886654479,
                  0.0,
                  95,
                  0
                ],
                [
                  0.17550274223034734,
                  0.0,
                  96,
                  0
                ],
                [
                  0.1773308957952468,
                  0.0,
                  97,
                  0
                ],
                [
                  0.17915904936014626,
                  0.01020408163265306,
                  98,
                  1
                ],
                [
                  0.1809872029250457,
                  0.010101010101010102,
                  99,
                  1
                ],
                [
                  0.18281535648994515,
                  0.01,
                  100,
                  1
                ],
                [
                  0.1846435100548446,
                  0.009900990099009901,
                  101,
                  1
                ],
                [
                  0.18647166361974407,
                  0.00980392156862745,
                  102,
                  1
                ],
                [
                  0.1882998171846435,
                  0.009708737864077669,
                  103,
                  1
                ],
                [
                  0.19012797074954296,
                  0.009615384615384616,
                  104,
                  1
                ],
                [
                  0.19195612431444242,
                  0.009523809523809525,
                  105,
                  1
                ],
                [
                  0.19378427787934185,
                  0.009433962264150943,
                  106,
                  1
                ],
                [
                  0.1956124314442413,
                  0.009345794392523364,
                  107,
                  1
                ],
                [
                  0.19744058500914077,
                  0.009259259259259259,
                  108,
                  1
                ],
                [
                  0.19926873857404023,
                  0.009174311926605505,
                  109,
                  1
                ],
                [
                  0.20109689213893966,
                  0.00909090909090909,
                  110,
                  1
                ],
                [
                  0.20292504570383912,
                  0.009009009009009009,
                  111,
                  1
                ],
                [
                  0.20475319926873858,
                  0.008928571428571428,
                  112,
                  1
                ],
                [
                  0.20658135283363802,
                  0.008849557522123894,
                  113,
                  1
                ],
                [
                  0.20840950639853748,
                  0.008771929824561403,
                  114,
                  1
                ],
                [
                  0.21023765996343693,
                  0.008695652173913044,
                  115,
                  1
                ],
                [
                  0.21206581352833637,
                  0.008620689655172414,
                  116,
                  1
                ],
                [
                  0.21389396709323583,
                  0.008547008547008548,
                  117,
                  1
                ],
                [
                  0.21572212065813529,
                  0.00847457627118644,
                  118,
                  1
                ],
                [
                  0.21755027422303475,
                  0.008403361344537815,
                  119,
                  1
                ],
                [
                  0.21937842778793418,
                  0.008333333333333333,
                  120,
                  1
                ],
                [
                  0.22120658135283364,
                  0.008264462809917356,
                  121,
                  1
                ],
                [
                  0.2230347349177331,
                  0.00819672131147541,
                  122,
                  1
                ],
                [
                  0.22486288848263253,
                  0.008130081300813009,
                  123,
                  1
                ],
                [
                  0.226691042047532,
                  0.008064516129032258,
                  124,
                  1
                ],
                [
                  0.22851919561243145,
                  0.008,
                  125,
                  1
                ],
                [
                  0.2303473491773309,
                  0.007936507936507936,
                  126,
                  1
                ],
                [
                  0.23217550274223034,
                  0.007874015748031496,
                  127,
                  1
                ],
                [
                  0.2340036563071298,
                  0.0078125,
                  128,
                  1
                ],
                [
                  0.23583180987202926,
                  0.007751937984496124,
                  129,
                  1
                ],
                [
                  0.2376599634369287,
                  0.007692307692307693,
                  130,
                  1
                ],
                [
                  0.23948811700182815,
                  0.007633587786259542,
                  131,
                  1
                ],
                [
                  0.2413162705667276,
                  0.007575757575757576,
                  132,
                  1
                ],
                [
                  0.24314442413162707,
                  0.007518796992481203,
                  133,
                  1
                ],
                [
                  0.2449725776965265,
                  0.007462686567164179,
                  134,
                  1
                ],
                [
                  0.24680073126142596,
                  0.007407407407407408,
                  135,
                  1
                ],
                [
                  0.24862888482632542,
                  0.007352941176470588,
                  136,
                  1
                ],
                [
                  0.25045703839122485,
                  0.0072992700729927005,
                  137,
                  1
                ],
                [
                  0.2522851919561243,
                  0.007246376811594203,
                  138,
                  1
                ],
                [
                  0.25411334552102377,
                  0.007194244604316547,
                  139,
                  1
                ],
                [
                  0.25594149908592323,
                  0.007142857142857143,
                  140,
                  1
                ],
                [
                  0.2577696526508227,
                  0.0070921985815602835,
                  141,
                  1
                ],
                [
                  0.2595978062157221,
                  0.007042253521126761,
                  142,
                  1
                ],
                [
                  0.26142595978062155,
                  0.006993006993006993,
                  143,
                  1
                ],
                [
                  0.263254113345521,
                  0.006944444444444444,
                  144,
                  1
                ],
                [
                  0.26508226691042047,
                  0.006896551724137931,
                  145,
                  1
                ],
                [
                  0.26691042047531993,
                  0.00684931506849315,
                  146,
                  1
                ],
                [
                  0.2687385740402194,
                  0.006802721088435374,
                  147,
                  1
                ],
                [
                  0.27056672760511885,
                  0.006756756756756757,
                  148,
                  1
                ],
                [
                  0.27239488117001825,
                  0.006711409395973154,
                  149,
                  1
                ],
                [
                  0.2742230347349177,
                  0.006666666666666667,
                  150,
                  1
                ],
                [
                  0.2760511882998172,
                  0.006622516556291391,
                  151,
                  1
                ],
                [
                  0.27787934186471663,
                  0.006578947368421052,
                  152,
                  1
                ],
                [
                  0.2797074954296161,
                  0.006535947712418301,
                  153,
                  1
                ],
                [
                  0.28153564899451555,
                  0.006493506493506494,
                  154,
                  1
                ],
                [
                  0.283363802559415,
                  0.0064516129032258064,
                  155,
                  1
                ],
                [
                  0.2851919561243144,
                  0.00641025641025641,
                  156,
                  1
                ],
                [
                  0.2870201096892139,
                  0.006369426751592357,
                  157,
                  1
                ],
                [
                  0.28884826325411334,
                  0.006329113924050633,
                  158,
                  1
                ],
                [
                  0.2906764168190128,
                  0.006289308176100629,
                  159,
                  1
                ],
                [
                  0.29250457038391225,
                  0.00625,
                  160,
                  1
                ],
                [
                  0.2943327239488117,
                  0.006211180124223602,
                  161,
                  1
                ],
                [
                  0.2961608775137112,
                  0.006172839506172839,
                  162,
                  1
                ],
                [
                  0.2979890310786106,
                  0.006134969325153374,
                  163,
                  1
                ],
                [
                  0.29981718464351004,
                  0.006097560975609756,
                  164,
                  1
                ],
                [
                  0.3016453382084095,
                  0.006060606060606061,
                  165,
                  1
                ],
                [
                  0.30347349177330896,
                  0.006024096385542169,
                  166,
                  1
                ],
                [
                  0.3053016453382084,
                  0.005988023952095809,
                  167,
                  1
                ],
                [
                  0.3071297989031079,
                  0.005952380952380952,
                  168,
                  1
                ],
                [
                  0.30895795246800734,
                  0.005917159763313609,
                  169,
                  1
                ],
                [
                  0.31078610603290674,
                  0.0058823529411764705,
                  170,
                  1
                ],
                [
                  0.3126142595978062,
                  0.005847953216374269,
                  171,
                  1
                ],
                [
                  0.31444241316270566,
                  0.005813953488372093,
                  172,
                  1
                ],
                [
                  0.3162705667276051,
                  0.005780346820809248,
                  173,
                  1
                ],
                [
                  0.3180987202925046,
                  0.005747126436781609,
                  174,
                  1
                ],
                [
                  0.31992687385740404,
                  0.005714285714285714,
                  175,
                  1
                ],
                [
                  0.3217550274223035,
                  0.005681818181818182,
                  176,
                  1
                ],
                [
                  0.3235831809872029,
                  0.005649717514124294,
                  177,
                  1
                ],
                [
                  0.32541133455210236,
                  0.0056179775280898875,
                  178,
                  1
                ],
                [
                  0.3272394881170018,
                  0.00558659217877095,
                  179,
                  1
                ],
                [
                  0.3290676416819013,
                  0.005555555555555556,
                  180,
                  1
                ],
                [
                  0.33089579524680074,
                  0.0055248618784530384,
                  181,
                  1
                ],
                [
                  0.3327239488117002,
                  0.005494505494505495,
                  182,
                  1
                ],
                [
                  0.33455210237659966,
                  0.00546448087431694,
                  183,
                  1
                ],
                [
                  0.33638025594149906,
                  0.005434782608695652,
                  184,
                  1
                ],
                [
                  0.3382084095063985,
                  0.005405405405405406,
                  185,
                  1
                ],
                [
                  0.340036563071298,
                  0.005376344086021506,
                  186,
                  1
                ],
                [
                  0.34186471663619744,
                  0.0053475935828877,
                  187,
                  1
                ],
                [
                  0.3436928702010969,
                  0.005319148936170213,
                  188,
                  1
                ],
                [
                  0.34552102376599636,
                  0.005291005291005291,
                  189,
                  1
                ],
                [
                  0.3473491773308958,
                  0.005263157894736842,
                  190,
                  1
                ],
                [
                  0.3491773308957952,
                  0.005235602094240838,
                  191,
                  1
                ],
                [
                  0.3510054844606947,
                  0.005208333333333333,
                  192,
                  1
                ],
                [
                  0.35283363802559414,
                  0.0051813471502590676,
                  193,
                  1
                ],
                [
                  0.3546617915904936,
                  0.005154639175257732,
                  194,
                  1
                ],
                [
                  0.35648994515539306,
                  0.005128205128205128,
                  195,
                  1
                ],
                [
                  0.3583180987202925,
                  0.00510204081632653,
                  196,
                  1
                ],
                [
                  0.360146252285192,
                  0.005076142131979695,
                  197,
                  1
                ],
                [
                  0.3619744058500914,
                  0.005050505050505051,
                  198,
                  1
                ],
                [
                  0.36380255941499084,
                  0.005025125628140704,
                  199,
                  1
                ],
                [
                  0.3656307129798903,
                  0.005,
                  200,
                  1
                ],
                [
                  0.36745886654478976,
                  0.004975124378109453,
                  201,
                  1
                ],
                [
                  0.3692870201096892,
                  0.0049504950495049506,
                  202,
                  1
                ],
                [
                  0.3711151736745887,
                  0.0049261083743842365,
                  203,
                  1
                ],
                [
                  0.37294332723948814,
                  0.004901960784313725,
                  204,
                  1
                ],
                [
                  0.37477148080438755,
                  0.004878048780487805,
                  205,
                  1
                ],
                [
                  0.376599634369287,
                  0.0048543689320388345,
                  206,
                  1
                ],
                [
                  0.37842778793418647,
                  0.004830917874396135,
                  207,
                  1
                ],
                [
                  0.3802559414990859,
                  0.004807692307692308,
                  208,
                  1
                ],
                [
                  0.3820840950639854,
                  0.004784688995215311,
                  209,
                  1
                ],
                [
                  0.38391224862888484,
                  0.004761904761904762,
                  210,
                  1
                ],
                [
                  0.3857404021937843,
                  0.004739336492890996,
                  211,
                  1
                ],
                [
                  0.3875685557586837,
                  0.0047169811320754715,
                  212,
                  1
                ],
                [
                  0.38939670932358317,
                  0.004694835680751174,
                  213,
                  1
                ],
                [
                  0.3912248628884826,
                  0.004672897196261682,
                  214,
                  1
                ],
                [
                  0.3930530164533821,
                  0.004651162790697674,
                  215,
                  1
                ],
                [
                  0.39488117001828155,
                  0.004629629629629629,
                  216,
                  1
                ],
                [
                  0.396709323583181,
                  0.004608294930875576,
                  217,
                  1
                ],
                [
                  0.39853747714808047,
                  0.0045871559633027525,
                  218,
                  1
                ],
                [
                  0.40036563071297987,
                  0.0045662100456621,
                  219,
                  1
                ],
                [
                  0.40219378427787933,
                  0.004545454545454545,
                  220,
                  1
                ],
                [
                  0.4040219378427788,
                  0.004524886877828055,
                  221,
                  1
                ],
                [
                  0.40585009140767825,
                  0.0045045045045045045,
                  222,
                  1
                ],
                [
                  0.4076782449725777,
                  0.004484304932735426,
                  223,
                  1
                ],
                [
                  0.40950639853747717,
                  0.004464285714285714,
                  224,
                  1
                ],
                [
                  0.4113345521023766,
                  0.0044444444444444444,
                  225,
                  1
                ],
                [
                  0.41316270566727603,
                  0.004424778761061947,
                  226,
                  1
                ],
                [
                  0.4149908592321755,
                  0.004405286343612335,
                  227,
                  1
                ],
                [
                  0.41681901279707495,
                  0.0043859649122807015,
                  228,
                  1
                ],
                [
                  0.4186471663619744,
                  0.004366812227074236,
                  229,
                  1
                ],
                [
                  0.42047531992687387,
                  0.004347826086956522,
                  230,
                  1
                ],
                [
                  0.42230347349177333,
                  0.004329004329004329,
                  231,
                  1
                ],
                [
                  0.42413162705667273,
                  0.004310344827586207,
                  232,
                  1
                ],
                [
                  0.4259597806215722,
                  0.004291845493562232,
                  233,
                  1
                ],
                [
                  0.42778793418647165,
                  0.004273504273504274,
                  234,
                  1
                ],
                [
                  0.4296160877513711,
                  0.00425531914893617,
                  235,
                  1
                ],
                [
                  0.43144424131627057,
                  0.00423728813559322,
                  236,
                  1
                ],
                [
                  0.43327239488117003,
                  0.004219409282700422,
                  237,
                  1
                ],
                [
                  0.4351005484460695,
                  0.004201680672268907,
                  238,
                  1
                ],
                [
                  0.4369287020109689,
                  0.0041841004184100415,
                  239,
                  1
                ],
                [
                  0.43875685557586835,
                  0.004166666666666667,
                  240,
                  1
                ],
                [
                  0.4405850091407678,
                  0.004149377593360996,
                  241,
                  1
                ],
                [
                  0.4424131627056673,
                  0.004132231404958678,
                  242,
                  1
                ],
                [
                  0.44424131627056673,
                  0.00411522633744856,
                  243,
                  1
                ],
                [
                  0.4460694698354662,
                  0.004098360655737705,
                  244,
                  1
                ],
                [
                  0.44789762340036565,
                  0.004081632653061225,
                  245,
                  1
                ],
                [
                  0.44972577696526506,
                  0.0040650406504065045,
                  246,
                  1
                ],
                [
                  0.4515539305301645,
                  0.004048582995951417,
                  247,
                  1
                ],
                [
                  0.453382084095064,
                  0.004032258064516129,
                  248,
                  1
                ],
                [
                  0.45521023765996343,
                  0.004016064257028112,
                  249,
                  1
                ],
                [
                  0.4570383912248629,
                  0.004,
                  250,
                  1
                ],
                [
                  0.45886654478976235,
                  0.00398406374501992,
                  251,
                  1
                ],
                [
                  0.4606946983546618,
                  0.003968253968253968,
                  252,
                  1
                ],
                [
                  0.4625228519195612,
                  0.003952569169960474,
                  253,
                  1
                ],
                [
                  0.4643510054844607,
                  0.003937007874015748,
                  254,
                  1
                ],
                [
                  0.46617915904936014,
                  0.00392156862745098,
                  255,
                  1
                ],
                [
                  0.4680073126142596,
                  0.00390625,
                  256,
                  1
                ],
                [
                  0.46983546617915906,
                  0.0038910505836575876,
                  257,
                  1
                ],
                [
                  0.4716636197440585,
                  0.003875968992248062,
                  258,
                  1
                ],
                [
                  0.473491773308958,
                  0.003861003861003861,
                  259,
                  1
                ],
                [
                  0.4753199268738574,
                  0.0038461538461538464,
                  260,
                  1
                ],
                [
                  0.47714808043875684,
                  0.0038314176245210726,
                  261,
                  1
                ],
                [
                  0.4789762340036563,
                  0.003816793893129771,
                  262,
                  1
                ],
                [
                  0.48080438756855576,
                  0.0038022813688212928,
                  263,
                  1
                ],
                [
                  0.4826325411334552,
                  0.003787878787878788,
                  264,
                  1
                ],
                [
                  0.4844606946983547,
                  0.0037735849056603774,
                  265,
                  1
                ],
                [
                  0.48628884826325414,
                  0.0037593984962406013,
                  266,
                  1
                ],
                [
                  0.48811700182815354,
                  0.003745318352059925,
                  267,
                  1
                ],
                [
                  0.489945155393053,
                  0.0037313432835820895,
                  268,
                  1
                ],
                [
                  0.49177330895795246,
                  0.0037174721189591076,
                  269,
                  1
                ],
                [
                  0.4936014625228519,
                  0.003703703703703704,
                  270,
                  1
                ],
                [
                  0.4954296160877514,
                  0.0036900369003690036,
                  271,
                  1
                ],
                [
                  0.49725776965265084,
                  0.003676470588235294,
                  272,
                  1
                ],
                [
                  0.4990859232175503,
                  0.003663003663003663,
                  273,
                  1
                ],
                [
                  0.5009140767824497,
                  0.0036496350364963502,
                  274,
                  1
                ],
                [
                  0.5027422303473492,
                  0.0036363636363636364,
                  275,
                  1
                ],
                [
                  0.5045703839122486,
                  0.0036231884057971015,
                  276,
                  1
                ],
                [
                  0.506398537477148,
                  0.0036101083032490976,
                  277,
                  1
                ],
                [
                  0.5082266910420475,
                  0.0035971223021582736,
                  278,
                  1
                ],
                [
                  0.5100548446069469,
                  0.0035842293906810036,
                  279,
                  1
                ],
                [
                  0.5118829981718465,
                  0.0035714285714285713,
                  280,
                  1
                ],
                [
                  0.5137111517367459,
                  0.0035587188612099642,
                  281,
                  1
                ],
                [
                  0.5155393053016454,
                  0.0035460992907801418,
                  282,
                  1
                ],
                [
                  0.5173674588665448,
                  0.0035335689045936395,
                  283,
                  1
                ],
                [
                  0.5191956124314442,
                  0.0035211267605633804,
                  284,
                  1
                ],
                [
                  0.5210237659963437,
                  0.0035087719298245615,
                  285,
                  1
                ],
                [
                  0.5228519195612431,
                  0.0034965034965034965,
                  286,
                  1
                ],
                [
                  0.5246800731261426,
                  0.003484320557491289,
                  287,
                  1
                ],
                [
                  0.526508226691042,
                  0.003472222222222222,
                  288,
                  1
                ],
                [
                  0.5283363802559415,
                  0.0034602076124567475,
                  289,
                  1
                ],
                [
                  0.5301645338208409,
                  0.0034482758620689655,
                  290,
                  1
                ],
                [
                  0.5319926873857403,
                  0.003436426116838488,
                  291,
                  1
                ],
                [
                  0.5338208409506399,
                  0.003424657534246575,
                  292,
                  1
                ],
                [
                  0.5356489945155393,
                  0.0034129692832764505,
                  293,
                  1
                ],
                [
                  0.5374771480804388,
                  0.003401360544217687,
                  294,
                  1
                ],
                [
                  0.5393053016453382,
                  0.003389830508474576,
                  295,
                  1
                ],
                [
                  0.5411334552102377,
                  0.0033783783783783786,
                  296,
                  1
                ],
                [
                  0.5429616087751371,
                  0.003367003367003367,
                  297,
                  1
                ],
                [
                  0.5447897623400365,
                  0.003355704697986577,
                  298,
                  1
                ],
                [
                  0.546617915904936,
                  0.0033444816053511705,
                  299,
                  1
                ],
                [
                  0.5484460694698354,
                  0.0033333333333333335,
                  300,
                  1
                ],
                [
                  0.5502742230347349,
                  0.0033222591362126247,
                  301,
                  1
                ],
                [
                  0.5521023765996343,
                  0.0033112582781456954,
                  302,
                  1
                ],
                [
                  0.5539305301645339,
                  0.0033003300330033004,
                  303,
                  1
                ],
                [
                  0.5557586837294333,
                  0.003289473684210526,
                  304,
                  1
                ],
                [
                  0.5575868372943327,
                  0.003278688524590164,
                  305,
                  1
                ],
                [
                  0.5594149908592322,
                  0.0032679738562091504,
                  306,
                  1
                ],
                [
                  0.5612431444241316,
                  0.003257328990228013,
                  307,
                  1
                ],
                [
                  0.5630712979890311,
                  0.003246753246753247,
                  308,
                  1
                ],
                [
                  0.5648994515539305,
                  0.003236245954692557,
                  309,
                  1
                ],
                [
                  0.56672760511883,
                  0.0032258064516129032,
                  310,
                  1
                ],
                [
                  0.5685557586837294,
                  0.003215434083601286,
                  311,
                  1
                ],
                [
                  0.5703839122486288,
                  0.003205128205128205,
                  312,
                  1
                ],
                [
                  0.5722120658135283,
                  0.003194888178913738,
                  313,
                  1
                ],
                [
                  0.5740402193784278,
                  0.0031847133757961785,
                  314,
                  1
                ],
                [
                  0.5758683729433273,
                  0.0031746031746031746,
                  315,
                  1
                ],
                [
                  0.5776965265082267,
                  0.0031645569620253164,
                  316,
                  1
                ],
                [
                  0.5795246800731262,
                  0.0031545741324921135,
                  317,
                  1
                ],
                [
                  0.5813528336380256,
                  0.0031446540880503146,
                  318,
                  1
                ],
                [
                  0.583180987202925,
                  0.003134796238244514,
                  319,
                  1
                ],
                [
                  0.5850091407678245,
                  0.003125,
                  320,
                  1
                ],
                [
                  0.5868372943327239,
                  0.003115264797507788,
                  321,
                  1
                ],
                [
                  0.5886654478976234,
                  0.003105590062111801,
                  322,
                  1
                ],
                [
                  0.5904936014625228,
                  0.0030959752321981426,
                  323,
                  1
                ],
                [
                  0.5923217550274223,
                  0.0030864197530864196,
                  324,
                  1
                ],
                [
                  0.5941499085923218,
                  0.003076923076923077,
                  325,
                  1
                ],
                [
                  0.5959780621572212,
                  0.003067484662576687,
                  326,
                  1
                ],
                [
                  0.5978062157221207,
                  0.0030581039755351682,
                  327,
                  1
                ],
                [
                  0.5996343692870201,
                  0.003048780487804878,
                  328,
                  1
                ],
                [
                  0.6014625228519196,
                  0.00303951367781155,
                  329,
                  1
                ],
                [
                  0.603290676416819,
                  0.0030303030303030303,
                  330,
                  1
                ],
                [
                  0.6051188299817185,
                  0.0030211480362537764,
                  331,
                  1
                ],
                [
                  0.6069469835466179,
                  0.0030120481927710845,
                  332,
                  1
                ],
                [
                  0.6087751371115173,
                  0.003003003003003003,
                  333,
                  1
                ],
                [
                  0.6106032906764168,
                  0.0029940119760479044,
                  334,
                  1
                ],
                [
                  0.6124314442413162,
                  0.0029850746268656717,
                  335,
                  1
                ],
                [
                  0.6142595978062158,
                  0.002976190476190476,
                  336,
                  1
                ],
                [
                  0.6160877513711152,
                  0.002967359050445104,
                  337,
                  1
                ],
                [
                  0.6179159049360147,
                  0.0029585798816568047,
                  338,
                  1
                ],
                [
                  0.6197440585009141,
                  0.0029498525073746312,
                  339,
                  1
                ],
                [
                  0.6215722120658135,
                  0.0029411764705882353,
                  340,
                  1
                ],
                [
                  0.623400365630713,
                  0.002932551319648094,
                  341,
                  1
                ],
                [
                  0.6252285191956124,
                  0.0029239766081871343,
                  342,
                  1
                ],
                [
                  0.6270566727605119,
                  0.0029154518950437317,
                  343,
                  1
                ],
                [
                  0.6288848263254113,
                  0.0029069767441860465,
                  344,
                  1
                ],
                [
                  0.6307129798903108,
                  0.002898550724637681,
                  345,
                  1
                ],
                [
                  0.6325411334552102,
                  0.002890173410404624,
                  346,
                  1
                ],
                [
                  0.6343692870201096,
                  0.002881844380403458,
                  347,
                  1
                ],
                [
                  0.6361974405850092,
                  0.0028735632183908046,
                  348,
                  1
                ],
                [
                  0.6380255941499086,
                  0.0028653295128939827,
                  349,
                  1
                ],
                [
                  0.6398537477148081,
                  0.002857142857142857,
                  350,
                  1
                ],
                [
                  0.6416819012797075,
                  0.002849002849002849,
                  351,
                  1
                ],
                [
                  0.643510054844607,
                  0.002840909090909091,
                  352,
                  1
                ],
                [
                  0.6453382084095064,
                  0.0028328611898017,
                  353,
                  1
                ],
                [
                  0.6471663619744058,
                  0.002824858757062147,
                  354,
                  1
                ],
                [
                  0.6489945155393053,
                  0.005633802816901409,
                  355,
                  2
                ],
                [
                  0.6508226691042047,
                  0.008426966292134831,
                  356,
                  3
                ],
                [
                  0.6526508226691042,
                  0.008403361344537815,
                  357,
                  3
                ],
                [
                  0.6544789762340036,
                  0.008379888268156424,
                  358,
                  3
                ],
                [
                  0.6563071297989032,
                  0.008356545961002786,
                  359,
                  3
                ],
                [
                  0.6581352833638026,
                  0.008333333333333333,
                  360,
                  3
                ],
                [
                  0.659963436928702,
                  0.008310249307479225,
                  361,
                  3
                ],
                [
                  0.6617915904936015,
                  0.008287292817679558,
                  362,
                  3
                ],
                [
                  0.6636197440585009,
                  0.008264462809917356,
                  363,
                  3
                ],
                [
                  0.6654478976234004,
                  0.008241758241758242,
                  364,
                  3
                ],
                [
                  0.6672760511882998,
                  0.00821917808219178,
                  365,
                  3
                ],
                [
                  0.6691042047531993,
                  0.00819672131147541,
                  366,
                  3
                ],
                [
                  0.6709323583180987,
                  0.008174386920980926,
                  367,
                  3
                ],
                [
                  0.6727605118829981,
                  0.008152173913043478,
                  368,
                  3
                ],
                [
                  0.6745886654478976,
                  0.008130081300813009,
                  369,
                  3
                ],
                [
                  0.676416819012797,
                  0.008108108108108109,
                  370,
                  3
                ],
                [
                  0.6782449725776966,
                  0.008086253369272238,
                  371,
                  3
                ],
                [
                  0.680073126142596,
                  0.008064516129032258,
                  372,
                  3
                ],
                [
                  0.6819012797074955,
                  0.00804289544235925,
                  373,
                  3
                ],
                [
                  0.6837294332723949,
                  0.008021390374331552,
                  374,
                  3
                ],
                [
                  0.6855575868372943,
                  0.008,
                  375,
                  3
                ],
                [
                  0.6873857404021938,
                  0.007978723404255319,
                  376,
                  3
                ],
                [
                  0.6892138939670932,
                  0.010610079575596816,
                  377,
                  4
                ],
                [
                  0.6910420475319927,
                  0.010582010582010581,
                  378,
                  4
                ],
                [
                  0.6928702010968921,
                  0.010554089709762533,
                  379,
                  4
                ],
                [
                  0.6946983546617916,
                  0.010526315789473684,
                  380,
                  4
                ],
                [
                  0.696526508226691,
                  0.010498687664041995,
                  381,
                  4
                ],
                [
                  0.6983546617915904,
                  0.010471204188481676,
                  382,
                  4
                ],
                [
                  0.70018281535649,
                  0.010443864229765013,
                  383,
                  4
                ],
                [
                  0.7020109689213894,
                  0.013020833333333334,
                  384,
                  5
                ],
                [
                  0.7038391224862889,
                  0.012987012987012988,
                  385,
                  5
                ],
                [
                  0.7056672760511883,
                  0.012953367875647668,
                  386,
                  5
                ],
                [
                  0.7074954296160878,
                  0.012919896640826873,
                  387,
                  5
                ],
                [
                  0.7093235831809872,
                  0.01288659793814433,
                  388,
                  5
                ],
                [
                  0.7111517367458866,
                  0.012853470437017995,
                  389,
                  5
                ],
                [
                  0.7129798903107861,
                  0.01282051282051282,
                  390,
                  5
                ],
                [
                  0.7148080438756855,
                  0.015345268542199489,
                  391,
                  6
                ],
                [
                  0.716636197440585,
                  0.017857142857142856,
                  392,
                  7
                ],
                [
                  0.7184643510054844,
                  0.017811704834605598,
                  393,
                  7
                ],
                [
                  0.720292504570384,
                  0.017766497461928935,
                  394,
                  7
                ],
                [
                  0.7221206581352834,
                  0.017721518987341773,
                  395,
                  7
                ],
                [
                  0.7239488117001828,
                  0.017676767676767676,
                  396,
                  7
                ],
                [
                  0.7257769652650823,
                  0.017632241813602016,
                  397,
                  7
                ],
                [
                  0.7276051188299817,
                  0.017587939698492462,
                  398,
                  7
                ],
                [
                  0.7294332723948812,
                  0.017543859649122806,
                  399,
                  7
                ],
                [
                  0.7312614259597806,
                  0.0175,
                  400,
                  7
                ],
                [
                  0.7330895795246801,
                  0.017456359102244388,
                  401,
                  7
                ],
                [
                  0.7349177330895795,
                  0.017412935323383085,
                  402,
                  7
                ],
                [
                  0.7367458866544789,
                  0.017369727047146403,
                  403,
                  7
                ],
                [
                  0.7385740402193784,
                  0.017326732673267328,
                  404,
                  7
                ],
                [
                  0.7404021937842779,
                  0.01728395061728395,
                  405,
                  7
                ],
                [
                  0.7422303473491774,
                  0.017241379310344827,
                  406,
                  7
                ],
                [
                  0.7440585009140768,
                  0.0171990171990172,
                  407,
                  7
                ],
                [
                  0.7458866544789763,
                  0.01715686274509804,
                  408,
                  7
                ],
                [
                  0.7477148080438757,
                  0.017114914425427872,
                  409,
                  7
                ],
                [
                  0.7495429616087751,
                  0.01707317073170732,
                  410,
                  7
                ],
                [
                  0.7513711151736746,
                  0.019464720194647202,
                  411,
                  8
                ],
                [
                  0.753199268738574,
                  0.019417475728155338,
                  412,
                  8
                ],
                [
                  0.7550274223034735,
                  0.01937046004842615,
                  413,
                  8
                ],
                [
                  0.7568555758683729,
                  0.01932367149758454,
                  414,
                  8
                ],
                [
                  0.7586837294332724,
                  0.021686746987951807,
                  415,
                  9
                ],
                [
                  0.7605118829981719,
                  0.021634615384615384,
                  416,
                  9
                ],
                [
                  0.7623400365630713,
                  0.02158273381294964,
                  417,
                  9
                ],
                [
                  0.7641681901279708,
                  0.0215311004784689,
                  418,
                  9
                ],
                [
                  0.7659963436928702,
                  0.021479713603818614,
                  419,
                  9
                ],
                [
                  0.7678244972577697,
                  0.02142857142857143,
                  420,
                  9
                ],
                [
                  0.7696526508226691,
                  0.021377672209026127,
                  421,
                  9
                ],
                [
                  0.7714808043875686,
                  0.02132701421800948,
                  422,
                  9
                ],
                [
                  0.773308957952468,
                  0.02127659574468085,
                  423,
                  9
                ],
                [
                  0.7751371115173674,
                  0.02122641509433962,
                  424,
                  9
                ],
                [
                  0.7769652650822669,
                  0.021176470588235293,
                  425,
                  9
                ],
                [
                  0.7787934186471663,
                  0.02112676056338028,
                  426,
                  9
                ],
                [
                  0.7806215722120659,
                  0.0234192037470726,
                  427,
                  10
                ],
                [
                  0.7824497257769653,
                  0.02336448598130841,
                  428,
                  10
                ],
                [
                  0.7842778793418648,
                  0.02564102564102564,
                  429,
                  11
                ],
                [
                  0.7861060329067642,
                  0.027906976744186046,
                  430,
                  12
                ],
                [
                  0.7879341864716636,
                  0.027842227378190254,
                  431,
                  12
                ],
                [
                  0.7897623400365631,
                  0.027777777777777776,
                  432,
                  12
                ],
                [
                  0.7915904936014625,
                  0.02771362586605081,
                  433,
                  12
                ],
                [
                  0.793418647166362,
                  0.027649769585253458,
                  434,
                  12
                ],
                [
                  0.7952468007312614,
                  0.027586206896551724,
                  435,
                  12
                ],
                [
                  0.7970749542961609,
                  0.02981651376146789,
                  436,
                  13
                ],
                [
                  0.7989031078610603,
                  0.029748283752860413,
                  437,
                  13
                ],
                [
                  0.8007312614259597,
                  0.02968036529680365,
                  438,
                  13
                ],
                [
                  0.8025594149908593,
                  0.029612756264236904,
                  439,
                  13
                ],
                [
                  0.8043875685557587,
                  0.029545454545454545,
                  440,
                  13
                ],
                [
                  0.8062157221206582,
                  0.02947845804988662,
                  441,
                  13
                ],
                [
                  0.8080438756855576,
                  0.029411764705882353,
                  442,
                  13
                ],
                [
                  0.8098720292504571,
                  0.029345372460496615,
                  443,
                  13
                ],
                [
                  0.8117001828153565,
                  0.02927927927927928,
                  444,
                  13
                ],
                [
                  0.8135283363802559,
                  0.029213483146067417,
                  445,
                  13
                ],
                [
                  0.8153564899451554,
                  0.02914798206278027,
                  446,
                  13
                ],
                [
                  0.8171846435100548,
                  0.029082774049217,
                  447,
                  13
                ],
                [
                  0.8190127970749543,
                  0.029017857142857144,
                  448,
                  13
                ],
                [
                  0.8208409506398537,
                  0.031180400890868598,
                  449,
                  14
                ],
                [
                  0.8226691042047533,
                  0.03111111111111111,
                  450,
                  14
                ],
                [
                  0.8244972577696527,
                  0.031042128603104215,
                  451,
                  14
                ],
                [
                  0.8263254113345521,
                  0.030973451327433628,
                  452,
                  14
                ],
                [
                  0.8281535648994516,
                  0.03090507726269316,
                  453,
                  14
                ],
                [
                  0.829981718464351,
                  0.03303964757709251,
                  454,
                  15
                ],
                [
                  0.8318098720292505,
                  0.03296703296703297,
                  455,
                  15
                ],
                [
                  0.8336380255941499,
                  0.03289473684210526,
                  456,
                  15
                ],
                [
                  0.8354661791590493,
                  0.0350109409190372,
                  457,
                  16
                ],
                [
                  0.8372943327239488,
                  0.034934497816593885,
                  458,
                  16
                ],
                [
                  0.8391224862888482,
                  0.034858387799564274,
                  459,
                  16
                ],
                [
                  0.8409506398537477,
                  0.034782608695652174,
                  460,
                  16
                ],
                [
                  0.8427787934186471,
                  0.03470715835140998,
                  461,
                  16
                ],
                [
                  0.8446069469835467,
                  0.03463203463203463,
                  462,
                  16
                ],
                [
                  0.8464351005484461,
                  0.03455723542116631,
                  463,
                  16
                ],
                [
                  0.8482632541133455,
                  0.036637931034482756,
                  464,
                  17
                ],
                [
                  0.850091407678245,
                  0.03870967741935484,
                  465,
                  18
                ],
                [
                  0.8519195612431444,
                  0.0407725321888412,
                  466,
                  19
                ],
                [
                  0.8537477148080439,
                  0.04068522483940043,
                  467,
                  19
                ],
                [
                  0.8555758683729433,
                  0.042735042735042736,
                  468,
                  20
                ],
                [
                  0.8574040219378428,
                  0.042643923240938165,
                  469,
                  20
                ],
                [
                  0.8592321755027422,
                  0.0425531914893617,
                  470,
                  20
                ],
                [
                  0.8610603290676416,
                  0.04246284501061571,
                  471,
                  20
                ],
                [
                  0.8628884826325411,
                  0.0423728813559322,
                  472,
                  20
                ],
                [
                  0.8647166361974405,
                  0.042283298097251586,
                  473,
                  20
                ],
                [
                  0.8665447897623401,
                  0.04219409282700422,
                  474,
                  20
                ],
                [
                  0.8683729433272395,
                  0.04421052631578947,
                  475,
                  21
                ],
                [
                  0.870201096892139,
                  0.046218487394957986,
                  476,
                  22
                ],
                [
                  0.8720292504570384,
                  0.04612159329140461,
                  477,
                  22
                ],
                [
                  0.8738574040219378,
                  0.04602510460251046,
                  478,
                  22
                ],
                [
                  0.8756855575868373,
                  0.04592901878914405,
                  479,
                  22
                ],
                [
                  0.8775137111517367,
                  0.04583333333333333,
                  480,
                  22
                ],
                [
                  0.8793418647166362,
                  0.04573804573804574,
                  481,
                  22
                ],
                [
                  0.8811700182815356,
                  0.04771784232365145,
                  482,
                  23
                ],
                [
                  0.8829981718464351,
                  0.047619047619047616,
                  483,
                  23
                ],
                [
                  0.8848263254113345,
                  0.047520661157024795,
                  484,
                  23
                ],
                [
                  0.886654478976234,
                  0.04742268041237113,
                  485,
                  23
                ],
                [
                  0.8884826325411335,
                  0.047325102880658436,
                  486,
                  23
                ],
                [
                  0.8903107861060329,
                  0.04722792607802875,
                  487,
                  23
                ],
                [
                  0.8921389396709324,
                  0.04918032786885246,
                  488,
                  24
                ],
                [
                  0.8939670932358318,
                  0.049079754601226995,
                  489,
                  24
                ],
                [
                  0.8957952468007313,
                  0.04897959183673469,
                  490,
                  24
                ],
                [
                  0.8976234003656307,
                  0.048879837067209775,
                  491,
                  24
                ],
                [
                  0.8994515539305301,
                  0.04878048780487805,
                  492,
                  24
                ],
                [
                  0.9012797074954296,
                  0.05070993914807302,
                  493,
                  25
                ],
                [
                  0.903107861060329,
                  0.05263157894736842,
                  494,
                  26
                ],
                [
                  0.9049360146252285,
                  0.052525252525252523,
                  495,
                  26
                ],
                [
                  0.906764168190128,
                  0.05241935483870968,
                  496,
                  26
                ],
                [
                  0.9085923217550275,
                  0.05432595573440644,
                  497,
                  27
                ],
                [
                  0.9104204753199269,
                  0.05622489959839357,
                  498,
                  28
                ],
                [
                  0.9122486288848263,
                  0.05811623246492986,
                  499,
                  29
                ],
                [
                  0.9140767824497258,
                  0.058,
                  500,
                  29
                ],
                [
                  0.9159049360146252,
                  0.05788423153692615,
                  501,
                  29
                ],
                [
                  0.9177330895795247,
                  0.05976095617529881,
                  502,
                  30
                ],
                [
                  0.9195612431444241,
                  0.061630218687872766,
                  503,
                  31
                ],
                [
                  0.9213893967093236,
                  0.061507936507936505,
                  504,
                  31
                ],
                [
                  0.923217550274223,
                  0.061386138613861385,
                  505,
                  31
                ],
                [
                  0.9250457038391224,
                  0.06126482213438735,
                  506,
                  31
                ],
                [
                  0.926873857404022,
                  0.0611439842209073,
                  507,
                  31
                ],
                [
                  0.9287020109689214,
                  0.06299212598425197,
                  508,
                  32
                ],
                [
                  0.9305301645338209,
                  0.06483300589390963,
                  509,
                  33
                ],
                [
                  0.9323583180987203,
                  0.06470588235294118,
                  510,
                  33
                ],
                [
                  0.9341864716636198,
                  0.06653620352250489,
                  511,
                  34
                ],
                [
                  0.9360146252285192,
                  0.06640625,
                  512,
                  34
                ],
                [
                  0.9378427787934186,
                  0.0682261208576998,
                  513,
                  35
                ],
                [
                  0.9396709323583181,
                  0.07003891050583658,
                  514,
                  36
                ],
                [
                  0.9414990859232175,
                  0.06990291262135923,
                  515,
                  36
                ],
                [
                  0.943327239488117,
                  0.06976744186046512,
                  516,
                  36
                ],
                [
                  0.9451553930530164,
                  0.07156673114119923,
                  517,
                  37
                ],
                [
                  0.946983546617916,
                  0.07335907335907337,
                  518,
                  38
                ],
                [
                  0.9488117001828154,
                  0.07514450867052024,
                  519,
                  39
                ],
                [
                  0.9506398537477148,
                  0.07692307692307693,
                  520,
                  40
                ],
                [
                  0.9524680073126143,
                  0.07677543186180422,
                  521,
                  40
                ],
                [
                  0.9542961608775137,
                  0.07662835249042145,
                  522,
                  40
                ],
                [
                  0.9561243144424132,
                  0.07839388145315487,
                  523,
                  41
                ],
                [
                  0.9579524680073126,
                  0.07824427480916031,
                  524,
                  41
                ],
                [
                  0.9597806215722121,
                  0.07809523809523809,
                  525,
                  41
                ],
                [
                  0.9616087751371115,
                  0.0779467680608365,
                  526,
                  41
                ],
                [
                  0.9634369287020109,
                  0.07969639468690702,
                  527,
                  42
                ],
                [
                  0.9652650822669104,
                  0.08143939393939394,
                  528,
                  43
                ],
                [
                  0.9670932358318098,
                  0.08128544423440454,
                  529,
                  43
                ],
                [
                  0.9689213893967094,
                  0.08113207547169811,
                  530,
                  43
                ],
                [
                  0.9707495429616088,
                  0.08286252354048965,
                  531,
                  44
                ],
                [
                  0.9725776965265083,
                  0.08458646616541353,
                  532,
                  45
                ],
                [
                  0.9744058500914077,
                  0.08630393996247655,
                  533,
                  46
                ],
                [
                  0.9762340036563071,
                  0.08614232209737828,
                  534,
                  46
                ],
                [
                  0.9780621572212066,
                  0.08785046728971962,
                  535,
                  47
                ],
                [
                  0.979890310786106,
                  0.08955223880597014,
                  536,
                  48
                ],
                [
                  0.9817184643510055,
                  0.09124767225325885,
                  537,
                  49
                ],
                [
                  0.9835466179159049,
                  0.09293680297397769,
                  538,
                  50
                ],
                [
                  0.9853747714808044,
                  0.09276437847866419,
                  539,
                  50
                ],
                [
                  0.9872029250457038,
                  0.09259259259259259,
                  540,
                  50
                ],
                [
                  0.9890310786106032,
                  0.09426987060998152,
                  541,
                  51
                ],
                [
                  0.9908592321755028,
                  0.0940959409594096,
                  542,
                  51
                ],
                [
                  0.9926873857404022,
                  0.09576427255985268,
                  543,
                  52
                ],
                [
                  0.9945155393053017,
                  0.09558823529411764,
                  544,
                  52
                ],
                [
                  0.9963436928702011,
                  0.09541284403669725,
                  545,
                  52
                ],
                [
                  0.9981718464351006,
                  0.09523809523809523,
                  546,
                  52
                ],
                [
                  1.0,
                  0.09689213893967093,
                  547,
                  53
                ]
              ]
            }
          ]
        },
        {
          "name": "Mammoth-1",
          "text_run": 374,
          "native": [
            {
              "language": "Turkish",
              "wording": "Bilmiyorum",
              "run": 386,
              "full_bank_restricted": true,
              "right": 490,
              "wrong": 33,
              "abstained": 24,
              "invalid": 0,
              "coverage": 0.9561243144424132,
              "risk": 0.06309751434034416
            },
            {
              "language": "Turkish",
              "wording": "I don't know",
              "run": 519,
              "full_bank_restricted": true,
              "right": 490,
              "wrong": 32,
              "abstained": 25,
              "invalid": 0,
              "coverage": 0.9542961608775137,
              "risk": 0.06130268199233716
            },
            {
              "language": "English",
              "wording": "I don't know",
              "run": 505,
              "full_bank_restricted": false,
              "right": 474,
              "wrong": 36,
              "abstained": 37,
              "invalid": 0,
              "coverage": 0.9323583180987203,
              "risk": 0.07058823529411765
            }
          ],
          "curves": [
            {
              "language": "Turkish",
              "run": 374,
              "points": [
                [
                  0.0018281535648994515,
                  0.0,
                  1,
                  0
                ],
                [
                  0.003656307129798903,
                  0.0,
                  2,
                  0
                ],
                [
                  0.005484460694698354,
                  0.0,
                  3,
                  0
                ],
                [
                  0.007312614259597806,
                  0.0,
                  4,
                  0
                ],
                [
                  0.009140767824497258,
                  0.0,
                  5,
                  0
                ],
                [
                  0.010968921389396709,
                  0.0,
                  6,
                  0
                ],
                [
                  0.012797074954296161,
                  0.0,
                  7,
                  0
                ],
                [
                  0.014625228519195612,
                  0.0,
                  8,
                  0
                ],
                [
                  0.016453382084095063,
                  0.0,
                  9,
                  0
                ],
                [
                  0.018281535648994516,
                  0.0,
                  10,
                  0
                ],
                [
                  0.02010968921389397,
                  0.0,
                  11,
                  0
                ],
                [
                  0.021937842778793418,
                  0.0,
                  12,
                  0
                ],
                [
                  0.02376599634369287,
                  0.0,
                  13,
                  0
                ],
                [
                  0.025594149908592323,
                  0.0,
                  14,
                  0
                ],
                [
                  0.027422303473491772,
                  0.0,
                  15,
                  0
                ],
                [
                  0.029250457038391225,
                  0.0,
                  16,
                  0
                ],
                [
                  0.031078610603290677,
                  0.0,
                  17,
                  0
                ],
                [
                  0.03290676416819013,
                  0.0,
                  18,
                  0
                ],
                [
                  0.03473491773308958,
                  0.0,
                  19,
                  0
                ],
                [
                  0.03656307129798903,
                  0.0,
                  20,
                  0
                ],
                [
                  0.038391224862888484,
                  0.0,
                  21,
                  0
                ],
                [
                  0.04021937842778794,
                  0.0,
                  22,
                  0
                ],
                [
                  0.04204753199268738,
                  0.0,
                  23,
                  0
                ],
                [
                  0.043875685557586835,
                  0.0,
                  24,
                  0
                ],
                [
                  0.04570383912248629,
                  0.0,
                  25,
                  0
                ],
                [
                  0.04936014625228519,
                  0.0,
                  27,
                  0
                ],
                [
                  0.051188299817184646,
                  0.0,
                  28,
                  0
                ],
                [
                  0.05301645338208409,
                  0.0,
                  29,
                  0
                ],
                [
                  0.054844606946983544,
                  0.0,
                  30,
                  0
                ],
                [
                  0.056672760511883,
                  0.0,
                  31,
                  0
                ],
                [
                  0.05850091407678245,
                  0.0,
                  32,
                  0
                ],
                [
                  0.0603290676416819,
                  0.0,
                  33,
                  0
                ],
                [
                  0.062157221206581355,
                  0.0,
                  34,
                  0
                ],
                [
                  0.06398537477148081,
                  0.0,
                  35,
                  0
                ],
                [
                  0.06581352833638025,
                  0.0,
                  36,
                  0
                ],
                [
                  0.06946983546617916,
                  0.0,
                  38,
                  0
                ],
                [
                  0.0712979890310786,
                  0.0,
                  39,
                  0
                ],
                [
                  0.07312614259597806,
                  0.0,
                  40,
                  0
                ],
                [
                  0.07495429616087751,
                  0.0,
                  41,
                  0
                ],
                [
                  0.07678244972577697,
                  0.0,
                  42,
                  0
                ],
                [
                  0.07861060329067641,
                  0.0,
                  43,
                  0
                ],
                [
                  0.08043875685557587,
                  0.0,
                  44,
                  0
                ],
                [
                  0.08226691042047532,
                  0.0,
                  45,
                  0
                ],
                [
                  0.08409506398537477,
                  0.0,
                  46,
                  0
                ],
                [
                  0.08592321755027423,
                  0.0,
                  47,
                  0
                ],
                [
                  0.08775137111517367,
                  0.0,
                  48,
                  0
                ],
                [
                  0.08957952468007313,
                  0.0,
                  49,
                  0
                ],
                [
                  0.09140767824497258,
                  0.0,
                  50,
                  0
                ],
                [
                  0.09323583180987204,
                  0.0,
                  51,
                  0
                ],
                [
                  0.09506398537477148,
                  0.0,
                  52,
                  0
                ],
                [
                  0.09689213893967093,
                  0.0,
                  53,
                  0
                ],
                [
                  0.09872029250457039,
                  0.0,
                  54,
                  0
                ],
                [
                  0.10054844606946983,
                  0.0,
                  55,
                  0
                ],
                [
                  0.10237659963436929,
                  0.0,
                  56,
                  0
                ],
                [
                  0.10420475319926874,
                  0.0,
                  57,
                  0
                ],
                [
                  0.10603290676416818,
                  0.0,
                  58,
                  0
                ],
                [
                  0.10786106032906764,
                  0.0,
                  59,
                  0
                ],
                [
                  0.10968921389396709,
                  0.0,
                  60,
                  0
                ],
                [
                  0.11151736745886655,
                  0.0,
                  61,
                  0
                ],
                [
                  0.113345521023766,
                  0.0,
                  62,
                  0
                ],
                [
                  0.11517367458866545,
                  0.0,
                  63,
                  0
                ],
                [
                  0.1170018281535649,
                  0.0,
                  64,
                  0
                ],
                [
                  0.11882998171846434,
                  0.0,
                  65,
                  0
                ],
                [
                  0.1206581352833638,
                  0.0,
                  66,
                  0
                ],
                [
                  0.12248628884826325,
                  0.0,
                  67,
                  0
                ],
                [
                  0.12431444241316271,
                  0.0,
                  68,
                  0
                ],
                [
                  0.12614259597806216,
                  0.0,
                  69,
                  0
                ],
                [
                  0.12797074954296161,
                  0.0,
                  70,
                  0
                ],
                [
                  0.12979890310786105,
                  0.0,
                  71,
                  0
                ],
                [
                  0.1316270566727605,
                  0.0,
                  72,
                  0
                ],
                [
                  0.13345521023765997,
                  0.0,
                  73,
                  0
                ],
                [
                  0.13528336380255943,
                  0.0,
                  74,
                  0
                ],
                [
                  0.13711151736745886,
                  0.0,
                  75,
                  0
                ],
                [
                  0.13893967093235832,
                  0.0,
                  76,
                  0
                ],
                [
                  0.14076782449725778,
                  0.0,
                  77,
                  0
                ],
                [
                  0.1425959780621572,
                  0.0,
                  78,
                  0
                ],
                [
                  0.14442413162705667,
                  0.0,
                  79,
                  0
                ],
                [
                  0.14625228519195613,
                  0.0,
                  80,
                  0
                ],
                [
                  0.1480804387568556,
                  0.0,
                  81,
                  0
                ],
                [
                  0.14990859232175502,
                  0.0,
                  82,
                  0
                ],
                [
                  0.15173674588665448,
                  0.0,
                  83,
                  0
                ],
                [
                  0.15356489945155394,
                  0.0,
                  84,
                  0
                ],
                [
                  0.15539305301645337,
                  0.0,
                  85,
                  0
                ],
                [
                  0.15722120658135283,
                  0.0,
                  86,
                  0
                ],
                [
                  0.1590493601462523,
                  0.0,
                  87,
                  0
                ],
                [
                  0.16087751371115175,
                  0.0,
                  88,
                  0
                ],
                [
                  0.16270566727605118,
                  0.0,
                  89,
                  0
                ],
                [
                  0.16453382084095064,
                  0.0,
                  90,
                  0
                ],
                [
                  0.1663619744058501,
                  0.0,
                  91,
                  0
                ],
                [
                  0.16819012797074953,
                  0.0,
                  92,
                  0
                ],
                [
                  0.170018281535649,
                  0.0,
                  93,
                  0
                ],
                [
                  0.17184643510054845,
                  0.0,
                  94,
                  0
                ],
                [
                  0.1736745886654479,
                  0.0,
                  95,
                  0
                ],
                [
                  0.17550274223034734,
                  0.0,
                  96,
                  0
                ],
                [
                  0.1773308957952468,
                  0.0,
                  97,
                  0
                ],
                [
                  0.17915904936014626,
                  0.0,
                  98,
                  0
                ],
                [
                  0.1809872029250457,
                  0.0,
                  99,
                  0
                ],
                [
                  0.18281535648994515,
                  0.0,
                  100,
                  0
                ],
                [
                  0.1846435100548446,
                  0.0,
                  101,
                  0
                ],
                [
                  0.18647166361974407,
                  0.0,
                  102,
                  0
                ],
                [
                  0.1882998171846435,
                  0.0,
                  103,
                  0
                ],
                [
                  0.19012797074954296,
                  0.0,
                  104,
                  0
                ],
                [
                  0.19195612431444242,
                  0.0,
                  105,
                  0
                ],
                [
                  0.19378427787934185,
                  0.0,
                  106,
                  0
                ],
                [
                  0.1956124314442413,
                  0.0,
                  107,
                  0
                ],
                [
                  0.19744058500914077,
                  0.0,
                  108,
                  0
                ],
                [
                  0.19926873857404023,
                  0.0,
                  109,
                  0
                ],
                [
                  0.20109689213893966,
                  0.0,
                  110,
                  0
                ],
                [
                  0.20292504570383912,
                  0.0,
                  111,
                  0
                ],
                [
                  0.20658135283363802,
                  0.0,
                  113,
                  0
                ],
                [
                  0.20840950639853748,
                  0.0,
                  114,
                  0
                ],
                [
                  0.21023765996343693,
                  0.0,
                  115,
                  0
                ],
                [
                  0.21206581352833637,
                  0.0,
                  116,
                  0
                ],
                [
                  0.21389396709323583,
                  0.0,
                  117,
                  0
                ],
                [
                  0.21572212065813529,
                  0.0,
                  118,
                  0
                ],
                [
                  0.21755027422303475,
                  0.0,
                  119,
                  0
                ],
                [
                  0.21937842778793418,
                  0.0,
                  120,
                  0
                ],
                [
                  0.22120658135283364,
                  0.0,
                  121,
                  0
                ],
                [
                  0.2230347349177331,
                  0.0,
                  122,
                  0
                ],
                [
                  0.22486288848263253,
                  0.0,
                  123,
                  0
                ],
                [
                  0.22851919561243145,
                  0.0,
                  125,
                  0
                ],
                [
                  0.2303473491773309,
                  0.0,
                  126,
                  0
                ],
                [
                  0.23217550274223034,
                  0.0,
                  127,
                  0
                ],
                [
                  0.2340036563071298,
                  0.0,
                  128,
                  0
                ],
                [
                  0.23583180987202926,
                  0.0,
                  129,
                  0
                ],
                [
                  0.2376599634369287,
                  0.0,
                  130,
                  0
                ],
                [
                  0.23948811700182815,
                  0.0,
                  131,
                  0
                ],
                [
                  0.2413162705667276,
                  0.0,
                  132,
                  0
                ],
                [
                  0.24314442413162707,
                  0.0,
                  133,
                  0
                ],
                [
                  0.2449725776965265,
                  0.0,
                  134,
                  0
                ],
                [
                  0.24680073126142596,
                  0.0,
                  135,
                  0
                ],
                [
                  0.24862888482632542,
                  0.0,
                  136,
                  0
                ],
                [
                  0.25045703839122485,
                  0.0,
                  137,
                  0
                ],
                [
                  0.2522851919561243,
                  0.0,
                  138,
                  0
                ],
                [
                  0.25411334552102377,
                  0.0,
                  139,
                  0
                ],
                [
                  0.25594149908592323,
                  0.0,
                  140,
                  0
                ],
                [
                  0.2577696526508227,
                  0.0,
                  141,
                  0
                ],
                [
                  0.2595978062157221,
                  0.0,
                  142,
                  0
                ],
                [
                  0.26142595978062155,
                  0.0,
                  143,
                  0
                ],
                [
                  0.263254113345521,
                  0.0,
                  144,
                  0
                ],
                [
                  0.26508226691042047,
                  0.0,
                  145,
                  0
                ],
                [
                  0.26691042047531993,
                  0.0,
                  146,
                  0
                ],
                [
                  0.2687385740402194,
                  0.0,
                  147,
                  0
                ],
                [
                  0.27056672760511885,
                  0.0,
                  148,
                  0
                ],
                [
                  0.27239488117001825,
                  0.0,
                  149,
                  0
                ],
                [
                  0.2742230347349177,
                  0.0,
                  150,
                  0
                ],
                [
                  0.2760511882998172,
                  0.0,
                  151,
                  0
                ],
                [
                  0.27787934186471663,
                  0.0,
                  152,
                  0
                ],
                [
                  0.2797074954296161,
                  0.0,
                  153,
                  0
                ],
                [
                  0.28153564899451555,
                  0.0,
                  154,
                  0
                ],
                [
                  0.283363802559415,
                  0.0,
                  155,
                  0
                ],
                [
                  0.2851919561243144,
                  0.0,
                  156,
                  0
                ],
                [
                  0.2870201096892139,
                  0.0,
                  157,
                  0
                ],
                [
                  0.28884826325411334,
                  0.0,
                  158,
                  0
                ],
                [
                  0.2906764168190128,
                  0.0,
                  159,
                  0
                ],
                [
                  0.29250457038391225,
                  0.0,
                  160,
                  0
                ],
                [
                  0.2943327239488117,
                  0.0,
                  161,
                  0
                ],
                [
                  0.2961608775137112,
                  0.0,
                  162,
                  0
                ],
                [
                  0.2979890310786106,
                  0.0,
                  163,
                  0
                ],
                [
                  0.29981718464351004,
                  0.0,
                  164,
                  0
                ],
                [
                  0.3016453382084095,
                  0.0,
                  165,
                  0
                ],
                [
                  0.30347349177330896,
                  0.0,
                  166,
                  0
                ],
                [
                  0.3053016453382084,
                  0.0,
                  167,
                  0
                ],
                [
                  0.3071297989031079,
                  0.0,
                  168,
                  0
                ],
                [
                  0.30895795246800734,
                  0.0,
                  169,
                  0
                ],
                [
                  0.31078610603290674,
                  0.0,
                  170,
                  0
                ],
                [
                  0.3126142595978062,
                  0.0,
                  171,
                  0
                ],
                [
                  0.31444241316270566,
                  0.0,
                  172,
                  0
                ],
                [
                  0.3162705667276051,
                  0.0,
                  173,
                  0
                ],
                [
                  0.3180987202925046,
                  0.0,
                  174,
                  0
                ],
                [
                  0.31992687385740404,
                  0.0,
                  175,
                  0
                ],
                [
                  0.3217550274223035,
                  0.0,
                  176,
                  0
                ],
                [
                  0.3235831809872029,
                  0.0,
                  177,
                  0
                ],
                [
                  0.32541133455210236,
                  0.0,
                  178,
                  0
                ],
                [
                  0.3272394881170018,
                  0.0,
                  179,
                  0
                ],
                [
                  0.3290676416819013,
                  0.0,
                  180,
                  0
                ],
                [
                  0.33089579524680074,
                  0.0,
                  181,
                  0
                ],
                [
                  0.3327239488117002,
                  0.0,
                  182,
                  0
                ],
                [
                  0.33455210237659966,
                  0.0,
                  183,
                  0
                ],
                [
                  0.33638025594149906,
                  0.0,
                  184,
                  0
                ],
                [
                  0.3382084095063985,
                  0.0,
                  185,
                  0
                ],
                [
                  0.340036563071298,
                  0.0,
                  186,
                  0
                ],
                [
                  0.34186471663619744,
                  0.0,
                  187,
                  0
                ],
                [
                  0.3436928702010969,
                  0.0,
                  188,
                  0
                ],
                [
                  0.34552102376599636,
                  0.0,
                  189,
                  0
                ],
                [
                  0.3473491773308958,
                  0.0,
                  190,
                  0
                ],
                [
                  0.3491773308957952,
                  0.0,
                  191,
                  0
                ],
                [
                  0.3510054844606947,
                  0.0,
                  192,
                  0
                ],
                [
                  0.35283363802559414,
                  0.0,
                  193,
                  0
                ],
                [
                  0.3546617915904936,
                  0.0,
                  194,
                  0
                ],
                [
                  0.35648994515539306,
                  0.0,
                  195,
                  0
                ],
                [
                  0.3583180987202925,
                  0.0,
                  196,
                  0
                ],
                [
                  0.360146252285192,
                  0.0,
                  197,
                  0
                ],
                [
                  0.3619744058500914,
                  0.0,
                  198,
                  0
                ],
                [
                  0.36380255941499084,
                  0.0,
                  199,
                  0
                ],
                [
                  0.3656307129798903,
                  0.0,
                  200,
                  0
                ],
                [
                  0.36745886654478976,
                  0.0,
                  201,
                  0
                ],
                [
                  0.3692870201096892,
                  0.0,
                  202,
                  0
                ],
                [
                  0.3711151736745887,
                  0.0,
                  203,
                  0
                ],
                [
                  0.37294332723948814,
                  0.0,
                  204,
                  0
                ],
                [
                  0.37477148080438755,
                  0.0,
                  205,
                  0
                ],
                [
                  0.376599634369287,
                  0.0,
                  206,
                  0
                ],
                [
                  0.37842778793418647,
                  0.0,
                  207,
                  0
                ],
                [
                  0.3802559414990859,
                  0.0,
                  208,
                  0
                ],
                [
                  0.3820840950639854,
                  0.0,
                  209,
                  0
                ],
                [
                  0.38391224862888484,
                  0.0,
                  210,
                  0
                ],
                [
                  0.3857404021937843,
                  0.0,
                  211,
                  0
                ],
                [
                  0.3875685557586837,
                  0.0,
                  212,
                  0
                ],
                [
                  0.38939670932358317,
                  0.0,
                  213,
                  0
                ],
                [
                  0.3912248628884826,
                  0.0,
                  214,
                  0
                ],
                [
                  0.3930530164533821,
                  0.0,
                  215,
                  0
                ],
                [
                  0.39488117001828155,
                  0.0,
                  216,
                  0
                ],
                [
                  0.396709323583181,
                  0.0,
                  217,
                  0
                ],
                [
                  0.39853747714808047,
                  0.0,
                  218,
                  0
                ],
                [
                  0.40036563071297987,
                  0.0,
                  219,
                  0
                ],
                [
                  0.40219378427787933,
                  0.0,
                  220,
                  0
                ],
                [
                  0.4040219378427788,
                  0.0,
                  221,
                  0
                ],
                [
                  0.40585009140767825,
                  0.0,
                  222,
                  0
                ],
                [
                  0.4076782449725777,
                  0.0,
                  223,
                  0
                ],
                [
                  0.40950639853747717,
                  0.0,
                  224,
                  0
                ],
                [
                  0.41316270566727603,
                  0.0,
                  226,
                  0
                ],
                [
                  0.41681901279707495,
                  0.0,
                  228,
                  0
                ],
                [
                  0.4186471663619744,
                  0.0,
                  229,
                  0
                ],
                [
                  0.42047531992687387,
                  0.0,
                  230,
                  0
                ],
                [
                  0.42230347349177333,
                  0.0,
                  231,
                  0
                ],
                [
                  0.42413162705667273,
                  0.0,
                  232,
                  0
                ],
                [
                  0.4259597806215722,
                  0.0,
                  233,
                  0
                ],
                [
                  0.42778793418647165,
                  0.0,
                  234,
                  0
                ],
                [
                  0.4296160877513711,
                  0.0,
                  235,
                  0
                ],
                [
                  0.43144424131627057,
                  0.0,
                  236,
                  0
                ],
                [
                  0.43327239488117003,
                  0.0,
                  237,
                  0
                ],
                [
                  0.4351005484460695,
                  0.0,
                  238,
                  0
                ],
                [
                  0.4369287020109689,
                  0.0,
                  239,
                  0
                ],
                [
                  0.43875685557586835,
                  0.0,
                  240,
                  0
                ],
                [
                  0.4405850091407678,
                  0.0,
                  241,
                  0
                ],
                [
                  0.4424131627056673,
                  0.0,
                  242,
                  0
                ],
                [
                  0.44424131627056673,
                  0.0,
                  243,
                  0
                ],
                [
                  0.4460694698354662,
                  0.0,
                  244,
                  0
                ],
                [
                  0.44789762340036565,
                  0.0,
                  245,
                  0
                ],
                [
                  0.44972577696526506,
                  0.0,
                  246,
                  0
                ],
                [
                  0.4515539305301645,
                  0.0,
                  247,
                  0
                ],
                [
                  0.453382084095064,
                  0.0,
                  248,
                  0
                ],
                [
                  0.45521023765996343,
                  0.0,
                  249,
                  0
                ],
                [
                  0.4570383912248629,
                  0.0,
                  250,
                  0
                ],
                [
                  0.45886654478976235,
                  0.0,
                  251,
                  0
                ],
                [
                  0.4606946983546618,
                  0.0,
                  252,
                  0
                ],
                [
                  0.4625228519195612,
                  0.0,
                  253,
                  0
                ],
                [
                  0.4643510054844607,
                  0.0,
                  254,
                  0
                ],
                [
                  0.46617915904936014,
                  0.0,
                  255,
                  0
                ],
                [
                  0.4680073126142596,
                  0.0,
                  256,
                  0
                ],
                [
                  0.46983546617915906,
                  0.0,
                  257,
                  0
                ],
                [
                  0.4716636197440585,
                  0.0,
                  258,
                  0
                ],
                [
                  0.473491773308958,
                  0.0,
                  259,
                  0
                ],
                [
                  0.4753199268738574,
                  0.0,
                  260,
                  0
                ],
                [
                  0.47714808043875684,
                  0.0,
                  261,
                  0
                ],
                [
                  0.4789762340036563,
                  0.0,
                  262,
                  0
                ],
                [
                  0.48080438756855576,
                  0.0,
                  263,
                  0
                ],
                [
                  0.4826325411334552,
                  0.0,
                  264,
                  0
                ],
                [
                  0.4844606946983547,
                  0.0,
                  265,
                  0
                ],
                [
                  0.48628884826325414,
                  0.0,
                  266,
                  0
                ],
                [
                  0.48811700182815354,
                  0.0,
                  267,
                  0
                ],
                [
                  0.489945155393053,
                  0.0,
                  268,
                  0
                ],
                [
                  0.49177330895795246,
                  0.0,
                  269,
                  0
                ],
                [
                  0.4936014625228519,
                  0.0,
                  270,
                  0
                ],
                [
                  0.4954296160877514,
                  0.0,
                  271,
                  0
                ],
                [
                  0.49725776965265084,
                  0.0,
                  272,
                  0
                ],
                [
                  0.4990859232175503,
                  0.0,
                  273,
                  0
                ],
                [
                  0.5009140767824497,
                  0.0,
                  274,
                  0
                ],
                [
                  0.5027422303473492,
                  0.0,
                  275,
                  0
                ],
                [
                  0.5045703839122486,
                  0.0,
                  276,
                  0
                ],
                [
                  0.506398537477148,
                  0.0,
                  277,
                  0
                ],
                [
                  0.5082266910420475,
                  0.0,
                  278,
                  0
                ],
                [
                  0.5100548446069469,
                  0.0,
                  279,
                  0
                ],
                [
                  0.5118829981718465,
                  0.0,
                  280,
                  0
                ],
                [
                  0.5137111517367459,
                  0.0,
                  281,
                  0
                ],
                [
                  0.5155393053016454,
                  0.0,
                  282,
                  0
                ],
                [
                  0.5173674588665448,
                  0.0,
                  283,
                  0
                ],
                [
                  0.5191956124314442,
                  0.0,
                  284,
                  0
                ],
                [
                  0.5210237659963437,
                  0.0,
                  285,
                  0
                ],
                [
                  0.5228519195612431,
                  0.0,
                  286,
                  0
                ],
                [
                  0.5246800731261426,
                  0.0,
                  287,
                  0
                ],
                [
                  0.526508226691042,
                  0.0,
                  288,
                  0
                ],
                [
                  0.5283363802559415,
                  0.0,
                  289,
                  0
                ],
                [
                  0.5301645338208409,
                  0.0034482758620689655,
                  290,
                  1
                ],
                [
                  0.5319926873857403,
                  0.003436426116838488,
                  291,
                  1
                ],
                [
                  0.5338208409506399,
                  0.003424657534246575,
                  292,
                  1
                ],
                [
                  0.5356489945155393,
                  0.0034129692832764505,
                  293,
                  1
                ],
                [
                  0.5374771480804388,
                  0.003401360544217687,
                  294,
                  1
                ],
                [
                  0.5393053016453382,
                  0.003389830508474576,
                  295,
                  1
                ],
                [
                  0.5411334552102377,
                  0.0033783783783783786,
                  296,
                  1
                ],
                [
                  0.5429616087751371,
                  0.003367003367003367,
                  297,
                  1
                ],
                [
                  0.5447897623400365,
                  0.003355704697986577,
                  298,
                  1
                ],
                [
                  0.546617915904936,
                  0.0033444816053511705,
                  299,
                  1
                ],
                [
                  0.5484460694698354,
                  0.0033333333333333335,
                  300,
                  1
                ],
                [
                  0.5502742230347349,
                  0.0033222591362126247,
                  301,
                  1
                ],
                [
                  0.5521023765996343,
                  0.0033112582781456954,
                  302,
                  1
                ],
                [
                  0.5539305301645339,
                  0.0033003300330033004,
                  303,
                  1
                ],
                [
                  0.5557586837294333,
                  0.003289473684210526,
                  304,
                  1
                ],
                [
                  0.5575868372943327,
                  0.003278688524590164,
                  305,
                  1
                ],
                [
                  0.5594149908592322,
                  0.0032679738562091504,
                  306,
                  1
                ],
                [
                  0.5612431444241316,
                  0.003257328990228013,
                  307,
                  1
                ],
                [
                  0.5630712979890311,
                  0.003246753246753247,
                  308,
                  1
                ],
                [
                  0.5648994515539305,
                  0.003236245954692557,
                  309,
                  1
                ],
                [
                  0.56672760511883,
                  0.0032258064516129032,
                  310,
                  1
                ],
                [
                  0.5685557586837294,
                  0.003215434083601286,
                  311,
                  1
                ],
                [
                  0.5703839122486288,
                  0.003205128205128205,
                  312,
                  1
                ],
                [
                  0.5722120658135283,
                  0.003194888178913738,
                  313,
                  1
                ],
                [
                  0.5740402193784278,
                  0.0031847133757961785,
                  314,
                  1
                ],
                [
                  0.5758683729433273,
                  0.0031746031746031746,
                  315,
                  1
                ],
                [
                  0.5776965265082267,
                  0.0031645569620253164,
                  316,
                  1
                ],
                [
                  0.5795246800731262,
                  0.0031545741324921135,
                  317,
                  1
                ],
                [
                  0.5813528336380256,
                  0.0031446540880503146,
                  318,
                  1
                ],
                [
                  0.583180987202925,
                  0.003134796238244514,
                  319,
                  1
                ],
                [
                  0.5850091407678245,
                  0.003125,
                  320,
                  1
                ],
                [
                  0.5868372943327239,
                  0.003115264797507788,
                  321,
                  1
                ],
                [
                  0.5886654478976234,
                  0.003105590062111801,
                  322,
                  1
                ],
                [
                  0.5904936014625228,
                  0.0030959752321981426,
                  323,
                  1
                ],
                [
                  0.5923217550274223,
                  0.0030864197530864196,
                  324,
                  1
                ],
                [
                  0.5941499085923218,
                  0.003076923076923077,
                  325,
                  1
                ],
                [
                  0.5959780621572212,
                  0.003067484662576687,
                  326,
                  1
                ],
                [
                  0.5978062157221207,
                  0.0030581039755351682,
                  327,
                  1
                ],
                [
                  0.5996343692870201,
                  0.003048780487804878,
                  328,
                  1
                ],
                [
                  0.6014625228519196,
                  0.00303951367781155,
                  329,
                  1
                ],
                [
                  0.603290676416819,
                  0.0030303030303030303,
                  330,
                  1
                ],
                [
                  0.6051188299817185,
                  0.0030211480362537764,
                  331,
                  1
                ],
                [
                  0.6069469835466179,
                  0.0030120481927710845,
                  332,
                  1
                ],
                [
                  0.6087751371115173,
                  0.003003003003003003,
                  333,
                  1
                ],
                [
                  0.6106032906764168,
                  0.0029940119760479044,
                  334,
                  1
                ],
                [
                  0.6124314442413162,
                  0.0029850746268656717,
                  335,
                  1
                ],
                [
                  0.6142595978062158,
                  0.002976190476190476,
                  336,
                  1
                ],
                [
                  0.6160877513711152,
                  0.002967359050445104,
                  337,
                  1
                ],
                [
                  0.6179159049360147,
                  0.0029585798816568047,
                  338,
                  1
                ],
                [
                  0.6197440585009141,
                  0.0029498525073746312,
                  339,
                  1
                ],
                [
                  0.6215722120658135,
                  0.0029411764705882353,
                  340,
                  1
                ],
                [
                  0.623400365630713,
                  0.002932551319648094,
                  341,
                  1
                ],
                [
                  0.6252285191956124,
                  0.0029239766081871343,
                  342,
                  1
                ],
                [
                  0.6270566727605119,
                  0.0029154518950437317,
                  343,
                  1
                ],
                [
                  0.6288848263254113,
                  0.0029069767441860465,
                  344,
                  1
                ],
                [
                  0.6307129798903108,
                  0.002898550724637681,
                  345,
                  1
                ],
                [
                  0.6325411334552102,
                  0.002890173410404624,
                  346,
                  1
                ],
                [
                  0.6343692870201096,
                  0.002881844380403458,
                  347,
                  1
                ],
                [
                  0.6361974405850092,
                  0.0028735632183908046,
                  348,
                  1
                ],
                [
                  0.6380255941499086,
                  0.0028653295128939827,
                  349,
                  1
                ],
                [
                  0.6398537477148081,
                  0.002857142857142857,
                  350,
                  1
                ],
                [
                  0.6416819012797075,
                  0.002849002849002849,
                  351,
                  1
                ],
                [
                  0.643510054844607,
                  0.002840909090909091,
                  352,
                  1
                ],
                [
                  0.6453382084095064,
                  0.0028328611898017,
                  353,
                  1
                ],
                [
                  0.6471663619744058,
                  0.002824858757062147,
                  354,
                  1
                ],
                [
                  0.6489945155393053,
                  0.0028169014084507044,
                  355,
                  1
                ],
                [
                  0.6508226691042047,
                  0.0028089887640449437,
                  356,
                  1
                ],
                [
                  0.6526508226691042,
                  0.0028011204481792717,
                  357,
                  1
                ],
                [
                  0.6544789762340036,
                  0.002793296089385475,
                  358,
                  1
                ],
                [
                  0.6563071297989032,
                  0.002785515320334262,
                  359,
                  1
                ],
                [
                  0.6581352833638026,
                  0.002777777777777778,
                  360,
                  1
                ],
                [
                  0.659963436928702,
                  0.002770083102493075,
                  361,
                  1
                ],
                [
                  0.6617915904936015,
                  0.0027624309392265192,
                  362,
                  1
                ],
                [
                  0.6636197440585009,
                  0.0027548209366391185,
                  363,
                  1
                ],
                [
                  0.6654478976234004,
                  0.0027472527472527475,
                  364,
                  1
                ],
                [
                  0.6672760511882998,
                  0.0027397260273972603,
                  365,
                  1
                ],
                [
                  0.6691042047531993,
                  0.00273224043715847,
                  366,
                  1
                ],
                [
                  0.6709323583180987,
                  0.0027247956403269754,
                  367,
                  1
                ],
                [
                  0.6727605118829981,
                  0.002717391304347826,
                  368,
                  1
                ],
                [
                  0.6745886654478976,
                  0.0027100271002710027,
                  369,
                  1
                ],
                [
                  0.676416819012797,
                  0.002702702702702703,
                  370,
                  1
                ],
                [
                  0.6782449725776966,
                  0.0026954177897574125,
                  371,
                  1
                ],
                [
                  0.680073126142596,
                  0.002688172043010753,
                  372,
                  1
                ],
                [
                  0.6819012797074955,
                  0.002680965147453083,
                  373,
                  1
                ],
                [
                  0.6837294332723949,
                  0.00267379679144385,
                  374,
                  1
                ],
                [
                  0.6855575868372943,
                  0.0026666666666666666,
                  375,
                  1
                ],
                [
                  0.6873857404021938,
                  0.0026595744680851063,
                  376,
                  1
                ],
                [
                  0.6892138939670932,
                  0.002652519893899204,
                  377,
                  1
                ],
                [
                  0.6910420475319927,
                  0.0026455026455026454,
                  378,
                  1
                ],
                [
                  0.6928702010968921,
                  0.002638522427440633,
                  379,
                  1
                ],
                [
                  0.6946983546617916,
                  0.002631578947368421,
                  380,
                  1
                ],
                [
                  0.696526508226691,
                  0.0026246719160104987,
                  381,
                  1
                ],
                [
                  0.6983546617915904,
                  0.002617801047120419,
                  382,
                  1
                ],
                [
                  0.70018281535649,
                  0.0026109660574412533,
                  383,
                  1
                ],
                [
                  0.7020109689213894,
                  0.0026041666666666665,
                  384,
                  1
                ],
                [
                  0.7038391224862889,
                  0.0025974025974025974,
                  385,
                  1
                ],
                [
                  0.7056672760511883,
                  0.0025906735751295338,
                  386,
                  1
                ],
                [
                  0.7074954296160878,
                  0.002583979328165375,
                  387,
                  1
                ],
                [
                  0.7093235831809872,
                  0.002577319587628866,
                  388,
                  1
                ],
                [
                  0.7111517367458866,
                  0.002570694087403599,
                  389,
                  1
                ],
                [
                  0.7129798903107861,
                  0.002564102564102564,
                  390,
                  1
                ],
                [
                  0.7148080438756855,
                  0.0025575447570332483,
                  391,
                  1
                ],
                [
                  0.716636197440585,
                  0.002551020408163265,
                  392,
                  1
                ],
                [
                  0.7184643510054844,
                  0.002544529262086514,
                  393,
                  1
                ],
                [
                  0.720292504570384,
                  0.0025380710659898475,
                  394,
                  1
                ],
                [
                  0.7221206581352834,
                  0.002531645569620253,
                  395,
                  1
                ],
                [
                  0.7239488117001828,
                  0.0025252525252525255,
                  396,
                  1
                ],
                [
                  0.7257769652650823,
                  0.0025188916876574307,
                  397,
                  1
                ],
                [
                  0.7276051188299817,
                  0.002512562814070352,
                  398,
                  1
                ],
                [
                  0.7294332723948812,
                  0.002506265664160401,
                  399,
                  1
                ],
                [
                  0.7312614259597806,
                  0.0025,
                  400,
                  1
                ],
                [
                  0.7330895795246801,
                  0.004987531172069825,
                  401,
                  2
                ],
                [
                  0.7349177330895795,
                  0.004975124378109453,
                  402,
                  2
                ],
                [
                  0.7367458866544789,
                  0.004962779156327543,
                  403,
                  2
                ],
                [
                  0.7385740402193784,
                  0.0049504950495049506,
                  404,
                  2
                ],
                [
                  0.7404021937842779,
                  0.0049382716049382715,
                  405,
                  2
                ],
                [
                  0.7422303473491774,
                  0.007389162561576354,
                  406,
                  3
                ],
                [
                  0.7440585009140768,
                  0.007371007371007371,
                  407,
                  3
                ],
                [
                  0.7458866544789763,
                  0.007352941176470588,
                  408,
                  3
                ],
                [
                  0.7477148080438757,
                  0.009779951100244499,
                  409,
                  4
                ],
                [
                  0.7495429616087751,
                  0.00975609756097561,
                  410,
                  4
                ],
                [
                  0.7513711151736746,
                  0.012165450121654502,
                  411,
                  5
                ],
                [
                  0.753199268738574,
                  0.012135922330097087,
                  412,
                  5
                ],
                [
                  0.7550274223034735,
                  0.012106537530266344,
                  413,
                  5
                ],
                [
                  0.7568555758683729,
                  0.012077294685990338,
                  414,
                  5
                ],
                [
                  0.7586837294332724,
                  0.012048192771084338,
                  415,
                  5
                ],
                [
                  0.7605118829981719,
                  0.01201923076923077,
                  416,
                  5
                ],
                [
                  0.7623400365630713,
                  0.011990407673860911,
                  417,
                  5
                ],
                [
                  0.7641681901279708,
                  0.011961722488038277,
                  418,
                  5
                ],
                [
                  0.7659963436928702,
                  0.011933174224343675,
                  419,
                  5
                ],
                [
                  0.7678244972577697,
                  0.011904761904761904,
                  420,
                  5
                ],
                [
                  0.7696526508226691,
                  0.014251781472684086,
                  421,
                  6
                ],
                [
                  0.7714808043875686,
                  0.014218009478672985,
                  422,
                  6
                ],
                [
                  0.773308957952468,
                  0.014184397163120567,
                  423,
                  6
                ],
                [
                  0.7751371115173674,
                  0.014150943396226415,
                  424,
                  6
                ],
                [
                  0.7769652650822669,
                  0.01411764705882353,
                  425,
                  6
                ],
                [
                  0.7787934186471663,
                  0.014084507042253521,
                  426,
                  6
                ],
                [
                  0.7806215722120659,
                  0.01639344262295082,
                  427,
                  7
                ],
                [
                  0.7824497257769653,
                  0.016355140186915886,
                  428,
                  7
                ],
                [
                  0.7842778793418648,
                  0.016317016317016316,
                  429,
                  7
                ],
                [
                  0.7861060329067642,
                  0.01627906976744186,
                  430,
                  7
                ],
                [
                  0.7879341864716636,
                  0.016241299303944315,
                  431,
                  7
                ],
                [
                  0.7897623400365631,
                  0.016203703703703703,
                  432,
                  7
                ],
                [
                  0.7915904936014625,
                  0.016166281755196306,
                  433,
                  7
                ],
                [
                  0.793418647166362,
                  0.016129032258064516,
                  434,
                  7
                ],
                [
                  0.7952468007312614,
                  0.016091954022988506,
                  435,
                  7
                ],
                [
                  0.7970749542961609,
                  0.016055045871559634,
                  436,
                  7
                ],
                [
                  0.7989031078610603,
                  0.018306636155606407,
                  437,
                  8
                ],
                [
                  0.8007312614259597,
                  0.0182648401826484,
                  438,
                  8
                ],
                [
                  0.8025594149908593,
                  0.018223234624145785,
                  439,
                  8
                ],
                [
                  0.8043875685557587,
                  0.01818181818181818,
                  440,
                  8
                ],
                [
                  0.8062157221206582,
                  0.018140589569160998,
                  441,
                  8
                ],
                [
                  0.8080438756855576,
                  0.01809954751131222,
                  442,
                  8
                ],
                [
                  0.8098720292504571,
                  0.01805869074492099,
                  443,
                  8
                ],
                [
                  0.8117001828153565,
                  0.018018018018018018,
                  444,
                  8
                ],
                [
                  0.8135283363802559,
                  0.017977528089887642,
                  445,
                  8
                ],
                [
                  0.8153564899451554,
                  0.017937219730941704,
                  446,
                  8
                ],
                [
                  0.8171846435100548,
                  0.020134228187919462,
                  447,
                  9
                ],
                [
                  0.8190127970749543,
                  0.020089285714285716,
                  448,
                  9
                ],
                [
                  0.8208409506398537,
                  0.022271714922048998,
                  449,
                  10
                ],
                [
                  0.8226691042047533,
                  0.022222222222222223,
                  450,
                  10
                ],
                [
                  0.8244972577696527,
                  0.022172949002217297,
                  451,
                  10
                ],
                [
                  0.8263254113345521,
                  0.022123893805309734,
                  452,
                  10
                ],
                [
                  0.8281535648994516,
                  0.02207505518763797,
                  453,
                  10
                ],
                [
                  0.829981718464351,
                  0.022026431718061675,
                  454,
                  10
                ],
                [
                  0.8318098720292505,
                  0.024175824175824177,
                  455,
                  11
                ],
                [
                  0.8336380255941499,
                  0.02412280701754386,
                  456,
                  11
                ],
                [
                  0.8354661791590493,
                  0.024070021881838075,
                  457,
                  11
                ],
                [
                  0.8372943327239488,
                  0.024017467248908297,
                  458,
                  11
                ],
                [
                  0.8391224862888482,
                  0.023965141612200435,
                  459,
                  11
                ],
                [
                  0.8409506398537477,
                  0.02391304347826087,
                  460,
                  11
                ],
                [
                  0.8427787934186471,
                  0.02386117136659436,
                  461,
                  11
                ],
                [
                  0.8446069469835467,
                  0.025974025974025976,
                  462,
                  12
                ],
                [
                  0.8464351005484461,
                  0.02591792656587473,
                  463,
                  12
                ],
                [
                  0.8482632541133455,
                  0.02586206896551724,
                  464,
                  12
                ],
                [
                  0.850091407678245,
                  0.025806451612903226,
                  465,
                  12
                ],
                [
                  0.8519195612431444,
                  0.02575107296137339,
                  466,
                  12
                ],
                [
                  0.8537477148080439,
                  0.02569593147751606,
                  467,
                  12
                ],
                [
                  0.8555758683729433,
                  0.027777777777777776,
                  468,
                  13
                ],
                [
                  0.8574040219378428,
                  0.02771855010660981,
                  469,
                  13
                ],
                [
                  0.8592321755027422,
                  0.027659574468085105,
                  470,
                  13
                ],
                [
                  0.8610603290676416,
                  0.027600849256900213,
                  471,
                  13
                ],
                [
                  0.8628884826325411,
                  0.02754237288135593,
                  472,
                  13
                ],
                [
                  0.8647166361974405,
                  0.02748414376321353,
                  473,
                  13
                ],
                [
                  0.8665447897623401,
                  0.027426160337552744,
                  474,
                  13
                ],
                [
                  0.8683729433272395,
                  0.02736842105263158,
                  475,
                  13
                ],
                [
                  0.870201096892139,
                  0.0273109243697479,
                  476,
                  13
                ],
                [
                  0.8720292504570384,
                  0.027253668763102725,
                  477,
                  13
                ],
                [
                  0.8738574040219378,
                  0.029288702928870293,
                  478,
                  14
                ],
                [
                  0.8756855575868373,
                  0.029227557411273485,
                  479,
                  14
                ],
                [
                  0.8775137111517367,
                  0.029166666666666667,
                  480,
                  14
                ],
                [
                  0.8793418647166362,
                  0.029106029106029108,
                  481,
                  14
                ],
                [
                  0.8811700182815356,
                  0.029045643153526972,
                  482,
                  14
                ],
                [
                  0.8829981718464351,
                  0.028985507246376812,
                  483,
                  14
                ],
                [
                  0.8848263254113345,
                  0.028925619834710745,
                  484,
                  14
                ],
                [
                  0.886654478976234,
                  0.0288659793814433,
                  485,
                  14
                ],
                [
                  0.8884826325411335,
                  0.030864197530864196,
                  486,
                  15
                ],
                [
                  0.8903107861060329,
                  0.030800821355236138,
                  487,
                  15
                ],
                [
                  0.8921389396709324,
                  0.030737704918032786,
                  488,
                  15
                ],
                [
                  0.8939670932358318,
                  0.032719836400818,
                  489,
                  16
                ],
                [
                  0.8957952468007313,
                  0.0326530612244898,
                  490,
                  16
                ],
                [
                  0.8976234003656307,
                  0.032586558044806514,
                  491,
                  16
                ],
                [
                  0.8994515539305301,
                  0.032520325203252036,
                  492,
                  16
                ],
                [
                  0.9012797074954296,
                  0.032454361054766734,
                  493,
                  16
                ],
                [
                  0.903107861060329,
                  0.03441295546558704,
                  494,
                  17
                ],
                [
                  0.9049360146252285,
                  0.03434343434343434,
                  495,
                  17
                ],
                [
                  0.906764168190128,
                  0.034274193548387094,
                  496,
                  17
                ],
                [
                  0.9085923217550275,
                  0.03621730382293763,
                  497,
                  18
                ],
                [
                  0.9104204753199269,
                  0.03815261044176707,
                  498,
                  19
                ],
                [
                  0.9122486288848263,
                  0.03807615230460922,
                  499,
                  19
                ],
                [
                  0.9140767824497258,
                  0.04,
                  500,
                  20
                ],
                [
                  0.9159049360146252,
                  0.03992015968063872,
                  501,
                  20
                ],
                [
                  0.9177330895795247,
                  0.0398406374501992,
                  502,
                  20
                ],
                [
                  0.9195612431444241,
                  0.039761431411530816,
                  503,
                  20
                ],
                [
                  0.9213893967093236,
                  0.03968253968253968,
                  504,
                  20
                ],
                [
                  0.923217550274223,
                  0.039603960396039604,
                  505,
                  20
                ],
                [
                  0.9250457038391224,
                  0.041501976284584984,
                  506,
                  21
                ],
                [
                  0.926873857404022,
                  0.04142011834319527,
                  507,
                  21
                ],
                [
                  0.9287020109689214,
                  0.04330708661417323,
                  508,
                  22
                ],
                [
                  0.9305301645338209,
                  0.043222003929273084,
                  509,
                  22
                ],
                [
                  0.9323583180987203,
                  0.043137254901960784,
                  510,
                  22
                ],
                [
                  0.9341864716636198,
                  0.04500978473581213,
                  511,
                  23
                ],
                [
                  0.9360146252285192,
                  0.044921875,
                  512,
                  23
                ],
                [
                  0.9378427787934186,
                  0.04483430799220273,
                  513,
                  23
                ],
                [
                  0.9396709323583181,
                  0.04669260700389105,
                  514,
                  24
                ],
                [
                  0.9414990859232175,
                  0.04660194174757282,
                  515,
                  24
                ],
                [
                  0.943327239488117,
                  0.046511627906976744,
                  516,
                  24
                ],
                [
                  0.9451553930530164,
                  0.048355899419729204,
                  517,
                  25
                ],
                [
                  0.946983546617916,
                  0.04826254826254826,
                  518,
                  25
                ],
                [
                  0.9488117001828154,
                  0.05009633911368015,
                  519,
                  26
                ],
                [
                  0.9506398537477148,
                  0.051923076923076926,
                  520,
                  27
                ],
                [
                  0.9524680073126143,
                  0.05182341650671785,
                  521,
                  27
                ],
                [
                  0.9542961608775137,
                  0.05172413793103448,
                  522,
                  27
                ],
                [
                  0.9561243144424132,
                  0.05162523900573614,
                  523,
                  27
                ],
                [
                  0.9579524680073126,
                  0.05152671755725191,
                  524,
                  27
                ],
                [
                  0.9597806215722121,
                  0.05142857142857143,
                  525,
                  27
                ],
                [
                  0.9616087751371115,
                  0.051330798479087454,
                  526,
                  27
                ],
                [
                  0.9634369287020109,
                  0.05313092979127135,
                  527,
                  28
                ],
                [
                  0.9652650822669104,
                  0.05303030303030303,
                  528,
                  28
                ],
                [
                  0.9670932358318098,
                  0.052930056710775046,
                  529,
                  28
                ],
                [
                  0.9689213893967094,
                  0.052830188679245285,
                  530,
                  28
                ],
                [
                  0.9707495429616088,
                  0.054613935969868174,
                  531,
                  29
                ],
                [
                  0.9725776965265083,
                  0.05639097744360902,
                  532,
                  30
                ],
                [
                  0.9744058500914077,
                  0.058161350844277676,
                  533,
                  31
                ],
                [
                  0.9762340036563071,
                  0.05805243445692884,
                  534,
                  31
                ],
                [
                  0.9780621572212066,
                  0.05794392523364486,
                  535,
                  31
                ],
                [
                  0.979890310786106,
                  0.05970149253731343,
                  536,
                  32
                ],
                [
                  0.9817184643510055,
                  0.061452513966480445,
                  537,
                  33
                ],
                [
                  0.9835466179159049,
                  0.06133828996282528,
                  538,
                  33
                ],
                [
                  0.9853747714808044,
                  0.06307977736549165,
                  539,
                  34
                ],
                [
                  0.9872029250457038,
                  0.06481481481481481,
                  540,
                  35
                ],
                [
                  0.9890310786106032,
                  0.066543438077634,
                  541,
                  36
                ],
                [
                  0.9908592321755028,
                  0.06826568265682657,
                  542,
                  37
                ],
                [
                  0.9926873857404022,
                  0.06998158379373849,
                  543,
                  38
                ],
                [
                  0.9945155393053017,
                  0.06985294117647059,
                  544,
                  38
                ],
                [
                  0.9963436928702011,
                  0.06972477064220184,
                  545,
                  38
                ],
                [
                  0.9981718464351006,
                  0.0695970695970696,
                  546,
                  38
                ],
                [
                  1.0,
                  0.0712979890310786,
                  547,
                  39
                ]
              ]
            },
            {
              "language": "English",
              "run": 503,
              "points": [
                [
                  0.0018281535648994515,
                  0.0,
                  1,
                  0
                ],
                [
                  0.003656307129798903,
                  0.0,
                  2,
                  0
                ],
                [
                  0.005484460694698354,
                  0.0,
                  3,
                  0
                ],
                [
                  0.007312614259597806,
                  0.0,
                  4,
                  0
                ],
                [
                  0.009140767824497258,
                  0.0,
                  5,
                  0
                ],
                [
                  0.010968921389396709,
                  0.0,
                  6,
                  0
                ],
                [
                  0.012797074954296161,
                  0.0,
                  7,
                  0
                ],
                [
                  0.014625228519195612,
                  0.0,
                  8,
                  0
                ],
                [
                  0.016453382084095063,
                  0.0,
                  9,
                  0
                ],
                [
                  0.018281535648994516,
                  0.0,
                  10,
                  0
                ],
                [
                  0.02010968921389397,
                  0.0,
                  11,
                  0
                ],
                [
                  0.021937842778793418,
                  0.0,
                  12,
                  0
                ],
                [
                  0.02376599634369287,
                  0.0,
                  13,
                  0
                ],
                [
                  0.025594149908592323,
                  0.0,
                  14,
                  0
                ],
                [
                  0.027422303473491772,
                  0.0,
                  15,
                  0
                ],
                [
                  0.029250457038391225,
                  0.0,
                  16,
                  0
                ],
                [
                  0.031078610603290677,
                  0.0,
                  17,
                  0
                ],
                [
                  0.03290676416819013,
                  0.0,
                  18,
                  0
                ],
                [
                  0.03473491773308958,
                  0.0,
                  19,
                  0
                ],
                [
                  0.03656307129798903,
                  0.0,
                  20,
                  0
                ],
                [
                  0.038391224862888484,
                  0.0,
                  21,
                  0
                ],
                [
                  0.04021937842778794,
                  0.0,
                  22,
                  0
                ],
                [
                  0.04204753199268738,
                  0.0,
                  23,
                  0
                ],
                [
                  0.043875685557586835,
                  0.0,
                  24,
                  0
                ],
                [
                  0.04570383912248629,
                  0.0,
                  25,
                  0
                ],
                [
                  0.04753199268738574,
                  0.0,
                  26,
                  0
                ],
                [
                  0.04936014625228519,
                  0.0,
                  27,
                  0
                ],
                [
                  0.051188299817184646,
                  0.0,
                  28,
                  0
                ],
                [
                  0.05301645338208409,
                  0.0,
                  29,
                  0
                ],
                [
                  0.054844606946983544,
                  0.0,
                  30,
                  0
                ],
                [
                  0.056672760511883,
                  0.0,
                  31,
                  0
                ],
                [
                  0.05850091407678245,
                  0.0,
                  32,
                  0
                ],
                [
                  0.0603290676416819,
                  0.0,
                  33,
                  0
                ],
                [
                  0.062157221206581355,
                  0.0,
                  34,
                  0
                ],
                [
                  0.06398537477148081,
                  0.0,
                  35,
                  0
                ],
                [
                  0.06581352833638025,
                  0.0,
                  36,
                  0
                ],
                [
                  0.06764168190127971,
                  0.0,
                  37,
                  0
                ],
                [
                  0.06946983546617916,
                  0.0,
                  38,
                  0
                ],
                [
                  0.0712979890310786,
                  0.0,
                  39,
                  0
                ],
                [
                  0.07495429616087751,
                  0.0,
                  41,
                  0
                ],
                [
                  0.07678244972577697,
                  0.0,
                  42,
                  0
                ],
                [
                  0.07861060329067641,
                  0.0,
                  43,
                  0
                ],
                [
                  0.08043875685557587,
                  0.0,
                  44,
                  0
                ],
                [
                  0.08226691042047532,
                  0.0,
                  45,
                  0
                ],
                [
                  0.08409506398537477,
                  0.0,
                  46,
                  0
                ],
                [
                  0.08592321755027423,
                  0.0,
                  47,
                  0
                ],
                [
                  0.08957952468007313,
                  0.0,
                  49,
                  0
                ],
                [
                  0.09140767824497258,
                  0.0,
                  50,
                  0
                ],
                [
                  0.09323583180987204,
                  0.0,
                  51,
                  0
                ],
                [
                  0.09506398537477148,
                  0.0,
                  52,
                  0
                ],
                [
                  0.09689213893967093,
                  0.0,
                  53,
                  0
                ],
                [
                  0.09872029250457039,
                  0.0,
                  54,
                  0
                ],
                [
                  0.10054844606946983,
                  0.0,
                  55,
                  0
                ],
                [
                  0.10237659963436929,
                  0.0,
                  56,
                  0
                ],
                [
                  0.10420475319926874,
                  0.0,
                  57,
                  0
                ],
                [
                  0.10603290676416818,
                  0.0,
                  58,
                  0
                ],
                [
                  0.10786106032906764,
                  0.0,
                  59,
                  0
                ],
                [
                  0.10968921389396709,
                  0.0,
                  60,
                  0
                ],
                [
                  0.11151736745886655,
                  0.0,
                  61,
                  0
                ],
                [
                  0.113345521023766,
                  0.0,
                  62,
                  0
                ],
                [
                  0.11517367458866545,
                  0.0,
                  63,
                  0
                ],
                [
                  0.1170018281535649,
                  0.0,
                  64,
                  0
                ],
                [
                  0.11882998171846434,
                  0.0,
                  65,
                  0
                ],
                [
                  0.1206581352833638,
                  0.0,
                  66,
                  0
                ],
                [
                  0.12248628884826325,
                  0.0,
                  67,
                  0
                ],
                [
                  0.12431444241316271,
                  0.0,
                  68,
                  0
                ],
                [
                  0.12614259597806216,
                  0.0,
                  69,
                  0
                ],
                [
                  0.12797074954296161,
                  0.0,
                  70,
                  0
                ],
                [
                  0.12979890310786105,
                  0.0,
                  71,
                  0
                ],
                [
                  0.1316270566727605,
                  0.0,
                  72,
                  0
                ],
                [
                  0.13345521023765997,
                  0.0,
                  73,
                  0
                ],
                [
                  0.13528336380255943,
                  0.0,
                  74,
                  0
                ],
                [
                  0.13711151736745886,
                  0.0,
                  75,
                  0
                ],
                [
                  0.13893967093235832,
                  0.0,
                  76,
                  0
                ],
                [
                  0.14076782449725778,
                  0.0,
                  77,
                  0
                ],
                [
                  0.1425959780621572,
                  0.0,
                  78,
                  0
                ],
                [
                  0.14442413162705667,
                  0.0,
                  79,
                  0
                ],
                [
                  0.14625228519195613,
                  0.0125,
                  80,
                  1
                ],
                [
                  0.1480804387568556,
                  0.012345679012345678,
                  81,
                  1
                ],
                [
                  0.14990859232175502,
                  0.012195121951219513,
                  82,
                  1
                ],
                [
                  0.15173674588665448,
                  0.012048192771084338,
                  83,
                  1
                ],
                [
                  0.15356489945155394,
                  0.011904761904761904,
                  84,
                  1
                ],
                [
                  0.15539305301645337,
                  0.011764705882352941,
                  85,
                  1
                ],
                [
                  0.15722120658135283,
                  0.011627906976744186,
                  86,
                  1
                ],
                [
                  0.1590493601462523,
                  0.011494252873563218,
                  87,
                  1
                ],
                [
                  0.16087751371115175,
                  0.011363636363636364,
                  88,
                  1
                ],
                [
                  0.16270566727605118,
                  0.011235955056179775,
                  89,
                  1
                ],
                [
                  0.16453382084095064,
                  0.011111111111111112,
                  90,
                  1
                ],
                [
                  0.1663619744058501,
                  0.01098901098901099,
                  91,
                  1
                ],
                [
                  0.16819012797074953,
                  0.010869565217391304,
                  92,
                  1
                ],
                [
                  0.170018281535649,
                  0.010752688172043012,
                  93,
                  1
                ],
                [
                  0.17184643510054845,
                  0.010638297872340425,
                  94,
                  1
                ],
                [
                  0.1736745886654479,
                  0.010526315789473684,
                  95,
                  1
                ],
                [
                  0.17550274223034734,
                  0.010416666666666666,
                  96,
                  1
                ],
                [
                  0.1773308957952468,
                  0.010309278350515464,
                  97,
                  1
                ],
                [
                  0.17915904936014626,
                  0.01020408163265306,
                  98,
                  1
                ],
                [
                  0.1809872029250457,
                  0.010101010101010102,
                  99,
                  1
                ],
                [
                  0.18281535648994515,
                  0.01,
                  100,
                  1
                ],
                [
                  0.1846435100548446,
                  0.009900990099009901,
                  101,
                  1
                ],
                [
                  0.18647166361974407,
                  0.00980392156862745,
                  102,
                  1
                ],
                [
                  0.1882998171846435,
                  0.009708737864077669,
                  103,
                  1
                ],
                [
                  0.19012797074954296,
                  0.009615384615384616,
                  104,
                  1
                ],
                [
                  0.19195612431444242,
                  0.009523809523809525,
                  105,
                  1
                ],
                [
                  0.19378427787934185,
                  0.009433962264150943,
                  106,
                  1
                ],
                [
                  0.1956124314442413,
                  0.009345794392523364,
                  107,
                  1
                ],
                [
                  0.19744058500914077,
                  0.009259259259259259,
                  108,
                  1
                ],
                [
                  0.19926873857404023,
                  0.009174311926605505,
                  109,
                  1
                ],
                [
                  0.20109689213893966,
                  0.00909090909090909,
                  110,
                  1
                ],
                [
                  0.20292504570383912,
                  0.009009009009009009,
                  111,
                  1
                ],
                [
                  0.20475319926873858,
                  0.008928571428571428,
                  112,
                  1
                ],
                [
                  0.20658135283363802,
                  0.008849557522123894,
                  113,
                  1
                ],
                [
                  0.20840950639853748,
                  0.008771929824561403,
                  114,
                  1
                ],
                [
                  0.21023765996343693,
                  0.008695652173913044,
                  115,
                  1
                ],
                [
                  0.21206581352833637,
                  0.008620689655172414,
                  116,
                  1
                ],
                [
                  0.21572212065813529,
                  0.00847457627118644,
                  118,
                  1
                ],
                [
                  0.21755027422303475,
                  0.008403361344537815,
                  119,
                  1
                ],
                [
                  0.21937842778793418,
                  0.008333333333333333,
                  120,
                  1
                ],
                [
                  0.22120658135283364,
                  0.008264462809917356,
                  121,
                  1
                ],
                [
                  0.2230347349177331,
                  0.00819672131147541,
                  122,
                  1
                ],
                [
                  0.22486288848263253,
                  0.008130081300813009,
                  123,
                  1
                ],
                [
                  0.226691042047532,
                  0.008064516129032258,
                  124,
                  1
                ],
                [
                  0.22851919561243145,
                  0.008,
                  125,
                  1
                ],
                [
                  0.2303473491773309,
                  0.007936507936507936,
                  126,
                  1
                ],
                [
                  0.23217550274223034,
                  0.007874015748031496,
                  127,
                  1
                ],
                [
                  0.2340036563071298,
                  0.0078125,
                  128,
                  1
                ],
                [
                  0.23583180987202926,
                  0.007751937984496124,
                  129,
                  1
                ],
                [
                  0.2376599634369287,
                  0.007692307692307693,
                  130,
                  1
                ],
                [
                  0.23948811700182815,
                  0.007633587786259542,
                  131,
                  1
                ],
                [
                  0.2413162705667276,
                  0.007575757575757576,
                  132,
                  1
                ],
                [
                  0.24314442413162707,
                  0.007518796992481203,
                  133,
                  1
                ],
                [
                  0.2449725776965265,
                  0.007462686567164179,
                  134,
                  1
                ],
                [
                  0.24680073126142596,
                  0.007407407407407408,
                  135,
                  1
                ],
                [
                  0.24862888482632542,
                  0.007352941176470588,
                  136,
                  1
                ],
                [
                  0.25045703839122485,
                  0.0072992700729927005,
                  137,
                  1
                ],
                [
                  0.2522851919561243,
                  0.007246376811594203,
                  138,
                  1
                ],
                [
                  0.25411334552102377,
                  0.007194244604316547,
                  139,
                  1
                ],
                [
                  0.25594149908592323,
                  0.007142857142857143,
                  140,
                  1
                ],
                [
                  0.2577696526508227,
                  0.0070921985815602835,
                  141,
                  1
                ],
                [
                  0.2595978062157221,
                  0.007042253521126761,
                  142,
                  1
                ],
                [
                  0.26142595978062155,
                  0.006993006993006993,
                  143,
                  1
                ],
                [
                  0.263254113345521,
                  0.006944444444444444,
                  144,
                  1
                ],
                [
                  0.26508226691042047,
                  0.006896551724137931,
                  145,
                  1
                ],
                [
                  0.26691042047531993,
                  0.00684931506849315,
                  146,
                  1
                ],
                [
                  0.2687385740402194,
                  0.006802721088435374,
                  147,
                  1
                ],
                [
                  0.27056672760511885,
                  0.006756756756756757,
                  148,
                  1
                ],
                [
                  0.27239488117001825,
                  0.006711409395973154,
                  149,
                  1
                ],
                [
                  0.2742230347349177,
                  0.006666666666666667,
                  150,
                  1
                ],
                [
                  0.2760511882998172,
                  0.006622516556291391,
                  151,
                  1
                ],
                [
                  0.27787934186471663,
                  0.006578947368421052,
                  152,
                  1
                ],
                [
                  0.2797074954296161,
                  0.006535947712418301,
                  153,
                  1
                ],
                [
                  0.28153564899451555,
                  0.006493506493506494,
                  154,
                  1
                ],
                [
                  0.283363802559415,
                  0.0064516129032258064,
                  155,
                  1
                ],
                [
                  0.2851919561243144,
                  0.00641025641025641,
                  156,
                  1
                ],
                [
                  0.2870201096892139,
                  0.006369426751592357,
                  157,
                  1
                ],
                [
                  0.28884826325411334,
                  0.006329113924050633,
                  158,
                  1
                ],
                [
                  0.2906764168190128,
                  0.006289308176100629,
                  159,
                  1
                ],
                [
                  0.29250457038391225,
                  0.00625,
                  160,
                  1
                ],
                [
                  0.2943327239488117,
                  0.006211180124223602,
                  161,
                  1
                ],
                [
                  0.2961608775137112,
                  0.006172839506172839,
                  162,
                  1
                ],
                [
                  0.2979890310786106,
                  0.006134969325153374,
                  163,
                  1
                ],
                [
                  0.29981718464351004,
                  0.006097560975609756,
                  164,
                  1
                ],
                [
                  0.3016453382084095,
                  0.006060606060606061,
                  165,
                  1
                ],
                [
                  0.30347349177330896,
                  0.006024096385542169,
                  166,
                  1
                ],
                [
                  0.3053016453382084,
                  0.005988023952095809,
                  167,
                  1
                ],
                [
                  0.3071297989031079,
                  0.005952380952380952,
                  168,
                  1
                ],
                [
                  0.30895795246800734,
                  0.005917159763313609,
                  169,
                  1
                ],
                [
                  0.31078610603290674,
                  0.0058823529411764705,
                  170,
                  1
                ],
                [
                  0.3126142595978062,
                  0.005847953216374269,
                  171,
                  1
                ],
                [
                  0.31444241316270566,
                  0.005813953488372093,
                  172,
                  1
                ],
                [
                  0.3162705667276051,
                  0.005780346820809248,
                  173,
                  1
                ],
                [
                  0.3180987202925046,
                  0.005747126436781609,
                  174,
                  1
                ],
                [
                  0.31992687385740404,
                  0.005714285714285714,
                  175,
                  1
                ],
                [
                  0.3217550274223035,
                  0.005681818181818182,
                  176,
                  1
                ],
                [
                  0.3235831809872029,
                  0.005649717514124294,
                  177,
                  1
                ],
                [
                  0.32541133455210236,
                  0.0056179775280898875,
                  178,
                  1
                ],
                [
                  0.3272394881170018,
                  0.00558659217877095,
                  179,
                  1
                ],
                [
                  0.3290676416819013,
                  0.005555555555555556,
                  180,
                  1
                ],
                [
                  0.33089579524680074,
                  0.0055248618784530384,
                  181,
                  1
                ],
                [
                  0.3327239488117002,
                  0.005494505494505495,
                  182,
                  1
                ],
                [
                  0.33455210237659966,
                  0.00546448087431694,
                  183,
                  1
                ],
                [
                  0.33638025594149906,
                  0.005434782608695652,
                  184,
                  1
                ],
                [
                  0.3382084095063985,
                  0.005405405405405406,
                  185,
                  1
                ],
                [
                  0.340036563071298,
                  0.005376344086021506,
                  186,
                  1
                ],
                [
                  0.34186471663619744,
                  0.0053475935828877,
                  187,
                  1
                ],
                [
                  0.3436928702010969,
                  0.005319148936170213,
                  188,
                  1
                ],
                [
                  0.34552102376599636,
                  0.005291005291005291,
                  189,
                  1
                ],
                [
                  0.3473491773308958,
                  0.005263157894736842,
                  190,
                  1
                ],
                [
                  0.3491773308957952,
                  0.005235602094240838,
                  191,
                  1
                ],
                [
                  0.3510054844606947,
                  0.005208333333333333,
                  192,
                  1
                ],
                [
                  0.35283363802559414,
                  0.0051813471502590676,
                  193,
                  1
                ],
                [
                  0.3546617915904936,
                  0.005154639175257732,
                  194,
                  1
                ],
                [
                  0.35648994515539306,
                  0.005128205128205128,
                  195,
                  1
                ],
                [
                  0.3583180987202925,
                  0.00510204081632653,
                  196,
                  1
                ],
                [
                  0.360146252285192,
                  0.005076142131979695,
                  197,
                  1
                ],
                [
                  0.3619744058500914,
                  0.005050505050505051,
                  198,
                  1
                ],
                [
                  0.36380255941499084,
                  0.005025125628140704,
                  199,
                  1
                ],
                [
                  0.3656307129798903,
                  0.005,
                  200,
                  1
                ],
                [
                  0.36745886654478976,
                  0.004975124378109453,
                  201,
                  1
                ],
                [
                  0.3692870201096892,
                  0.0049504950495049506,
                  202,
                  1
                ],
                [
                  0.3711151736745887,
                  0.0049261083743842365,
                  203,
                  1
                ],
                [
                  0.37294332723948814,
                  0.004901960784313725,
                  204,
                  1
                ],
                [
                  0.37477148080438755,
                  0.004878048780487805,
                  205,
                  1
                ],
                [
                  0.376599634369287,
                  0.0048543689320388345,
                  206,
                  1
                ],
                [
                  0.37842778793418647,
                  0.004830917874396135,
                  207,
                  1
                ],
                [
                  0.3820840950639854,
                  0.004784688995215311,
                  209,
                  1
                ],
                [
                  0.38391224862888484,
                  0.004761904761904762,
                  210,
                  1
                ],
                [
                  0.3857404021937843,
                  0.004739336492890996,
                  211,
                  1
                ],
                [
                  0.3875685557586837,
                  0.0047169811320754715,
                  212,
                  1
                ],
                [
                  0.38939670932358317,
                  0.004694835680751174,
                  213,
                  1
                ],
                [
                  0.3912248628884826,
                  0.004672897196261682,
                  214,
                  1
                ],
                [
                  0.3930530164533821,
                  0.004651162790697674,
                  215,
                  1
                ],
                [
                  0.39488117001828155,
                  0.004629629629629629,
                  216,
                  1
                ],
                [
                  0.396709323583181,
                  0.004608294930875576,
                  217,
                  1
                ],
                [
                  0.39853747714808047,
                  0.0045871559633027525,
                  218,
                  1
                ],
                [
                  0.40036563071297987,
                  0.0045662100456621,
                  219,
                  1
                ],
                [
                  0.40219378427787933,
                  0.004545454545454545,
                  220,
                  1
                ],
                [
                  0.4040219378427788,
                  0.004524886877828055,
                  221,
                  1
                ],
                [
                  0.40585009140767825,
                  0.0045045045045045045,
                  222,
                  1
                ],
                [
                  0.4076782449725777,
                  0.004484304932735426,
                  223,
                  1
                ],
                [
                  0.40950639853747717,
                  0.004464285714285714,
                  224,
                  1
                ],
                [
                  0.4113345521023766,
                  0.0044444444444444444,
                  225,
                  1
                ],
                [
                  0.41316270566727603,
                  0.004424778761061947,
                  226,
                  1
                ],
                [
                  0.4149908592321755,
                  0.004405286343612335,
                  227,
                  1
                ],
                [
                  0.41681901279707495,
                  0.0043859649122807015,
                  228,
                  1
                ],
                [
                  0.4186471663619744,
                  0.004366812227074236,
                  229,
                  1
                ],
                [
                  0.42047531992687387,
                  0.004347826086956522,
                  230,
                  1
                ],
                [
                  0.42230347349177333,
                  0.004329004329004329,
                  231,
                  1
                ],
                [
                  0.42413162705667273,
                  0.004310344827586207,
                  232,
                  1
                ],
                [
                  0.4259597806215722,
                  0.004291845493562232,
                  233,
                  1
                ],
                [
                  0.42778793418647165,
                  0.004273504273504274,
                  234,
                  1
                ],
                [
                  0.4296160877513711,
                  0.00425531914893617,
                  235,
                  1
                ],
                [
                  0.43144424131627057,
                  0.00423728813559322,
                  236,
                  1
                ],
                [
                  0.43327239488117003,
                  0.004219409282700422,
                  237,
                  1
                ],
                [
                  0.4351005484460695,
                  0.004201680672268907,
                  238,
                  1
                ],
                [
                  0.4369287020109689,
                  0.0041841004184100415,
                  239,
                  1
                ],
                [
                  0.43875685557586835,
                  0.004166666666666667,
                  240,
                  1
                ],
                [
                  0.4405850091407678,
                  0.004149377593360996,
                  241,
                  1
                ],
                [
                  0.4424131627056673,
                  0.004132231404958678,
                  242,
                  1
                ],
                [
                  0.44424131627056673,
                  0.00411522633744856,
                  243,
                  1
                ],
                [
                  0.4460694698354662,
                  0.004098360655737705,
                  244,
                  1
                ],
                [
                  0.44789762340036565,
                  0.004081632653061225,
                  245,
                  1
                ],
                [
                  0.44972577696526506,
                  0.0040650406504065045,
                  246,
                  1
                ],
                [
                  0.4515539305301645,
                  0.004048582995951417,
                  247,
                  1
                ],
                [
                  0.453382084095064,
                  0.004032258064516129,
                  248,
                  1
                ],
                [
                  0.45521023765996343,
                  0.004016064257028112,
                  249,
                  1
                ],
                [
                  0.4570383912248629,
                  0.004,
                  250,
                  1
                ],
                [
                  0.45886654478976235,
                  0.00398406374501992,
                  251,
                  1
                ],
                [
                  0.4606946983546618,
                  0.003968253968253968,
                  252,
                  1
                ],
                [
                  0.4625228519195612,
                  0.003952569169960474,
                  253,
                  1
                ],
                [
                  0.4643510054844607,
                  0.003937007874015748,
                  254,
                  1
                ],
                [
                  0.46617915904936014,
                  0.00392156862745098,
                  255,
                  1
                ],
                [
                  0.4680073126142596,
                  0.00390625,
                  256,
                  1
                ],
                [
                  0.46983546617915906,
                  0.0038910505836575876,
                  257,
                  1
                ],
                [
                  0.4716636197440585,
                  0.003875968992248062,
                  258,
                  1
                ],
                [
                  0.473491773308958,
                  0.003861003861003861,
                  259,
                  1
                ],
                [
                  0.4753199268738574,
                  0.0038461538461538464,
                  260,
                  1
                ],
                [
                  0.47714808043875684,
                  0.0038314176245210726,
                  261,
                  1
                ],
                [
                  0.4789762340036563,
                  0.003816793893129771,
                  262,
                  1
                ],
                [
                  0.48080438756855576,
                  0.0038022813688212928,
                  263,
                  1
                ],
                [
                  0.4826325411334552,
                  0.003787878787878788,
                  264,
                  1
                ],
                [
                  0.4844606946983547,
                  0.0037735849056603774,
                  265,
                  1
                ],
                [
                  0.48628884826325414,
                  0.0037593984962406013,
                  266,
                  1
                ],
                [
                  0.48811700182815354,
                  0.003745318352059925,
                  267,
                  1
                ],
                [
                  0.489945155393053,
                  0.0037313432835820895,
                  268,
                  1
                ],
                [
                  0.49177330895795246,
                  0.0037174721189591076,
                  269,
                  1
                ],
                [
                  0.4936014625228519,
                  0.003703703703703704,
                  270,
                  1
                ],
                [
                  0.4954296160877514,
                  0.0036900369003690036,
                  271,
                  1
                ],
                [
                  0.49725776965265084,
                  0.003676470588235294,
                  272,
                  1
                ],
                [
                  0.4990859232175503,
                  0.003663003663003663,
                  273,
                  1
                ],
                [
                  0.5009140767824497,
                  0.0036496350364963502,
                  274,
                  1
                ],
                [
                  0.5027422303473492,
                  0.0036363636363636364,
                  275,
                  1
                ],
                [
                  0.5045703839122486,
                  0.0036231884057971015,
                  276,
                  1
                ],
                [
                  0.506398537477148,
                  0.0036101083032490976,
                  277,
                  1
                ],
                [
                  0.5082266910420475,
                  0.0035971223021582736,
                  278,
                  1
                ],
                [
                  0.5100548446069469,
                  0.0035842293906810036,
                  279,
                  1
                ],
                [
                  0.5118829981718465,
                  0.0035714285714285713,
                  280,
                  1
                ],
                [
                  0.5137111517367459,
                  0.0035587188612099642,
                  281,
                  1
                ],
                [
                  0.5155393053016454,
                  0.0035460992907801418,
                  282,
                  1
                ],
                [
                  0.5173674588665448,
                  0.0035335689045936395,
                  283,
                  1
                ],
                [
                  0.5191956124314442,
                  0.0035211267605633804,
                  284,
                  1
                ],
                [
                  0.5210237659963437,
                  0.0035087719298245615,
                  285,
                  1
                ],
                [
                  0.5228519195612431,
                  0.0034965034965034965,
                  286,
                  1
                ],
                [
                  0.5246800731261426,
                  0.003484320557491289,
                  287,
                  1
                ],
                [
                  0.526508226691042,
                  0.003472222222222222,
                  288,
                  1
                ],
                [
                  0.5283363802559415,
                  0.0034602076124567475,
                  289,
                  1
                ],
                [
                  0.5301645338208409,
                  0.0034482758620689655,
                  290,
                  1
                ],
                [
                  0.5319926873857403,
                  0.003436426116838488,
                  291,
                  1
                ],
                [
                  0.5338208409506399,
                  0.003424657534246575,
                  292,
                  1
                ],
                [
                  0.5356489945155393,
                  0.0034129692832764505,
                  293,
                  1
                ],
                [
                  0.5374771480804388,
                  0.003401360544217687,
                  294,
                  1
                ],
                [
                  0.5393053016453382,
                  0.003389830508474576,
                  295,
                  1
                ],
                [
                  0.5411334552102377,
                  0.0033783783783783786,
                  296,
                  1
                ],
                [
                  0.5429616087751371,
                  0.003367003367003367,
                  297,
                  1
                ],
                [
                  0.5447897623400365,
                  0.003355704697986577,
                  298,
                  1
                ],
                [
                  0.546617915904936,
                  0.0033444816053511705,
                  299,
                  1
                ],
                [
                  0.5484460694698354,
                  0.0033333333333333335,
                  300,
                  1
                ],
                [
                  0.5502742230347349,
                  0.0033222591362126247,
                  301,
                  1
                ],
                [
                  0.5521023765996343,
                  0.0033112582781456954,
                  302,
                  1
                ],
                [
                  0.5539305301645339,
                  0.0033003300330033004,
                  303,
                  1
                ],
                [
                  0.5557586837294333,
                  0.003289473684210526,
                  304,
                  1
                ],
                [
                  0.5575868372943327,
                  0.003278688524590164,
                  305,
                  1
                ],
                [
                  0.5594149908592322,
                  0.0032679738562091504,
                  306,
                  1
                ],
                [
                  0.5612431444241316,
                  0.003257328990228013,
                  307,
                  1
                ],
                [
                  0.5630712979890311,
                  0.003246753246753247,
                  308,
                  1
                ],
                [
                  0.5648994515539305,
                  0.003236245954692557,
                  309,
                  1
                ],
                [
                  0.56672760511883,
                  0.0032258064516129032,
                  310,
                  1
                ],
                [
                  0.5685557586837294,
                  0.003215434083601286,
                  311,
                  1
                ],
                [
                  0.5703839122486288,
                  0.003205128205128205,
                  312,
                  1
                ],
                [
                  0.5722120658135283,
                  0.003194888178913738,
                  313,
                  1
                ],
                [
                  0.5740402193784278,
                  0.0031847133757961785,
                  314,
                  1
                ],
                [
                  0.5758683729433273,
                  0.0031746031746031746,
                  315,
                  1
                ],
                [
                  0.5776965265082267,
                  0.0031645569620253164,
                  316,
                  1
                ],
                [
                  0.5795246800731262,
                  0.0031545741324921135,
                  317,
                  1
                ],
                [
                  0.5813528336380256,
                  0.0031446540880503146,
                  318,
                  1
                ],
                [
                  0.583180987202925,
                  0.003134796238244514,
                  319,
                  1
                ],
                [
                  0.5850091407678245,
                  0.003125,
                  320,
                  1
                ],
                [
                  0.5868372943327239,
                  0.003115264797507788,
                  321,
                  1
                ],
                [
                  0.5886654478976234,
                  0.003105590062111801,
                  322,
                  1
                ],
                [
                  0.5904936014625228,
                  0.0030959752321981426,
                  323,
                  1
                ],
                [
                  0.5923217550274223,
                  0.0030864197530864196,
                  324,
                  1
                ],
                [
                  0.5941499085923218,
                  0.003076923076923077,
                  325,
                  1
                ],
                [
                  0.5959780621572212,
                  0.003067484662576687,
                  326,
                  1
                ],
                [
                  0.5978062157221207,
                  0.0030581039755351682,
                  327,
                  1
                ],
                [
                  0.5996343692870201,
                  0.003048780487804878,
                  328,
                  1
                ],
                [
                  0.6014625228519196,
                  0.00303951367781155,
                  329,
                  1
                ],
                [
                  0.603290676416819,
                  0.0030303030303030303,
                  330,
                  1
                ],
                [
                  0.6051188299817185,
                  0.0030211480362537764,
                  331,
                  1
                ],
                [
                  0.6069469835466179,
                  0.0030120481927710845,
                  332,
                  1
                ],
                [
                  0.6087751371115173,
                  0.003003003003003003,
                  333,
                  1
                ],
                [
                  0.6106032906764168,
                  0.0029940119760479044,
                  334,
                  1
                ],
                [
                  0.6124314442413162,
                  0.0029850746268656717,
                  335,
                  1
                ],
                [
                  0.6142595978062158,
                  0.002976190476190476,
                  336,
                  1
                ],
                [
                  0.6160877513711152,
                  0.002967359050445104,
                  337,
                  1
                ],
                [
                  0.6179159049360147,
                  0.0029585798816568047,
                  338,
                  1
                ],
                [
                  0.6197440585009141,
                  0.0029498525073746312,
                  339,
                  1
                ],
                [
                  0.6215722120658135,
                  0.0029411764705882353,
                  340,
                  1
                ],
                [
                  0.623400365630713,
                  0.002932551319648094,
                  341,
                  1
                ],
                [
                  0.6252285191956124,
                  0.0029239766081871343,
                  342,
                  1
                ],
                [
                  0.6270566727605119,
                  0.0029154518950437317,
                  343,
                  1
                ],
                [
                  0.6288848263254113,
                  0.0029069767441860465,
                  344,
                  1
                ],
                [
                  0.6307129798903108,
                  0.002898550724637681,
                  345,
                  1
                ],
                [
                  0.6325411334552102,
                  0.002890173410404624,
                  346,
                  1
                ],
                [
                  0.6343692870201096,
                  0.002881844380403458,
                  347,
                  1
                ],
                [
                  0.6361974405850092,
                  0.0028735632183908046,
                  348,
                  1
                ],
                [
                  0.6380255941499086,
                  0.0028653295128939827,
                  349,
                  1
                ],
                [
                  0.6398537477148081,
                  0.002857142857142857,
                  350,
                  1
                ],
                [
                  0.6416819012797075,
                  0.002849002849002849,
                  351,
                  1
                ],
                [
                  0.643510054844607,
                  0.002840909090909091,
                  352,
                  1
                ],
                [
                  0.6453382084095064,
                  0.0028328611898017,
                  353,
                  1
                ],
                [
                  0.6471663619744058,
                  0.002824858757062147,
                  354,
                  1
                ],
                [
                  0.6489945155393053,
                  0.0028169014084507044,
                  355,
                  1
                ],
                [
                  0.6508226691042047,
                  0.0028089887640449437,
                  356,
                  1
                ],
                [
                  0.6526508226691042,
                  0.0028011204481792717,
                  357,
                  1
                ],
                [
                  0.6544789762340036,
                  0.002793296089385475,
                  358,
                  1
                ],
                [
                  0.6563071297989032,
                  0.005571030640668524,
                  359,
                  2
                ],
                [
                  0.6581352833638026,
                  0.005555555555555556,
                  360,
                  2
                ],
                [
                  0.659963436928702,
                  0.008310249307479225,
                  361,
                  3
                ],
                [
                  0.6617915904936015,
                  0.008287292817679558,
                  362,
                  3
                ],
                [
                  0.6636197440585009,
                  0.008264462809917356,
                  363,
                  3
                ],
                [
                  0.6654478976234004,
                  0.008241758241758242,
                  364,
                  3
                ],
                [
                  0.6672760511882998,
                  0.00821917808219178,
                  365,
                  3
                ],
                [
                  0.6691042047531993,
                  0.00819672131147541,
                  366,
                  3
                ],
                [
                  0.6709323583180987,
                  0.008174386920980926,
                  367,
                  3
                ],
                [
                  0.6727605118829981,
                  0.008152173913043478,
                  368,
                  3
                ],
                [
                  0.6745886654478976,
                  0.008130081300813009,
                  369,
                  3
                ],
                [
                  0.676416819012797,
                  0.008108108108108109,
                  370,
                  3
                ],
                [
                  0.6782449725776966,
                  0.008086253369272238,
                  371,
                  3
                ],
                [
                  0.680073126142596,
                  0.008064516129032258,
                  372,
                  3
                ],
                [
                  0.6819012797074955,
                  0.00804289544235925,
                  373,
                  3
                ],
                [
                  0.6837294332723949,
                  0.008021390374331552,
                  374,
                  3
                ],
                [
                  0.6855575868372943,
                  0.008,
                  375,
                  3
                ],
                [
                  0.6873857404021938,
                  0.007978723404255319,
                  376,
                  3
                ],
                [
                  0.6892138939670932,
                  0.010610079575596816,
                  377,
                  4
                ],
                [
                  0.6910420475319927,
                  0.010582010582010581,
                  378,
                  4
                ],
                [
                  0.6928702010968921,
                  0.010554089709762533,
                  379,
                  4
                ],
                [
                  0.6946983546617916,
                  0.010526315789473684,
                  380,
                  4
                ],
                [
                  0.696526508226691,
                  0.010498687664041995,
                  381,
                  4
                ],
                [
                  0.6983546617915904,
                  0.010471204188481676,
                  382,
                  4
                ],
                [
                  0.70018281535649,
                  0.010443864229765013,
                  383,
                  4
                ],
                [
                  0.7020109689213894,
                  0.010416666666666666,
                  384,
                  4
                ],
                [
                  0.7038391224862889,
                  0.01038961038961039,
                  385,
                  4
                ],
                [
                  0.7056672760511883,
                  0.010362694300518135,
                  386,
                  4
                ],
                [
                  0.7074954296160878,
                  0.0103359173126615,
                  387,
                  4
                ],
                [
                  0.7093235831809872,
                  0.01288659793814433,
                  388,
                  5
                ],
                [
                  0.7111517367458866,
                  0.012853470437017995,
                  389,
                  5
                ],
                [
                  0.7129798903107861,
                  0.01282051282051282,
                  390,
                  5
                ],
                [
                  0.7148080438756855,
                  0.01278772378516624,
                  391,
                  5
                ],
                [
                  0.716636197440585,
                  0.012755102040816327,
                  392,
                  5
                ],
                [
                  0.7184643510054844,
                  0.01272264631043257,
                  393,
                  5
                ],
                [
                  0.720292504570384,
                  0.012690355329949238,
                  394,
                  5
                ],
                [
                  0.7221206581352834,
                  0.012658227848101266,
                  395,
                  5
                ],
                [
                  0.7239488117001828,
                  0.012626262626262626,
                  396,
                  5
                ],
                [
                  0.7257769652650823,
                  0.012594458438287154,
                  397,
                  5
                ],
                [
                  0.7276051188299817,
                  0.01256281407035176,
                  398,
                  5
                ],
                [
                  0.7294332723948812,
                  0.012531328320802004,
                  399,
                  5
                ],
                [
                  0.7312614259597806,
                  0.0125,
                  400,
                  5
                ],
                [
                  0.7330895795246801,
                  0.012468827930174564,
                  401,
                  5
                ],
                [
                  0.7349177330895795,
                  0.014925373134328358,
                  402,
                  6
                ],
                [
                  0.7367458866544789,
                  0.01488833746898263,
                  403,
                  6
                ],
                [
                  0.7385740402193784,
                  0.01485148514851485,
                  404,
                  6
                ],
                [
                  0.7404021937842779,
                  0.014814814814814815,
                  405,
                  6
                ],
                [
                  0.7422303473491774,
                  0.017241379310344827,
                  406,
                  7
                ],
                [
                  0.7440585009140768,
                  0.0171990171990172,
                  407,
                  7
                ],
                [
                  0.7458866544789763,
                  0.01715686274509804,
                  408,
                  7
                ],
                [
                  0.7477148080438757,
                  0.017114914425427872,
                  409,
                  7
                ],
                [
                  0.7495429616087751,
                  0.01951219512195122,
                  410,
                  8
                ],
                [
                  0.7513711151736746,
                  0.019464720194647202,
                  411,
                  8
                ],
                [
                  0.753199268738574,
                  0.019417475728155338,
                  412,
                  8
                ],
                [
                  0.7550274223034735,
                  0.01937046004842615,
                  413,
                  8
                ],
                [
                  0.7568555758683729,
                  0.01932367149758454,
                  414,
                  8
                ],
                [
                  0.7586837294332724,
                  0.021686746987951807,
                  415,
                  9
                ],
                [
                  0.7605118829981719,
                  0.021634615384615384,
                  416,
                  9
                ],
                [
                  0.7623400365630713,
                  0.02158273381294964,
                  417,
                  9
                ],
                [
                  0.7641681901279708,
                  0.0215311004784689,
                  418,
                  9
                ],
                [
                  0.7659963436928702,
                  0.02386634844868735,
                  419,
                  10
                ],
                [
                  0.7678244972577697,
                  0.023809523809523808,
                  420,
                  10
                ],
                [
                  0.7696526508226691,
                  0.023752969121140142,
                  421,
                  10
                ],
                [
                  0.7714808043875686,
                  0.023696682464454975,
                  422,
                  10
                ],
                [
                  0.773308957952468,
                  0.026004728132387706,
                  423,
                  11
                ],
                [
                  0.7751371115173674,
                  0.025943396226415096,
                  424,
                  11
                ],
                [
                  0.7769652650822669,
                  0.02588235294117647,
                  425,
                  11
                ],
                [
                  0.7787934186471663,
                  0.028169014084507043,
                  426,
                  12
                ],
                [
                  0.7806215722120659,
                  0.02810304449648712,
                  427,
                  12
                ],
                [
                  0.7824497257769653,
                  0.028037383177570093,
                  428,
                  12
                ],
                [
                  0.7842778793418648,
                  0.027972027972027972,
                  429,
                  12
                ],
                [
                  0.7861060329067642,
                  0.030232558139534883,
                  430,
                  13
                ],
                [
                  0.7879341864716636,
                  0.030162412993039442,
                  431,
                  13
                ],
                [
                  0.7897623400365631,
                  0.03009259259259259,
                  432,
                  13
                ],
                [
                  0.7915904936014625,
                  0.03002309468822171,
                  433,
                  13
                ],
                [
                  0.793418647166362,
                  0.029953917050691243,
                  434,
                  13
                ],
                [
                  0.7952468007312614,
                  0.029885057471264367,
                  435,
                  13
                ],
                [
                  0.7970749542961609,
                  0.02981651376146789,
                  436,
                  13
                ],
                [
                  0.7989031078610603,
                  0.029748283752860413,
                  437,
                  13
                ],
                [
                  0.8007312614259597,
                  0.02968036529680365,
                  438,
                  13
                ],
                [
                  0.8025594149908593,
                  0.029612756264236904,
                  439,
                  13
                ],
                [
                  0.8043875685557587,
                  0.029545454545454545,
                  440,
                  13
                ],
                [
                  0.8062157221206582,
                  0.02947845804988662,
                  441,
                  13
                ],
                [
                  0.8080438756855576,
                  0.029411764705882353,
                  442,
                  13
                ],
                [
                  0.8098720292504571,
                  0.029345372460496615,
                  443,
                  13
                ],
                [
                  0.8117001828153565,
                  0.02927927927927928,
                  444,
                  13
                ],
                [
                  0.8135283363802559,
                  0.03146067415730337,
                  445,
                  14
                ],
                [
                  0.8153564899451554,
                  0.03139013452914798,
                  446,
                  14
                ],
                [
                  0.8171846435100548,
                  0.03131991051454139,
                  447,
                  14
                ],
                [
                  0.8190127970749543,
                  0.03125,
                  448,
                  14
                ],
                [
                  0.8208409506398537,
                  0.031180400890868598,
                  449,
                  14
                ],
                [
                  0.8226691042047533,
                  0.03111111111111111,
                  450,
                  14
                ],
                [
                  0.8244972577696527,
                  0.03325942350332594,
                  451,
                  15
                ],
                [
                  0.8263254113345521,
                  0.033185840707964605,
                  452,
                  15
                ],
                [
                  0.8281535648994516,
                  0.033112582781456956,
                  453,
                  15
                ],
                [
                  0.829981718464351,
                  0.03303964757709251,
                  454,
                  15
                ],
                [
                  0.8318098720292505,
                  0.035164835164835165,
                  455,
                  16
                ],
                [
                  0.8336380255941499,
                  0.03508771929824561,
                  456,
                  16
                ],
                [
                  0.8354661791590493,
                  0.0350109409190372,
                  457,
                  16
                ],
                [
                  0.8372943327239488,
                  0.034934497816593885,
                  458,
                  16
                ],
                [
                  0.8391224862888482,
                  0.034858387799564274,
                  459,
                  16
                ],
                [
                  0.8409506398537477,
                  0.034782608695652174,
                  460,
                  16
                ],
                [
                  0.8427787934186471,
                  0.03470715835140998,
                  461,
                  16
                ],
                [
                  0.8446069469835467,
                  0.0367965367965368,
                  462,
                  17
                ],
                [
                  0.8464351005484461,
                  0.0367170626349892,
                  463,
                  17
                ],
                [
                  0.8482632541133455,
                  0.036637931034482756,
                  464,
                  17
                ],
                [
                  0.850091407678245,
                  0.03655913978494624,
                  465,
                  17
                ],
                [
                  0.8519195612431444,
                  0.03648068669527897,
                  466,
                  17
                ],
                [
                  0.8537477148080439,
                  0.03640256959314775,
                  467,
                  17
                ],
                [
                  0.8555758683729433,
                  0.038461538461538464,
                  468,
                  18
                ],
                [
                  0.8574040219378428,
                  0.03837953091684435,
                  469,
                  18
                ],
                [
                  0.8592321755027422,
                  0.03829787234042553,
                  470,
                  18
                ],
                [
                  0.8610603290676416,
                  0.03821656050955414,
                  471,
                  18
                ],
                [
                  0.8628884826325411,
                  0.04025423728813559,
                  472,
                  19
                ],
                [
                  0.8647166361974405,
                  0.042283298097251586,
                  473,
                  20
                ],
                [
                  0.8665447897623401,
                  0.04430379746835443,
                  474,
                  21
                ],
                [
                  0.8683729433272395,
                  0.04631578947368421,
                  475,
                  22
                ],
                [
                  0.870201096892139,
                  0.046218487394957986,
                  476,
                  22
                ],
                [
                  0.8720292504570384,
                  0.04612159329140461,
                  477,
                  22
                ],
                [
                  0.8738574040219378,
                  0.04811715481171548,
                  478,
                  23
                ],
                [
                  0.8756855575868373,
                  0.05010438413361169,
                  479,
                  24
                ],
                [
                  0.8775137111517367,
                  0.052083333333333336,
                  480,
                  25
                ],
                [
                  0.8793418647166362,
                  0.05405405405405406,
                  481,
                  26
                ],
                [
                  0.8811700182815356,
                  0.05394190871369295,
                  482,
                  26
                ],
                [
                  0.8829981718464351,
                  0.055900621118012424,
                  483,
                  27
                ],
                [
                  0.8848263254113345,
                  0.05785123966942149,
                  484,
                  28
                ],
                [
                  0.886654478976234,
                  0.0577319587628866,
                  485,
                  28
                ],
                [
                  0.8884826325411335,
                  0.05761316872427984,
                  486,
                  28
                ],
                [
                  0.8903107861060329,
                  0.057494866529774126,
                  487,
                  28
                ],
                [
                  0.8921389396709324,
                  0.05737704918032787,
                  488,
                  28
                ],
                [
                  0.8939670932358318,
                  0.05725971370143149,
                  489,
                  28
                ],
                [
                  0.8957952468007313,
                  0.05918367346938776,
                  490,
                  29
                ],
                [
                  0.8976234003656307,
                  0.059063136456211814,
                  491,
                  29
                ],
                [
                  0.8994515539305301,
                  0.05894308943089431,
                  492,
                  29
                ],
                [
                  0.9012797074954296,
                  0.060851926977687626,
                  493,
                  30
                ],
                [
                  0.903107861060329,
                  0.06072874493927125,
                  494,
                  30
                ],
                [
                  0.9049360146252285,
                  0.06060606060606061,
                  495,
                  30
                ],
                [
                  0.906764168190128,
                  0.0625,
                  496,
                  31
                ],
                [
                  0.9085923217550275,
                  0.06237424547283702,
                  497,
                  31
                ],
                [
                  0.9104204753199269,
                  0.06224899598393574,
                  498,
                  31
                ],
                [
                  0.9122486288848263,
                  0.06412825651302605,
                  499,
                  32
                ],
                [
                  0.9140767824497258,
                  0.064,
                  500,
                  32
                ],
                [
                  0.9159049360146252,
                  0.06387225548902195,
                  501,
                  32
                ],
                [
                  0.9177330895795247,
                  0.06374501992031872,
                  502,
                  32
                ],
                [
                  0.9195612431444241,
                  0.0636182902584493,
                  503,
                  32
                ],
                [
                  0.9213893967093236,
                  0.06349206349206349,
                  504,
                  32
                ],
                [
                  0.923217550274223,
                  0.06336633663366337,
                  505,
                  32
                ],
                [
                  0.9250457038391224,
                  0.06521739130434782,
                  506,
                  33
                ],
                [
                  0.926873857404022,
                  0.0650887573964497,
                  507,
                  33
                ],
                [
                  0.9287020109689214,
                  0.06692913385826772,
                  508,
                  34
                ],
                [
                  0.9305301645338209,
                  0.068762278978389,
                  509,
                  35
                ],
                [
                  0.9323583180987203,
                  0.07058823529411765,
                  510,
                  36
                ],
                [
                  0.9341864716636198,
                  0.07240704500978473,
                  511,
                  37
                ],
                [
                  0.9360146252285192,
                  0.072265625,
                  512,
                  37
                ],
                [
                  0.9378427787934186,
                  0.07407407407407407,
                  513,
                  38
                ],
                [
                  0.9396709323583181,
                  0.07392996108949416,
                  514,
                  38
                ],
                [
                  0.9414990859232175,
                  0.07378640776699029,
                  515,
                  38
                ],
                [
                  0.943327239488117,
                  0.07364341085271318,
                  516,
                  38
                ],
                [
                  0.9451553930530164,
                  0.07543520309477757,
                  517,
                  39
                ],
                [
                  0.946983546617916,
                  0.07528957528957529,
                  518,
                  39
                ],
                [
                  0.9488117001828154,
                  0.07707129094412331,
                  519,
                  40
                ],
                [
                  0.9506398537477148,
                  0.07884615384615384,
                  520,
                  41
                ],
                [
                  0.9524680073126143,
                  0.08061420345489444,
                  521,
                  42
                ],
                [
                  0.9542961608775137,
                  0.08045977011494253,
                  522,
                  42
                ],
                [
                  0.9561243144424132,
                  0.08030592734225621,
                  523,
                  42
                ],
                [
                  0.9579524680073126,
                  0.08015267175572519,
                  524,
                  42
                ],
                [
                  0.9597806215722121,
                  0.08,
                  525,
                  42
                ],
                [
                  0.9616087751371115,
                  0.07984790874524715,
                  526,
                  42
                ],
                [
                  0.9634369287020109,
                  0.07969639468690702,
                  527,
                  42
                ],
                [
                  0.9652650822669104,
                  0.08143939393939394,
                  528,
                  43
                ],
                [
                  0.9670932358318098,
                  0.0831758034026465,
                  529,
                  44
                ],
                [
                  0.9689213893967094,
                  0.08490566037735849,
                  530,
                  45
                ],
                [
                  0.9707495429616088,
                  0.08662900188323917,
                  531,
                  46
                ],
                [
                  0.9725776965265083,
                  0.08646616541353383,
                  532,
                  46
                ],
                [
                  0.9744058500914077,
                  0.08630393996247655,
                  533,
                  46
                ],
                [
                  0.9762340036563071,
                  0.08801498127340825,
                  534,
                  47
                ],
                [
                  0.9780621572212066,
                  0.08971962616822429,
                  535,
                  48
                ],
                [
                  0.979890310786106,
                  0.0914179104477612,
                  536,
                  49
                ],
                [
                  0.9817184643510055,
                  0.0931098696461825,
                  537,
                  50
                ],
                [
                  0.9835466179159049,
                  0.09479553903345725,
                  538,
                  51
                ],
                [
                  0.9853747714808044,
                  0.09647495361781076,
                  539,
                  52
                ],
                [
                  0.9872029250457038,
                  0.09814814814814815,
                  540,
                  53
                ],
                [
                  0.9890310786106032,
                  0.09796672828096119,
                  541,
                  53
                ],
                [
                  0.9908592321755028,
                  0.09778597785977859,
                  542,
                  53
                ],
                [
                  0.9926873857404022,
                  0.09760589318600368,
                  543,
                  53
                ],
                [
                  0.9945155393053017,
                  0.0974264705882353,
                  544,
                  53
                ],
                [
                  0.9963436928702011,
                  0.09908256880733946,
                  545,
                  54
                ],
                [
                  0.9981718464351006,
                  0.0989010989010989,
                  546,
                  54
                ],
                [
                  1.0,
                  0.09872029250457039,
                  547,
                  54
                ]
              ]
            }
          ]
        },
        {
          "name": "vLLM-SR d3",
          "text_run": 259,
          "native": [
            {
              "language": "Turkish",
              "wording": "Bilmiyorum",
              "run": 358,
              "full_bank_restricted": true,
              "right": 494,
              "wrong": 53,
              "abstained": 0,
              "invalid": 0,
              "coverage": 1.0,
              "risk": 0.09689213893967093
            },
            {
              "language": "Turkish",
              "wording": "I don't know",
              "run": 544,
              "full_bank_restricted": true,
              "right": 495,
              "wrong": 52,
              "abstained": 0,
              "invalid": 0,
              "coverage": 1.0,
              "risk": 0.09506398537477148
            },
            {
              "language": "English",
              "wording": "I don't know",
              "run": null
            }
          ],
          "curves": [
            {
              "language": "Turkish",
              "run": 259,
              "points": [
                [
                  0.0018281535648994515,
                  0.0,
                  1,
                  0
                ],
                [
                  0.003656307129798903,
                  0.0,
                  2,
                  0
                ],
                [
                  0.005484460694698354,
                  0.0,
                  3,
                  0
                ],
                [
                  0.007312614259597806,
                  0.0,
                  4,
                  0
                ],
                [
                  0.009140767824497258,
                  0.0,
                  5,
                  0
                ],
                [
                  0.010968921389396709,
                  0.0,
                  6,
                  0
                ],
                [
                  0.012797074954296161,
                  0.0,
                  7,
                  0
                ],
                [
                  0.014625228519195612,
                  0.0,
                  8,
                  0
                ],
                [
                  0.016453382084095063,
                  0.0,
                  9,
                  0
                ],
                [
                  0.018281535648994516,
                  0.0,
                  10,
                  0
                ],
                [
                  0.02010968921389397,
                  0.0,
                  11,
                  0
                ],
                [
                  0.021937842778793418,
                  0.0,
                  12,
                  0
                ],
                [
                  0.02376599634369287,
                  0.0,
                  13,
                  0
                ],
                [
                  0.025594149908592323,
                  0.0,
                  14,
                  0
                ],
                [
                  0.027422303473491772,
                  0.0,
                  15,
                  0
                ],
                [
                  0.029250457038391225,
                  0.0,
                  16,
                  0
                ],
                [
                  0.031078610603290677,
                  0.0,
                  17,
                  0
                ],
                [
                  0.03290676416819013,
                  0.0,
                  18,
                  0
                ],
                [
                  0.03473491773308958,
                  0.0,
                  19,
                  0
                ],
                [
                  0.03656307129798903,
                  0.0,
                  20,
                  0
                ],
                [
                  0.038391224862888484,
                  0.0,
                  21,
                  0
                ],
                [
                  0.04021937842778794,
                  0.0,
                  22,
                  0
                ],
                [
                  0.04204753199268738,
                  0.0,
                  23,
                  0
                ],
                [
                  0.043875685557586835,
                  0.0,
                  24,
                  0
                ],
                [
                  0.04570383912248629,
                  0.0,
                  25,
                  0
                ],
                [
                  0.04753199268738574,
                  0.0,
                  26,
                  0
                ],
                [
                  0.04936014625228519,
                  0.0,
                  27,
                  0
                ],
                [
                  0.051188299817184646,
                  0.0,
                  28,
                  0
                ],
                [
                  0.05301645338208409,
                  0.0,
                  29,
                  0
                ],
                [
                  0.054844606946983544,
                  0.0,
                  30,
                  0
                ],
                [
                  0.056672760511883,
                  0.0,
                  31,
                  0
                ],
                [
                  0.05850091407678245,
                  0.0,
                  32,
                  0
                ],
                [
                  0.0603290676416819,
                  0.0,
                  33,
                  0
                ],
                [
                  0.062157221206581355,
                  0.0,
                  34,
                  0
                ],
                [
                  0.06398537477148081,
                  0.0,
                  35,
                  0
                ],
                [
                  0.06581352833638025,
                  0.0,
                  36,
                  0
                ],
                [
                  0.06764168190127971,
                  0.0,
                  37,
                  0
                ],
                [
                  0.06946983546617916,
                  0.0,
                  38,
                  0
                ],
                [
                  0.0712979890310786,
                  0.0,
                  39,
                  0
                ],
                [
                  0.07312614259597806,
                  0.0,
                  40,
                  0
                ],
                [
                  0.07495429616087751,
                  0.0,
                  41,
                  0
                ],
                [
                  0.07678244972577697,
                  0.0,
                  42,
                  0
                ],
                [
                  0.07861060329067641,
                  0.0,
                  43,
                  0
                ],
                [
                  0.08043875685557587,
                  0.0,
                  44,
                  0
                ],
                [
                  0.08226691042047532,
                  0.0,
                  45,
                  0
                ],
                [
                  0.08409506398537477,
                  0.0,
                  46,
                  0
                ],
                [
                  0.08592321755027423,
                  0.0,
                  47,
                  0
                ],
                [
                  0.08775137111517367,
                  0.0,
                  48,
                  0
                ],
                [
                  0.08957952468007313,
                  0.0,
                  49,
                  0
                ],
                [
                  0.09140767824497258,
                  0.0,
                  50,
                  0
                ],
                [
                  0.09323583180987204,
                  0.0,
                  51,
                  0
                ],
                [
                  0.09506398537477148,
                  0.0,
                  52,
                  0
                ],
                [
                  0.09689213893967093,
                  0.0,
                  53,
                  0
                ],
                [
                  0.09872029250457039,
                  0.0,
                  54,
                  0
                ],
                [
                  0.10054844606946983,
                  0.0,
                  55,
                  0
                ],
                [
                  0.10237659963436929,
                  0.0,
                  56,
                  0
                ],
                [
                  0.10420475319926874,
                  0.0,
                  57,
                  0
                ],
                [
                  0.10603290676416818,
                  0.0,
                  58,
                  0
                ],
                [
                  0.10786106032906764,
                  0.0,
                  59,
                  0
                ],
                [
                  0.10968921389396709,
                  0.0,
                  60,
                  0
                ],
                [
                  0.11151736745886655,
                  0.0,
                  61,
                  0
                ],
                [
                  0.113345521023766,
                  0.0,
                  62,
                  0
                ],
                [
                  0.11517367458866545,
                  0.0,
                  63,
                  0
                ],
                [
                  0.1170018281535649,
                  0.0,
                  64,
                  0
                ],
                [
                  0.11882998171846434,
                  0.0,
                  65,
                  0
                ],
                [
                  0.1206581352833638,
                  0.0,
                  66,
                  0
                ],
                [
                  0.12248628884826325,
                  0.0,
                  67,
                  0
                ],
                [
                  0.12431444241316271,
                  0.0,
                  68,
                  0
                ],
                [
                  0.12614259597806216,
                  0.0,
                  69,
                  0
                ],
                [
                  0.12797074954296161,
                  0.0,
                  70,
                  0
                ],
                [
                  0.12979890310786105,
                  0.0,
                  71,
                  0
                ],
                [
                  0.1316270566727605,
                  0.0,
                  72,
                  0
                ],
                [
                  0.13345521023765997,
                  0.0,
                  73,
                  0
                ],
                [
                  0.13528336380255943,
                  0.0,
                  74,
                  0
                ],
                [
                  0.13711151736745886,
                  0.0,
                  75,
                  0
                ],
                [
                  0.13893967093235832,
                  0.0,
                  76,
                  0
                ],
                [
                  0.14076782449725778,
                  0.0,
                  77,
                  0
                ],
                [
                  0.1425959780621572,
                  0.0,
                  78,
                  0
                ],
                [
                  0.14442413162705667,
                  0.0,
                  79,
                  0
                ],
                [
                  0.14625228519195613,
                  0.0,
                  80,
                  0
                ],
                [
                  0.1480804387568556,
                  0.0,
                  81,
                  0
                ],
                [
                  0.14990859232175502,
                  0.0,
                  82,
                  0
                ],
                [
                  0.15173674588665448,
                  0.0,
                  83,
                  0
                ],
                [
                  0.15356489945155394,
                  0.0,
                  84,
                  0
                ],
                [
                  0.15539305301645337,
                  0.0,
                  85,
                  0
                ],
                [
                  0.15722120658135283,
                  0.0,
                  86,
                  0
                ],
                [
                  0.1590493601462523,
                  0.0,
                  87,
                  0
                ],
                [
                  0.16087751371115175,
                  0.0,
                  88,
                  0
                ],
                [
                  0.16270566727605118,
                  0.0,
                  89,
                  0
                ],
                [
                  0.16453382084095064,
                  0.0,
                  90,
                  0
                ],
                [
                  0.1663619744058501,
                  0.0,
                  91,
                  0
                ],
                [
                  0.16819012797074953,
                  0.0,
                  92,
                  0
                ],
                [
                  0.170018281535649,
                  0.0,
                  93,
                  0
                ],
                [
                  0.17184643510054845,
                  0.0,
                  94,
                  0
                ],
                [
                  0.1736745886654479,
                  0.0,
                  95,
                  0
                ],
                [
                  0.17550274223034734,
                  0.0,
                  96,
                  0
                ],
                [
                  0.1773308957952468,
                  0.0,
                  97,
                  0
                ],
                [
                  0.17915904936014626,
                  0.0,
                  98,
                  0
                ],
                [
                  0.1809872029250457,
                  0.0,
                  99,
                  0
                ],
                [
                  0.18281535648994515,
                  0.0,
                  100,
                  0
                ],
                [
                  0.1846435100548446,
                  0.0,
                  101,
                  0
                ],
                [
                  0.18647166361974407,
                  0.0,
                  102,
                  0
                ],
                [
                  0.1882998171846435,
                  0.0,
                  103,
                  0
                ],
                [
                  0.19012797074954296,
                  0.0,
                  104,
                  0
                ],
                [
                  0.19195612431444242,
                  0.0,
                  105,
                  0
                ],
                [
                  0.19378427787934185,
                  0.0,
                  106,
                  0
                ],
                [
                  0.1956124314442413,
                  0.0,
                  107,
                  0
                ],
                [
                  0.19744058500914077,
                  0.0,
                  108,
                  0
                ],
                [
                  0.19926873857404023,
                  0.0,
                  109,
                  0
                ],
                [
                  0.20109689213893966,
                  0.0,
                  110,
                  0
                ],
                [
                  0.20292504570383912,
                  0.0,
                  111,
                  0
                ],
                [
                  0.20475319926873858,
                  0.0,
                  112,
                  0
                ],
                [
                  0.20658135283363802,
                  0.0,
                  113,
                  0
                ],
                [
                  0.20840950639853748,
                  0.0,
                  114,
                  0
                ],
                [
                  0.21023765996343693,
                  0.0,
                  115,
                  0
                ],
                [
                  0.21206581352833637,
                  0.0,
                  116,
                  0
                ],
                [
                  0.21389396709323583,
                  0.0,
                  117,
                  0
                ],
                [
                  0.21572212065813529,
                  0.0,
                  118,
                  0
                ],
                [
                  0.21755027422303475,
                  0.0,
                  119,
                  0
                ],
                [
                  0.21937842778793418,
                  0.0,
                  120,
                  0
                ],
                [
                  0.22120658135283364,
                  0.0,
                  121,
                  0
                ],
                [
                  0.2230347349177331,
                  0.0,
                  122,
                  0
                ],
                [
                  0.22486288848263253,
                  0.0,
                  123,
                  0
                ],
                [
                  0.226691042047532,
                  0.0,
                  124,
                  0
                ],
                [
                  0.22851919561243145,
                  0.0,
                  125,
                  0
                ],
                [
                  0.2303473491773309,
                  0.0,
                  126,
                  0
                ],
                [
                  0.23217550274223034,
                  0.0,
                  127,
                  0
                ],
                [
                  0.2340036563071298,
                  0.0,
                  128,
                  0
                ],
                [
                  0.23583180987202926,
                  0.0,
                  129,
                  0
                ],
                [
                  0.2376599634369287,
                  0.0,
                  130,
                  0
                ],
                [
                  0.23948811700182815,
                  0.0,
                  131,
                  0
                ],
                [
                  0.2413162705667276,
                  0.0,
                  132,
                  0
                ],
                [
                  0.24314442413162707,
                  0.0,
                  133,
                  0
                ],
                [
                  0.2449725776965265,
                  0.0,
                  134,
                  0
                ],
                [
                  0.24680073126142596,
                  0.0,
                  135,
                  0
                ],
                [
                  0.24862888482632542,
                  0.0,
                  136,
                  0
                ],
                [
                  0.25045703839122485,
                  0.0,
                  137,
                  0
                ],
                [
                  0.2522851919561243,
                  0.0,
                  138,
                  0
                ],
                [
                  0.25411334552102377,
                  0.0,
                  139,
                  0
                ],
                [
                  0.25594149908592323,
                  0.0,
                  140,
                  0
                ],
                [
                  0.2577696526508227,
                  0.0,
                  141,
                  0
                ],
                [
                  0.2595978062157221,
                  0.0,
                  142,
                  0
                ],
                [
                  0.26142595978062155,
                  0.0,
                  143,
                  0
                ],
                [
                  0.263254113345521,
                  0.0,
                  144,
                  0
                ],
                [
                  0.26508226691042047,
                  0.0,
                  145,
                  0
                ],
                [
                  0.26691042047531993,
                  0.0,
                  146,
                  0
                ],
                [
                  0.2687385740402194,
                  0.0,
                  147,
                  0
                ],
                [
                  0.27056672760511885,
                  0.0,
                  148,
                  0
                ],
                [
                  0.27239488117001825,
                  0.0,
                  149,
                  0
                ],
                [
                  0.2742230347349177,
                  0.0,
                  150,
                  0
                ],
                [
                  0.2760511882998172,
                  0.0,
                  151,
                  0
                ],
                [
                  0.27787934186471663,
                  0.0,
                  152,
                  0
                ],
                [
                  0.2797074954296161,
                  0.0,
                  153,
                  0
                ],
                [
                  0.28153564899451555,
                  0.0,
                  154,
                  0
                ],
                [
                  0.283363802559415,
                  0.0,
                  155,
                  0
                ],
                [
                  0.2851919561243144,
                  0.0,
                  156,
                  0
                ],
                [
                  0.2870201096892139,
                  0.0,
                  157,
                  0
                ],
                [
                  0.28884826325411334,
                  0.0,
                  158,
                  0
                ],
                [
                  0.2906764168190128,
                  0.0,
                  159,
                  0
                ],
                [
                  0.29250457038391225,
                  0.0,
                  160,
                  0
                ],
                [
                  0.2943327239488117,
                  0.0,
                  161,
                  0
                ],
                [
                  0.2961608775137112,
                  0.0,
                  162,
                  0
                ],
                [
                  0.2979890310786106,
                  0.0,
                  163,
                  0
                ],
                [
                  0.29981718464351004,
                  0.0,
                  164,
                  0
                ],
                [
                  0.3016453382084095,
                  0.0,
                  165,
                  0
                ],
                [
                  0.30347349177330896,
                  0.0,
                  166,
                  0
                ],
                [
                  0.3053016453382084,
                  0.0,
                  167,
                  0
                ],
                [
                  0.3071297989031079,
                  0.0,
                  168,
                  0
                ],
                [
                  0.30895795246800734,
                  0.0,
                  169,
                  0
                ],
                [
                  0.31078610603290674,
                  0.0,
                  170,
                  0
                ],
                [
                  0.3126142595978062,
                  0.0,
                  171,
                  0
                ],
                [
                  0.31444241316270566,
                  0.0,
                  172,
                  0
                ],
                [
                  0.3162705667276051,
                  0.0,
                  173,
                  0
                ],
                [
                  0.3180987202925046,
                  0.0,
                  174,
                  0
                ],
                [
                  0.31992687385740404,
                  0.0,
                  175,
                  0
                ],
                [
                  0.3217550274223035,
                  0.0,
                  176,
                  0
                ],
                [
                  0.3235831809872029,
                  0.0,
                  177,
                  0
                ],
                [
                  0.32541133455210236,
                  0.0,
                  178,
                  0
                ],
                [
                  0.3272394881170018,
                  0.0,
                  179,
                  0
                ],
                [
                  0.3290676416819013,
                  0.0,
                  180,
                  0
                ],
                [
                  0.33089579524680074,
                  0.0,
                  181,
                  0
                ],
                [
                  0.3327239488117002,
                  0.0,
                  182,
                  0
                ],
                [
                  0.33455210237659966,
                  0.0,
                  183,
                  0
                ],
                [
                  0.33638025594149906,
                  0.0,
                  184,
                  0
                ],
                [
                  0.3382084095063985,
                  0.0,
                  185,
                  0
                ],
                [
                  0.340036563071298,
                  0.0,
                  186,
                  0
                ],
                [
                  0.34186471663619744,
                  0.0,
                  187,
                  0
                ],
                [
                  0.3436928702010969,
                  0.0,
                  188,
                  0
                ],
                [
                  0.34552102376599636,
                  0.0,
                  189,
                  0
                ],
                [
                  0.3473491773308958,
                  0.0,
                  190,
                  0
                ],
                [
                  0.3491773308957952,
                  0.0,
                  191,
                  0
                ],
                [
                  0.3510054844606947,
                  0.0,
                  192,
                  0
                ],
                [
                  0.35283363802559414,
                  0.0,
                  193,
                  0
                ],
                [
                  0.3546617915904936,
                  0.0,
                  194,
                  0
                ],
                [
                  0.35648994515539306,
                  0.0,
                  195,
                  0
                ],
                [
                  0.3583180987202925,
                  0.0,
                  196,
                  0
                ],
                [
                  0.360146252285192,
                  0.0,
                  197,
                  0
                ],
                [
                  0.3619744058500914,
                  0.0,
                  198,
                  0
                ],
                [
                  0.36380255941499084,
                  0.0,
                  199,
                  0
                ],
                [
                  0.3656307129798903,
                  0.0,
                  200,
                  0
                ],
                [
                  0.36745886654478976,
                  0.0,
                  201,
                  0
                ],
                [
                  0.3692870201096892,
                  0.0,
                  202,
                  0
                ],
                [
                  0.3711151736745887,
                  0.0,
                  203,
                  0
                ],
                [
                  0.37294332723948814,
                  0.0,
                  204,
                  0
                ],
                [
                  0.37477148080438755,
                  0.0,
                  205,
                  0
                ],
                [
                  0.376599634369287,
                  0.0,
                  206,
                  0
                ],
                [
                  0.37842778793418647,
                  0.0,
                  207,
                  0
                ],
                [
                  0.3802559414990859,
                  0.0,
                  208,
                  0
                ],
                [
                  0.3820840950639854,
                  0.0,
                  209,
                  0
                ],
                [
                  0.38391224862888484,
                  0.0,
                  210,
                  0
                ],
                [
                  0.3857404021937843,
                  0.0,
                  211,
                  0
                ],
                [
                  0.3875685557586837,
                  0.0,
                  212,
                  0
                ],
                [
                  0.38939670932358317,
                  0.0,
                  213,
                  0
                ],
                [
                  0.3912248628884826,
                  0.0,
                  214,
                  0
                ],
                [
                  0.3930530164533821,
                  0.0,
                  215,
                  0
                ],
                [
                  0.39488117001828155,
                  0.0,
                  216,
                  0
                ],
                [
                  0.396709323583181,
                  0.0,
                  217,
                  0
                ],
                [
                  0.39853747714808047,
                  0.0,
                  218,
                  0
                ],
                [
                  0.40036563071297987,
                  0.0,
                  219,
                  0
                ],
                [
                  0.40219378427787933,
                  0.0,
                  220,
                  0
                ],
                [
                  0.4040219378427788,
                  0.0,
                  221,
                  0
                ],
                [
                  0.40585009140767825,
                  0.0,
                  222,
                  0
                ],
                [
                  0.4076782449725777,
                  0.0,
                  223,
                  0
                ],
                [
                  0.40950639853747717,
                  0.0,
                  224,
                  0
                ],
                [
                  0.4113345521023766,
                  0.0,
                  225,
                  0
                ],
                [
                  0.41316270566727603,
                  0.0,
                  226,
                  0
                ],
                [
                  0.4149908592321755,
                  0.0,
                  227,
                  0
                ],
                [
                  0.41681901279707495,
                  0.0,
                  228,
                  0
                ],
                [
                  0.4186471663619744,
                  0.0,
                  229,
                  0
                ],
                [
                  0.42047531992687387,
                  0.0,
                  230,
                  0
                ],
                [
                  0.42230347349177333,
                  0.0,
                  231,
                  0
                ],
                [
                  0.42413162705667273,
                  0.0,
                  232,
                  0
                ],
                [
                  0.4259597806215722,
                  0.0,
                  233,
                  0
                ],
                [
                  0.42778793418647165,
                  0.0,
                  234,
                  0
                ],
                [
                  0.4296160877513711,
                  0.0,
                  235,
                  0
                ],
                [
                  0.43144424131627057,
                  0.0,
                  236,
                  0
                ],
                [
                  0.43327239488117003,
                  0.0,
                  237,
                  0
                ],
                [
                  0.4351005484460695,
                  0.0,
                  238,
                  0
                ],
                [
                  0.4369287020109689,
                  0.0,
                  239,
                  0
                ],
                [
                  0.43875685557586835,
                  0.0,
                  240,
                  0
                ],
                [
                  0.4405850091407678,
                  0.0,
                  241,
                  0
                ],
                [
                  0.4424131627056673,
                  0.0,
                  242,
                  0
                ],
                [
                  0.44424131627056673,
                  0.0,
                  243,
                  0
                ],
                [
                  0.4460694698354662,
                  0.0,
                  244,
                  0
                ],
                [
                  0.44789762340036565,
                  0.0,
                  245,
                  0
                ],
                [
                  0.44972577696526506,
                  0.0,
                  246,
                  0
                ],
                [
                  0.4515539305301645,
                  0.0,
                  247,
                  0
                ],
                [
                  0.453382084095064,
                  0.0,
                  248,
                  0
                ],
                [
                  0.45521023765996343,
                  0.0,
                  249,
                  0
                ],
                [
                  0.4570383912248629,
                  0.0,
                  250,
                  0
                ],
                [
                  0.45886654478976235,
                  0.0,
                  251,
                  0
                ],
                [
                  0.4606946983546618,
                  0.0,
                  252,
                  0
                ],
                [
                  0.4625228519195612,
                  0.0,
                  253,
                  0
                ],
                [
                  0.4643510054844607,
                  0.0,
                  254,
                  0
                ],
                [
                  0.46617915904936014,
                  0.0,
                  255,
                  0
                ],
                [
                  0.4680073126142596,
                  0.0,
                  256,
                  0
                ],
                [
                  0.46983546617915906,
                  0.0,
                  257,
                  0
                ],
                [
                  0.4716636197440585,
                  0.0,
                  258,
                  0
                ],
                [
                  0.473491773308958,
                  0.0,
                  259,
                  0
                ],
                [
                  0.4753199268738574,
                  0.0,
                  260,
                  0
                ],
                [
                  0.47714808043875684,
                  0.0,
                  261,
                  0
                ],
                [
                  0.4789762340036563,
                  0.0,
                  262,
                  0
                ],
                [
                  0.48080438756855576,
                  0.0,
                  263,
                  0
                ],
                [
                  0.4826325411334552,
                  0.0,
                  264,
                  0
                ],
                [
                  0.4844606946983547,
                  0.0,
                  265,
                  0
                ],
                [
                  0.48628884826325414,
                  0.0,
                  266,
                  0
                ],
                [
                  0.48811700182815354,
                  0.0,
                  267,
                  0
                ],
                [
                  0.489945155393053,
                  0.0,
                  268,
                  0
                ],
                [
                  0.49177330895795246,
                  0.0,
                  269,
                  0
                ],
                [
                  0.4936014625228519,
                  0.0,
                  270,
                  0
                ],
                [
                  0.4954296160877514,
                  0.0,
                  271,
                  0
                ],
                [
                  0.49725776965265084,
                  0.0,
                  272,
                  0
                ],
                [
                  0.4990859232175503,
                  0.0,
                  273,
                  0
                ],
                [
                  0.5009140767824497,
                  0.0,
                  274,
                  0
                ],
                [
                  0.5027422303473492,
                  0.0,
                  275,
                  0
                ],
                [
                  0.5045703839122486,
                  0.0,
                  276,
                  0
                ],
                [
                  0.506398537477148,
                  0.0,
                  277,
                  0
                ],
                [
                  0.5082266910420475,
                  0.0,
                  278,
                  0
                ],
                [
                  0.5100548446069469,
                  0.0,
                  279,
                  0
                ],
                [
                  0.5118829981718465,
                  0.0,
                  280,
                  0
                ],
                [
                  0.5137111517367459,
                  0.0,
                  281,
                  0
                ],
                [
                  0.5155393053016454,
                  0.0,
                  282,
                  0
                ],
                [
                  0.5173674588665448,
                  0.0,
                  283,
                  0
                ],
                [
                  0.5191956124314442,
                  0.0,
                  284,
                  0
                ],
                [
                  0.5210237659963437,
                  0.0,
                  285,
                  0
                ],
                [
                  0.5228519195612431,
                  0.0,
                  286,
                  0
                ],
                [
                  0.5246800731261426,
                  0.0,
                  287,
                  0
                ],
                [
                  0.526508226691042,
                  0.0,
                  288,
                  0
                ],
                [
                  0.5283363802559415,
                  0.0,
                  289,
                  0
                ],
                [
                  0.5301645338208409,
                  0.0,
                  290,
                  0
                ],
                [
                  0.5319926873857403,
                  0.0,
                  291,
                  0
                ],
                [
                  0.5338208409506399,
                  0.0,
                  292,
                  0
                ],
                [
                  0.5356489945155393,
                  0.0,
                  293,
                  0
                ],
                [
                  0.5374771480804388,
                  0.0,
                  294,
                  0
                ],
                [
                  0.5393053016453382,
                  0.0,
                  295,
                  0
                ],
                [
                  0.5411334552102377,
                  0.0,
                  296,
                  0
                ],
                [
                  0.5429616087751371,
                  0.0,
                  297,
                  0
                ],
                [
                  0.5447897623400365,
                  0.0,
                  298,
                  0
                ],
                [
                  0.546617915904936,
                  0.0,
                  299,
                  0
                ],
                [
                  0.5484460694698354,
                  0.0,
                  300,
                  0
                ],
                [
                  0.5502742230347349,
                  0.0,
                  301,
                  0
                ],
                [
                  0.5521023765996343,
                  0.0,
                  302,
                  0
                ],
                [
                  0.5539305301645339,
                  0.0,
                  303,
                  0
                ],
                [
                  0.5557586837294333,
                  0.0,
                  304,
                  0
                ],
                [
                  0.5575868372943327,
                  0.0,
                  305,
                  0
                ],
                [
                  0.5594149908592322,
                  0.0,
                  306,
                  0
                ],
                [
                  0.5612431444241316,
                  0.0,
                  307,
                  0
                ],
                [
                  0.5630712979890311,
                  0.0,
                  308,
                  0
                ],
                [
                  0.5648994515539305,
                  0.0,
                  309,
                  0
                ],
                [
                  0.56672760511883,
                  0.0,
                  310,
                  0
                ],
                [
                  0.5685557586837294,
                  0.0,
                  311,
                  0
                ],
                [
                  0.5703839122486288,
                  0.0,
                  312,
                  0
                ],
                [
                  0.5722120658135283,
                  0.0,
                  313,
                  0
                ],
                [
                  0.5740402193784278,
                  0.0,
                  314,
                  0
                ],
                [
                  0.5758683729433273,
                  0.0,
                  315,
                  0
                ],
                [
                  0.5776965265082267,
                  0.0,
                  316,
                  0
                ],
                [
                  0.5795246800731262,
                  0.0,
                  317,
                  0
                ],
                [
                  0.5813528336380256,
                  0.0,
                  318,
                  0
                ],
                [
                  0.583180987202925,
                  0.0,
                  319,
                  0
                ],
                [
                  0.5850091407678245,
                  0.0,
                  320,
                  0
                ],
                [
                  0.5868372943327239,
                  0.0,
                  321,
                  0
                ],
                [
                  0.5886654478976234,
                  0.0,
                  322,
                  0
                ],
                [
                  0.5904936014625228,
                  0.0,
                  323,
                  0
                ],
                [
                  0.5923217550274223,
                  0.0,
                  324,
                  0
                ],
                [
                  0.5941499085923218,
                  0.0,
                  325,
                  0
                ],
                [
                  0.5959780621572212,
                  0.0,
                  326,
                  0
                ],
                [
                  0.5978062157221207,
                  0.0,
                  327,
                  0
                ],
                [
                  0.5996343692870201,
                  0.0,
                  328,
                  0
                ],
                [
                  0.6014625228519196,
                  0.0,
                  329,
                  0
                ],
                [
                  0.603290676416819,
                  0.0,
                  330,
                  0
                ],
                [
                  0.6051188299817185,
                  0.0,
                  331,
                  0
                ],
                [
                  0.6069469835466179,
                  0.0,
                  332,
                  0
                ],
                [
                  0.6087751371115173,
                  0.0,
                  333,
                  0
                ],
                [
                  0.6106032906764168,
                  0.0,
                  334,
                  0
                ],
                [
                  0.6124314442413162,
                  0.0,
                  335,
                  0
                ],
                [
                  0.6142595978062158,
                  0.0,
                  336,
                  0
                ],
                [
                  0.6160877513711152,
                  0.0,
                  337,
                  0
                ],
                [
                  0.6179159049360147,
                  0.0,
                  338,
                  0
                ],
                [
                  0.6197440585009141,
                  0.0,
                  339,
                  0
                ],
                [
                  0.6215722120658135,
                  0.0,
                  340,
                  0
                ],
                [
                  0.623400365630713,
                  0.0,
                  341,
                  0
                ],
                [
                  0.6252285191956124,
                  0.0,
                  342,
                  0
                ],
                [
                  0.6270566727605119,
                  0.0,
                  343,
                  0
                ],
                [
                  0.6288848263254113,
                  0.0,
                  344,
                  0
                ],
                [
                  0.6307129798903108,
                  0.0,
                  345,
                  0
                ],
                [
                  0.6325411334552102,
                  0.0,
                  346,
                  0
                ],
                [
                  0.6343692870201096,
                  0.0,
                  347,
                  0
                ],
                [
                  0.6361974405850092,
                  0.0,
                  348,
                  0
                ],
                [
                  0.6380255941499086,
                  0.0,
                  349,
                  0
                ],
                [
                  0.6398537477148081,
                  0.0,
                  350,
                  0
                ],
                [
                  0.6416819012797075,
                  0.0,
                  351,
                  0
                ],
                [
                  0.643510054844607,
                  0.0,
                  352,
                  0
                ],
                [
                  0.6453382084095064,
                  0.0,
                  353,
                  0
                ],
                [
                  0.6471663619744058,
                  0.0,
                  354,
                  0
                ],
                [
                  0.6489945155393053,
                  0.0,
                  355,
                  0
                ],
                [
                  0.6508226691042047,
                  0.0,
                  356,
                  0
                ],
                [
                  0.6526508226691042,
                  0.0,
                  357,
                  0
                ],
                [
                  0.6544789762340036,
                  0.0,
                  358,
                  0
                ],
                [
                  0.6563071297989032,
                  0.0,
                  359,
                  0
                ],
                [
                  0.6581352833638026,
                  0.0,
                  360,
                  0
                ],
                [
                  0.659963436928702,
                  0.0,
                  361,
                  0
                ],
                [
                  0.6617915904936015,
                  0.0,
                  362,
                  0
                ],
                [
                  0.6636197440585009,
                  0.0,
                  363,
                  0
                ],
                [
                  0.6654478976234004,
                  0.0,
                  364,
                  0
                ],
                [
                  0.6672760511882998,
                  0.0,
                  365,
                  0
                ],
                [
                  0.6691042047531993,
                  0.0,
                  366,
                  0
                ],
                [
                  0.6709323583180987,
                  0.0,
                  367,
                  0
                ],
                [
                  0.6727605118829981,
                  0.0,
                  368,
                  0
                ],
                [
                  0.6745886654478976,
                  0.0,
                  369,
                  0
                ],
                [
                  0.676416819012797,
                  0.0,
                  370,
                  0
                ],
                [
                  0.6782449725776966,
                  0.0,
                  371,
                  0
                ],
                [
                  0.680073126142596,
                  0.0,
                  372,
                  0
                ],
                [
                  0.6819012797074955,
                  0.0,
                  373,
                  0
                ],
                [
                  0.6837294332723949,
                  0.0,
                  374,
                  0
                ],
                [
                  0.6855575868372943,
                  0.0,
                  375,
                  0
                ],
                [
                  0.6873857404021938,
                  0.0,
                  376,
                  0
                ],
                [
                  0.6892138939670932,
                  0.0,
                  377,
                  0
                ],
                [
                  0.6910420475319927,
                  0.0,
                  378,
                  0
                ],
                [
                  0.6928702010968921,
                  0.0,
                  379,
                  0
                ],
                [
                  0.6946983546617916,
                  0.0,
                  380,
                  0
                ],
                [
                  0.696526508226691,
                  0.0,
                  381,
                  0
                ],
                [
                  0.6983546617915904,
                  0.0,
                  382,
                  0
                ],
                [
                  0.70018281535649,
                  0.0,
                  383,
                  0
                ],
                [
                  0.7020109689213894,
                  0.0,
                  384,
                  0
                ],
                [
                  0.7038391224862889,
                  0.0,
                  385,
                  0
                ],
                [
                  0.7056672760511883,
                  0.0,
                  386,
                  0
                ],
                [
                  0.7074954296160878,
                  0.0,
                  387,
                  0
                ],
                [
                  0.7093235831809872,
                  0.0,
                  388,
                  0
                ],
                [
                  0.7111517367458866,
                  0.0,
                  389,
                  0
                ],
                [
                  0.7129798903107861,
                  0.0,
                  390,
                  0
                ],
                [
                  0.7148080438756855,
                  0.0,
                  391,
                  0
                ],
                [
                  0.716636197440585,
                  0.0,
                  392,
                  0
                ],
                [
                  0.7184643510054844,
                  0.0,
                  393,
                  0
                ],
                [
                  0.720292504570384,
                  0.0,
                  394,
                  0
                ],
                [
                  0.7221206581352834,
                  0.0,
                  395,
                  0
                ],
                [
                  0.7239488117001828,
                  0.0025252525252525255,
                  396,
                  1
                ],
                [
                  0.7257769652650823,
                  0.0025188916876574307,
                  397,
                  1
                ],
                [
                  0.7276051188299817,
                  0.002512562814070352,
                  398,
                  1
                ],
                [
                  0.7294332723948812,
                  0.002506265664160401,
                  399,
                  1
                ],
                [
                  0.7312614259597806,
                  0.0025,
                  400,
                  1
                ],
                [
                  0.7330895795246801,
                  0.0024937655860349127,
                  401,
                  1
                ],
                [
                  0.7349177330895795,
                  0.0024875621890547263,
                  402,
                  1
                ],
                [
                  0.7367458866544789,
                  0.0024813895781637717,
                  403,
                  1
                ],
                [
                  0.7385740402193784,
                  0.0024752475247524753,
                  404,
                  1
                ],
                [
                  0.7404021937842779,
                  0.0024691358024691358,
                  405,
                  1
                ],
                [
                  0.7422303473491774,
                  0.0024630541871921183,
                  406,
                  1
                ],
                [
                  0.7440585009140768,
                  0.002457002457002457,
                  407,
                  1
                ],
                [
                  0.7458866544789763,
                  0.0024509803921568627,
                  408,
                  1
                ],
                [
                  0.7477148080438757,
                  0.0024449877750611247,
                  409,
                  1
                ],
                [
                  0.7495429616087751,
                  0.0024390243902439024,
                  410,
                  1
                ],
                [
                  0.7513711151736746,
                  0.0024330900243309003,
                  411,
                  1
                ],
                [
                  0.753199268738574,
                  0.0024271844660194173,
                  412,
                  1
                ],
                [
                  0.7550274223034735,
                  0.002421307506053269,
                  413,
                  1
                ],
                [
                  0.7568555758683729,
                  0.0024154589371980675,
                  414,
                  1
                ],
                [
                  0.7586837294332724,
                  0.004819277108433735,
                  415,
                  2
                ],
                [
                  0.7605118829981719,
                  0.004807692307692308,
                  416,
                  2
                ],
                [
                  0.7623400365630713,
                  0.004796163069544364,
                  417,
                  2
                ],
                [
                  0.7641681901279708,
                  0.004784688995215311,
                  418,
                  2
                ],
                [
                  0.7659963436928702,
                  0.00477326968973747,
                  419,
                  2
                ],
                [
                  0.7678244972577697,
                  0.004761904761904762,
                  420,
                  2
                ],
                [
                  0.7696526508226691,
                  0.004750593824228029,
                  421,
                  2
                ],
                [
                  0.7714808043875686,
                  0.004739336492890996,
                  422,
                  2
                ],
                [
                  0.773308957952468,
                  0.004728132387706856,
                  423,
                  2
                ],
                [
                  0.7751371115173674,
                  0.0047169811320754715,
                  424,
                  2
                ],
                [
                  0.7769652650822669,
                  0.004705882352941176,
                  425,
                  2
                ],
                [
                  0.7787934186471663,
                  0.004694835680751174,
                  426,
                  2
                ],
                [
                  0.7806215722120659,
                  0.00468384074941452,
                  427,
                  2
                ],
                [
                  0.7824497257769653,
                  0.004672897196261682,
                  428,
                  2
                ],
                [
                  0.7842778793418648,
                  0.004662004662004662,
                  429,
                  2
                ],
                [
                  0.7861060329067642,
                  0.0069767441860465115,
                  430,
                  3
                ],
                [
                  0.7879341864716636,
                  0.0069605568445475635,
                  431,
                  3
                ],
                [
                  0.7897623400365631,
                  0.009259259259259259,
                  432,
                  4
                ],
                [
                  0.7915904936014625,
                  0.009237875288683603,
                  433,
                  4
                ],
                [
                  0.793418647166362,
                  0.009216589861751152,
                  434,
                  4
                ],
                [
                  0.7952468007312614,
                  0.009195402298850575,
                  435,
                  4
                ],
                [
                  0.7970749542961609,
                  0.011467889908256881,
                  436,
                  5
                ],
                [
                  0.7989031078610603,
                  0.013729977116704805,
                  437,
                  6
                ],
                [
                  0.8007312614259597,
                  0.0136986301369863,
                  438,
                  6
                ],
                [
                  0.8025594149908593,
                  0.01366742596810934,
                  439,
                  6
                ],
                [
                  0.8043875685557587,
                  0.013636363636363636,
                  440,
                  6
                ],
                [
                  0.8062157221206582,
                  0.013605442176870748,
                  441,
                  6
                ],
                [
                  0.8080438756855576,
                  0.013574660633484163,
                  442,
                  6
                ],
                [
                  0.8098720292504571,
                  0.013544018058690745,
                  443,
                  6
                ],
                [
                  0.8117001828153565,
                  0.013513513513513514,
                  444,
                  6
                ],
                [
                  0.8135283363802559,
                  0.01348314606741573,
                  445,
                  6
                ],
                [
                  0.8153564899451554,
                  0.013452914798206279,
                  446,
                  6
                ],
                [
                  0.8171846435100548,
                  0.013422818791946308,
                  447,
                  6
                ],
                [
                  0.8190127970749543,
                  0.013392857142857142,
                  448,
                  6
                ],
                [
                  0.8208409506398537,
                  0.015590200445434299,
                  449,
                  7
                ],
                [
                  0.8226691042047533,
                  0.015555555555555555,
                  450,
                  7
                ],
                [
                  0.8244972577696527,
                  0.015521064301552107,
                  451,
                  7
                ],
                [
                  0.8263254113345521,
                  0.015486725663716814,
                  452,
                  7
                ],
                [
                  0.8281535648994516,
                  0.01545253863134658,
                  453,
                  7
                ],
                [
                  0.829981718464351,
                  0.015418502202643172,
                  454,
                  7
                ],
                [
                  0.8318098720292505,
                  0.015384615384615385,
                  455,
                  7
                ],
                [
                  0.8336380255941499,
                  0.015350877192982455,
                  456,
                  7
                ],
                [
                  0.8354661791590493,
                  0.015317286652078774,
                  457,
                  7
                ],
                [
                  0.8372943327239488,
                  0.015283842794759825,
                  458,
                  7
                ],
                [
                  0.8391224862888482,
                  0.015250544662309368,
                  459,
                  7
                ],
                [
                  0.8409506398537477,
                  0.017391304347826087,
                  460,
                  8
                ],
                [
                  0.8427787934186471,
                  0.01735357917570499,
                  461,
                  8
                ],
                [
                  0.8446069469835467,
                  0.017316017316017316,
                  462,
                  8
                ],
                [
                  0.8464351005484461,
                  0.019438444924406047,
                  463,
                  9
                ],
                [
                  0.8482632541133455,
                  0.01939655172413793,
                  464,
                  9
                ],
                [
                  0.850091407678245,
                  0.021505376344086023,
                  465,
                  10
                ],
                [
                  0.8519195612431444,
                  0.02145922746781116,
                  466,
                  10
                ],
                [
                  0.8537477148080439,
                  0.023554603854389723,
                  467,
                  11
                ],
                [
                  0.8555758683729433,
                  0.02564102564102564,
                  468,
                  12
                ],
                [
                  0.8574040219378428,
                  0.02771855010660981,
                  469,
                  13
                ],
                [
                  0.8592321755027422,
                  0.029787234042553193,
                  470,
                  14
                ],
                [
                  0.8610603290676416,
                  0.029723991507430998,
                  471,
                  14
                ],
                [
                  0.8628884826325411,
                  0.029661016949152543,
                  472,
                  14
                ],
                [
                  0.8647166361974405,
                  0.03171247357293869,
                  473,
                  15
                ],
                [
                  0.8665447897623401,
                  0.03164556962025317,
                  474,
                  15
                ],
                [
                  0.8683729433272395,
                  0.03368421052631579,
                  475,
                  16
                ],
                [
                  0.870201096892139,
                  0.03571428571428571,
                  476,
                  17
                ],
                [
                  0.8720292504570384,
                  0.03773584905660377,
                  477,
                  18
                ],
                [
                  0.8738574040219378,
                  0.03765690376569038,
                  478,
                  18
                ],
                [
                  0.8756855575868373,
                  0.03966597077244259,
                  479,
                  19
                ],
                [
                  0.8775137111517367,
                  0.03958333333333333,
                  480,
                  19
                ],
                [
                  0.8793418647166362,
                  0.04158004158004158,
                  481,
                  20
                ],
                [
                  0.8811700182815356,
                  0.04149377593360996,
                  482,
                  20
                ],
                [
                  0.8829981718464351,
                  0.041407867494824016,
                  483,
                  20
                ],
                [
                  0.8848263254113345,
                  0.04132231404958678,
                  484,
                  20
                ],
                [
                  0.886654478976234,
                  0.041237113402061855,
                  485,
                  20
                ],
                [
                  0.8884826325411335,
                  0.0411522633744856,
                  486,
                  20
                ],
                [
                  0.8903107861060329,
                  0.043121149897330596,
                  487,
                  21
                ],
                [
                  0.8921389396709324,
                  0.045081967213114756,
                  488,
                  22
                ],
                [
                  0.8939670932358318,
                  0.044989775051124746,
                  489,
                  22
                ],
                [
                  0.8957952468007313,
                  0.04693877551020408,
                  490,
                  23
                ],
                [
                  0.8976234003656307,
                  0.04684317718940937,
                  491,
                  23
                ],
                [
                  0.8994515539305301,
                  0.046747967479674794,
                  492,
                  23
                ],
                [
                  0.9012797074954296,
                  0.04665314401622718,
                  493,
                  23
                ],
                [
                  0.903107861060329,
                  0.0465587044534413,
                  494,
                  23
                ],
                [
                  0.9049360146252285,
                  0.048484848484848485,
                  495,
                  24
                ],
                [
                  0.906764168190128,
                  0.05040322580645161,
                  496,
                  25
                ],
                [
                  0.9085923217550275,
                  0.05030181086519115,
                  497,
                  25
                ],
                [
                  0.9104204753199269,
                  0.050200803212851405,
                  498,
                  25
                ],
                [
                  0.9122486288848263,
                  0.050100200400801605,
                  499,
                  25
                ],
                [
                  0.9140767824497258,
                  0.052,
                  500,
                  26
                ],
                [
                  0.9159049360146252,
                  0.05389221556886228,
                  501,
                  27
                ],
                [
                  0.9177330895795247,
                  0.053784860557768925,
                  502,
                  27
                ],
                [
                  0.9195612431444241,
                  0.055666003976143144,
                  503,
                  28
                ],
                [
                  0.9213893967093236,
                  0.05555555555555555,
                  504,
                  28
                ],
                [
                  0.923217550274223,
                  0.055445544554455446,
                  505,
                  28
                ],
                [
                  0.9250457038391224,
                  0.05731225296442688,
                  506,
                  29
                ],
                [
                  0.926873857404022,
                  0.05719921104536489,
                  507,
                  29
                ],
                [
                  0.9287020109689214,
                  0.05905511811023622,
                  508,
                  30
                ],
                [
                  0.9305301645338209,
                  0.05893909626719057,
                  509,
                  30
                ],
                [
                  0.9323583180987203,
                  0.058823529411764705,
                  510,
                  30
                ],
                [
                  0.9341864716636198,
                  0.05870841487279843,
                  511,
                  30
                ],
                [
                  0.9360146252285192,
                  0.05859375,
                  512,
                  30
                ],
                [
                  0.9378427787934186,
                  0.06042884990253411,
                  513,
                  31
                ],
                [
                  0.9396709323583181,
                  0.0622568093385214,
                  514,
                  32
                ],
                [
                  0.9414990859232175,
                  0.062135922330097085,
                  515,
                  32
                ],
                [
                  0.943327239488117,
                  0.06201550387596899,
                  516,
                  32
                ],
                [
                  0.9451553930530164,
                  0.06382978723404255,
                  517,
                  33
                ],
                [
                  0.946983546617916,
                  0.06563706563706563,
                  518,
                  34
                ],
                [
                  0.9488117001828154,
                  0.06551059730250482,
                  519,
                  34
                ],
                [
                  0.9506398537477148,
                  0.0673076923076923,
                  520,
                  35
                ],
                [
                  0.9524680073126143,
                  0.0690978886756238,
                  521,
                  36
                ],
                [
                  0.9542961608775137,
                  0.07088122605363985,
                  522,
                  37
                ],
                [
                  0.9561243144424132,
                  0.07265774378585087,
                  523,
                  38
                ],
                [
                  0.9579524680073126,
                  0.07442748091603053,
                  524,
                  39
                ],
                [
                  0.9597806215722121,
                  0.07428571428571429,
                  525,
                  39
                ],
                [
                  0.9616087751371115,
                  0.07604562737642585,
                  526,
                  40
                ],
                [
                  0.9634369287020109,
                  0.07590132827324478,
                  527,
                  40
                ],
                [
                  0.9652650822669104,
                  0.07575757575757576,
                  528,
                  40
                ],
                [
                  0.9670932358318098,
                  0.07750472589792061,
                  529,
                  41
                ],
                [
                  0.9689213893967094,
                  0.07735849056603773,
                  530,
                  41
                ],
                [
                  0.9707495429616088,
                  0.07721280602636535,
                  531,
                  41
                ],
                [
                  0.9725776965265083,
                  0.07894736842105263,
                  532,
                  42
                ],
                [
                  0.9744058500914077,
                  0.08067542213883677,
                  533,
                  43
                ],
                [
                  0.9762340036563071,
                  0.08052434456928839,
                  534,
                  43
                ],
                [
                  0.9780621572212066,
                  0.08037383177570094,
                  535,
                  43
                ],
                [
                  0.979890310786106,
                  0.08208955223880597,
                  536,
                  44
                ],
                [
                  0.9817184643510055,
                  0.08379888268156424,
                  537,
                  45
                ],
                [
                  0.9835466179159049,
                  0.08364312267657993,
                  538,
                  45
                ],
                [
                  0.9853747714808044,
                  0.08534322820037106,
                  539,
                  46
                ],
                [
                  0.9872029250457038,
                  0.08703703703703704,
                  540,
                  47
                ],
                [
                  0.9890310786106032,
                  0.08687615526802218,
                  541,
                  47
                ],
                [
                  0.9908592321755028,
                  0.08671586715867159,
                  542,
                  47
                ],
                [
                  0.9926873857404022,
                  0.08839779005524862,
                  543,
                  48
                ],
                [
                  0.9945155393053017,
                  0.08823529411764706,
                  544,
                  48
                ],
                [
                  0.9963436928702011,
                  0.08990825688073395,
                  545,
                  49
                ],
                [
                  0.9981718464351006,
                  0.08974358974358974,
                  546,
                  49
                ],
                [
                  1.0,
                  0.08957952468007313,
                  547,
                  49
                ]
              ]
            },
            {
              "language": "English",
              "run": 554,
              "points": [
                [
                  0.0018281535648994515,
                  0.0,
                  1,
                  0
                ],
                [
                  0.003656307129798903,
                  0.0,
                  2,
                  0
                ],
                [
                  0.005484460694698354,
                  0.0,
                  3,
                  0
                ],
                [
                  0.007312614259597806,
                  0.0,
                  4,
                  0
                ],
                [
                  0.009140767824497258,
                  0.0,
                  5,
                  0
                ],
                [
                  0.010968921389396709,
                  0.0,
                  6,
                  0
                ],
                [
                  0.012797074954296161,
                  0.0,
                  7,
                  0
                ],
                [
                  0.014625228519195612,
                  0.0,
                  8,
                  0
                ],
                [
                  0.016453382084095063,
                  0.0,
                  9,
                  0
                ],
                [
                  0.018281535648994516,
                  0.0,
                  10,
                  0
                ],
                [
                  0.02010968921389397,
                  0.0,
                  11,
                  0
                ],
                [
                  0.021937842778793418,
                  0.0,
                  12,
                  0
                ],
                [
                  0.02376599634369287,
                  0.0,
                  13,
                  0
                ],
                [
                  0.025594149908592323,
                  0.0,
                  14,
                  0
                ],
                [
                  0.027422303473491772,
                  0.0,
                  15,
                  0
                ],
                [
                  0.029250457038391225,
                  0.0,
                  16,
                  0
                ],
                [
                  0.031078610603290677,
                  0.0,
                  17,
                  0
                ],
                [
                  0.03290676416819013,
                  0.0,
                  18,
                  0
                ],
                [
                  0.03473491773308958,
                  0.0,
                  19,
                  0
                ],
                [
                  0.03656307129798903,
                  0.0,
                  20,
                  0
                ],
                [
                  0.038391224862888484,
                  0.0,
                  21,
                  0
                ],
                [
                  0.04021937842778794,
                  0.0,
                  22,
                  0
                ],
                [
                  0.04204753199268738,
                  0.0,
                  23,
                  0
                ],
                [
                  0.043875685557586835,
                  0.0,
                  24,
                  0
                ],
                [
                  0.04570383912248629,
                  0.0,
                  25,
                  0
                ],
                [
                  0.04753199268738574,
                  0.0,
                  26,
                  0
                ],
                [
                  0.04936014625228519,
                  0.0,
                  27,
                  0
                ],
                [
                  0.051188299817184646,
                  0.0,
                  28,
                  0
                ],
                [
                  0.05301645338208409,
                  0.0,
                  29,
                  0
                ],
                [
                  0.054844606946983544,
                  0.0,
                  30,
                  0
                ],
                [
                  0.056672760511883,
                  0.0,
                  31,
                  0
                ],
                [
                  0.05850091407678245,
                  0.0,
                  32,
                  0
                ],
                [
                  0.0603290676416819,
                  0.0,
                  33,
                  0
                ],
                [
                  0.062157221206581355,
                  0.0,
                  34,
                  0
                ],
                [
                  0.06398537477148081,
                  0.0,
                  35,
                  0
                ],
                [
                  0.06581352833638025,
                  0.0,
                  36,
                  0
                ],
                [
                  0.06764168190127971,
                  0.0,
                  37,
                  0
                ],
                [
                  0.06946983546617916,
                  0.0,
                  38,
                  0
                ],
                [
                  0.0712979890310786,
                  0.0,
                  39,
                  0
                ],
                [
                  0.07312614259597806,
                  0.0,
                  40,
                  0
                ],
                [
                  0.07495429616087751,
                  0.0,
                  41,
                  0
                ],
                [
                  0.07678244972577697,
                  0.0,
                  42,
                  0
                ],
                [
                  0.07861060329067641,
                  0.0,
                  43,
                  0
                ],
                [
                  0.08043875685557587,
                  0.0,
                  44,
                  0
                ],
                [
                  0.08226691042047532,
                  0.0,
                  45,
                  0
                ],
                [
                  0.08409506398537477,
                  0.0,
                  46,
                  0
                ],
                [
                  0.08592321755027423,
                  0.0,
                  47,
                  0
                ],
                [
                  0.08775137111517367,
                  0.0,
                  48,
                  0
                ],
                [
                  0.08957952468007313,
                  0.0,
                  49,
                  0
                ],
                [
                  0.09140767824497258,
                  0.0,
                  50,
                  0
                ],
                [
                  0.09323583180987204,
                  0.0,
                  51,
                  0
                ],
                [
                  0.09506398537477148,
                  0.0,
                  52,
                  0
                ],
                [
                  0.09689213893967093,
                  0.0,
                  53,
                  0
                ],
                [
                  0.09872029250457039,
                  0.0,
                  54,
                  0
                ],
                [
                  0.10054844606946983,
                  0.0,
                  55,
                  0
                ],
                [
                  0.10237659963436929,
                  0.0,
                  56,
                  0
                ],
                [
                  0.10420475319926874,
                  0.0,
                  57,
                  0
                ],
                [
                  0.10603290676416818,
                  0.0,
                  58,
                  0
                ],
                [
                  0.10786106032906764,
                  0.0,
                  59,
                  0
                ],
                [
                  0.10968921389396709,
                  0.0,
                  60,
                  0
                ],
                [
                  0.11151736745886655,
                  0.0,
                  61,
                  0
                ],
                [
                  0.113345521023766,
                  0.0,
                  62,
                  0
                ],
                [
                  0.11517367458866545,
                  0.0,
                  63,
                  0
                ],
                [
                  0.1170018281535649,
                  0.0,
                  64,
                  0
                ],
                [
                  0.11882998171846434,
                  0.0,
                  65,
                  0
                ],
                [
                  0.1206581352833638,
                  0.0,
                  66,
                  0
                ],
                [
                  0.12248628884826325,
                  0.0,
                  67,
                  0
                ],
                [
                  0.12431444241316271,
                  0.0,
                  68,
                  0
                ],
                [
                  0.12614259597806216,
                  0.0,
                  69,
                  0
                ],
                [
                  0.12797074954296161,
                  0.0,
                  70,
                  0
                ],
                [
                  0.12979890310786105,
                  0.0,
                  71,
                  0
                ],
                [
                  0.1316270566727605,
                  0.0,
                  72,
                  0
                ],
                [
                  0.13345521023765997,
                  0.0,
                  73,
                  0
                ],
                [
                  0.13528336380255943,
                  0.0,
                  74,
                  0
                ],
                [
                  0.13711151736745886,
                  0.0,
                  75,
                  0
                ],
                [
                  0.13893967093235832,
                  0.0,
                  76,
                  0
                ],
                [
                  0.14076782449725778,
                  0.0,
                  77,
                  0
                ],
                [
                  0.1425959780621572,
                  0.0,
                  78,
                  0
                ],
                [
                  0.14442413162705667,
                  0.0,
                  79,
                  0
                ],
                [
                  0.14625228519195613,
                  0.0,
                  80,
                  0
                ],
                [
                  0.1480804387568556,
                  0.0,
                  81,
                  0
                ],
                [
                  0.14990859232175502,
                  0.0,
                  82,
                  0
                ],
                [
                  0.15173674588665448,
                  0.0,
                  83,
                  0
                ],
                [
                  0.15356489945155394,
                  0.0,
                  84,
                  0
                ],
                [
                  0.15539305301645337,
                  0.0,
                  85,
                  0
                ],
                [
                  0.15722120658135283,
                  0.0,
                  86,
                  0
                ],
                [
                  0.1590493601462523,
                  0.0,
                  87,
                  0
                ],
                [
                  0.16087751371115175,
                  0.0,
                  88,
                  0
                ],
                [
                  0.16270566727605118,
                  0.0,
                  89,
                  0
                ],
                [
                  0.16453382084095064,
                  0.0,
                  90,
                  0
                ],
                [
                  0.1663619744058501,
                  0.0,
                  91,
                  0
                ],
                [
                  0.16819012797074953,
                  0.0,
                  92,
                  0
                ],
                [
                  0.170018281535649,
                  0.0,
                  93,
                  0
                ],
                [
                  0.17184643510054845,
                  0.0,
                  94,
                  0
                ],
                [
                  0.1736745886654479,
                  0.0,
                  95,
                  0
                ],
                [
                  0.17550274223034734,
                  0.0,
                  96,
                  0
                ],
                [
                  0.1773308957952468,
                  0.0,
                  97,
                  0
                ],
                [
                  0.17915904936014626,
                  0.0,
                  98,
                  0
                ],
                [
                  0.1809872029250457,
                  0.0,
                  99,
                  0
                ],
                [
                  0.18281535648994515,
                  0.0,
                  100,
                  0
                ],
                [
                  0.1846435100548446,
                  0.0,
                  101,
                  0
                ],
                [
                  0.18647166361974407,
                  0.0,
                  102,
                  0
                ],
                [
                  0.1882998171846435,
                  0.0,
                  103,
                  0
                ],
                [
                  0.19012797074954296,
                  0.0,
                  104,
                  0
                ],
                [
                  0.19195612431444242,
                  0.0,
                  105,
                  0
                ],
                [
                  0.19378427787934185,
                  0.0,
                  106,
                  0
                ],
                [
                  0.1956124314442413,
                  0.0,
                  107,
                  0
                ],
                [
                  0.19744058500914077,
                  0.0,
                  108,
                  0
                ],
                [
                  0.19926873857404023,
                  0.0,
                  109,
                  0
                ],
                [
                  0.20109689213893966,
                  0.0,
                  110,
                  0
                ],
                [
                  0.20292504570383912,
                  0.0,
                  111,
                  0
                ],
                [
                  0.20475319926873858,
                  0.0,
                  112,
                  0
                ],
                [
                  0.20658135283363802,
                  0.0,
                  113,
                  0
                ],
                [
                  0.20840950639853748,
                  0.0,
                  114,
                  0
                ],
                [
                  0.21023765996343693,
                  0.0,
                  115,
                  0
                ],
                [
                  0.21206581352833637,
                  0.0,
                  116,
                  0
                ],
                [
                  0.21389396709323583,
                  0.0,
                  117,
                  0
                ],
                [
                  0.21572212065813529,
                  0.0,
                  118,
                  0
                ],
                [
                  0.21755027422303475,
                  0.0,
                  119,
                  0
                ],
                [
                  0.21937842778793418,
                  0.0,
                  120,
                  0
                ],
                [
                  0.22120658135283364,
                  0.0,
                  121,
                  0
                ],
                [
                  0.2230347349177331,
                  0.0,
                  122,
                  0
                ],
                [
                  0.22486288848263253,
                  0.0,
                  123,
                  0
                ],
                [
                  0.226691042047532,
                  0.0,
                  124,
                  0
                ],
                [
                  0.22851919561243145,
                  0.0,
                  125,
                  0
                ],
                [
                  0.2303473491773309,
                  0.0,
                  126,
                  0
                ],
                [
                  0.23217550274223034,
                  0.0,
                  127,
                  0
                ],
                [
                  0.2340036563071298,
                  0.0,
                  128,
                  0
                ],
                [
                  0.23583180987202926,
                  0.0,
                  129,
                  0
                ],
                [
                  0.2376599634369287,
                  0.0,
                  130,
                  0
                ],
                [
                  0.23948811700182815,
                  0.0,
                  131,
                  0
                ],
                [
                  0.2413162705667276,
                  0.0,
                  132,
                  0
                ],
                [
                  0.24314442413162707,
                  0.0,
                  133,
                  0
                ],
                [
                  0.2449725776965265,
                  0.0,
                  134,
                  0
                ],
                [
                  0.24680073126142596,
                  0.0,
                  135,
                  0
                ],
                [
                  0.24862888482632542,
                  0.0,
                  136,
                  0
                ],
                [
                  0.25045703839122485,
                  0.0,
                  137,
                  0
                ],
                [
                  0.2522851919561243,
                  0.0,
                  138,
                  0
                ],
                [
                  0.25411334552102377,
                  0.0,
                  139,
                  0
                ],
                [
                  0.25594149908592323,
                  0.0,
                  140,
                  0
                ],
                [
                  0.2577696526508227,
                  0.0,
                  141,
                  0
                ],
                [
                  0.2595978062157221,
                  0.0,
                  142,
                  0
                ],
                [
                  0.26142595978062155,
                  0.0,
                  143,
                  0
                ],
                [
                  0.263254113345521,
                  0.0,
                  144,
                  0
                ],
                [
                  0.26508226691042047,
                  0.0,
                  145,
                  0
                ],
                [
                  0.26691042047531993,
                  0.0,
                  146,
                  0
                ],
                [
                  0.2687385740402194,
                  0.0,
                  147,
                  0
                ],
                [
                  0.27056672760511885,
                  0.0,
                  148,
                  0
                ],
                [
                  0.27239488117001825,
                  0.0,
                  149,
                  0
                ],
                [
                  0.2742230347349177,
                  0.0,
                  150,
                  0
                ],
                [
                  0.2760511882998172,
                  0.0,
                  151,
                  0
                ],
                [
                  0.27787934186471663,
                  0.0,
                  152,
                  0
                ],
                [
                  0.2797074954296161,
                  0.0,
                  153,
                  0
                ],
                [
                  0.28153564899451555,
                  0.0,
                  154,
                  0
                ],
                [
                  0.283363802559415,
                  0.0,
                  155,
                  0
                ],
                [
                  0.2851919561243144,
                  0.0,
                  156,
                  0
                ],
                [
                  0.2870201096892139,
                  0.0,
                  157,
                  0
                ],
                [
                  0.28884826325411334,
                  0.0,
                  158,
                  0
                ],
                [
                  0.2906764168190128,
                  0.0,
                  159,
                  0
                ],
                [
                  0.29250457038391225,
                  0.0,
                  160,
                  0
                ],
                [
                  0.2943327239488117,
                  0.0,
                  161,
                  0
                ],
                [
                  0.2961608775137112,
                  0.0,
                  162,
                  0
                ],
                [
                  0.2979890310786106,
                  0.0,
                  163,
                  0
                ],
                [
                  0.29981718464351004,
                  0.0,
                  164,
                  0
                ],
                [
                  0.3016453382084095,
                  0.0,
                  165,
                  0
                ],
                [
                  0.30347349177330896,
                  0.0,
                  166,
                  0
                ],
                [
                  0.3053016453382084,
                  0.0,
                  167,
                  0
                ],
                [
                  0.3071297989031079,
                  0.0,
                  168,
                  0
                ],
                [
                  0.30895795246800734,
                  0.0,
                  169,
                  0
                ],
                [
                  0.31078610603290674,
                  0.0,
                  170,
                  0
                ],
                [
                  0.3126142595978062,
                  0.0,
                  171,
                  0
                ],
                [
                  0.31444241316270566,
                  0.0,
                  172,
                  0
                ],
                [
                  0.3162705667276051,
                  0.0,
                  173,
                  0
                ],
                [
                  0.3180987202925046,
                  0.0,
                  174,
                  0
                ],
                [
                  0.31992687385740404,
                  0.0,
                  175,
                  0
                ],
                [
                  0.3217550274223035,
                  0.0,
                  176,
                  0
                ],
                [
                  0.3235831809872029,
                  0.0,
                  177,
                  0
                ],
                [
                  0.32541133455210236,
                  0.0,
                  178,
                  0
                ],
                [
                  0.3272394881170018,
                  0.0,
                  179,
                  0
                ],
                [
                  0.3290676416819013,
                  0.0,
                  180,
                  0
                ],
                [
                  0.33089579524680074,
                  0.0,
                  181,
                  0
                ],
                [
                  0.3327239488117002,
                  0.0,
                  182,
                  0
                ],
                [
                  0.33455210237659966,
                  0.0,
                  183,
                  0
                ],
                [
                  0.33638025594149906,
                  0.0,
                  184,
                  0
                ],
                [
                  0.3382084095063985,
                  0.0,
                  185,
                  0
                ],
                [
                  0.340036563071298,
                  0.0,
                  186,
                  0
                ],
                [
                  0.34186471663619744,
                  0.0,
                  187,
                  0
                ],
                [
                  0.3436928702010969,
                  0.0,
                  188,
                  0
                ],
                [
                  0.34552102376599636,
                  0.0,
                  189,
                  0
                ],
                [
                  0.3473491773308958,
                  0.0,
                  190,
                  0
                ],
                [
                  0.3491773308957952,
                  0.0,
                  191,
                  0
                ],
                [
                  0.3510054844606947,
                  0.0,
                  192,
                  0
                ],
                [
                  0.35283363802559414,
                  0.0,
                  193,
                  0
                ],
                [
                  0.3546617915904936,
                  0.0,
                  194,
                  0
                ],
                [
                  0.35648994515539306,
                  0.0,
                  195,
                  0
                ],
                [
                  0.3583180987202925,
                  0.0,
                  196,
                  0
                ],
                [
                  0.360146252285192,
                  0.0,
                  197,
                  0
                ],
                [
                  0.3619744058500914,
                  0.0,
                  198,
                  0
                ],
                [
                  0.36380255941499084,
                  0.0,
                  199,
                  0
                ],
                [
                  0.3656307129798903,
                  0.0,
                  200,
                  0
                ],
                [
                  0.36745886654478976,
                  0.0,
                  201,
                  0
                ],
                [
                  0.3692870201096892,
                  0.0,
                  202,
                  0
                ],
                [
                  0.3711151736745887,
                  0.0,
                  203,
                  0
                ],
                [
                  0.37294332723948814,
                  0.0,
                  204,
                  0
                ],
                [
                  0.37477148080438755,
                  0.0,
                  205,
                  0
                ],
                [
                  0.376599634369287,
                  0.0,
                  206,
                  0
                ],
                [
                  0.37842778793418647,
                  0.0,
                  207,
                  0
                ],
                [
                  0.3802559414990859,
                  0.0,
                  208,
                  0
                ],
                [
                  0.3820840950639854,
                  0.0,
                  209,
                  0
                ],
                [
                  0.38391224862888484,
                  0.0,
                  210,
                  0
                ],
                [
                  0.3857404021937843,
                  0.0,
                  211,
                  0
                ],
                [
                  0.3875685557586837,
                  0.0,
                  212,
                  0
                ],
                [
                  0.38939670932358317,
                  0.0,
                  213,
                  0
                ],
                [
                  0.3912248628884826,
                  0.0,
                  214,
                  0
                ],
                [
                  0.3930530164533821,
                  0.0,
                  215,
                  0
                ],
                [
                  0.39488117001828155,
                  0.0,
                  216,
                  0
                ],
                [
                  0.396709323583181,
                  0.0,
                  217,
                  0
                ],
                [
                  0.39853747714808047,
                  0.0,
                  218,
                  0
                ],
                [
                  0.40036563071297987,
                  0.0,
                  219,
                  0
                ],
                [
                  0.40219378427787933,
                  0.0,
                  220,
                  0
                ],
                [
                  0.4040219378427788,
                  0.0,
                  221,
                  0
                ],
                [
                  0.40585009140767825,
                  0.0,
                  222,
                  0
                ],
                [
                  0.4076782449725777,
                  0.0,
                  223,
                  0
                ],
                [
                  0.40950639853747717,
                  0.0,
                  224,
                  0
                ],
                [
                  0.4113345521023766,
                  0.0044444444444444444,
                  225,
                  1
                ],
                [
                  0.41316270566727603,
                  0.004424778761061947,
                  226,
                  1
                ],
                [
                  0.4149908592321755,
                  0.004405286343612335,
                  227,
                  1
                ],
                [
                  0.41681901279707495,
                  0.0043859649122807015,
                  228,
                  1
                ],
                [
                  0.4186471663619744,
                  0.004366812227074236,
                  229,
                  1
                ],
                [
                  0.42047531992687387,
                  0.004347826086956522,
                  230,
                  1
                ],
                [
                  0.42230347349177333,
                  0.004329004329004329,
                  231,
                  1
                ],
                [
                  0.42413162705667273,
                  0.004310344827586207,
                  232,
                  1
                ],
                [
                  0.4259597806215722,
                  0.004291845493562232,
                  233,
                  1
                ],
                [
                  0.42778793418647165,
                  0.004273504273504274,
                  234,
                  1
                ],
                [
                  0.4296160877513711,
                  0.00425531914893617,
                  235,
                  1
                ],
                [
                  0.43144424131627057,
                  0.00423728813559322,
                  236,
                  1
                ],
                [
                  0.43327239488117003,
                  0.004219409282700422,
                  237,
                  1
                ],
                [
                  0.4351005484460695,
                  0.004201680672268907,
                  238,
                  1
                ],
                [
                  0.4369287020109689,
                  0.0041841004184100415,
                  239,
                  1
                ],
                [
                  0.43875685557586835,
                  0.004166666666666667,
                  240,
                  1
                ],
                [
                  0.4405850091407678,
                  0.004149377593360996,
                  241,
                  1
                ],
                [
                  0.4424131627056673,
                  0.004132231404958678,
                  242,
                  1
                ],
                [
                  0.44424131627056673,
                  0.00411522633744856,
                  243,
                  1
                ],
                [
                  0.4460694698354662,
                  0.004098360655737705,
                  244,
                  1
                ],
                [
                  0.44789762340036565,
                  0.004081632653061225,
                  245,
                  1
                ],
                [
                  0.44972577696526506,
                  0.0040650406504065045,
                  246,
                  1
                ],
                [
                  0.4515539305301645,
                  0.004048582995951417,
                  247,
                  1
                ],
                [
                  0.453382084095064,
                  0.004032258064516129,
                  248,
                  1
                ],
                [
                  0.45521023765996343,
                  0.004016064257028112,
                  249,
                  1
                ],
                [
                  0.4570383912248629,
                  0.004,
                  250,
                  1
                ],
                [
                  0.45886654478976235,
                  0.00398406374501992,
                  251,
                  1
                ],
                [
                  0.4606946983546618,
                  0.003968253968253968,
                  252,
                  1
                ],
                [
                  0.4625228519195612,
                  0.003952569169960474,
                  253,
                  1
                ],
                [
                  0.4643510054844607,
                  0.003937007874015748,
                  254,
                  1
                ],
                [
                  0.46617915904936014,
                  0.00392156862745098,
                  255,
                  1
                ],
                [
                  0.4680073126142596,
                  0.00390625,
                  256,
                  1
                ],
                [
                  0.46983546617915906,
                  0.0038910505836575876,
                  257,
                  1
                ],
                [
                  0.4716636197440585,
                  0.003875968992248062,
                  258,
                  1
                ],
                [
                  0.473491773308958,
                  0.003861003861003861,
                  259,
                  1
                ],
                [
                  0.4753199268738574,
                  0.0038461538461538464,
                  260,
                  1
                ],
                [
                  0.47714808043875684,
                  0.0038314176245210726,
                  261,
                  1
                ],
                [
                  0.4789762340036563,
                  0.003816793893129771,
                  262,
                  1
                ],
                [
                  0.48080438756855576,
                  0.0038022813688212928,
                  263,
                  1
                ],
                [
                  0.4826325411334552,
                  0.003787878787878788,
                  264,
                  1
                ],
                [
                  0.4844606946983547,
                  0.0037735849056603774,
                  265,
                  1
                ],
                [
                  0.48628884826325414,
                  0.0037593984962406013,
                  266,
                  1
                ],
                [
                  0.48811700182815354,
                  0.003745318352059925,
                  267,
                  1
                ],
                [
                  0.489945155393053,
                  0.0037313432835820895,
                  268,
                  1
                ],
                [
                  0.49177330895795246,
                  0.0037174721189591076,
                  269,
                  1
                ],
                [
                  0.4936014625228519,
                  0.003703703703703704,
                  270,
                  1
                ],
                [
                  0.4954296160877514,
                  0.0036900369003690036,
                  271,
                  1
                ],
                [
                  0.49725776965265084,
                  0.003676470588235294,
                  272,
                  1
                ],
                [
                  0.4990859232175503,
                  0.003663003663003663,
                  273,
                  1
                ],
                [
                  0.5009140767824497,
                  0.0036496350364963502,
                  274,
                  1
                ],
                [
                  0.5027422303473492,
                  0.0036363636363636364,
                  275,
                  1
                ],
                [
                  0.5045703839122486,
                  0.0036231884057971015,
                  276,
                  1
                ],
                [
                  0.506398537477148,
                  0.0036101083032490976,
                  277,
                  1
                ],
                [
                  0.5082266910420475,
                  0.0035971223021582736,
                  278,
                  1
                ],
                [
                  0.5100548446069469,
                  0.0035842293906810036,
                  279,
                  1
                ],
                [
                  0.5118829981718465,
                  0.0035714285714285713,
                  280,
                  1
                ],
                [
                  0.5137111517367459,
                  0.0035587188612099642,
                  281,
                  1
                ],
                [
                  0.5155393053016454,
                  0.0035460992907801418,
                  282,
                  1
                ],
                [
                  0.5173674588665448,
                  0.007067137809187279,
                  283,
                  2
                ],
                [
                  0.5191956124314442,
                  0.007042253521126761,
                  284,
                  2
                ],
                [
                  0.5210237659963437,
                  0.007017543859649123,
                  285,
                  2
                ],
                [
                  0.5228519195612431,
                  0.006993006993006993,
                  286,
                  2
                ],
                [
                  0.5246800731261426,
                  0.006968641114982578,
                  287,
                  2
                ],
                [
                  0.526508226691042,
                  0.006944444444444444,
                  288,
                  2
                ],
                [
                  0.5283363802559415,
                  0.006920415224913495,
                  289,
                  2
                ],
                [
                  0.5301645338208409,
                  0.006896551724137931,
                  290,
                  2
                ],
                [
                  0.5319926873857403,
                  0.006872852233676976,
                  291,
                  2
                ],
                [
                  0.5338208409506399,
                  0.00684931506849315,
                  292,
                  2
                ],
                [
                  0.5356489945155393,
                  0.006825938566552901,
                  293,
                  2
                ],
                [
                  0.5374771480804388,
                  0.006802721088435374,
                  294,
                  2
                ],
                [
                  0.5393053016453382,
                  0.006779661016949152,
                  295,
                  2
                ],
                [
                  0.5411334552102377,
                  0.006756756756756757,
                  296,
                  2
                ],
                [
                  0.5429616087751371,
                  0.006734006734006734,
                  297,
                  2
                ],
                [
                  0.5447897623400365,
                  0.006711409395973154,
                  298,
                  2
                ],
                [
                  0.546617915904936,
                  0.006688963210702341,
                  299,
                  2
                ],
                [
                  0.5484460694698354,
                  0.006666666666666667,
                  300,
                  2
                ],
                [
                  0.5502742230347349,
                  0.006644518272425249,
                  301,
                  2
                ],
                [
                  0.5521023765996343,
                  0.006622516556291391,
                  302,
                  2
                ],
                [
                  0.5539305301645339,
                  0.006600660066006601,
                  303,
                  2
                ],
                [
                  0.5557586837294333,
                  0.006578947368421052,
                  304,
                  2
                ],
                [
                  0.5575868372943327,
                  0.006557377049180328,
                  305,
                  2
                ],
                [
                  0.5594149908592322,
                  0.006535947712418301,
                  306,
                  2
                ],
                [
                  0.5612431444241316,
                  0.006514657980456026,
                  307,
                  2
                ],
                [
                  0.5630712979890311,
                  0.006493506493506494,
                  308,
                  2
                ],
                [
                  0.5648994515539305,
                  0.006472491909385114,
                  309,
                  2
                ],
                [
                  0.56672760511883,
                  0.0064516129032258064,
                  310,
                  2
                ],
                [
                  0.5685557586837294,
                  0.006430868167202572,
                  311,
                  2
                ],
                [
                  0.5703839122486288,
                  0.00641025641025641,
                  312,
                  2
                ],
                [
                  0.5722120658135283,
                  0.006389776357827476,
                  313,
                  2
                ],
                [
                  0.5740402193784278,
                  0.006369426751592357,
                  314,
                  2
                ],
                [
                  0.5758683729433273,
                  0.006349206349206349,
                  315,
                  2
                ],
                [
                  0.5776965265082267,
                  0.006329113924050633,
                  316,
                  2
                ],
                [
                  0.5795246800731262,
                  0.006309148264984227,
                  317,
                  2
                ],
                [
                  0.5813528336380256,
                  0.006289308176100629,
                  318,
                  2
                ],
                [
                  0.583180987202925,
                  0.006269592476489028,
                  319,
                  2
                ],
                [
                  0.5850091407678245,
                  0.00625,
                  320,
                  2
                ],
                [
                  0.5868372943327239,
                  0.006230529595015576,
                  321,
                  2
                ],
                [
                  0.5886654478976234,
                  0.006211180124223602,
                  322,
                  2
                ],
                [
                  0.5904936014625228,
                  0.006191950464396285,
                  323,
                  2
                ],
                [
                  0.5923217550274223,
                  0.006172839506172839,
                  324,
                  2
                ],
                [
                  0.5941499085923218,
                  0.006153846153846154,
                  325,
                  2
                ],
                [
                  0.5959780621572212,
                  0.006134969325153374,
                  326,
                  2
                ],
                [
                  0.5978062157221207,
                  0.0061162079510703364,
                  327,
                  2
                ],
                [
                  0.5996343692870201,
                  0.006097560975609756,
                  328,
                  2
                ],
                [
                  0.6014625228519196,
                  0.0060790273556231,
                  329,
                  2
                ],
                [
                  0.603290676416819,
                  0.006060606060606061,
                  330,
                  2
                ],
                [
                  0.6051188299817185,
                  0.006042296072507553,
                  331,
                  2
                ],
                [
                  0.6069469835466179,
                  0.006024096385542169,
                  332,
                  2
                ],
                [
                  0.6087751371115173,
                  0.006006006006006006,
                  333,
                  2
                ],
                [
                  0.6106032906764168,
                  0.005988023952095809,
                  334,
                  2
                ],
                [
                  0.6124314442413162,
                  0.005970149253731343,
                  335,
                  2
                ],
                [
                  0.6142595978062158,
                  0.005952380952380952,
                  336,
                  2
                ],
                [
                  0.6160877513711152,
                  0.005934718100890208,
                  337,
                  2
                ],
                [
                  0.6179159049360147,
                  0.005917159763313609,
                  338,
                  2
                ],
                [
                  0.6197440585009141,
                  0.0058997050147492625,
                  339,
                  2
                ],
                [
                  0.6215722120658135,
                  0.0058823529411764705,
                  340,
                  2
                ],
                [
                  0.623400365630713,
                  0.005865102639296188,
                  341,
                  2
                ],
                [
                  0.6252285191956124,
                  0.005847953216374269,
                  342,
                  2
                ],
                [
                  0.6270566727605119,
                  0.0058309037900874635,
                  343,
                  2
                ],
                [
                  0.6288848263254113,
                  0.005813953488372093,
                  344,
                  2
                ],
                [
                  0.6307129798903108,
                  0.005797101449275362,
                  345,
                  2
                ],
                [
                  0.6325411334552102,
                  0.005780346820809248,
                  346,
                  2
                ],
                [
                  0.6343692870201096,
                  0.005763688760806916,
                  347,
                  2
                ],
                [
                  0.6361974405850092,
                  0.005747126436781609,
                  348,
                  2
                ],
                [
                  0.6380255941499086,
                  0.0057306590257879654,
                  349,
                  2
                ],
                [
                  0.6398537477148081,
                  0.005714285714285714,
                  350,
                  2
                ],
                [
                  0.6416819012797075,
                  0.005698005698005698,
                  351,
                  2
                ],
                [
                  0.643510054844607,
                  0.005681818181818182,
                  352,
                  2
                ],
                [
                  0.6453382084095064,
                  0.0056657223796034,
                  353,
                  2
                ],
                [
                  0.6471663619744058,
                  0.005649717514124294,
                  354,
                  2
                ],
                [
                  0.6489945155393053,
                  0.005633802816901409,
                  355,
                  2
                ],
                [
                  0.6508226691042047,
                  0.0056179775280898875,
                  356,
                  2
                ],
                [
                  0.6526508226691042,
                  0.0056022408963585435,
                  357,
                  2
                ],
                [
                  0.6544789762340036,
                  0.00558659217877095,
                  358,
                  2
                ],
                [
                  0.6563071297989032,
                  0.005571030640668524,
                  359,
                  2
                ],
                [
                  0.6581352833638026,
                  0.005555555555555556,
                  360,
                  2
                ],
                [
                  0.659963436928702,
                  0.00554016620498615,
                  361,
                  2
                ],
                [
                  0.6617915904936015,
                  0.0055248618784530384,
                  362,
                  2
                ],
                [
                  0.6636197440585009,
                  0.005509641873278237,
                  363,
                  2
                ],
                [
                  0.6654478976234004,
                  0.005494505494505495,
                  364,
                  2
                ],
                [
                  0.6672760511882998,
                  0.005479452054794521,
                  365,
                  2
                ],
                [
                  0.6691042047531993,
                  0.00546448087431694,
                  366,
                  2
                ],
                [
                  0.6709323583180987,
                  0.005449591280653951,
                  367,
                  2
                ],
                [
                  0.6727605118829981,
                  0.005434782608695652,
                  368,
                  2
                ],
                [
                  0.6745886654478976,
                  0.005420054200542005,
                  369,
                  2
                ],
                [
                  0.676416819012797,
                  0.008108108108108109,
                  370,
                  3
                ],
                [
                  0.6782449725776966,
                  0.008086253369272238,
                  371,
                  3
                ],
                [
                  0.680073126142596,
                  0.008064516129032258,
                  372,
                  3
                ],
                [
                  0.6819012797074955,
                  0.00804289544235925,
                  373,
                  3
                ],
                [
                  0.6837294332723949,
                  0.008021390374331552,
                  374,
                  3
                ],
                [
                  0.6855575868372943,
                  0.008,
                  375,
                  3
                ],
                [
                  0.6873857404021938,
                  0.010638297872340425,
                  376,
                  4
                ],
                [
                  0.6892138939670932,
                  0.010610079575596816,
                  377,
                  4
                ],
                [
                  0.6910420475319927,
                  0.010582010582010581,
                  378,
                  4
                ],
                [
                  0.6928702010968921,
                  0.010554089709762533,
                  379,
                  4
                ],
                [
                  0.6946983546617916,
                  0.010526315789473684,
                  380,
                  4
                ],
                [
                  0.696526508226691,
                  0.010498687664041995,
                  381,
                  4
                ],
                [
                  0.6983546617915904,
                  0.010471204188481676,
                  382,
                  4
                ],
                [
                  0.70018281535649,
                  0.010443864229765013,
                  383,
                  4
                ],
                [
                  0.7020109689213894,
                  0.010416666666666666,
                  384,
                  4
                ],
                [
                  0.7038391224862889,
                  0.01038961038961039,
                  385,
                  4
                ],
                [
                  0.7056672760511883,
                  0.012953367875647668,
                  386,
                  5
                ],
                [
                  0.7074954296160878,
                  0.015503875968992248,
                  387,
                  6
                ],
                [
                  0.7093235831809872,
                  0.015463917525773196,
                  388,
                  6
                ],
                [
                  0.7111517367458866,
                  0.015424164524421594,
                  389,
                  6
                ],
                [
                  0.7129798903107861,
                  0.015384615384615385,
                  390,
                  6
                ],
                [
                  0.7148080438756855,
                  0.015345268542199489,
                  391,
                  6
                ],
                [
                  0.716636197440585,
                  0.015306122448979591,
                  392,
                  6
                ],
                [
                  0.7184643510054844,
                  0.015267175572519083,
                  393,
                  6
                ],
                [
                  0.720292504570384,
                  0.015228426395939087,
                  394,
                  6
                ],
                [
                  0.7221206581352834,
                  0.015189873417721518,
                  395,
                  6
                ],
                [
                  0.7239488117001828,
                  0.015151515151515152,
                  396,
                  6
                ],
                [
                  0.7257769652650823,
                  0.015113350125944584,
                  397,
                  6
                ],
                [
                  0.7276051188299817,
                  0.01507537688442211,
                  398,
                  6
                ],
                [
                  0.7294332723948812,
                  0.015037593984962405,
                  399,
                  6
                ],
                [
                  0.7312614259597806,
                  0.0175,
                  400,
                  7
                ],
                [
                  0.7330895795246801,
                  0.017456359102244388,
                  401,
                  7
                ],
                [
                  0.7349177330895795,
                  0.017412935323383085,
                  402,
                  7
                ],
                [
                  0.7367458866544789,
                  0.017369727047146403,
                  403,
                  7
                ],
                [
                  0.7385740402193784,
                  0.017326732673267328,
                  404,
                  7
                ],
                [
                  0.7404021937842779,
                  0.01728395061728395,
                  405,
                  7
                ],
                [
                  0.7422303473491774,
                  0.017241379310344827,
                  406,
                  7
                ],
                [
                  0.7440585009140768,
                  0.0171990171990172,
                  407,
                  7
                ],
                [
                  0.7458866544789763,
                  0.01715686274509804,
                  408,
                  7
                ],
                [
                  0.7477148080438757,
                  0.017114914425427872,
                  409,
                  7
                ],
                [
                  0.7495429616087751,
                  0.01707317073170732,
                  410,
                  7
                ],
                [
                  0.7513711151736746,
                  0.0170316301703163,
                  411,
                  7
                ],
                [
                  0.753199268738574,
                  0.019417475728155338,
                  412,
                  8
                ],
                [
                  0.7550274223034735,
                  0.021791767554479417,
                  413,
                  9
                ],
                [
                  0.7568555758683729,
                  0.021739130434782608,
                  414,
                  9
                ],
                [
                  0.7586837294332724,
                  0.021686746987951807,
                  415,
                  9
                ],
                [
                  0.7605118829981719,
                  0.021634615384615384,
                  416,
                  9
                ],
                [
                  0.7623400365630713,
                  0.02158273381294964,
                  417,
                  9
                ],
                [
                  0.7641681901279708,
                  0.0215311004784689,
                  418,
                  9
                ],
                [
                  0.7659963436928702,
                  0.021479713603818614,
                  419,
                  9
                ],
                [
                  0.7678244972577697,
                  0.02142857142857143,
                  420,
                  9
                ],
                [
                  0.7696526508226691,
                  0.021377672209026127,
                  421,
                  9
                ],
                [
                  0.7714808043875686,
                  0.02132701421800948,
                  422,
                  9
                ],
                [
                  0.773308957952468,
                  0.02127659574468085,
                  423,
                  9
                ],
                [
                  0.7751371115173674,
                  0.02122641509433962,
                  424,
                  9
                ],
                [
                  0.7769652650822669,
                  0.021176470588235293,
                  425,
                  9
                ],
                [
                  0.7787934186471663,
                  0.023474178403755867,
                  426,
                  10
                ],
                [
                  0.7806215722120659,
                  0.0234192037470726,
                  427,
                  10
                ],
                [
                  0.7824497257769653,
                  0.02336448598130841,
                  428,
                  10
                ],
                [
                  0.7842778793418648,
                  0.023310023310023312,
                  429,
                  10
                ],
                [
                  0.7861060329067642,
                  0.023255813953488372,
                  430,
                  10
                ],
                [
                  0.7879341864716636,
                  0.02320185614849188,
                  431,
                  10
                ],
                [
                  0.7897623400365631,
                  0.023148148148148147,
                  432,
                  10
                ],
                [
                  0.7915904936014625,
                  0.025404157043879907,
                  433,
                  11
                ],
                [
                  0.793418647166362,
                  0.02534562211981567,
                  434,
                  11
                ],
                [
                  0.7952468007312614,
                  0.02528735632183908,
                  435,
                  11
                ],
                [
                  0.7970749542961609,
                  0.027522935779816515,
                  436,
                  12
                ],
                [
                  0.7989031078610603,
                  0.02745995423340961,
                  437,
                  12
                ],
                [
                  0.8007312614259597,
                  0.0273972602739726,
                  438,
                  12
                ],
                [
                  0.8025594149908593,
                  0.02733485193621868,
                  439,
                  12
                ],
                [
                  0.8043875685557587,
                  0.02727272727272727,
                  440,
                  12
                ],
                [
                  0.8062157221206582,
                  0.027210884353741496,
                  441,
                  12
                ],
                [
                  0.8080438756855576,
                  0.027149321266968326,
                  442,
                  12
                ],
                [
                  0.8098720292504571,
                  0.02708803611738149,
                  443,
                  12
                ],
                [
                  0.8117001828153565,
                  0.02927927927927928,
                  444,
                  13
                ],
                [
                  0.8135283363802559,
                  0.03146067415730337,
                  445,
                  14
                ],
                [
                  0.8153564899451554,
                  0.03139013452914798,
                  446,
                  14
                ],
                [
                  0.8171846435100548,
                  0.03131991051454139,
                  447,
                  14
                ],
                [
                  0.8190127970749543,
                  0.03125,
                  448,
                  14
                ],
                [
                  0.8208409506398537,
                  0.0334075723830735,
                  449,
                  15
                ],
                [
                  0.8226691042047533,
                  0.03333333333333333,
                  450,
                  15
                ],
                [
                  0.8244972577696527,
                  0.03547671840354767,
                  451,
                  16
                ],
                [
                  0.8263254113345521,
                  0.03761061946902655,
                  452,
                  17
                ],
                [
                  0.8281535648994516,
                  0.039735099337748346,
                  453,
                  18
                ],
                [
                  0.829981718464351,
                  0.04185022026431718,
                  454,
                  19
                ],
                [
                  0.8318098720292505,
                  0.041758241758241756,
                  455,
                  19
                ],
                [
                  0.8336380255941499,
                  0.041666666666666664,
                  456,
                  19
                ],
                [
                  0.8354661791590493,
                  0.04157549234135667,
                  457,
                  19
                ],
                [
                  0.8372943327239488,
                  0.04148471615720524,
                  458,
                  19
                ],
                [
                  0.8391224862888482,
                  0.04139433551198257,
                  459,
                  19
                ],
                [
                  0.8409506398537477,
                  0.041304347826086954,
                  460,
                  19
                ],
                [
                  0.8427787934186471,
                  0.04121475054229935,
                  461,
                  19
                ],
                [
                  0.8446069469835467,
                  0.04329004329004329,
                  462,
                  20
                ],
                [
                  0.8464351005484461,
                  0.04535637149028078,
                  463,
                  21
                ],
                [
                  0.8482632541133455,
                  0.04525862068965517,
                  464,
                  21
                ],
                [
                  0.850091407678245,
                  0.04516129032258064,
                  465,
                  21
                ],
                [
                  0.8519195612431444,
                  0.045064377682403435,
                  466,
                  21
                ],
                [
                  0.8537477148080439,
                  0.044967880085653104,
                  467,
                  21
                ],
                [
                  0.8555758683729433,
                  0.04700854700854701,
                  468,
                  22
                ],
                [
                  0.8574040219378428,
                  0.046908315565031986,
                  469,
                  22
                ],
                [
                  0.8592321755027422,
                  0.04680851063829787,
                  470,
                  22
                ],
                [
                  0.8610603290676416,
                  0.04670912951167728,
                  471,
                  22
                ],
                [
                  0.8628884826325411,
                  0.046610169491525424,
                  472,
                  22
                ],
                [
                  0.8647166361974405,
                  0.048625792811839326,
                  473,
                  23
                ],
                [
                  0.8665447897623401,
                  0.04852320675105485,
                  474,
                  23
                ],
                [
                  0.8683729433272395,
                  0.04842105263157895,
                  475,
                  23
                ],
                [
                  0.870201096892139,
                  0.04831932773109244,
                  476,
                  23
                ],
                [
                  0.8720292504570384,
                  0.04821802935010482,
                  477,
                  23
                ],
                [
                  0.8738574040219378,
                  0.04811715481171548,
                  478,
                  23
                ],
                [
                  0.8756855575868373,
                  0.05010438413361169,
                  479,
                  24
                ],
                [
                  0.8775137111517367,
                  0.05,
                  480,
                  24
                ],
                [
                  0.8793418647166362,
                  0.05197505197505198,
                  481,
                  25
                ],
                [
                  0.8811700182815356,
                  0.05394190871369295,
                  482,
                  26
                ],
                [
                  0.8829981718464351,
                  0.053830227743271224,
                  483,
                  26
                ],
                [
                  0.8848263254113345,
                  0.05371900826446281,
                  484,
                  26
                ],
                [
                  0.886654478976234,
                  0.05567010309278351,
                  485,
                  27
                ],
                [
                  0.8884826325411335,
                  0.05555555555555555,
                  486,
                  27
                ],
                [
                  0.8903107861060329,
                  0.055441478439425054,
                  487,
                  27
                ],
                [
                  0.8921389396709324,
                  0.055327868852459015,
                  488,
                  27
                ],
                [
                  0.8939670932358318,
                  0.05725971370143149,
                  489,
                  28
                ],
                [
                  0.8957952468007313,
                  0.05714285714285714,
                  490,
                  28
                ],
                [
                  0.8976234003656307,
                  0.05702647657841141,
                  491,
                  28
                ],
                [
                  0.8994515539305301,
                  0.056910569105691054,
                  492,
                  28
                ],
                [
                  0.9012797074954296,
                  0.056795131845841784,
                  493,
                  28
                ],
                [
                  0.903107861060329,
                  0.05668016194331984,
                  494,
                  28
                ],
                [
                  0.9049360146252285,
                  0.05858585858585859,
                  495,
                  29
                ],
                [
                  0.906764168190128,
                  0.05846774193548387,
                  496,
                  29
                ],
                [
                  0.9085923217550275,
                  0.060362173038229376,
                  497,
                  30
                ],
                [
                  0.9104204753199269,
                  0.060240963855421686,
                  498,
                  30
                ],
                [
                  0.9122486288848263,
                  0.06212424849699399,
                  499,
                  31
                ],
                [
                  0.9140767824497258,
                  0.064,
                  500,
                  32
                ],
                [
                  0.9159049360146252,
                  0.0658682634730539,
                  501,
                  33
                ],
                [
                  0.9177330895795247,
                  0.06573705179282868,
                  502,
                  33
                ],
                [
                  0.9195612431444241,
                  0.06560636182902585,
                  503,
                  33
                ],
                [
                  0.9213893967093236,
                  0.06547619047619048,
                  504,
                  33
                ],
                [
                  0.923217550274223,
                  0.06534653465346535,
                  505,
                  33
                ],
                [
                  0.9250457038391224,
                  0.06521739130434782,
                  506,
                  33
                ],
                [
                  0.926873857404022,
                  0.0650887573964497,
                  507,
                  33
                ],
                [
                  0.9287020109689214,
                  0.06692913385826772,
                  508,
                  34
                ],
                [
                  0.9305301645338209,
                  0.06679764243614932,
                  509,
                  34
                ],
                [
                  0.9323583180987203,
                  0.06862745098039216,
                  510,
                  35
                ],
                [
                  0.9341864716636198,
                  0.0684931506849315,
                  511,
                  35
                ],
                [
                  0.9360146252285192,
                  0.068359375,
                  512,
                  35
                ],
                [
                  0.9378427787934186,
                  0.07017543859649122,
                  513,
                  36
                ],
                [
                  0.9396709323583181,
                  0.07198443579766536,
                  514,
                  37
                ],
                [
                  0.9414990859232175,
                  0.07184466019417475,
                  515,
                  37
                ],
                [
                  0.943327239488117,
                  0.07170542635658915,
                  516,
                  37
                ],
                [
                  0.9451553930530164,
                  0.07156673114119923,
                  517,
                  37
                ],
                [
                  0.946983546617916,
                  0.07142857142857142,
                  518,
                  37
                ],
                [
                  0.9488117001828154,
                  0.07129094412331406,
                  519,
                  37
                ],
                [
                  0.9506398537477148,
                  0.07115384615384615,
                  520,
                  37
                ],
                [
                  0.9524680073126143,
                  0.0710172744721689,
                  521,
                  37
                ],
                [
                  0.9542961608775137,
                  0.07279693486590039,
                  522,
                  38
                ],
                [
                  0.9561243144424132,
                  0.0745697896749522,
                  523,
                  39
                ],
                [
                  0.9579524680073126,
                  0.07442748091603053,
                  524,
                  39
                ],
                [
                  0.9597806215722121,
                  0.0761904761904762,
                  525,
                  40
                ],
                [
                  0.9616087751371115,
                  0.07604562737642585,
                  526,
                  40
                ],
                [
                  0.9634369287020109,
                  0.0777988614800759,
                  527,
                  41
                ],
                [
                  0.9652650822669104,
                  0.07765151515151515,
                  528,
                  41
                ],
                [
                  0.9670932358318098,
                  0.07939508506616257,
                  529,
                  42
                ],
                [
                  0.9689213893967094,
                  0.08113207547169811,
                  530,
                  43
                ],
                [
                  0.9707495429616088,
                  0.08097928436911488,
                  531,
                  43
                ],
                [
                  0.9725776965265083,
                  0.08082706766917293,
                  532,
                  43
                ],
                [
                  0.9744058500914077,
                  0.0825515947467167,
                  533,
                  44
                ],
                [
                  0.9762340036563071,
                  0.08426966292134831,
                  534,
                  45
                ],
                [
                  0.9780621572212066,
                  0.08411214953271028,
                  535,
                  45
                ],
                [
                  0.979890310786106,
                  0.08582089552238806,
                  536,
                  46
                ],
                [
                  0.9817184643510055,
                  0.08752327746741155,
                  537,
                  47
                ],
                [
                  0.9835466179159049,
                  0.08921933085501858,
                  538,
                  48
                ],
                [
                  0.9853747714808044,
                  0.09090909090909091,
                  539,
                  49
                ],
                [
                  0.9872029250457038,
                  0.09259259259259259,
                  540,
                  50
                ],
                [
                  0.9890310786106032,
                  0.09426987060998152,
                  541,
                  51
                ],
                [
                  0.9908592321755028,
                  0.0959409594095941,
                  542,
                  52
                ],
                [
                  0.9926873857404022,
                  0.09760589318600368,
                  543,
                  53
                ],
                [
                  0.9945155393053017,
                  0.09926470588235294,
                  544,
                  54
                ],
                [
                  0.9963436928702011,
                  0.10091743119266056,
                  545,
                  55
                ],
                [
                  0.9981718464351006,
                  0.10256410256410256,
                  546,
                  56
                ],
                [
                  1.0,
                  0.10237659963436929,
                  547,
                  56
                ]
              ]
            }
          ]
        },
        {
          "name": "Drex Max v1",
          "text_run": 322,
          "native": [
            {
              "language": "Turkish",
              "wording": "Bilmiyorum",
              "run": 367,
              "full_bank_restricted": true,
              "right": 492,
              "wrong": 53,
              "abstained": 2,
              "invalid": 0,
              "coverage": 0.9963436928702011,
              "risk": 0.09724770642201835
            },
            {
              "language": "Turkish",
              "wording": "I don't know",
              "run": 542,
              "full_bank_restricted": true,
              "right": 497,
              "wrong": 49,
              "abstained": 1,
              "invalid": 0,
              "coverage": 0.9981718464351006,
              "risk": 0.08974358974358974
            },
            {
              "language": "English",
              "wording": "I don't know",
              "run": null
            }
          ],
          "curves": [
            {
              "language": "Turkish",
              "run": 322,
              "points": [
                [
                  0.0018281535648994515,
                  0.0,
                  1,
                  0
                ],
                [
                  0.003656307129798903,
                  0.0,
                  2,
                  0
                ],
                [
                  0.005484460694698354,
                  0.0,
                  3,
                  0
                ],
                [
                  0.007312614259597806,
                  0.0,
                  4,
                  0
                ],
                [
                  0.009140767824497258,
                  0.0,
                  5,
                  0
                ],
                [
                  0.010968921389396709,
                  0.0,
                  6,
                  0
                ],
                [
                  0.012797074954296161,
                  0.0,
                  7,
                  0
                ],
                [
                  0.014625228519195612,
                  0.0,
                  8,
                  0
                ],
                [
                  0.016453382084095063,
                  0.0,
                  9,
                  0
                ],
                [
                  0.018281535648994516,
                  0.0,
                  10,
                  0
                ],
                [
                  0.02010968921389397,
                  0.0,
                  11,
                  0
                ],
                [
                  0.021937842778793418,
                  0.0,
                  12,
                  0
                ],
                [
                  0.02376599634369287,
                  0.0,
                  13,
                  0
                ],
                [
                  0.025594149908592323,
                  0.0,
                  14,
                  0
                ],
                [
                  0.027422303473491772,
                  0.0,
                  15,
                  0
                ],
                [
                  0.029250457038391225,
                  0.0,
                  16,
                  0
                ],
                [
                  0.031078610603290677,
                  0.0,
                  17,
                  0
                ],
                [
                  0.03290676416819013,
                  0.0,
                  18,
                  0
                ],
                [
                  0.03473491773308958,
                  0.0,
                  19,
                  0
                ],
                [
                  0.03656307129798903,
                  0.0,
                  20,
                  0
                ],
                [
                  0.038391224862888484,
                  0.0,
                  21,
                  0
                ],
                [
                  0.04021937842778794,
                  0.0,
                  22,
                  0
                ],
                [
                  0.04204753199268738,
                  0.0,
                  23,
                  0
                ],
                [
                  0.043875685557586835,
                  0.0,
                  24,
                  0
                ],
                [
                  0.04570383912248629,
                  0.0,
                  25,
                  0
                ],
                [
                  0.04753199268738574,
                  0.0,
                  26,
                  0
                ],
                [
                  0.04936014625228519,
                  0.0,
                  27,
                  0
                ],
                [
                  0.051188299817184646,
                  0.0,
                  28,
                  0
                ],
                [
                  0.05301645338208409,
                  0.0,
                  29,
                  0
                ],
                [
                  0.054844606946983544,
                  0.0,
                  30,
                  0
                ],
                [
                  0.056672760511883,
                  0.0,
                  31,
                  0
                ],
                [
                  0.05850091407678245,
                  0.0,
                  32,
                  0
                ],
                [
                  0.0603290676416819,
                  0.0,
                  33,
                  0
                ],
                [
                  0.062157221206581355,
                  0.0,
                  34,
                  0
                ],
                [
                  0.06398537477148081,
                  0.0,
                  35,
                  0
                ],
                [
                  0.06581352833638025,
                  0.0,
                  36,
                  0
                ],
                [
                  0.06764168190127971,
                  0.0,
                  37,
                  0
                ],
                [
                  0.06946983546617916,
                  0.0,
                  38,
                  0
                ],
                [
                  0.0712979890310786,
                  0.0,
                  39,
                  0
                ],
                [
                  0.07312614259597806,
                  0.0,
                  40,
                  0
                ],
                [
                  0.07495429616087751,
                  0.0,
                  41,
                  0
                ],
                [
                  0.07678244972577697,
                  0.0,
                  42,
                  0
                ],
                [
                  0.07861060329067641,
                  0.0,
                  43,
                  0
                ],
                [
                  0.08043875685557587,
                  0.0,
                  44,
                  0
                ],
                [
                  0.08226691042047532,
                  0.0,
                  45,
                  0
                ],
                [
                  0.08409506398537477,
                  0.0,
                  46,
                  0
                ],
                [
                  0.08592321755027423,
                  0.0,
                  47,
                  0
                ],
                [
                  0.08775137111517367,
                  0.0,
                  48,
                  0
                ],
                [
                  0.08957952468007313,
                  0.0,
                  49,
                  0
                ],
                [
                  0.09140767824497258,
                  0.0,
                  50,
                  0
                ],
                [
                  0.09323583180987204,
                  0.0,
                  51,
                  0
                ],
                [
                  0.09506398537477148,
                  0.0,
                  52,
                  0
                ],
                [
                  0.09689213893967093,
                  0.0,
                  53,
                  0
                ],
                [
                  0.09872029250457039,
                  0.0,
                  54,
                  0
                ],
                [
                  0.10054844606946983,
                  0.0,
                  55,
                  0
                ],
                [
                  0.10237659963436929,
                  0.0,
                  56,
                  0
                ],
                [
                  0.10420475319926874,
                  0.0,
                  57,
                  0
                ],
                [
                  0.10603290676416818,
                  0.0,
                  58,
                  0
                ],
                [
                  0.10786106032906764,
                  0.0,
                  59,
                  0
                ],
                [
                  0.10968921389396709,
                  0.0,
                  60,
                  0
                ],
                [
                  0.11151736745886655,
                  0.0,
                  61,
                  0
                ],
                [
                  0.113345521023766,
                  0.0,
                  62,
                  0
                ],
                [
                  0.11517367458866545,
                  0.0,
                  63,
                  0
                ],
                [
                  0.1170018281535649,
                  0.0,
                  64,
                  0
                ],
                [
                  0.11882998171846434,
                  0.0,
                  65,
                  0
                ],
                [
                  0.1206581352833638,
                  0.0,
                  66,
                  0
                ],
                [
                  0.12248628884826325,
                  0.0,
                  67,
                  0
                ],
                [
                  0.12431444241316271,
                  0.0,
                  68,
                  0
                ],
                [
                  0.12614259597806216,
                  0.0,
                  69,
                  0
                ],
                [
                  0.12797074954296161,
                  0.0,
                  70,
                  0
                ],
                [
                  0.12979890310786105,
                  0.0,
                  71,
                  0
                ],
                [
                  0.1316270566727605,
                  0.0,
                  72,
                  0
                ],
                [
                  0.13345521023765997,
                  0.0,
                  73,
                  0
                ],
                [
                  0.13528336380255943,
                  0.0,
                  74,
                  0
                ],
                [
                  0.13711151736745886,
                  0.0,
                  75,
                  0
                ],
                [
                  0.13893967093235832,
                  0.0,
                  76,
                  0
                ],
                [
                  0.14076782449725778,
                  0.0,
                  77,
                  0
                ],
                [
                  0.1425959780621572,
                  0.0,
                  78,
                  0
                ],
                [
                  0.14442413162705667,
                  0.0,
                  79,
                  0
                ],
                [
                  0.14625228519195613,
                  0.0,
                  80,
                  0
                ],
                [
                  0.1480804387568556,
                  0.0,
                  81,
                  0
                ],
                [
                  0.14990859232175502,
                  0.0,
                  82,
                  0
                ],
                [
                  0.15173674588665448,
                  0.0,
                  83,
                  0
                ],
                [
                  0.15356489945155394,
                  0.0,
                  84,
                  0
                ],
                [
                  0.15539305301645337,
                  0.0,
                  85,
                  0
                ],
                [
                  0.15722120658135283,
                  0.0,
                  86,
                  0
                ],
                [
                  0.1590493601462523,
                  0.0,
                  87,
                  0
                ],
                [
                  0.16087751371115175,
                  0.0,
                  88,
                  0
                ],
                [
                  0.16270566727605118,
                  0.0,
                  89,
                  0
                ],
                [
                  0.16453382084095064,
                  0.0,
                  90,
                  0
                ],
                [
                  0.1663619744058501,
                  0.0,
                  91,
                  0
                ],
                [
                  0.16819012797074953,
                  0.0,
                  92,
                  0
                ],
                [
                  0.170018281535649,
                  0.0,
                  93,
                  0
                ],
                [
                  0.17184643510054845,
                  0.0,
                  94,
                  0
                ],
                [
                  0.1736745886654479,
                  0.0,
                  95,
                  0
                ],
                [
                  0.17550274223034734,
                  0.0,
                  96,
                  0
                ],
                [
                  0.1773308957952468,
                  0.0,
                  97,
                  0
                ],
                [
                  0.17915904936014626,
                  0.0,
                  98,
                  0
                ],
                [
                  0.1809872029250457,
                  0.0,
                  99,
                  0
                ],
                [
                  0.18281535648994515,
                  0.0,
                  100,
                  0
                ],
                [
                  0.1846435100548446,
                  0.0,
                  101,
                  0
                ],
                [
                  0.18647166361974407,
                  0.0,
                  102,
                  0
                ],
                [
                  0.1882998171846435,
                  0.0,
                  103,
                  0
                ],
                [
                  0.19012797074954296,
                  0.0,
                  104,
                  0
                ],
                [
                  0.19195612431444242,
                  0.0,
                  105,
                  0
                ],
                [
                  0.19378427787934185,
                  0.0,
                  106,
                  0
                ],
                [
                  0.1956124314442413,
                  0.0,
                  107,
                  0
                ],
                [
                  0.19744058500914077,
                  0.0,
                  108,
                  0
                ],
                [
                  0.19926873857404023,
                  0.0,
                  109,
                  0
                ],
                [
                  0.20109689213893966,
                  0.0,
                  110,
                  0
                ],
                [
                  0.20292504570383912,
                  0.0,
                  111,
                  0
                ],
                [
                  0.20475319926873858,
                  0.0,
                  112,
                  0
                ],
                [
                  0.20658135283363802,
                  0.0,
                  113,
                  0
                ],
                [
                  0.20840950639853748,
                  0.0,
                  114,
                  0
                ],
                [
                  0.21023765996343693,
                  0.0,
                  115,
                  0
                ],
                [
                  0.21206581352833637,
                  0.0,
                  116,
                  0
                ],
                [
                  0.21389396709323583,
                  0.0,
                  117,
                  0
                ],
                [
                  0.21572212065813529,
                  0.0,
                  118,
                  0
                ],
                [
                  0.21755027422303475,
                  0.0,
                  119,
                  0
                ],
                [
                  0.21937842778793418,
                  0.0,
                  120,
                  0
                ],
                [
                  0.22120658135283364,
                  0.0,
                  121,
                  0
                ],
                [
                  0.2230347349177331,
                  0.0,
                  122,
                  0
                ],
                [
                  0.22486288848263253,
                  0.0,
                  123,
                  0
                ],
                [
                  0.226691042047532,
                  0.0,
                  124,
                  0
                ],
                [
                  0.22851919561243145,
                  0.0,
                  125,
                  0
                ],
                [
                  0.2303473491773309,
                  0.0,
                  126,
                  0
                ],
                [
                  0.23217550274223034,
                  0.0,
                  127,
                  0
                ],
                [
                  0.2340036563071298,
                  0.0,
                  128,
                  0
                ],
                [
                  0.23583180987202926,
                  0.0,
                  129,
                  0
                ],
                [
                  0.2376599634369287,
                  0.0,
                  130,
                  0
                ],
                [
                  0.23948811700182815,
                  0.0,
                  131,
                  0
                ],
                [
                  0.2413162705667276,
                  0.0,
                  132,
                  0
                ],
                [
                  0.24314442413162707,
                  0.0,
                  133,
                  0
                ],
                [
                  0.2449725776965265,
                  0.0,
                  134,
                  0
                ],
                [
                  0.24680073126142596,
                  0.0,
                  135,
                  0
                ],
                [
                  0.24862888482632542,
                  0.0,
                  136,
                  0
                ],
                [
                  0.25045703839122485,
                  0.0,
                  137,
                  0
                ],
                [
                  0.2522851919561243,
                  0.0,
                  138,
                  0
                ],
                [
                  0.25411334552102377,
                  0.0,
                  139,
                  0
                ],
                [
                  0.25594149908592323,
                  0.0,
                  140,
                  0
                ],
                [
                  0.2577696526508227,
                  0.0,
                  141,
                  0
                ],
                [
                  0.2595978062157221,
                  0.0,
                  142,
                  0
                ],
                [
                  0.26142595978062155,
                  0.0,
                  143,
                  0
                ],
                [
                  0.263254113345521,
                  0.0,
                  144,
                  0
                ],
                [
                  0.26508226691042047,
                  0.0,
                  145,
                  0
                ],
                [
                  0.26691042047531993,
                  0.0,
                  146,
                  0
                ],
                [
                  0.2687385740402194,
                  0.0,
                  147,
                  0
                ],
                [
                  0.27056672760511885,
                  0.0,
                  148,
                  0
                ],
                [
                  0.27239488117001825,
                  0.0,
                  149,
                  0
                ],
                [
                  0.2742230347349177,
                  0.0,
                  150,
                  0
                ],
                [
                  0.2760511882998172,
                  0.0,
                  151,
                  0
                ],
                [
                  0.27787934186471663,
                  0.0,
                  152,
                  0
                ],
                [
                  0.2797074954296161,
                  0.0,
                  153,
                  0
                ],
                [
                  0.28153564899451555,
                  0.0,
                  154,
                  0
                ],
                [
                  0.283363802559415,
                  0.0,
                  155,
                  0
                ],
                [
                  0.2851919561243144,
                  0.0,
                  156,
                  0
                ],
                [
                  0.2870201096892139,
                  0.0,
                  157,
                  0
                ],
                [
                  0.28884826325411334,
                  0.0,
                  158,
                  0
                ],
                [
                  0.2906764168190128,
                  0.0,
                  159,
                  0
                ],
                [
                  0.29250457038391225,
                  0.0,
                  160,
                  0
                ],
                [
                  0.2943327239488117,
                  0.0,
                  161,
                  0
                ],
                [
                  0.2961608775137112,
                  0.0,
                  162,
                  0
                ],
                [
                  0.2979890310786106,
                  0.0,
                  163,
                  0
                ],
                [
                  0.29981718464351004,
                  0.0,
                  164,
                  0
                ],
                [
                  0.3016453382084095,
                  0.0,
                  165,
                  0
                ],
                [
                  0.30347349177330896,
                  0.0,
                  166,
                  0
                ],
                [
                  0.3053016453382084,
                  0.0,
                  167,
                  0
                ],
                [
                  0.3071297989031079,
                  0.0,
                  168,
                  0
                ],
                [
                  0.30895795246800734,
                  0.0,
                  169,
                  0
                ],
                [
                  0.31078610603290674,
                  0.0,
                  170,
                  0
                ],
                [
                  0.3126142595978062,
                  0.0,
                  171,
                  0
                ],
                [
                  0.31444241316270566,
                  0.0,
                  172,
                  0
                ],
                [
                  0.3162705667276051,
                  0.0,
                  173,
                  0
                ],
                [
                  0.3180987202925046,
                  0.0,
                  174,
                  0
                ],
                [
                  0.31992687385740404,
                  0.0,
                  175,
                  0
                ],
                [
                  0.3217550274223035,
                  0.0,
                  176,
                  0
                ],
                [
                  0.3235831809872029,
                  0.0,
                  177,
                  0
                ],
                [
                  0.32541133455210236,
                  0.0,
                  178,
                  0
                ],
                [
                  0.3272394881170018,
                  0.0,
                  179,
                  0
                ],
                [
                  0.3290676416819013,
                  0.0,
                  180,
                  0
                ],
                [
                  0.33089579524680074,
                  0.0,
                  181,
                  0
                ],
                [
                  0.3327239488117002,
                  0.0,
                  182,
                  0
                ],
                [
                  0.33455210237659966,
                  0.0,
                  183,
                  0
                ],
                [
                  0.33638025594149906,
                  0.0,
                  184,
                  0
                ],
                [
                  0.3382084095063985,
                  0.0,
                  185,
                  0
                ],
                [
                  0.340036563071298,
                  0.0,
                  186,
                  0
                ],
                [
                  0.34186471663619744,
                  0.0,
                  187,
                  0
                ],
                [
                  0.3436928702010969,
                  0.0,
                  188,
                  0
                ],
                [
                  0.34552102376599636,
                  0.0,
                  189,
                  0
                ],
                [
                  0.3473491773308958,
                  0.0,
                  190,
                  0
                ],
                [
                  0.3491773308957952,
                  0.0,
                  191,
                  0
                ],
                [
                  0.3510054844606947,
                  0.0,
                  192,
                  0
                ],
                [
                  0.35283363802559414,
                  0.0,
                  193,
                  0
                ],
                [
                  0.3546617915904936,
                  0.0,
                  194,
                  0
                ],
                [
                  0.35648994515539306,
                  0.0,
                  195,
                  0
                ],
                [
                  0.3583180987202925,
                  0.0,
                  196,
                  0
                ],
                [
                  0.360146252285192,
                  0.0,
                  197,
                  0
                ],
                [
                  0.3619744058500914,
                  0.0,
                  198,
                  0
                ],
                [
                  0.36380255941499084,
                  0.0,
                  199,
                  0
                ],
                [
                  0.3656307129798903,
                  0.0,
                  200,
                  0
                ],
                [
                  0.36745886654478976,
                  0.0,
                  201,
                  0
                ],
                [
                  0.3692870201096892,
                  0.0,
                  202,
                  0
                ],
                [
                  0.3711151736745887,
                  0.0,
                  203,
                  0
                ],
                [
                  0.37294332723948814,
                  0.0,
                  204,
                  0
                ],
                [
                  0.37477148080438755,
                  0.0,
                  205,
                  0
                ],
                [
                  0.376599634369287,
                  0.0,
                  206,
                  0
                ],
                [
                  0.37842778793418647,
                  0.0,
                  207,
                  0
                ],
                [
                  0.3802559414990859,
                  0.0,
                  208,
                  0
                ],
                [
                  0.3820840950639854,
                  0.0,
                  209,
                  0
                ],
                [
                  0.38391224862888484,
                  0.0,
                  210,
                  0
                ],
                [
                  0.3857404021937843,
                  0.0,
                  211,
                  0
                ],
                [
                  0.3875685557586837,
                  0.0,
                  212,
                  0
                ],
                [
                  0.38939670932358317,
                  0.0,
                  213,
                  0
                ],
                [
                  0.3912248628884826,
                  0.0,
                  214,
                  0
                ],
                [
                  0.3930530164533821,
                  0.0,
                  215,
                  0
                ],
                [
                  0.39488117001828155,
                  0.0,
                  216,
                  0
                ],
                [
                  0.396709323583181,
                  0.0,
                  217,
                  0
                ],
                [
                  0.39853747714808047,
                  0.0,
                  218,
                  0
                ],
                [
                  0.40036563071297987,
                  0.0,
                  219,
                  0
                ],
                [
                  0.40219378427787933,
                  0.0,
                  220,
                  0
                ],
                [
                  0.4040219378427788,
                  0.0,
                  221,
                  0
                ],
                [
                  0.40585009140767825,
                  0.0,
                  222,
                  0
                ],
                [
                  0.4076782449725777,
                  0.0,
                  223,
                  0
                ],
                [
                  0.40950639853747717,
                  0.0,
                  224,
                  0
                ],
                [
                  0.4113345521023766,
                  0.0,
                  225,
                  0
                ],
                [
                  0.41316270566727603,
                  0.0,
                  226,
                  0
                ],
                [
                  0.4149908592321755,
                  0.0,
                  227,
                  0
                ],
                [
                  0.41681901279707495,
                  0.0,
                  228,
                  0
                ],
                [
                  0.4186471663619744,
                  0.0,
                  229,
                  0
                ],
                [
                  0.42047531992687387,
                  0.0,
                  230,
                  0
                ],
                [
                  0.42230347349177333,
                  0.0,
                  231,
                  0
                ],
                [
                  0.42413162705667273,
                  0.0,
                  232,
                  0
                ],
                [
                  0.4259597806215722,
                  0.0,
                  233,
                  0
                ],
                [
                  0.42778793418647165,
                  0.0,
                  234,
                  0
                ],
                [
                  0.4296160877513711,
                  0.0,
                  235,
                  0
                ],
                [
                  0.43144424131627057,
                  0.0,
                  236,
                  0
                ],
                [
                  0.43327239488117003,
                  0.0,
                  237,
                  0
                ],
                [
                  0.4351005484460695,
                  0.0,
                  238,
                  0
                ],
                [
                  0.4369287020109689,
                  0.0,
                  239,
                  0
                ],
                [
                  0.43875685557586835,
                  0.0,
                  240,
                  0
                ],
                [
                  0.4405850091407678,
                  0.0,
                  241,
                  0
                ],
                [
                  0.4424131627056673,
                  0.0,
                  242,
                  0
                ],
                [
                  0.44424131627056673,
                  0.0,
                  243,
                  0
                ],
                [
                  0.4460694698354662,
                  0.0,
                  244,
                  0
                ],
                [
                  0.44789762340036565,
                  0.0,
                  245,
                  0
                ],
                [
                  0.44972577696526506,
                  0.0,
                  246,
                  0
                ],
                [
                  0.4515539305301645,
                  0.0,
                  247,
                  0
                ],
                [
                  0.453382084095064,
                  0.0,
                  248,
                  0
                ],
                [
                  0.45521023765996343,
                  0.0,
                  249,
                  0
                ],
                [
                  0.4570383912248629,
                  0.0,
                  250,
                  0
                ],
                [
                  0.45886654478976235,
                  0.0,
                  251,
                  0
                ],
                [
                  0.4606946983546618,
                  0.0,
                  252,
                  0
                ],
                [
                  0.4625228519195612,
                  0.0,
                  253,
                  0
                ],
                [
                  0.4643510054844607,
                  0.0,
                  254,
                  0
                ],
                [
                  0.46617915904936014,
                  0.0,
                  255,
                  0
                ],
                [
                  0.4680073126142596,
                  0.0,
                  256,
                  0
                ],
                [
                  0.46983546617915906,
                  0.0,
                  257,
                  0
                ],
                [
                  0.4716636197440585,
                  0.0,
                  258,
                  0
                ],
                [
                  0.473491773308958,
                  0.0,
                  259,
                  0
                ],
                [
                  0.4753199268738574,
                  0.0,
                  260,
                  0
                ],
                [
                  0.47714808043875684,
                  0.0,
                  261,
                  0
                ],
                [
                  0.4789762340036563,
                  0.0,
                  262,
                  0
                ],
                [
                  0.48080438756855576,
                  0.0,
                  263,
                  0
                ],
                [
                  0.4826325411334552,
                  0.0,
                  264,
                  0
                ],
                [
                  0.4844606946983547,
                  0.0,
                  265,
                  0
                ],
                [
                  0.48628884826325414,
                  0.0,
                  266,
                  0
                ],
                [
                  0.48811700182815354,
                  0.0,
                  267,
                  0
                ],
                [
                  0.489945155393053,
                  0.0,
                  268,
                  0
                ],
                [
                  0.49177330895795246,
                  0.0,
                  269,
                  0
                ],
                [
                  0.4936014625228519,
                  0.0,
                  270,
                  0
                ],
                [
                  0.4954296160877514,
                  0.0,
                  271,
                  0
                ],
                [
                  0.49725776965265084,
                  0.0,
                  272,
                  0
                ],
                [
                  0.4990859232175503,
                  0.0,
                  273,
                  0
                ],
                [
                  0.5009140767824497,
                  0.0,
                  274,
                  0
                ],
                [
                  0.5027422303473492,
                  0.0,
                  275,
                  0
                ],
                [
                  0.5045703839122486,
                  0.0,
                  276,
                  0
                ],
                [
                  0.506398537477148,
                  0.0,
                  277,
                  0
                ],
                [
                  0.5082266910420475,
                  0.0,
                  278,
                  0
                ],
                [
                  0.5100548446069469,
                  0.0,
                  279,
                  0
                ],
                [
                  0.5118829981718465,
                  0.0,
                  280,
                  0
                ],
                [
                  0.5137111517367459,
                  0.0,
                  281,
                  0
                ],
                [
                  0.5155393053016454,
                  0.0,
                  282,
                  0
                ],
                [
                  0.5173674588665448,
                  0.0,
                  283,
                  0
                ],
                [
                  0.5191956124314442,
                  0.0,
                  284,
                  0
                ],
                [
                  0.5210237659963437,
                  0.0,
                  285,
                  0
                ],
                [
                  0.5228519195612431,
                  0.0,
                  286,
                  0
                ],
                [
                  0.5246800731261426,
                  0.0,
                  287,
                  0
                ],
                [
                  0.526508226691042,
                  0.0,
                  288,
                  0
                ],
                [
                  0.5283363802559415,
                  0.0,
                  289,
                  0
                ],
                [
                  0.5301645338208409,
                  0.0,
                  290,
                  0
                ],
                [
                  0.5319926873857403,
                  0.0,
                  291,
                  0
                ],
                [
                  0.5338208409506399,
                  0.0,
                  292,
                  0
                ],
                [
                  0.5356489945155393,
                  0.0,
                  293,
                  0
                ],
                [
                  0.5374771480804388,
                  0.0,
                  294,
                  0
                ],
                [
                  0.5393053016453382,
                  0.0,
                  295,
                  0
                ],
                [
                  0.5411334552102377,
                  0.0,
                  296,
                  0
                ],
                [
                  0.5429616087751371,
                  0.0,
                  297,
                  0
                ],
                [
                  0.5447897623400365,
                  0.0,
                  298,
                  0
                ],
                [
                  0.546617915904936,
                  0.0,
                  299,
                  0
                ],
                [
                  0.5484460694698354,
                  0.0,
                  300,
                  0
                ],
                [
                  0.5502742230347349,
                  0.0,
                  301,
                  0
                ],
                [
                  0.5521023765996343,
                  0.0,
                  302,
                  0
                ],
                [
                  0.5539305301645339,
                  0.0033003300330033004,
                  303,
                  1
                ],
                [
                  0.5557586837294333,
                  0.003289473684210526,
                  304,
                  1
                ],
                [
                  0.5575868372943327,
                  0.003278688524590164,
                  305,
                  1
                ],
                [
                  0.5594149908592322,
                  0.0032679738562091504,
                  306,
                  1
                ],
                [
                  0.5612431444241316,
                  0.003257328990228013,
                  307,
                  1
                ],
                [
                  0.5630712979890311,
                  0.003246753246753247,
                  308,
                  1
                ],
                [
                  0.5648994515539305,
                  0.003236245954692557,
                  309,
                  1
                ],
                [
                  0.56672760511883,
                  0.0032258064516129032,
                  310,
                  1
                ],
                [
                  0.5685557586837294,
                  0.003215434083601286,
                  311,
                  1
                ],
                [
                  0.5703839122486288,
                  0.003205128205128205,
                  312,
                  1
                ],
                [
                  0.5722120658135283,
                  0.003194888178913738,
                  313,
                  1
                ],
                [
                  0.5740402193784278,
                  0.0031847133757961785,
                  314,
                  1
                ],
                [
                  0.5758683729433273,
                  0.0031746031746031746,
                  315,
                  1
                ],
                [
                  0.5776965265082267,
                  0.0031645569620253164,
                  316,
                  1
                ],
                [
                  0.5795246800731262,
                  0.0031545741324921135,
                  317,
                  1
                ],
                [
                  0.5813528336380256,
                  0.0031446540880503146,
                  318,
                  1
                ],
                [
                  0.583180987202925,
                  0.003134796238244514,
                  319,
                  1
                ],
                [
                  0.5850091407678245,
                  0.003125,
                  320,
                  1
                ],
                [
                  0.5868372943327239,
                  0.003115264797507788,
                  321,
                  1
                ],
                [
                  0.5886654478976234,
                  0.003105590062111801,
                  322,
                  1
                ],
                [
                  0.5904936014625228,
                  0.0030959752321981426,
                  323,
                  1
                ],
                [
                  0.5923217550274223,
                  0.0030864197530864196,
                  324,
                  1
                ],
                [
                  0.5941499085923218,
                  0.003076923076923077,
                  325,
                  1
                ],
                [
                  0.5959780621572212,
                  0.003067484662576687,
                  326,
                  1
                ],
                [
                  0.5978062157221207,
                  0.0030581039755351682,
                  327,
                  1
                ],
                [
                  0.5996343692870201,
                  0.003048780487804878,
                  328,
                  1
                ],
                [
                  0.6014625228519196,
                  0.00303951367781155,
                  329,
                  1
                ],
                [
                  0.603290676416819,
                  0.0030303030303030303,
                  330,
                  1
                ],
                [
                  0.6051188299817185,
                  0.0030211480362537764,
                  331,
                  1
                ],
                [
                  0.6069469835466179,
                  0.0030120481927710845,
                  332,
                  1
                ],
                [
                  0.6087751371115173,
                  0.003003003003003003,
                  333,
                  1
                ],
                [
                  0.6106032906764168,
                  0.0029940119760479044,
                  334,
                  1
                ],
                [
                  0.6124314442413162,
                  0.0029850746268656717,
                  335,
                  1
                ],
                [
                  0.6142595978062158,
                  0.002976190476190476,
                  336,
                  1
                ],
                [
                  0.6160877513711152,
                  0.002967359050445104,
                  337,
                  1
                ],
                [
                  0.6179159049360147,
                  0.0029585798816568047,
                  338,
                  1
                ],
                [
                  0.6197440585009141,
                  0.0029498525073746312,
                  339,
                  1
                ],
                [
                  0.6215722120658135,
                  0.0029411764705882353,
                  340,
                  1
                ],
                [
                  0.623400365630713,
                  0.002932551319648094,
                  341,
                  1
                ],
                [
                  0.6252285191956124,
                  0.0029239766081871343,
                  342,
                  1
                ],
                [
                  0.6270566727605119,
                  0.0029154518950437317,
                  343,
                  1
                ],
                [
                  0.6288848263254113,
                  0.0029069767441860465,
                  344,
                  1
                ],
                [
                  0.6307129798903108,
                  0.002898550724637681,
                  345,
                  1
                ],
                [
                  0.6325411334552102,
                  0.002890173410404624,
                  346,
                  1
                ],
                [
                  0.6343692870201096,
                  0.002881844380403458,
                  347,
                  1
                ],
                [
                  0.6361974405850092,
                  0.0028735632183908046,
                  348,
                  1
                ],
                [
                  0.6380255941499086,
                  0.0028653295128939827,
                  349,
                  1
                ],
                [
                  0.6398537477148081,
                  0.002857142857142857,
                  350,
                  1
                ],
                [
                  0.6416819012797075,
                  0.002849002849002849,
                  351,
                  1
                ],
                [
                  0.643510054844607,
                  0.002840909090909091,
                  352,
                  1
                ],
                [
                  0.6453382084095064,
                  0.0028328611898017,
                  353,
                  1
                ],
                [
                  0.6471663619744058,
                  0.002824858757062147,
                  354,
                  1
                ],
                [
                  0.6489945155393053,
                  0.0028169014084507044,
                  355,
                  1
                ],
                [
                  0.6508226691042047,
                  0.0028089887640449437,
                  356,
                  1
                ],
                [
                  0.6526508226691042,
                  0.0028011204481792717,
                  357,
                  1
                ],
                [
                  0.6544789762340036,
                  0.002793296089385475,
                  358,
                  1
                ],
                [
                  0.6563071297989032,
                  0.002785515320334262,
                  359,
                  1
                ],
                [
                  0.6581352833638026,
                  0.002777777777777778,
                  360,
                  1
                ],
                [
                  0.659963436928702,
                  0.002770083102493075,
                  361,
                  1
                ],
                [
                  0.6617915904936015,
                  0.0027624309392265192,
                  362,
                  1
                ],
                [
                  0.6636197440585009,
                  0.0027548209366391185,
                  363,
                  1
                ],
                [
                  0.6654478976234004,
                  0.0027472527472527475,
                  364,
                  1
                ],
                [
                  0.6672760511882998,
                  0.0027397260273972603,
                  365,
                  1
                ],
                [
                  0.6691042047531993,
                  0.00273224043715847,
                  366,
                  1
                ],
                [
                  0.6709323583180987,
                  0.0027247956403269754,
                  367,
                  1
                ],
                [
                  0.6727605118829981,
                  0.002717391304347826,
                  368,
                  1
                ],
                [
                  0.6745886654478976,
                  0.0027100271002710027,
                  369,
                  1
                ],
                [
                  0.676416819012797,
                  0.002702702702702703,
                  370,
                  1
                ],
                [
                  0.6782449725776966,
                  0.0026954177897574125,
                  371,
                  1
                ],
                [
                  0.680073126142596,
                  0.002688172043010753,
                  372,
                  1
                ],
                [
                  0.6819012797074955,
                  0.002680965147453083,
                  373,
                  1
                ],
                [
                  0.6837294332723949,
                  0.00267379679144385,
                  374,
                  1
                ],
                [
                  0.6855575868372943,
                  0.0026666666666666666,
                  375,
                  1
                ],
                [
                  0.6873857404021938,
                  0.0026595744680851063,
                  376,
                  1
                ],
                [
                  0.6892138939670932,
                  0.002652519893899204,
                  377,
                  1
                ],
                [
                  0.6910420475319927,
                  0.0026455026455026454,
                  378,
                  1
                ],
                [
                  0.6928702010968921,
                  0.002638522427440633,
                  379,
                  1
                ],
                [
                  0.6946983546617916,
                  0.002631578947368421,
                  380,
                  1
                ],
                [
                  0.696526508226691,
                  0.0026246719160104987,
                  381,
                  1
                ],
                [
                  0.6983546617915904,
                  0.002617801047120419,
                  382,
                  1
                ],
                [
                  0.70018281535649,
                  0.0026109660574412533,
                  383,
                  1
                ],
                [
                  0.7020109689213894,
                  0.0026041666666666665,
                  384,
                  1
                ],
                [
                  0.7038391224862889,
                  0.0025974025974025974,
                  385,
                  1
                ],
                [
                  0.7056672760511883,
                  0.0051813471502590676,
                  386,
                  2
                ],
                [
                  0.7074954296160878,
                  0.00516795865633075,
                  387,
                  2
                ],
                [
                  0.7093235831809872,
                  0.005154639175257732,
                  388,
                  2
                ],
                [
                  0.7111517367458866,
                  0.005141388174807198,
                  389,
                  2
                ],
                [
                  0.7129798903107861,
                  0.005128205128205128,
                  390,
                  2
                ],
                [
                  0.7148080438756855,
                  0.005115089514066497,
                  391,
                  2
                ],
                [
                  0.716636197440585,
                  0.00510204081632653,
                  392,
                  2
                ],
                [
                  0.7184643510054844,
                  0.005089058524173028,
                  393,
                  2
                ],
                [
                  0.720292504570384,
                  0.005076142131979695,
                  394,
                  2
                ],
                [
                  0.7221206581352834,
                  0.005063291139240506,
                  395,
                  2
                ],
                [
                  0.7239488117001828,
                  0.005050505050505051,
                  396,
                  2
                ],
                [
                  0.7257769652650823,
                  0.005037783375314861,
                  397,
                  2
                ],
                [
                  0.7276051188299817,
                  0.005025125628140704,
                  398,
                  2
                ],
                [
                  0.7294332723948812,
                  0.007518796992481203,
                  399,
                  3
                ],
                [
                  0.7312614259597806,
                  0.0075,
                  400,
                  3
                ],
                [
                  0.7330895795246801,
                  0.007481296758104738,
                  401,
                  3
                ],
                [
                  0.7349177330895795,
                  0.007462686567164179,
                  402,
                  3
                ],
                [
                  0.7367458866544789,
                  0.007444168734491315,
                  403,
                  3
                ],
                [
                  0.7385740402193784,
                  0.007425742574257425,
                  404,
                  3
                ],
                [
                  0.7404021937842779,
                  0.007407407407407408,
                  405,
                  3
                ],
                [
                  0.7422303473491774,
                  0.007389162561576354,
                  406,
                  3
                ],
                [
                  0.7440585009140768,
                  0.007371007371007371,
                  407,
                  3
                ],
                [
                  0.7458866544789763,
                  0.00980392156862745,
                  408,
                  4
                ],
                [
                  0.7477148080438757,
                  0.009779951100244499,
                  409,
                  4
                ],
                [
                  0.7495429616087751,
                  0.00975609756097561,
                  410,
                  4
                ],
                [
                  0.7513711151736746,
                  0.012165450121654502,
                  411,
                  5
                ],
                [
                  0.753199268738574,
                  0.012135922330097087,
                  412,
                  5
                ],
                [
                  0.7550274223034735,
                  0.012106537530266344,
                  413,
                  5
                ],
                [
                  0.7568555758683729,
                  0.012077294685990338,
                  414,
                  5
                ],
                [
                  0.7586837294332724,
                  0.014457831325301205,
                  415,
                  6
                ],
                [
                  0.7605118829981719,
                  0.014423076923076924,
                  416,
                  6
                ],
                [
                  0.7623400365630713,
                  0.014388489208633094,
                  417,
                  6
                ],
                [
                  0.7641681901279708,
                  0.014354066985645933,
                  418,
                  6
                ],
                [
                  0.7659963436928702,
                  0.014319809069212411,
                  419,
                  6
                ],
                [
                  0.7678244972577697,
                  0.014285714285714285,
                  420,
                  6
                ],
                [
                  0.7696526508226691,
                  0.014251781472684086,
                  421,
                  6
                ],
                [
                  0.7714808043875686,
                  0.014218009478672985,
                  422,
                  6
                ],
                [
                  0.773308957952468,
                  0.014184397163120567,
                  423,
                  6
                ],
                [
                  0.7751371115173674,
                  0.014150943396226415,
                  424,
                  6
                ],
                [
                  0.7769652650822669,
                  0.01411764705882353,
                  425,
                  6
                ],
                [
                  0.7787934186471663,
                  0.014084507042253521,
                  426,
                  6
                ],
                [
                  0.7806215722120659,
                  0.01405152224824356,
                  427,
                  6
                ],
                [
                  0.7824497257769653,
                  0.014018691588785047,
                  428,
                  6
                ],
                [
                  0.7842778793418648,
                  0.013986013986013986,
                  429,
                  6
                ],
                [
                  0.7861060329067642,
                  0.013953488372093023,
                  430,
                  6
                ],
                [
                  0.7879341864716636,
                  0.016241299303944315,
                  431,
                  7
                ],
                [
                  0.7897623400365631,
                  0.018518518518518517,
                  432,
                  8
                ],
                [
                  0.7915904936014625,
                  0.018475750577367205,
                  433,
                  8
                ],
                [
                  0.793418647166362,
                  0.018433179723502304,
                  434,
                  8
                ],
                [
                  0.7952468007312614,
                  0.01839080459770115,
                  435,
                  8
                ],
                [
                  0.7970749542961609,
                  0.020642201834862386,
                  436,
                  9
                ],
                [
                  0.7989031078610603,
                  0.020594965675057208,
                  437,
                  9
                ],
                [
                  0.8007312614259597,
                  0.02054794520547945,
                  438,
                  9
                ],
                [
                  0.8025594149908593,
                  0.02050113895216401,
                  439,
                  9
                ],
                [
                  0.8043875685557587,
                  0.020454545454545454,
                  440,
                  9
                ],
                [
                  0.8062157221206582,
                  0.02040816326530612,
                  441,
                  9
                ],
                [
                  0.8080438756855576,
                  0.020361990950226245,
                  442,
                  9
                ],
                [
                  0.8098720292504571,
                  0.020316027088036117,
                  443,
                  9
                ],
                [
                  0.8117001828153565,
                  0.02252252252252252,
                  444,
                  10
                ],
                [
                  0.8135283363802559,
                  0.02247191011235955,
                  445,
                  10
                ],
                [
                  0.8153564899451554,
                  0.02466367713004484,
                  446,
                  11
                ],
                [
                  0.8171846435100548,
                  0.024608501118568233,
                  447,
                  11
                ],
                [
                  0.8190127970749543,
                  0.024553571428571428,
                  448,
                  11
                ],
                [
                  0.8208409506398537,
                  0.024498886414253896,
                  449,
                  11
                ],
                [
                  0.8226691042047533,
                  0.024444444444444446,
                  450,
                  11
                ],
                [
                  0.8244972577696527,
                  0.024390243902439025,
                  451,
                  11
                ],
                [
                  0.8263254113345521,
                  0.024336283185840708,
                  452,
                  11
                ],
                [
                  0.8281535648994516,
                  0.024282560706401765,
                  453,
                  11
                ],
                [
                  0.829981718464351,
                  0.02643171806167401,
                  454,
                  12
                ],
                [
                  0.8318098720292505,
                  0.026373626373626374,
                  455,
                  12
                ],
                [
                  0.8336380255941499,
                  0.02631578947368421,
                  456,
                  12
                ],
                [
                  0.8354661791590493,
                  0.0262582056892779,
                  457,
                  12
                ],
                [
                  0.8372943327239488,
                  0.026200873362445413,
                  458,
                  12
                ],
                [
                  0.8391224862888482,
                  0.026143790849673203,
                  459,
                  12
                ],
                [
                  0.8409506398537477,
                  0.02608695652173913,
                  460,
                  12
                ],
                [
                  0.8427787934186471,
                  0.026030368763557483,
                  461,
                  12
                ],
                [
                  0.8446069469835467,
                  0.025974025974025976,
                  462,
                  12
                ],
                [
                  0.8464351005484461,
                  0.02591792656587473,
                  463,
                  12
                ],
                [
                  0.8482632541133455,
                  0.02586206896551724,
                  464,
                  12
                ],
                [
                  0.850091407678245,
                  0.025806451612903226,
                  465,
                  12
                ],
                [
                  0.8519195612431444,
                  0.02575107296137339,
                  466,
                  12
                ],
                [
                  0.8537477148080439,
                  0.027837259100642397,
                  467,
                  13
                ],
                [
                  0.8555758683729433,
                  0.027777777777777776,
                  468,
                  13
                ],
                [
                  0.8574040219378428,
                  0.02771855010660981,
                  469,
                  13
                ],
                [
                  0.8592321755027422,
                  0.027659574468085105,
                  470,
                  13
                ],
                [
                  0.8610603290676416,
                  0.029723991507430998,
                  471,
                  14
                ],
                [
                  0.8628884826325411,
                  0.03177966101694915,
                  472,
                  15
                ],
                [
                  0.8647166361974405,
                  0.03171247357293869,
                  473,
                  15
                ],
                [
                  0.8665447897623401,
                  0.03164556962025317,
                  474,
                  15
                ],
                [
                  0.8683729433272395,
                  0.031578947368421054,
                  475,
                  15
                ],
                [
                  0.870201096892139,
                  0.031512605042016806,
                  476,
                  15
                ],
                [
                  0.8720292504570384,
                  0.033542976939203356,
                  477,
                  16
                ],
                [
                  0.8738574040219378,
                  0.03556485355648536,
                  478,
                  17
                ],
                [
                  0.8756855575868373,
                  0.037578288100208766,
                  479,
                  18
                ],
                [
                  0.8775137111517367,
                  0.0375,
                  480,
                  18
                ],
                [
                  0.8793418647166362,
                  0.0395010395010395,
                  481,
                  19
                ],
                [
                  0.8811700182815356,
                  0.03941908713692946,
                  482,
                  19
                ],
                [
                  0.8829981718464351,
                  0.039337474120082816,
                  483,
                  19
                ],
                [
                  0.8848263254113345,
                  0.03925619834710744,
                  484,
                  19
                ],
                [
                  0.886654478976234,
                  0.041237113402061855,
                  485,
                  20
                ],
                [
                  0.8884826325411335,
                  0.0411522633744856,
                  486,
                  20
                ],
                [
                  0.8903107861060329,
                  0.04106776180698152,
                  487,
                  20
                ],
                [
                  0.8921389396709324,
                  0.0430327868852459,
                  488,
                  21
                ],
                [
                  0.8939670932358318,
                  0.04294478527607362,
                  489,
                  21
                ],
                [
                  0.8957952468007313,
                  0.04285714285714286,
                  490,
                  21
                ],
                [
                  0.8976234003656307,
                  0.04276985743380855,
                  491,
                  21
                ],
                [
                  0.8994515539305301,
                  0.044715447154471545,
                  492,
                  22
                ],
                [
                  0.9012797074954296,
                  0.04462474645030426,
                  493,
                  22
                ],
                [
                  0.903107861060329,
                  0.044534412955465584,
                  494,
                  22
                ],
                [
                  0.9049360146252285,
                  0.044444444444444446,
                  495,
                  22
                ],
                [
                  0.906764168190128,
                  0.04435483870967742,
                  496,
                  22
                ],
                [
                  0.9085923217550275,
                  0.04627766599597585,
                  497,
                  23
                ],
                [
                  0.9104204753199269,
                  0.04819277108433735,
                  498,
                  24
                ],
                [
                  0.9122486288848263,
                  0.04809619238476954,
                  499,
                  24
                ],
                [
                  0.9140767824497258,
                  0.048,
                  500,
                  24
                ],
                [
                  0.9159049360146252,
                  0.0499001996007984,
                  501,
                  25
                ],
                [
                  0.9177330895795247,
                  0.049800796812749,
                  502,
                  25
                ],
                [
                  0.9195612431444241,
                  0.04970178926441352,
                  503,
                  25
                ],
                [
                  0.9213893967093236,
                  0.051587301587301584,
                  504,
                  26
                ],
                [
                  0.923217550274223,
                  0.053465346534653464,
                  505,
                  27
                ],
                [
                  0.9250457038391224,
                  0.0533596837944664,
                  506,
                  27
                ],
                [
                  0.926873857404022,
                  0.05325443786982249,
                  507,
                  27
                ],
                [
                  0.9287020109689214,
                  0.05511811023622047,
                  508,
                  28
                ],
                [
                  0.9305301645338209,
                  0.05697445972495088,
                  509,
                  29
                ],
                [
                  0.9323583180987203,
                  0.058823529411764705,
                  510,
                  30
                ],
                [
                  0.9341864716636198,
                  0.060665362035225046,
                  511,
                  31
                ],
                [
                  0.9360146252285192,
                  0.0625,
                  512,
                  32
                ],
                [
                  0.9378427787934186,
                  0.06432748538011696,
                  513,
                  33
                ],
                [
                  0.9396709323583181,
                  0.06420233463035019,
                  514,
                  33
                ],
                [
                  0.9414990859232175,
                  0.06407766990291262,
                  515,
                  33
                ],
                [
                  0.943327239488117,
                  0.06395348837209303,
                  516,
                  33
                ],
                [
                  0.9451553930530164,
                  0.06382978723404255,
                  517,
                  33
                ],
                [
                  0.946983546617916,
                  0.06563706563706563,
                  518,
                  34
                ],
                [
                  0.9488117001828154,
                  0.0674373795761079,
                  519,
                  35
                ],
                [
                  0.9506398537477148,
                  0.0673076923076923,
                  520,
                  35
                ],
                [
                  0.9524680073126143,
                  0.0671785028790787,
                  521,
                  35
                ],
                [
                  0.9542961608775137,
                  0.06896551724137931,
                  522,
                  36
                ],
                [
                  0.9561243144424132,
                  0.07074569789674952,
                  523,
                  37
                ],
                [
                  0.9579524680073126,
                  0.07061068702290077,
                  524,
                  37
                ],
                [
                  0.9597806215722121,
                  0.07238095238095238,
                  525,
                  38
                ],
                [
                  0.9616087751371115,
                  0.07224334600760456,
                  526,
                  38
                ],
                [
                  0.9634369287020109,
                  0.07210626185958255,
                  527,
                  38
                ],
                [
                  0.9652650822669104,
                  0.07386363636363637,
                  528,
                  39
                ],
                [
                  0.9670932358318098,
                  0.07372400756143667,
                  529,
                  39
                ],
                [
                  0.9689213893967094,
                  0.07547169811320754,
                  530,
                  40
                ],
                [
                  0.9707495429616088,
                  0.07721280602636535,
                  531,
                  41
                ],
                [
                  0.9725776965265083,
                  0.07706766917293233,
                  532,
                  41
                ],
                [
                  0.9744058500914077,
                  0.07879924953095685,
                  533,
                  42
                ],
                [
                  0.9762340036563071,
                  0.07865168539325842,
                  534,
                  42
                ],
                [
                  0.9780621572212066,
                  0.07850467289719626,
                  535,
                  42
                ],
                [
                  0.979890310786106,
                  0.07835820895522388,
                  536,
                  42
                ],
                [
                  0.9817184643510055,
                  0.0782122905027933,
                  537,
                  42
                ],
                [
                  0.9835466179159049,
                  0.07992565055762081,
                  538,
                  43
                ],
                [
                  0.9853747714808044,
                  0.08163265306122448,
                  539,
                  44
                ],
                [
                  0.9872029250457038,
                  0.08148148148148149,
                  540,
                  44
                ],
                [
                  0.9890310786106032,
                  0.08133086876155268,
                  541,
                  44
                ],
                [
                  0.9908592321755028,
                  0.08118081180811808,
                  542,
                  44
                ],
                [
                  0.9926873857404022,
                  0.08287292817679558,
                  543,
                  45
                ],
                [
                  0.9945155393053017,
                  0.08272058823529412,
                  544,
                  45
                ],
                [
                  0.9963436928702011,
                  0.08440366972477065,
                  545,
                  46
                ],
                [
                  0.9981718464351006,
                  0.08608058608058608,
                  546,
                  47
                ],
                [
                  1.0,
                  0.08592321755027423,
                  547,
                  47
                ]
              ]
            },
            {
              "language": "English",
              "run": 552,
              "points": [
                [
                  0.0018281535648994515,
                  0.0,
                  1,
                  0
                ],
                [
                  0.003656307129798903,
                  0.0,
                  2,
                  0
                ],
                [
                  0.005484460694698354,
                  0.0,
                  3,
                  0
                ],
                [
                  0.007312614259597806,
                  0.0,
                  4,
                  0
                ],
                [
                  0.009140767824497258,
                  0.0,
                  5,
                  0
                ],
                [
                  0.010968921389396709,
                  0.0,
                  6,
                  0
                ],
                [
                  0.012797074954296161,
                  0.0,
                  7,
                  0
                ],
                [
                  0.014625228519195612,
                  0.0,
                  8,
                  0
                ],
                [
                  0.016453382084095063,
                  0.0,
                  9,
                  0
                ],
                [
                  0.018281535648994516,
                  0.0,
                  10,
                  0
                ],
                [
                  0.02010968921389397,
                  0.0,
                  11,
                  0
                ],
                [
                  0.021937842778793418,
                  0.0,
                  12,
                  0
                ],
                [
                  0.02376599634369287,
                  0.0,
                  13,
                  0
                ],
                [
                  0.025594149908592323,
                  0.0,
                  14,
                  0
                ],
                [
                  0.027422303473491772,
                  0.0,
                  15,
                  0
                ],
                [
                  0.029250457038391225,
                  0.0,
                  16,
                  0
                ],
                [
                  0.031078610603290677,
                  0.0,
                  17,
                  0
                ],
                [
                  0.03290676416819013,
                  0.0,
                  18,
                  0
                ],
                [
                  0.03473491773308958,
                  0.0,
                  19,
                  0
                ],
                [
                  0.03656307129798903,
                  0.0,
                  20,
                  0
                ],
                [
                  0.038391224862888484,
                  0.0,
                  21,
                  0
                ],
                [
                  0.04021937842778794,
                  0.0,
                  22,
                  0
                ],
                [
                  0.04204753199268738,
                  0.0,
                  23,
                  0
                ],
                [
                  0.043875685557586835,
                  0.0,
                  24,
                  0
                ],
                [
                  0.04570383912248629,
                  0.0,
                  25,
                  0
                ],
                [
                  0.04753199268738574,
                  0.0,
                  26,
                  0
                ],
                [
                  0.04936014625228519,
                  0.0,
                  27,
                  0
                ],
                [
                  0.051188299817184646,
                  0.0,
                  28,
                  0
                ],
                [
                  0.05301645338208409,
                  0.0,
                  29,
                  0
                ],
                [
                  0.054844606946983544,
                  0.0,
                  30,
                  0
                ],
                [
                  0.056672760511883,
                  0.0,
                  31,
                  0
                ],
                [
                  0.05850091407678245,
                  0.0,
                  32,
                  0
                ],
                [
                  0.0603290676416819,
                  0.0,
                  33,
                  0
                ],
                [
                  0.062157221206581355,
                  0.0,
                  34,
                  0
                ],
                [
                  0.06398537477148081,
                  0.0,
                  35,
                  0
                ],
                [
                  0.06581352833638025,
                  0.0,
                  36,
                  0
                ],
                [
                  0.06764168190127971,
                  0.0,
                  37,
                  0
                ],
                [
                  0.06946983546617916,
                  0.0,
                  38,
                  0
                ],
                [
                  0.0712979890310786,
                  0.0,
                  39,
                  0
                ],
                [
                  0.07312614259597806,
                  0.0,
                  40,
                  0
                ],
                [
                  0.07495429616087751,
                  0.0,
                  41,
                  0
                ],
                [
                  0.07678244972577697,
                  0.0,
                  42,
                  0
                ],
                [
                  0.07861060329067641,
                  0.0,
                  43,
                  0
                ],
                [
                  0.08043875685557587,
                  0.0,
                  44,
                  0
                ],
                [
                  0.08226691042047532,
                  0.0,
                  45,
                  0
                ],
                [
                  0.08409506398537477,
                  0.0,
                  46,
                  0
                ],
                [
                  0.08592321755027423,
                  0.0,
                  47,
                  0
                ],
                [
                  0.08775137111517367,
                  0.0,
                  48,
                  0
                ],
                [
                  0.08957952468007313,
                  0.0,
                  49,
                  0
                ],
                [
                  0.09140767824497258,
                  0.0,
                  50,
                  0
                ],
                [
                  0.09323583180987204,
                  0.0,
                  51,
                  0
                ],
                [
                  0.09506398537477148,
                  0.0,
                  52,
                  0
                ],
                [
                  0.09689213893967093,
                  0.0,
                  53,
                  0
                ],
                [
                  0.09872029250457039,
                  0.0,
                  54,
                  0
                ],
                [
                  0.10054844606946983,
                  0.0,
                  55,
                  0
                ],
                [
                  0.10237659963436929,
                  0.0,
                  56,
                  0
                ],
                [
                  0.10420475319926874,
                  0.0,
                  57,
                  0
                ],
                [
                  0.10603290676416818,
                  0.0,
                  58,
                  0
                ],
                [
                  0.10786106032906764,
                  0.0,
                  59,
                  0
                ],
                [
                  0.10968921389396709,
                  0.0,
                  60,
                  0
                ],
                [
                  0.11151736745886655,
                  0.0,
                  61,
                  0
                ],
                [
                  0.113345521023766,
                  0.0,
                  62,
                  0
                ],
                [
                  0.11517367458866545,
                  0.0,
                  63,
                  0
                ],
                [
                  0.1170018281535649,
                  0.0,
                  64,
                  0
                ],
                [
                  0.11882998171846434,
                  0.0,
                  65,
                  0
                ],
                [
                  0.1206581352833638,
                  0.0,
                  66,
                  0
                ],
                [
                  0.12248628884826325,
                  0.0,
                  67,
                  0
                ],
                [
                  0.12431444241316271,
                  0.0,
                  68,
                  0
                ],
                [
                  0.12614259597806216,
                  0.0,
                  69,
                  0
                ],
                [
                  0.12797074954296161,
                  0.0,
                  70,
                  0
                ],
                [
                  0.12979890310786105,
                  0.0,
                  71,
                  0
                ],
                [
                  0.1316270566727605,
                  0.0,
                  72,
                  0
                ],
                [
                  0.13345521023765997,
                  0.0,
                  73,
                  0
                ],
                [
                  0.13528336380255943,
                  0.0,
                  74,
                  0
                ],
                [
                  0.13711151736745886,
                  0.0,
                  75,
                  0
                ],
                [
                  0.13893967093235832,
                  0.0,
                  76,
                  0
                ],
                [
                  0.14076782449725778,
                  0.0,
                  77,
                  0
                ],
                [
                  0.1425959780621572,
                  0.0,
                  78,
                  0
                ],
                [
                  0.14442413162705667,
                  0.0,
                  79,
                  0
                ],
                [
                  0.14625228519195613,
                  0.0,
                  80,
                  0
                ],
                [
                  0.1480804387568556,
                  0.0,
                  81,
                  0
                ],
                [
                  0.14990859232175502,
                  0.0,
                  82,
                  0
                ],
                [
                  0.15173674588665448,
                  0.0,
                  83,
                  0
                ],
                [
                  0.15356489945155394,
                  0.0,
                  84,
                  0
                ],
                [
                  0.15539305301645337,
                  0.0,
                  85,
                  0
                ],
                [
                  0.15722120658135283,
                  0.0,
                  86,
                  0
                ],
                [
                  0.1590493601462523,
                  0.0,
                  87,
                  0
                ],
                [
                  0.16087751371115175,
                  0.0,
                  88,
                  0
                ],
                [
                  0.16270566727605118,
                  0.0,
                  89,
                  0
                ],
                [
                  0.16453382084095064,
                  0.0,
                  90,
                  0
                ],
                [
                  0.1663619744058501,
                  0.0,
                  91,
                  0
                ],
                [
                  0.16819012797074953,
                  0.0,
                  92,
                  0
                ],
                [
                  0.170018281535649,
                  0.0,
                  93,
                  0
                ],
                [
                  0.17184643510054845,
                  0.0,
                  94,
                  0
                ],
                [
                  0.1736745886654479,
                  0.0,
                  95,
                  0
                ],
                [
                  0.17550274223034734,
                  0.0,
                  96,
                  0
                ],
                [
                  0.1773308957952468,
                  0.0,
                  97,
                  0
                ],
                [
                  0.17915904936014626,
                  0.0,
                  98,
                  0
                ],
                [
                  0.1809872029250457,
                  0.0,
                  99,
                  0
                ],
                [
                  0.18281535648994515,
                  0.0,
                  100,
                  0
                ],
                [
                  0.1846435100548446,
                  0.0,
                  101,
                  0
                ],
                [
                  0.18647166361974407,
                  0.0,
                  102,
                  0
                ],
                [
                  0.1882998171846435,
                  0.0,
                  103,
                  0
                ],
                [
                  0.19012797074954296,
                  0.0,
                  104,
                  0
                ],
                [
                  0.19195612431444242,
                  0.0,
                  105,
                  0
                ],
                [
                  0.19378427787934185,
                  0.0,
                  106,
                  0
                ],
                [
                  0.1956124314442413,
                  0.0,
                  107,
                  0
                ],
                [
                  0.19744058500914077,
                  0.0,
                  108,
                  0
                ],
                [
                  0.19926873857404023,
                  0.0,
                  109,
                  0
                ],
                [
                  0.20109689213893966,
                  0.0,
                  110,
                  0
                ],
                [
                  0.20292504570383912,
                  0.0,
                  111,
                  0
                ],
                [
                  0.20475319926873858,
                  0.0,
                  112,
                  0
                ],
                [
                  0.20658135283363802,
                  0.0,
                  113,
                  0
                ],
                [
                  0.20840950639853748,
                  0.0,
                  114,
                  0
                ],
                [
                  0.21023765996343693,
                  0.0,
                  115,
                  0
                ],
                [
                  0.21206581352833637,
                  0.0,
                  116,
                  0
                ],
                [
                  0.21389396709323583,
                  0.0,
                  117,
                  0
                ],
                [
                  0.21572212065813529,
                  0.0,
                  118,
                  0
                ],
                [
                  0.21755027422303475,
                  0.0,
                  119,
                  0
                ],
                [
                  0.21937842778793418,
                  0.0,
                  120,
                  0
                ],
                [
                  0.22120658135283364,
                  0.0,
                  121,
                  0
                ],
                [
                  0.2230347349177331,
                  0.0,
                  122,
                  0
                ],
                [
                  0.22486288848263253,
                  0.0,
                  123,
                  0
                ],
                [
                  0.226691042047532,
                  0.0,
                  124,
                  0
                ],
                [
                  0.22851919561243145,
                  0.0,
                  125,
                  0
                ],
                [
                  0.2303473491773309,
                  0.0,
                  126,
                  0
                ],
                [
                  0.23217550274223034,
                  0.0,
                  127,
                  0
                ],
                [
                  0.2340036563071298,
                  0.0,
                  128,
                  0
                ],
                [
                  0.23583180987202926,
                  0.0,
                  129,
                  0
                ],
                [
                  0.2376599634369287,
                  0.0,
                  130,
                  0
                ],
                [
                  0.23948811700182815,
                  0.0,
                  131,
                  0
                ],
                [
                  0.2413162705667276,
                  0.0,
                  132,
                  0
                ],
                [
                  0.24314442413162707,
                  0.0,
                  133,
                  0
                ],
                [
                  0.2449725776965265,
                  0.0,
                  134,
                  0
                ],
                [
                  0.24680073126142596,
                  0.0,
                  135,
                  0
                ],
                [
                  0.24862888482632542,
                  0.0,
                  136,
                  0
                ],
                [
                  0.25045703839122485,
                  0.0,
                  137,
                  0
                ],
                [
                  0.2522851919561243,
                  0.0,
                  138,
                  0
                ],
                [
                  0.25411334552102377,
                  0.0,
                  139,
                  0
                ],
                [
                  0.25594149908592323,
                  0.0,
                  140,
                  0
                ],
                [
                  0.2577696526508227,
                  0.0,
                  141,
                  0
                ],
                [
                  0.2595978062157221,
                  0.0,
                  142,
                  0
                ],
                [
                  0.26142595978062155,
                  0.0,
                  143,
                  0
                ],
                [
                  0.263254113345521,
                  0.0,
                  144,
                  0
                ],
                [
                  0.26508226691042047,
                  0.0,
                  145,
                  0
                ],
                [
                  0.26691042047531993,
                  0.0,
                  146,
                  0
                ],
                [
                  0.2687385740402194,
                  0.0,
                  147,
                  0
                ],
                [
                  0.27056672760511885,
                  0.0,
                  148,
                  0
                ],
                [
                  0.27239488117001825,
                  0.0,
                  149,
                  0
                ],
                [
                  0.2742230347349177,
                  0.0,
                  150,
                  0
                ],
                [
                  0.2760511882998172,
                  0.0,
                  151,
                  0
                ],
                [
                  0.27787934186471663,
                  0.0,
                  152,
                  0
                ],
                [
                  0.2797074954296161,
                  0.0,
                  153,
                  0
                ],
                [
                  0.28153564899451555,
                  0.0,
                  154,
                  0
                ],
                [
                  0.283363802559415,
                  0.0,
                  155,
                  0
                ],
                [
                  0.2851919561243144,
                  0.0,
                  156,
                  0
                ],
                [
                  0.2870201096892139,
                  0.0,
                  157,
                  0
                ],
                [
                  0.28884826325411334,
                  0.0,
                  158,
                  0
                ],
                [
                  0.2906764168190128,
                  0.0,
                  159,
                  0
                ],
                [
                  0.29250457038391225,
                  0.0,
                  160,
                  0
                ],
                [
                  0.2943327239488117,
                  0.0,
                  161,
                  0
                ],
                [
                  0.2961608775137112,
                  0.0,
                  162,
                  0
                ],
                [
                  0.2979890310786106,
                  0.0,
                  163,
                  0
                ],
                [
                  0.29981718464351004,
                  0.0,
                  164,
                  0
                ],
                [
                  0.3016453382084095,
                  0.0,
                  165,
                  0
                ],
                [
                  0.30347349177330896,
                  0.0,
                  166,
                  0
                ],
                [
                  0.3053016453382084,
                  0.0,
                  167,
                  0
                ],
                [
                  0.3071297989031079,
                  0.0,
                  168,
                  0
                ],
                [
                  0.30895795246800734,
                  0.0,
                  169,
                  0
                ],
                [
                  0.31078610603290674,
                  0.0,
                  170,
                  0
                ],
                [
                  0.3126142595978062,
                  0.0,
                  171,
                  0
                ],
                [
                  0.31444241316270566,
                  0.0,
                  172,
                  0
                ],
                [
                  0.3162705667276051,
                  0.0,
                  173,
                  0
                ],
                [
                  0.3180987202925046,
                  0.0,
                  174,
                  0
                ],
                [
                  0.31992687385740404,
                  0.0,
                  175,
                  0
                ],
                [
                  0.3217550274223035,
                  0.0,
                  176,
                  0
                ],
                [
                  0.3235831809872029,
                  0.0,
                  177,
                  0
                ],
                [
                  0.32541133455210236,
                  0.0,
                  178,
                  0
                ],
                [
                  0.3272394881170018,
                  0.0,
                  179,
                  0
                ],
                [
                  0.3290676416819013,
                  0.0,
                  180,
                  0
                ],
                [
                  0.33089579524680074,
                  0.0,
                  181,
                  0
                ],
                [
                  0.3327239488117002,
                  0.0,
                  182,
                  0
                ],
                [
                  0.33455210237659966,
                  0.0,
                  183,
                  0
                ],
                [
                  0.33638025594149906,
                  0.0,
                  184,
                  0
                ],
                [
                  0.3382084095063985,
                  0.0,
                  185,
                  0
                ],
                [
                  0.340036563071298,
                  0.0,
                  186,
                  0
                ],
                [
                  0.34186471663619744,
                  0.0,
                  187,
                  0
                ],
                [
                  0.3436928702010969,
                  0.0,
                  188,
                  0
                ],
                [
                  0.34552102376599636,
                  0.0,
                  189,
                  0
                ],
                [
                  0.3473491773308958,
                  0.0,
                  190,
                  0
                ],
                [
                  0.3491773308957952,
                  0.0,
                  191,
                  0
                ],
                [
                  0.3510054844606947,
                  0.0,
                  192,
                  0
                ],
                [
                  0.35283363802559414,
                  0.0,
                  193,
                  0
                ],
                [
                  0.3546617915904936,
                  0.0,
                  194,
                  0
                ],
                [
                  0.35648994515539306,
                  0.0,
                  195,
                  0
                ],
                [
                  0.3583180987202925,
                  0.0,
                  196,
                  0
                ],
                [
                  0.360146252285192,
                  0.0,
                  197,
                  0
                ],
                [
                  0.3619744058500914,
                  0.0,
                  198,
                  0
                ],
                [
                  0.36380255941499084,
                  0.0,
                  199,
                  0
                ],
                [
                  0.3656307129798903,
                  0.0,
                  200,
                  0
                ],
                [
                  0.36745886654478976,
                  0.0,
                  201,
                  0
                ],
                [
                  0.3692870201096892,
                  0.0,
                  202,
                  0
                ],
                [
                  0.3711151736745887,
                  0.0,
                  203,
                  0
                ],
                [
                  0.37294332723948814,
                  0.0,
                  204,
                  0
                ],
                [
                  0.37477148080438755,
                  0.0,
                  205,
                  0
                ],
                [
                  0.376599634369287,
                  0.0,
                  206,
                  0
                ],
                [
                  0.37842778793418647,
                  0.0,
                  207,
                  0
                ],
                [
                  0.3802559414990859,
                  0.0,
                  208,
                  0
                ],
                [
                  0.3820840950639854,
                  0.0,
                  209,
                  0
                ],
                [
                  0.38391224862888484,
                  0.004761904761904762,
                  210,
                  1
                ],
                [
                  0.3857404021937843,
                  0.004739336492890996,
                  211,
                  1
                ],
                [
                  0.3875685557586837,
                  0.0047169811320754715,
                  212,
                  1
                ],
                [
                  0.38939670932358317,
                  0.004694835680751174,
                  213,
                  1
                ],
                [
                  0.3912248628884826,
                  0.004672897196261682,
                  214,
                  1
                ],
                [
                  0.3930530164533821,
                  0.004651162790697674,
                  215,
                  1
                ],
                [
                  0.39488117001828155,
                  0.004629629629629629,
                  216,
                  1
                ],
                [
                  0.396709323583181,
                  0.004608294930875576,
                  217,
                  1
                ],
                [
                  0.39853747714808047,
                  0.0045871559633027525,
                  218,
                  1
                ],
                [
                  0.40036563071297987,
                  0.0045662100456621,
                  219,
                  1
                ],
                [
                  0.40219378427787933,
                  0.004545454545454545,
                  220,
                  1
                ],
                [
                  0.4040219378427788,
                  0.004524886877828055,
                  221,
                  1
                ],
                [
                  0.40585009140767825,
                  0.0045045045045045045,
                  222,
                  1
                ],
                [
                  0.4076782449725777,
                  0.004484304932735426,
                  223,
                  1
                ],
                [
                  0.40950639853747717,
                  0.004464285714285714,
                  224,
                  1
                ],
                [
                  0.4113345521023766,
                  0.0044444444444444444,
                  225,
                  1
                ],
                [
                  0.41316270566727603,
                  0.004424778761061947,
                  226,
                  1
                ],
                [
                  0.4149908592321755,
                  0.004405286343612335,
                  227,
                  1
                ],
                [
                  0.41681901279707495,
                  0.0043859649122807015,
                  228,
                  1
                ],
                [
                  0.4186471663619744,
                  0.004366812227074236,
                  229,
                  1
                ],
                [
                  0.42047531992687387,
                  0.004347826086956522,
                  230,
                  1
                ],
                [
                  0.42230347349177333,
                  0.004329004329004329,
                  231,
                  1
                ],
                [
                  0.42413162705667273,
                  0.004310344827586207,
                  232,
                  1
                ],
                [
                  0.4259597806215722,
                  0.004291845493562232,
                  233,
                  1
                ],
                [
                  0.42778793418647165,
                  0.004273504273504274,
                  234,
                  1
                ],
                [
                  0.4296160877513711,
                  0.00425531914893617,
                  235,
                  1
                ],
                [
                  0.43144424131627057,
                  0.00423728813559322,
                  236,
                  1
                ],
                [
                  0.43327239488117003,
                  0.004219409282700422,
                  237,
                  1
                ],
                [
                  0.4351005484460695,
                  0.004201680672268907,
                  238,
                  1
                ],
                [
                  0.4369287020109689,
                  0.0041841004184100415,
                  239,
                  1
                ],
                [
                  0.43875685557586835,
                  0.004166666666666667,
                  240,
                  1
                ],
                [
                  0.4405850091407678,
                  0.004149377593360996,
                  241,
                  1
                ],
                [
                  0.4424131627056673,
                  0.004132231404958678,
                  242,
                  1
                ],
                [
                  0.44424131627056673,
                  0.00411522633744856,
                  243,
                  1
                ],
                [
                  0.4460694698354662,
                  0.004098360655737705,
                  244,
                  1
                ],
                [
                  0.44789762340036565,
                  0.004081632653061225,
                  245,
                  1
                ],
                [
                  0.44972577696526506,
                  0.0040650406504065045,
                  246,
                  1
                ],
                [
                  0.4515539305301645,
                  0.004048582995951417,
                  247,
                  1
                ],
                [
                  0.453382084095064,
                  0.004032258064516129,
                  248,
                  1
                ],
                [
                  0.45521023765996343,
                  0.004016064257028112,
                  249,
                  1
                ],
                [
                  0.4570383912248629,
                  0.004,
                  250,
                  1
                ],
                [
                  0.45886654478976235,
                  0.00398406374501992,
                  251,
                  1
                ],
                [
                  0.4606946983546618,
                  0.003968253968253968,
                  252,
                  1
                ],
                [
                  0.4625228519195612,
                  0.003952569169960474,
                  253,
                  1
                ],
                [
                  0.4643510054844607,
                  0.003937007874015748,
                  254,
                  1
                ],
                [
                  0.46617915904936014,
                  0.00392156862745098,
                  255,
                  1
                ],
                [
                  0.4680073126142596,
                  0.00390625,
                  256,
                  1
                ],
                [
                  0.46983546617915906,
                  0.0038910505836575876,
                  257,
                  1
                ],
                [
                  0.4716636197440585,
                  0.007751937984496124,
                  258,
                  2
                ],
                [
                  0.473491773308958,
                  0.007722007722007722,
                  259,
                  2
                ],
                [
                  0.4753199268738574,
                  0.007692307692307693,
                  260,
                  2
                ],
                [
                  0.47714808043875684,
                  0.007662835249042145,
                  261,
                  2
                ],
                [
                  0.4789762340036563,
                  0.007633587786259542,
                  262,
                  2
                ],
                [
                  0.48080438756855576,
                  0.0076045627376425855,
                  263,
                  2
                ],
                [
                  0.4826325411334552,
                  0.007575757575757576,
                  264,
                  2
                ],
                [
                  0.4844606946983547,
                  0.007547169811320755,
                  265,
                  2
                ],
                [
                  0.48628884826325414,
                  0.007518796992481203,
                  266,
                  2
                ],
                [
                  0.48811700182815354,
                  0.00749063670411985,
                  267,
                  2
                ],
                [
                  0.489945155393053,
                  0.007462686567164179,
                  268,
                  2
                ],
                [
                  0.49177330895795246,
                  0.007434944237918215,
                  269,
                  2
                ],
                [
                  0.4936014625228519,
                  0.007407407407407408,
                  270,
                  2
                ],
                [
                  0.4954296160877514,
                  0.007380073800738007,
                  271,
                  2
                ],
                [
                  0.49725776965265084,
                  0.007352941176470588,
                  272,
                  2
                ],
                [
                  0.4990859232175503,
                  0.007326007326007326,
                  273,
                  2
                ],
                [
                  0.5009140767824497,
                  0.0072992700729927005,
                  274,
                  2
                ],
                [
                  0.5027422303473492,
                  0.007272727272727273,
                  275,
                  2
                ],
                [
                  0.5045703839122486,
                  0.007246376811594203,
                  276,
                  2
                ],
                [
                  0.506398537477148,
                  0.007220216606498195,
                  277,
                  2
                ],
                [
                  0.5082266910420475,
                  0.007194244604316547,
                  278,
                  2
                ],
                [
                  0.5100548446069469,
                  0.007168458781362007,
                  279,
                  2
                ],
                [
                  0.5118829981718465,
                  0.007142857142857143,
                  280,
                  2
                ],
                [
                  0.5137111517367459,
                  0.0071174377224199285,
                  281,
                  2
                ],
                [
                  0.5155393053016454,
                  0.0070921985815602835,
                  282,
                  2
                ],
                [
                  0.5173674588665448,
                  0.007067137809187279,
                  283,
                  2
                ],
                [
                  0.5191956124314442,
                  0.007042253521126761,
                  284,
                  2
                ],
                [
                  0.5210237659963437,
                  0.007017543859649123,
                  285,
                  2
                ],
                [
                  0.5228519195612431,
                  0.006993006993006993,
                  286,
                  2
                ],
                [
                  0.5246800731261426,
                  0.006968641114982578,
                  287,
                  2
                ],
                [
                  0.526508226691042,
                  0.006944444444444444,
                  288,
                  2
                ],
                [
                  0.5283363802559415,
                  0.006920415224913495,
                  289,
                  2
                ],
                [
                  0.5301645338208409,
                  0.006896551724137931,
                  290,
                  2
                ],
                [
                  0.5319926873857403,
                  0.006872852233676976,
                  291,
                  2
                ],
                [
                  0.5338208409506399,
                  0.00684931506849315,
                  292,
                  2
                ],
                [
                  0.5356489945155393,
                  0.006825938566552901,
                  293,
                  2
                ],
                [
                  0.5374771480804388,
                  0.006802721088435374,
                  294,
                  2
                ],
                [
                  0.5393053016453382,
                  0.006779661016949152,
                  295,
                  2
                ],
                [
                  0.5411334552102377,
                  0.006756756756756757,
                  296,
                  2
                ],
                [
                  0.5429616087751371,
                  0.006734006734006734,
                  297,
                  2
                ],
                [
                  0.5447897623400365,
                  0.006711409395973154,
                  298,
                  2
                ],
                [
                  0.546617915904936,
                  0.006688963210702341,
                  299,
                  2
                ],
                [
                  0.5484460694698354,
                  0.006666666666666667,
                  300,
                  2
                ],
                [
                  0.5502742230347349,
                  0.006644518272425249,
                  301,
                  2
                ],
                [
                  0.5521023765996343,
                  0.006622516556291391,
                  302,
                  2
                ],
                [
                  0.5539305301645339,
                  0.006600660066006601,
                  303,
                  2
                ],
                [
                  0.5557586837294333,
                  0.006578947368421052,
                  304,
                  2
                ],
                [
                  0.5575868372943327,
                  0.006557377049180328,
                  305,
                  2
                ],
                [
                  0.5594149908592322,
                  0.006535947712418301,
                  306,
                  2
                ],
                [
                  0.5612431444241316,
                  0.006514657980456026,
                  307,
                  2
                ],
                [
                  0.5630712979890311,
                  0.006493506493506494,
                  308,
                  2
                ],
                [
                  0.5648994515539305,
                  0.006472491909385114,
                  309,
                  2
                ],
                [
                  0.56672760511883,
                  0.0064516129032258064,
                  310,
                  2
                ],
                [
                  0.5685557586837294,
                  0.006430868167202572,
                  311,
                  2
                ],
                [
                  0.5703839122486288,
                  0.00641025641025641,
                  312,
                  2
                ],
                [
                  0.5722120658135283,
                  0.006389776357827476,
                  313,
                  2
                ],
                [
                  0.5740402193784278,
                  0.006369426751592357,
                  314,
                  2
                ],
                [
                  0.5758683729433273,
                  0.006349206349206349,
                  315,
                  2
                ],
                [
                  0.5776965265082267,
                  0.006329113924050633,
                  316,
                  2
                ],
                [
                  0.5795246800731262,
                  0.006309148264984227,
                  317,
                  2
                ],
                [
                  0.5813528336380256,
                  0.006289308176100629,
                  318,
                  2
                ],
                [
                  0.583180987202925,
                  0.006269592476489028,
                  319,
                  2
                ],
                [
                  0.5850091407678245,
                  0.00625,
                  320,
                  2
                ],
                [
                  0.5868372943327239,
                  0.006230529595015576,
                  321,
                  2
                ],
                [
                  0.5886654478976234,
                  0.006211180124223602,
                  322,
                  2
                ],
                [
                  0.5904936014625228,
                  0.006191950464396285,
                  323,
                  2
                ],
                [
                  0.5923217550274223,
                  0.006172839506172839,
                  324,
                  2
                ],
                [
                  0.5941499085923218,
                  0.006153846153846154,
                  325,
                  2
                ],
                [
                  0.5959780621572212,
                  0.006134969325153374,
                  326,
                  2
                ],
                [
                  0.5978062157221207,
                  0.0061162079510703364,
                  327,
                  2
                ],
                [
                  0.5996343692870201,
                  0.006097560975609756,
                  328,
                  2
                ],
                [
                  0.6014625228519196,
                  0.0060790273556231,
                  329,
                  2
                ],
                [
                  0.603290676416819,
                  0.00909090909090909,
                  330,
                  3
                ],
                [
                  0.6051188299817185,
                  0.00906344410876133,
                  331,
                  3
                ],
                [
                  0.6069469835466179,
                  0.009036144578313253,
                  332,
                  3
                ],
                [
                  0.6087751371115173,
                  0.009009009009009009,
                  333,
                  3
                ],
                [
                  0.6106032906764168,
                  0.008982035928143712,
                  334,
                  3
                ],
                [
                  0.6124314442413162,
                  0.008955223880597015,
                  335,
                  3
                ],
                [
                  0.6142595978062158,
                  0.008928571428571428,
                  336,
                  3
                ],
                [
                  0.6160877513711152,
                  0.008902077151335312,
                  337,
                  3
                ],
                [
                  0.6179159049360147,
                  0.008875739644970414,
                  338,
                  3
                ],
                [
                  0.6197440585009141,
                  0.008849557522123894,
                  339,
                  3
                ],
                [
                  0.6215722120658135,
                  0.008823529411764706,
                  340,
                  3
                ],
                [
                  0.623400365630713,
                  0.008797653958944282,
                  341,
                  3
                ],
                [
                  0.6252285191956124,
                  0.008771929824561403,
                  342,
                  3
                ],
                [
                  0.6270566727605119,
                  0.008746355685131196,
                  343,
                  3
                ],
                [
                  0.6288848263254113,
                  0.00872093023255814,
                  344,
                  3
                ],
                [
                  0.6307129798903108,
                  0.008695652173913044,
                  345,
                  3
                ],
                [
                  0.6325411334552102,
                  0.008670520231213872,
                  346,
                  3
                ],
                [
                  0.6343692870201096,
                  0.008645533141210375,
                  347,
                  3
                ],
                [
                  0.6361974405850092,
                  0.008620689655172414,
                  348,
                  3
                ],
                [
                  0.6380255941499086,
                  0.008595988538681949,
                  349,
                  3
                ],
                [
                  0.6398537477148081,
                  0.008571428571428572,
                  350,
                  3
                ],
                [
                  0.6416819012797075,
                  0.008547008547008548,
                  351,
                  3
                ],
                [
                  0.643510054844607,
                  0.008522727272727272,
                  352,
                  3
                ],
                [
                  0.6453382084095064,
                  0.0084985835694051,
                  353,
                  3
                ],
                [
                  0.6471663619744058,
                  0.00847457627118644,
                  354,
                  3
                ],
                [
                  0.6489945155393053,
                  0.008450704225352112,
                  355,
                  3
                ],
                [
                  0.6508226691042047,
                  0.008426966292134831,
                  356,
                  3
                ],
                [
                  0.6526508226691042,
                  0.008403361344537815,
                  357,
                  3
                ],
                [
                  0.6544789762340036,
                  0.008379888268156424,
                  358,
                  3
                ],
                [
                  0.6563071297989032,
                  0.008356545961002786,
                  359,
                  3
                ],
                [
                  0.6581352833638026,
                  0.008333333333333333,
                  360,
                  3
                ],
                [
                  0.659963436928702,
                  0.008310249307479225,
                  361,
                  3
                ],
                [
                  0.6617915904936015,
                  0.008287292817679558,
                  362,
                  3
                ],
                [
                  0.6636197440585009,
                  0.008264462809917356,
                  363,
                  3
                ],
                [
                  0.6654478976234004,
                  0.008241758241758242,
                  364,
                  3
                ],
                [
                  0.6672760511882998,
                  0.00821917808219178,
                  365,
                  3
                ],
                [
                  0.6691042047531993,
                  0.00819672131147541,
                  366,
                  3
                ],
                [
                  0.6709323583180987,
                  0.010899182561307902,
                  367,
                  4
                ],
                [
                  0.6727605118829981,
                  0.01358695652173913,
                  368,
                  5
                ],
                [
                  0.6745886654478976,
                  0.013550135501355014,
                  369,
                  5
                ],
                [
                  0.676416819012797,
                  0.013513513513513514,
                  370,
                  5
                ],
                [
                  0.6782449725776966,
                  0.013477088948787063,
                  371,
                  5
                ],
                [
                  0.680073126142596,
                  0.013440860215053764,
                  372,
                  5
                ],
                [
                  0.6819012797074955,
                  0.013404825737265416,
                  373,
                  5
                ],
                [
                  0.6837294332723949,
                  0.013368983957219251,
                  374,
                  5
                ],
                [
                  0.6855575868372943,
                  0.016,
                  375,
                  6
                ],
                [
                  0.6873857404021938,
                  0.015957446808510637,
                  376,
                  6
                ],
                [
                  0.6892138939670932,
                  0.015915119363395226,
                  377,
                  6
                ],
                [
                  0.6910420475319927,
                  0.015873015873015872,
                  378,
                  6
                ],
                [
                  0.6928702010968921,
                  0.0158311345646438,
                  379,
                  6
                ],
                [
                  0.6946983546617916,
                  0.015789473684210527,
                  380,
                  6
                ],
                [
                  0.696526508226691,
                  0.015748031496062992,
                  381,
                  6
                ],
                [
                  0.6983546617915904,
                  0.015706806282722512,
                  382,
                  6
                ],
                [
                  0.70018281535649,
                  0.015665796344647518,
                  383,
                  6
                ],
                [
                  0.7020109689213894,
                  0.015625,
                  384,
                  6
                ],
                [
                  0.7038391224862889,
                  0.015584415584415584,
                  385,
                  6
                ],
                [
                  0.7056672760511883,
                  0.015544041450777202,
                  386,
                  6
                ],
                [
                  0.7074954296160878,
                  0.015503875968992248,
                  387,
                  6
                ],
                [
                  0.7093235831809872,
                  0.015463917525773196,
                  388,
                  6
                ],
                [
                  0.7111517367458866,
                  0.015424164524421594,
                  389,
                  6
                ],
                [
                  0.7129798903107861,
                  0.015384615384615385,
                  390,
                  6
                ],
                [
                  0.7148080438756855,
                  0.015345268542199489,
                  391,
                  6
                ],
                [
                  0.716636197440585,
                  0.017857142857142856,
                  392,
                  7
                ],
                [
                  0.7184643510054844,
                  0.017811704834605598,
                  393,
                  7
                ],
                [
                  0.720292504570384,
                  0.02030456852791878,
                  394,
                  8
                ],
                [
                  0.7221206581352834,
                  0.020253164556962026,
                  395,
                  8
                ],
                [
                  0.7239488117001828,
                  0.020202020202020204,
                  396,
                  8
                ],
                [
                  0.7257769652650823,
                  0.020151133501259445,
                  397,
                  8
                ],
                [
                  0.7276051188299817,
                  0.020100502512562814,
                  398,
                  8
                ],
                [
                  0.7294332723948812,
                  0.020050125313283207,
                  399,
                  8
                ],
                [
                  0.7312614259597806,
                  0.02,
                  400,
                  8
                ],
                [
                  0.7330895795246801,
                  0.0199501246882793,
                  401,
                  8
                ],
                [
                  0.7349177330895795,
                  0.01990049751243781,
                  402,
                  8
                ],
                [
                  0.7367458866544789,
                  0.019851116625310174,
                  403,
                  8
                ],
                [
                  0.7385740402193784,
                  0.022277227722772276,
                  404,
                  9
                ],
                [
                  0.7404021937842779,
                  0.024691358024691357,
                  405,
                  10
                ],
                [
                  0.7422303473491774,
                  0.024630541871921183,
                  406,
                  10
                ],
                [
                  0.7440585009140768,
                  0.02702702702702703,
                  407,
                  11
                ],
                [
                  0.7458866544789763,
                  0.02696078431372549,
                  408,
                  11
                ],
                [
                  0.7477148080438757,
                  0.02689486552567237,
                  409,
                  11
                ],
                [
                  0.7495429616087751,
                  0.026829268292682926,
                  410,
                  11
                ],
                [
                  0.7513711151736746,
                  0.0267639902676399,
                  411,
                  11
                ],
                [
                  0.753199268738574,
                  0.02669902912621359,
                  412,
                  11
                ],
                [
                  0.7550274223034735,
                  0.026634382566585957,
                  413,
                  11
                ],
                [
                  0.7568555758683729,
                  0.026570048309178744,
                  414,
                  11
                ],
                [
                  0.7586837294332724,
                  0.02650602409638554,
                  415,
                  11
                ],
                [
                  0.7605118829981719,
                  0.026442307692307692,
                  416,
                  11
                ],
                [
                  0.7623400365630713,
                  0.026378896882494004,
                  417,
                  11
                ],
                [
                  0.7641681901279708,
                  0.02631578947368421,
                  418,
                  11
                ],
                [
                  0.7659963436928702,
                  0.026252983293556086,
                  419,
                  11
                ],
                [
                  0.7678244972577697,
                  0.02619047619047619,
                  420,
                  11
                ],
                [
                  0.7696526508226691,
                  0.026128266033254157,
                  421,
                  11
                ],
                [
                  0.7714808043875686,
                  0.026066350710900472,
                  422,
                  11
                ],
                [
                  0.773308957952468,
                  0.026004728132387706,
                  423,
                  11
                ],
                [
                  0.7751371115173674,
                  0.025943396226415096,
                  424,
                  11
                ],
                [
                  0.7769652650822669,
                  0.02588235294117647,
                  425,
                  11
                ],
                [
                  0.7787934186471663,
                  0.025821596244131457,
                  426,
                  11
                ],
                [
                  0.7806215722120659,
                  0.02576112412177986,
                  427,
                  11
                ],
                [
                  0.7824497257769653,
                  0.02570093457943925,
                  428,
                  11
                ],
                [
                  0.7842778793418648,
                  0.02564102564102564,
                  429,
                  11
                ],
                [
                  0.7861060329067642,
                  0.02558139534883721,
                  430,
                  11
                ],
                [
                  0.7879341864716636,
                  0.027842227378190254,
                  431,
                  12
                ],
                [
                  0.7897623400365631,
                  0.027777777777777776,
                  432,
                  12
                ],
                [
                  0.7915904936014625,
                  0.02771362586605081,
                  433,
                  12
                ],
                [
                  0.793418647166362,
                  0.027649769585253458,
                  434,
                  12
                ],
                [
                  0.7952468007312614,
                  0.027586206896551724,
                  435,
                  12
                ],
                [
                  0.7970749542961609,
                  0.027522935779816515,
                  436,
                  12
                ],
                [
                  0.7989031078610603,
                  0.02745995423340961,
                  437,
                  12
                ],
                [
                  0.8007312614259597,
                  0.0273972602739726,
                  438,
                  12
                ],
                [
                  0.8025594149908593,
                  0.02733485193621868,
                  439,
                  12
                ],
                [
                  0.8043875685557587,
                  0.02727272727272727,
                  440,
                  12
                ],
                [
                  0.8062157221206582,
                  0.027210884353741496,
                  441,
                  12
                ],
                [
                  0.8080438756855576,
                  0.029411764705882353,
                  442,
                  13
                ],
                [
                  0.8098720292504571,
                  0.029345372460496615,
                  443,
                  13
                ],
                [
                  0.8117001828153565,
                  0.03153153153153153,
                  444,
                  14
                ],
                [
                  0.8135283363802559,
                  0.03146067415730337,
                  445,
                  14
                ],
                [
                  0.8153564899451554,
                  0.03139013452914798,
                  446,
                  14
                ],
                [
                  0.8171846435100548,
                  0.03355704697986577,
                  447,
                  15
                ],
                [
                  0.8190127970749543,
                  0.033482142857142856,
                  448,
                  15
                ],
                [
                  0.8208409506398537,
                  0.035634743875278395,
                  449,
                  16
                ],
                [
                  0.8226691042047533,
                  0.035555555555555556,
                  450,
                  16
                ],
                [
                  0.8244972577696527,
                  0.03547671840354767,
                  451,
                  16
                ],
                [
                  0.8263254113345521,
                  0.035398230088495575,
                  452,
                  16
                ],
                [
                  0.8281535648994516,
                  0.03532008830022075,
                  453,
                  16
                ],
                [
                  0.829981718464351,
                  0.03524229074889868,
                  454,
                  16
                ],
                [
                  0.8318098720292505,
                  0.035164835164835165,
                  455,
                  16
                ],
                [
                  0.8336380255941499,
                  0.03508771929824561,
                  456,
                  16
                ],
                [
                  0.8354661791590493,
                  0.0350109409190372,
                  457,
                  16
                ],
                [
                  0.8372943327239488,
                  0.034934497816593885,
                  458,
                  16
                ],
                [
                  0.8391224862888482,
                  0.034858387799564274,
                  459,
                  16
                ],
                [
                  0.8409506398537477,
                  0.03695652173913044,
                  460,
                  17
                ],
                [
                  0.8427787934186471,
                  0.039045553145336226,
                  461,
                  18
                ],
                [
                  0.8446069469835467,
                  0.03896103896103896,
                  462,
                  18
                ],
                [
                  0.8464351005484461,
                  0.04103671706263499,
                  463,
                  19
                ],
                [
                  0.8482632541133455,
                  0.040948275862068964,
                  464,
                  19
                ],
                [
                  0.850091407678245,
                  0.04086021505376344,
                  465,
                  19
                ],
                [
                  0.8519195612431444,
                  0.0407725321888412,
                  466,
                  19
                ],
                [
                  0.8537477148080439,
                  0.04068522483940043,
                  467,
                  19
                ],
                [
                  0.8555758683729433,
                  0.0405982905982906,
                  468,
                  19
                ],
                [
                  0.8574040219378428,
                  0.04051172707889126,
                  469,
                  19
                ],
                [
                  0.8592321755027422,
                  0.04042553191489362,
                  470,
                  19
                ],
                [
                  0.8610603290676416,
                  0.040339702760084924,
                  471,
                  19
                ],
                [
                  0.8628884826325411,
                  0.04025423728813559,
                  472,
                  19
                ],
                [
                  0.8647166361974405,
                  0.042283298097251586,
                  473,
                  20
                ],
                [
                  0.8665447897623401,
                  0.04219409282700422,
                  474,
                  20
                ],
                [
                  0.8683729433272395,
                  0.04421052631578947,
                  475,
                  21
                ],
                [
                  0.870201096892139,
                  0.04411764705882353,
                  476,
                  21
                ],
                [
                  0.8720292504570384,
                  0.0440251572327044,
                  477,
                  21
                ],
                [
                  0.8738574040219378,
                  0.043933054393305436,
                  478,
                  21
                ],
                [
                  0.8756855575868373,
                  0.04384133611691023,
                  479,
                  21
                ],
                [
                  0.8775137111517367,
                  0.04583333333333333,
                  480,
                  22
                ],
                [
                  0.8793418647166362,
                  0.04573804573804574,
                  481,
                  22
                ],
                [
                  0.8811700182815356,
                  0.04771784232365145,
                  482,
                  23
                ],
                [
                  0.8829981718464351,
                  0.047619047619047616,
                  483,
                  23
                ],
                [
                  0.8848263254113345,
                  0.047520661157024795,
                  484,
                  23
                ],
                [
                  0.886654478976234,
                  0.04742268041237113,
                  485,
                  23
                ],
                [
                  0.8884826325411335,
                  0.04938271604938271,
                  486,
                  24
                ],
                [
                  0.8903107861060329,
                  0.0513347022587269,
                  487,
                  25
                ],
                [
                  0.8921389396709324,
                  0.05122950819672131,
                  488,
                  25
                ],
                [
                  0.8939670932358318,
                  0.05112474437627812,
                  489,
                  25
                ],
                [
                  0.8957952468007313,
                  0.05102040816326531,
                  490,
                  25
                ],
                [
                  0.8976234003656307,
                  0.05091649694501019,
                  491,
                  25
                ],
                [
                  0.8994515539305301,
                  0.0508130081300813,
                  492,
                  25
                ],
                [
                  0.9012797074954296,
                  0.05273833671399594,
                  493,
                  26
                ],
                [
                  0.903107861060329,
                  0.05465587044534413,
                  494,
                  27
                ],
                [
                  0.9049360146252285,
                  0.05454545454545454,
                  495,
                  27
                ],
                [
                  0.906764168190128,
                  0.056451612903225805,
                  496,
                  28
                ],
                [
                  0.9085923217550275,
                  0.05835010060362173,
                  497,
                  29
                ],
                [
                  0.9104204753199269,
                  0.060240963855421686,
                  498,
                  30
                ],
                [
                  0.9122486288848263,
                  0.06012024048096192,
                  499,
                  30
                ],
                [
                  0.9140767824497258,
                  0.06,
                  500,
                  30
                ],
                [
                  0.9159049360146252,
                  0.06187624750499002,
                  501,
                  31
                ],
                [
                  0.9177330895795247,
                  0.061752988047808766,
                  502,
                  31
                ],
                [
                  0.9195612431444241,
                  0.0636182902584493,
                  503,
                  32
                ],
                [
                  0.9213893967093236,
                  0.06547619047619048,
                  504,
                  33
                ],
                [
                  0.923217550274223,
                  0.06534653465346535,
                  505,
                  33
                ],
                [
                  0.9250457038391224,
                  0.06719367588932806,
                  506,
                  34
                ],
                [
                  0.926873857404022,
                  0.0670611439842209,
                  507,
                  34
                ],
                [
                  0.9287020109689214,
                  0.06692913385826772,
                  508,
                  34
                ],
                [
                  0.9305301645338209,
                  0.068762278978389,
                  509,
                  35
                ],
                [
                  0.9323583180987203,
                  0.07058823529411765,
                  510,
                  36
                ],
                [
                  0.9341864716636198,
                  0.07240704500978473,
                  511,
                  37
                ],
                [
                  0.9360146252285192,
                  0.07421875,
                  512,
                  38
                ],
                [
                  0.9378427787934186,
                  0.07602339181286549,
                  513,
                  39
                ],
                [
                  0.9396709323583181,
                  0.07587548638132295,
                  514,
                  39
                ],
                [
                  0.9414990859232175,
                  0.07572815533980583,
                  515,
                  39
                ],
                [
                  0.943327239488117,
                  0.07751937984496124,
                  516,
                  40
                ],
                [
                  0.9451553930530164,
                  0.07736943907156674,
                  517,
                  40
                ],
                [
                  0.946983546617916,
                  0.07915057915057915,
                  518,
                  41
                ],
                [
                  0.9488117001828154,
                  0.0789980732177264,
                  519,
                  41
                ],
                [
                  0.9506398537477148,
                  0.08076923076923077,
                  520,
                  42
                ],
                [
                  0.9524680073126143,
                  0.08253358925143954,
                  521,
                  43
                ],
                [
                  0.9542961608775137,
                  0.08237547892720307,
                  522,
                  43
                ],
                [
                  0.9561243144424132,
                  0.08221797323135756,
                  523,
                  43
                ],
                [
                  0.9579524680073126,
                  0.08206106870229007,
                  524,
                  43
                ],
                [
                  0.9597806215722121,
                  0.0838095238095238,
                  525,
                  44
                ],
                [
                  0.9616087751371115,
                  0.08555133079847908,
                  526,
                  45
                ],
                [
                  0.9634369287020109,
                  0.0872865275142315,
                  527,
                  46
                ],
                [
                  0.9652650822669104,
                  0.08901515151515152,
                  528,
                  47
                ],
                [
                  0.9670932358318098,
                  0.09073724007561437,
                  529,
                  48
                ],
                [
                  0.9689213893967094,
                  0.09245283018867924,
                  530,
                  49
                ],
                [
                  0.9707495429616088,
                  0.09416195856873823,
                  531,
                  50
                ],
                [
                  0.9725776965265083,
                  0.09398496240601503,
                  532,
                  50
                ],
                [
                  0.9744058500914077,
                  0.09568480300187618,
                  533,
                  51
                ],
                [
                  0.9762340036563071,
                  0.09550561797752809,
                  534,
                  51
                ],
                [
                  0.9780621572212066,
                  0.09719626168224299,
                  535,
                  52
                ],
                [
                  0.979890310786106,
                  0.09701492537313433,
                  536,
                  52
                ],
                [
                  0.9817184643510055,
                  0.09683426443202979,
                  537,
                  52
                ],
                [
                  0.9835466179159049,
                  0.09851301115241635,
                  538,
                  53
                ],
                [
                  0.9853747714808044,
                  0.09833024118738404,
                  539,
                  53
                ],
                [
                  0.9872029250457038,
                  0.09814814814814815,
                  540,
                  53
                ],
                [
                  0.9890310786106032,
                  0.09796672828096119,
                  541,
                  53
                ],
                [
                  0.9908592321755028,
                  0.09778597785977859,
                  542,
                  53
                ],
                [
                  0.9926873857404022,
                  0.09944751381215469,
                  543,
                  54
                ],
                [
                  0.9945155393053017,
                  0.10110294117647059,
                  544,
                  55
                ],
                [
                  0.9963436928702011,
                  0.10091743119266056,
                  545,
                  55
                ],
                [
                  0.9981718464351006,
                  0.10256410256410256,
                  546,
                  56
                ],
                [
                  1.0,
                  0.10420475319926874,
                  547,
                  57
                ]
              ]
            }
          ]
        },
        {
          "name": "vLLM-SR d3-ultra",
          "text_run": 269,
          "native": [
            {
              "language": "Turkish",
              "wording": "Bilmiyorum",
              "run": 363,
              "full_bank_restricted": true,
              "right": 500,
              "wrong": 46,
              "abstained": 1,
              "invalid": 0,
              "coverage": 0.9981718464351006,
              "risk": 0.08424908424908426
            },
            {
              "language": "Turkish",
              "wording": "I don't know",
              "run": 540,
              "full_bank_restricted": true,
              "right": 494,
              "wrong": 51,
              "abstained": 2,
              "invalid": 0,
              "coverage": 0.9963436928702011,
              "risk": 0.09357798165137615
            },
            {
              "language": "English",
              "wording": "I don't know",
              "run": null
            }
          ],
          "curves": [
            {
              "language": "Turkish",
              "run": 269,
              "points": [
                [
                  0.0018281535648994515,
                  0.0,
                  1,
                  0
                ],
                [
                  0.003656307129798903,
                  0.0,
                  2,
                  0
                ],
                [
                  0.005484460694698354,
                  0.0,
                  3,
                  0
                ],
                [
                  0.007312614259597806,
                  0.0,
                  4,
                  0
                ],
                [
                  0.009140767824497258,
                  0.0,
                  5,
                  0
                ],
                [
                  0.010968921389396709,
                  0.0,
                  6,
                  0
                ],
                [
                  0.012797074954296161,
                  0.0,
                  7,
                  0
                ],
                [
                  0.014625228519195612,
                  0.0,
                  8,
                  0
                ],
                [
                  0.016453382084095063,
                  0.0,
                  9,
                  0
                ],
                [
                  0.018281535648994516,
                  0.0,
                  10,
                  0
                ],
                [
                  0.02010968921389397,
                  0.0,
                  11,
                  0
                ],
                [
                  0.021937842778793418,
                  0.0,
                  12,
                  0
                ],
                [
                  0.02376599634369287,
                  0.0,
                  13,
                  0
                ],
                [
                  0.025594149908592323,
                  0.0,
                  14,
                  0
                ],
                [
                  0.027422303473491772,
                  0.0,
                  15,
                  0
                ],
                [
                  0.029250457038391225,
                  0.0,
                  16,
                  0
                ],
                [
                  0.031078610603290677,
                  0.0,
                  17,
                  0
                ],
                [
                  0.03290676416819013,
                  0.0,
                  18,
                  0
                ],
                [
                  0.03473491773308958,
                  0.0,
                  19,
                  0
                ],
                [
                  0.03656307129798903,
                  0.0,
                  20,
                  0
                ],
                [
                  0.038391224862888484,
                  0.0,
                  21,
                  0
                ],
                [
                  0.04021937842778794,
                  0.0,
                  22,
                  0
                ],
                [
                  0.04204753199268738,
                  0.0,
                  23,
                  0
                ],
                [
                  0.043875685557586835,
                  0.0,
                  24,
                  0
                ],
                [
                  0.04570383912248629,
                  0.0,
                  25,
                  0
                ],
                [
                  0.04753199268738574,
                  0.0,
                  26,
                  0
                ],
                [
                  0.04936014625228519,
                  0.0,
                  27,
                  0
                ],
                [
                  0.051188299817184646,
                  0.0,
                  28,
                  0
                ],
                [
                  0.05301645338208409,
                  0.0,
                  29,
                  0
                ],
                [
                  0.054844606946983544,
                  0.0,
                  30,
                  0
                ],
                [
                  0.056672760511883,
                  0.0,
                  31,
                  0
                ],
                [
                  0.05850091407678245,
                  0.0,
                  32,
                  0
                ],
                [
                  0.0603290676416819,
                  0.0,
                  33,
                  0
                ],
                [
                  0.062157221206581355,
                  0.0,
                  34,
                  0
                ],
                [
                  0.06398537477148081,
                  0.0,
                  35,
                  0
                ],
                [
                  0.06581352833638025,
                  0.0,
                  36,
                  0
                ],
                [
                  0.06764168190127971,
                  0.0,
                  37,
                  0
                ],
                [
                  0.06946983546617916,
                  0.0,
                  38,
                  0
                ],
                [
                  0.0712979890310786,
                  0.0,
                  39,
                  0
                ],
                [
                  0.07312614259597806,
                  0.0,
                  40,
                  0
                ],
                [
                  0.07495429616087751,
                  0.0,
                  41,
                  0
                ],
                [
                  0.07678244972577697,
                  0.0,
                  42,
                  0
                ],
                [
                  0.07861060329067641,
                  0.0,
                  43,
                  0
                ],
                [
                  0.08043875685557587,
                  0.0,
                  44,
                  0
                ],
                [
                  0.08226691042047532,
                  0.0,
                  45,
                  0
                ],
                [
                  0.08409506398537477,
                  0.0,
                  46,
                  0
                ],
                [
                  0.08592321755027423,
                  0.0,
                  47,
                  0
                ],
                [
                  0.08775137111517367,
                  0.0,
                  48,
                  0
                ],
                [
                  0.08957952468007313,
                  0.0,
                  49,
                  0
                ],
                [
                  0.09140767824497258,
                  0.0,
                  50,
                  0
                ],
                [
                  0.09323583180987204,
                  0.0,
                  51,
                  0
                ],
                [
                  0.09506398537477148,
                  0.0,
                  52,
                  0
                ],
                [
                  0.09689213893967093,
                  0.0,
                  53,
                  0
                ],
                [
                  0.09872029250457039,
                  0.0,
                  54,
                  0
                ],
                [
                  0.10054844606946983,
                  0.0,
                  55,
                  0
                ],
                [
                  0.10237659963436929,
                  0.0,
                  56,
                  0
                ],
                [
                  0.10420475319926874,
                  0.0,
                  57,
                  0
                ],
                [
                  0.10603290676416818,
                  0.0,
                  58,
                  0
                ],
                [
                  0.10786106032906764,
                  0.0,
                  59,
                  0
                ],
                [
                  0.10968921389396709,
                  0.0,
                  60,
                  0
                ],
                [
                  0.11151736745886655,
                  0.0,
                  61,
                  0
                ],
                [
                  0.113345521023766,
                  0.0,
                  62,
                  0
                ],
                [
                  0.11517367458866545,
                  0.0,
                  63,
                  0
                ],
                [
                  0.1170018281535649,
                  0.0,
                  64,
                  0
                ],
                [
                  0.11882998171846434,
                  0.0,
                  65,
                  0
                ],
                [
                  0.1206581352833638,
                  0.0,
                  66,
                  0
                ],
                [
                  0.12248628884826325,
                  0.0,
                  67,
                  0
                ],
                [
                  0.12431444241316271,
                  0.0,
                  68,
                  0
                ],
                [
                  0.12614259597806216,
                  0.0,
                  69,
                  0
                ],
                [
                  0.12797074954296161,
                  0.0,
                  70,
                  0
                ],
                [
                  0.12979890310786105,
                  0.0,
                  71,
                  0
                ],
                [
                  0.1316270566727605,
                  0.0,
                  72,
                  0
                ],
                [
                  0.13345521023765997,
                  0.0,
                  73,
                  0
                ],
                [
                  0.13528336380255943,
                  0.0,
                  74,
                  0
                ],
                [
                  0.13711151736745886,
                  0.0,
                  75,
                  0
                ],
                [
                  0.13893967093235832,
                  0.0,
                  76,
                  0
                ],
                [
                  0.14076782449725778,
                  0.0,
                  77,
                  0
                ],
                [
                  0.1425959780621572,
                  0.0,
                  78,
                  0
                ],
                [
                  0.14442413162705667,
                  0.0,
                  79,
                  0
                ],
                [
                  0.14625228519195613,
                  0.0,
                  80,
                  0
                ],
                [
                  0.1480804387568556,
                  0.0,
                  81,
                  0
                ],
                [
                  0.14990859232175502,
                  0.0,
                  82,
                  0
                ],
                [
                  0.15173674588665448,
                  0.0,
                  83,
                  0
                ],
                [
                  0.15356489945155394,
                  0.0,
                  84,
                  0
                ],
                [
                  0.15539305301645337,
                  0.0,
                  85,
                  0
                ],
                [
                  0.15722120658135283,
                  0.0,
                  86,
                  0
                ],
                [
                  0.1590493601462523,
                  0.0,
                  87,
                  0
                ],
                [
                  0.16087751371115175,
                  0.0,
                  88,
                  0
                ],
                [
                  0.16270566727605118,
                  0.0,
                  89,
                  0
                ],
                [
                  0.16453382084095064,
                  0.0,
                  90,
                  0
                ],
                [
                  0.1663619744058501,
                  0.0,
                  91,
                  0
                ],
                [
                  0.16819012797074953,
                  0.0,
                  92,
                  0
                ],
                [
                  0.170018281535649,
                  0.0,
                  93,
                  0
                ],
                [
                  0.17184643510054845,
                  0.0,
                  94,
                  0
                ],
                [
                  0.1736745886654479,
                  0.0,
                  95,
                  0
                ],
                [
                  0.17550274223034734,
                  0.0,
                  96,
                  0
                ],
                [
                  0.1773308957952468,
                  0.0,
                  97,
                  0
                ],
                [
                  0.17915904936014626,
                  0.0,
                  98,
                  0
                ],
                [
                  0.1809872029250457,
                  0.0,
                  99,
                  0
                ],
                [
                  0.18281535648994515,
                  0.0,
                  100,
                  0
                ],
                [
                  0.1846435100548446,
                  0.0,
                  101,
                  0
                ],
                [
                  0.18647166361974407,
                  0.0,
                  102,
                  0
                ],
                [
                  0.1882998171846435,
                  0.0,
                  103,
                  0
                ],
                [
                  0.19012797074954296,
                  0.0,
                  104,
                  0
                ],
                [
                  0.19195612431444242,
                  0.0,
                  105,
                  0
                ],
                [
                  0.19378427787934185,
                  0.0,
                  106,
                  0
                ],
                [
                  0.1956124314442413,
                  0.0,
                  107,
                  0
                ],
                [
                  0.19744058500914077,
                  0.0,
                  108,
                  0
                ],
                [
                  0.19926873857404023,
                  0.0,
                  109,
                  0
                ],
                [
                  0.20109689213893966,
                  0.0,
                  110,
                  0
                ],
                [
                  0.20292504570383912,
                  0.0,
                  111,
                  0
                ],
                [
                  0.20475319926873858,
                  0.0,
                  112,
                  0
                ],
                [
                  0.20658135283363802,
                  0.0,
                  113,
                  0
                ],
                [
                  0.20840950639853748,
                  0.0,
                  114,
                  0
                ],
                [
                  0.21023765996343693,
                  0.0,
                  115,
                  0
                ],
                [
                  0.21206581352833637,
                  0.0,
                  116,
                  0
                ],
                [
                  0.21389396709323583,
                  0.0,
                  117,
                  0
                ],
                [
                  0.21572212065813529,
                  0.0,
                  118,
                  0
                ],
                [
                  0.21755027422303475,
                  0.0,
                  119,
                  0
                ],
                [
                  0.21937842778793418,
                  0.0,
                  120,
                  0
                ],
                [
                  0.22120658135283364,
                  0.0,
                  121,
                  0
                ],
                [
                  0.2230347349177331,
                  0.0,
                  122,
                  0
                ],
                [
                  0.22486288848263253,
                  0.0,
                  123,
                  0
                ],
                [
                  0.226691042047532,
                  0.0,
                  124,
                  0
                ],
                [
                  0.22851919561243145,
                  0.0,
                  125,
                  0
                ],
                [
                  0.2303473491773309,
                  0.0,
                  126,
                  0
                ],
                [
                  0.23217550274223034,
                  0.0,
                  127,
                  0
                ],
                [
                  0.2340036563071298,
                  0.0,
                  128,
                  0
                ],
                [
                  0.23583180987202926,
                  0.0,
                  129,
                  0
                ],
                [
                  0.2376599634369287,
                  0.0,
                  130,
                  0
                ],
                [
                  0.23948811700182815,
                  0.0,
                  131,
                  0
                ],
                [
                  0.2413162705667276,
                  0.0,
                  132,
                  0
                ],
                [
                  0.24314442413162707,
                  0.0,
                  133,
                  0
                ],
                [
                  0.2449725776965265,
                  0.0,
                  134,
                  0
                ],
                [
                  0.24680073126142596,
                  0.0,
                  135,
                  0
                ],
                [
                  0.24862888482632542,
                  0.0,
                  136,
                  0
                ],
                [
                  0.25045703839122485,
                  0.0,
                  137,
                  0
                ],
                [
                  0.2522851919561243,
                  0.0,
                  138,
                  0
                ],
                [
                  0.25411334552102377,
                  0.0,
                  139,
                  0
                ],
                [
                  0.25594149908592323,
                  0.0,
                  140,
                  0
                ],
                [
                  0.2577696526508227,
                  0.0,
                  141,
                  0
                ],
                [
                  0.2595978062157221,
                  0.0,
                  142,
                  0
                ],
                [
                  0.26142595978062155,
                  0.0,
                  143,
                  0
                ],
                [
                  0.263254113345521,
                  0.0,
                  144,
                  0
                ],
                [
                  0.26508226691042047,
                  0.0,
                  145,
                  0
                ],
                [
                  0.26691042047531993,
                  0.0,
                  146,
                  0
                ],
                [
                  0.2687385740402194,
                  0.0,
                  147,
                  0
                ],
                [
                  0.27056672760511885,
                  0.0,
                  148,
                  0
                ],
                [
                  0.27239488117001825,
                  0.0,
                  149,
                  0
                ],
                [
                  0.2742230347349177,
                  0.0,
                  150,
                  0
                ],
                [
                  0.2760511882998172,
                  0.0,
                  151,
                  0
                ],
                [
                  0.27787934186471663,
                  0.0,
                  152,
                  0
                ],
                [
                  0.2797074954296161,
                  0.0,
                  153,
                  0
                ],
                [
                  0.28153564899451555,
                  0.0,
                  154,
                  0
                ],
                [
                  0.283363802559415,
                  0.0,
                  155,
                  0
                ],
                [
                  0.2851919561243144,
                  0.0,
                  156,
                  0
                ],
                [
                  0.2870201096892139,
                  0.0,
                  157,
                  0
                ],
                [
                  0.28884826325411334,
                  0.0,
                  158,
                  0
                ],
                [
                  0.2906764168190128,
                  0.0,
                  159,
                  0
                ],
                [
                  0.29250457038391225,
                  0.0,
                  160,
                  0
                ],
                [
                  0.2943327239488117,
                  0.0,
                  161,
                  0
                ],
                [
                  0.2961608775137112,
                  0.0,
                  162,
                  0
                ],
                [
                  0.2979890310786106,
                  0.0,
                  163,
                  0
                ],
                [
                  0.29981718464351004,
                  0.0,
                  164,
                  0
                ],
                [
                  0.3016453382084095,
                  0.0,
                  165,
                  0
                ],
                [
                  0.30347349177330896,
                  0.0,
                  166,
                  0
                ],
                [
                  0.3053016453382084,
                  0.0,
                  167,
                  0
                ],
                [
                  0.3071297989031079,
                  0.0,
                  168,
                  0
                ],
                [
                  0.30895795246800734,
                  0.0,
                  169,
                  0
                ],
                [
                  0.31078610603290674,
                  0.0,
                  170,
                  0
                ],
                [
                  0.3126142595978062,
                  0.0,
                  171,
                  0
                ],
                [
                  0.31444241316270566,
                  0.0,
                  172,
                  0
                ],
                [
                  0.3162705667276051,
                  0.0,
                  173,
                  0
                ],
                [
                  0.3180987202925046,
                  0.0,
                  174,
                  0
                ],
                [
                  0.31992687385740404,
                  0.0,
                  175,
                  0
                ],
                [
                  0.3217550274223035,
                  0.0,
                  176,
                  0
                ],
                [
                  0.3235831809872029,
                  0.0,
                  177,
                  0
                ],
                [
                  0.32541133455210236,
                  0.0,
                  178,
                  0
                ],
                [
                  0.3272394881170018,
                  0.0,
                  179,
                  0
                ],
                [
                  0.3290676416819013,
                  0.0,
                  180,
                  0
                ],
                [
                  0.33089579524680074,
                  0.0,
                  181,
                  0
                ],
                [
                  0.3327239488117002,
                  0.0,
                  182,
                  0
                ],
                [
                  0.33455210237659966,
                  0.0,
                  183,
                  0
                ],
                [
                  0.33638025594149906,
                  0.0,
                  184,
                  0
                ],
                [
                  0.3382084095063985,
                  0.0,
                  185,
                  0
                ],
                [
                  0.340036563071298,
                  0.0,
                  186,
                  0
                ],
                [
                  0.34186471663619744,
                  0.0,
                  187,
                  0
                ],
                [
                  0.3436928702010969,
                  0.0,
                  188,
                  0
                ],
                [
                  0.34552102376599636,
                  0.0,
                  189,
                  0
                ],
                [
                  0.3473491773308958,
                  0.0,
                  190,
                  0
                ],
                [
                  0.3491773308957952,
                  0.0,
                  191,
                  0
                ],
                [
                  0.3510054844606947,
                  0.0,
                  192,
                  0
                ],
                [
                  0.35283363802559414,
                  0.0,
                  193,
                  0
                ],
                [
                  0.3546617915904936,
                  0.0,
                  194,
                  0
                ],
                [
                  0.35648994515539306,
                  0.0,
                  195,
                  0
                ],
                [
                  0.3583180987202925,
                  0.0,
                  196,
                  0
                ],
                [
                  0.360146252285192,
                  0.0,
                  197,
                  0
                ],
                [
                  0.3619744058500914,
                  0.0,
                  198,
                  0
                ],
                [
                  0.36380255941499084,
                  0.0,
                  199,
                  0
                ],
                [
                  0.3656307129798903,
                  0.0,
                  200,
                  0
                ],
                [
                  0.36745886654478976,
                  0.0,
                  201,
                  0
                ],
                [
                  0.3692870201096892,
                  0.0,
                  202,
                  0
                ],
                [
                  0.3711151736745887,
                  0.0,
                  203,
                  0
                ],
                [
                  0.37294332723948814,
                  0.0,
                  204,
                  0
                ],
                [
                  0.37477148080438755,
                  0.0,
                  205,
                  0
                ],
                [
                  0.376599634369287,
                  0.0,
                  206,
                  0
                ],
                [
                  0.37842778793418647,
                  0.0,
                  207,
                  0
                ],
                [
                  0.3802559414990859,
                  0.0,
                  208,
                  0
                ],
                [
                  0.3820840950639854,
                  0.0,
                  209,
                  0
                ],
                [
                  0.38391224862888484,
                  0.0,
                  210,
                  0
                ],
                [
                  0.3857404021937843,
                  0.0,
                  211,
                  0
                ],
                [
                  0.3875685557586837,
                  0.0,
                  212,
                  0
                ],
                [
                  0.38939670932358317,
                  0.0,
                  213,
                  0
                ],
                [
                  0.3912248628884826,
                  0.0,
                  214,
                  0
                ],
                [
                  0.3930530164533821,
                  0.0,
                  215,
                  0
                ],
                [
                  0.39488117001828155,
                  0.0,
                  216,
                  0
                ],
                [
                  0.396709323583181,
                  0.0,
                  217,
                  0
                ],
                [
                  0.39853747714808047,
                  0.0,
                  218,
                  0
                ],
                [
                  0.40036563071297987,
                  0.0,
                  219,
                  0
                ],
                [
                  0.40219378427787933,
                  0.0,
                  220,
                  0
                ],
                [
                  0.4040219378427788,
                  0.0,
                  221,
                  0
                ],
                [
                  0.40585009140767825,
                  0.0,
                  222,
                  0
                ],
                [
                  0.4076782449725777,
                  0.0,
                  223,
                  0
                ],
                [
                  0.40950639853747717,
                  0.0,
                  224,
                  0
                ],
                [
                  0.4113345521023766,
                  0.0,
                  225,
                  0
                ],
                [
                  0.41316270566727603,
                  0.0,
                  226,
                  0
                ],
                [
                  0.4149908592321755,
                  0.0,
                  227,
                  0
                ],
                [
                  0.41681901279707495,
                  0.0,
                  228,
                  0
                ],
                [
                  0.4186471663619744,
                  0.0,
                  229,
                  0
                ],
                [
                  0.42047531992687387,
                  0.0,
                  230,
                  0
                ],
                [
                  0.42230347349177333,
                  0.0,
                  231,
                  0
                ],
                [
                  0.42413162705667273,
                  0.0,
                  232,
                  0
                ],
                [
                  0.4259597806215722,
                  0.0,
                  233,
                  0
                ],
                [
                  0.42778793418647165,
                  0.0,
                  234,
                  0
                ],
                [
                  0.4296160877513711,
                  0.0,
                  235,
                  0
                ],
                [
                  0.43144424131627057,
                  0.0,
                  236,
                  0
                ],
                [
                  0.43327239488117003,
                  0.0,
                  237,
                  0
                ],
                [
                  0.4351005484460695,
                  0.0,
                  238,
                  0
                ],
                [
                  0.4369287020109689,
                  0.0,
                  239,
                  0
                ],
                [
                  0.43875685557586835,
                  0.0,
                  240,
                  0
                ],
                [
                  0.4405850091407678,
                  0.0,
                  241,
                  0
                ],
                [
                  0.4424131627056673,
                  0.0,
                  242,
                  0
                ],
                [
                  0.44424131627056673,
                  0.0,
                  243,
                  0
                ],
                [
                  0.4460694698354662,
                  0.0,
                  244,
                  0
                ],
                [
                  0.44789762340036565,
                  0.0,
                  245,
                  0
                ],
                [
                  0.44972577696526506,
                  0.0,
                  246,
                  0
                ],
                [
                  0.4515539305301645,
                  0.0,
                  247,
                  0
                ],
                [
                  0.453382084095064,
                  0.0,
                  248,
                  0
                ],
                [
                  0.45521023765996343,
                  0.0,
                  249,
                  0
                ],
                [
                  0.4570383912248629,
                  0.0,
                  250,
                  0
                ],
                [
                  0.45886654478976235,
                  0.0,
                  251,
                  0
                ],
                [
                  0.4606946983546618,
                  0.0,
                  252,
                  0
                ],
                [
                  0.4625228519195612,
                  0.0,
                  253,
                  0
                ],
                [
                  0.4643510054844607,
                  0.0,
                  254,
                  0
                ],
                [
                  0.46617915904936014,
                  0.0,
                  255,
                  0
                ],
                [
                  0.4680073126142596,
                  0.0,
                  256,
                  0
                ],
                [
                  0.46983546617915906,
                  0.0,
                  257,
                  0
                ],
                [
                  0.4716636197440585,
                  0.0,
                  258,
                  0
                ],
                [
                  0.473491773308958,
                  0.0,
                  259,
                  0
                ],
                [
                  0.4753199268738574,
                  0.0,
                  260,
                  0
                ],
                [
                  0.47714808043875684,
                  0.0,
                  261,
                  0
                ],
                [
                  0.4789762340036563,
                  0.0,
                  262,
                  0
                ],
                [
                  0.48080438756855576,
                  0.0,
                  263,
                  0
                ],
                [
                  0.4826325411334552,
                  0.0,
                  264,
                  0
                ],
                [
                  0.4844606946983547,
                  0.0,
                  265,
                  0
                ],
                [
                  0.48628884826325414,
                  0.0,
                  266,
                  0
                ],
                [
                  0.48811700182815354,
                  0.0,
                  267,
                  0
                ],
                [
                  0.489945155393053,
                  0.0,
                  268,
                  0
                ],
                [
                  0.49177330895795246,
                  0.0,
                  269,
                  0
                ],
                [
                  0.4936014625228519,
                  0.0,
                  270,
                  0
                ],
                [
                  0.4954296160877514,
                  0.0,
                  271,
                  0
                ],
                [
                  0.49725776965265084,
                  0.0,
                  272,
                  0
                ],
                [
                  0.4990859232175503,
                  0.0,
                  273,
                  0
                ],
                [
                  0.5009140767824497,
                  0.0,
                  274,
                  0
                ],
                [
                  0.5027422303473492,
                  0.0,
                  275,
                  0
                ],
                [
                  0.5045703839122486,
                  0.0,
                  276,
                  0
                ],
                [
                  0.506398537477148,
                  0.0,
                  277,
                  0
                ],
                [
                  0.5082266910420475,
                  0.0,
                  278,
                  0
                ],
                [
                  0.5100548446069469,
                  0.0,
                  279,
                  0
                ],
                [
                  0.5118829981718465,
                  0.0,
                  280,
                  0
                ],
                [
                  0.5137111517367459,
                  0.0,
                  281,
                  0
                ],
                [
                  0.5155393053016454,
                  0.0,
                  282,
                  0
                ],
                [
                  0.5173674588665448,
                  0.0,
                  283,
                  0
                ],
                [
                  0.5191956124314442,
                  0.0,
                  284,
                  0
                ],
                [
                  0.5210237659963437,
                  0.0,
                  285,
                  0
                ],
                [
                  0.5228519195612431,
                  0.0,
                  286,
                  0
                ],
                [
                  0.5246800731261426,
                  0.0,
                  287,
                  0
                ],
                [
                  0.526508226691042,
                  0.0,
                  288,
                  0
                ],
                [
                  0.5283363802559415,
                  0.0,
                  289,
                  0
                ],
                [
                  0.5301645338208409,
                  0.0,
                  290,
                  0
                ],
                [
                  0.5319926873857403,
                  0.0,
                  291,
                  0
                ],
                [
                  0.5338208409506399,
                  0.0,
                  292,
                  0
                ],
                [
                  0.5356489945155393,
                  0.0,
                  293,
                  0
                ],
                [
                  0.5374771480804388,
                  0.0,
                  294,
                  0
                ],
                [
                  0.5393053016453382,
                  0.0,
                  295,
                  0
                ],
                [
                  0.5411334552102377,
                  0.0,
                  296,
                  0
                ],
                [
                  0.5429616087751371,
                  0.0,
                  297,
                  0
                ],
                [
                  0.5447897623400365,
                  0.0,
                  298,
                  0
                ],
                [
                  0.546617915904936,
                  0.0,
                  299,
                  0
                ],
                [
                  0.5484460694698354,
                  0.0,
                  300,
                  0
                ],
                [
                  0.5502742230347349,
                  0.0,
                  301,
                  0
                ],
                [
                  0.5521023765996343,
                  0.0,
                  302,
                  0
                ],
                [
                  0.5539305301645339,
                  0.0033003300330033004,
                  303,
                  1
                ],
                [
                  0.5557586837294333,
                  0.003289473684210526,
                  304,
                  1
                ],
                [
                  0.5575868372943327,
                  0.003278688524590164,
                  305,
                  1
                ],
                [
                  0.5594149908592322,
                  0.0032679738562091504,
                  306,
                  1
                ],
                [
                  0.5612431444241316,
                  0.003257328990228013,
                  307,
                  1
                ],
                [
                  0.5630712979890311,
                  0.003246753246753247,
                  308,
                  1
                ],
                [
                  0.5648994515539305,
                  0.003236245954692557,
                  309,
                  1
                ],
                [
                  0.56672760511883,
                  0.0032258064516129032,
                  310,
                  1
                ],
                [
                  0.5685557586837294,
                  0.003215434083601286,
                  311,
                  1
                ],
                [
                  0.5703839122486288,
                  0.003205128205128205,
                  312,
                  1
                ],
                [
                  0.5722120658135283,
                  0.003194888178913738,
                  313,
                  1
                ],
                [
                  0.5740402193784278,
                  0.0031847133757961785,
                  314,
                  1
                ],
                [
                  0.5758683729433273,
                  0.0031746031746031746,
                  315,
                  1
                ],
                [
                  0.5776965265082267,
                  0.0031645569620253164,
                  316,
                  1
                ],
                [
                  0.5795246800731262,
                  0.0031545741324921135,
                  317,
                  1
                ],
                [
                  0.5813528336380256,
                  0.0031446540880503146,
                  318,
                  1
                ],
                [
                  0.583180987202925,
                  0.003134796238244514,
                  319,
                  1
                ],
                [
                  0.5850091407678245,
                  0.003125,
                  320,
                  1
                ],
                [
                  0.5868372943327239,
                  0.003115264797507788,
                  321,
                  1
                ],
                [
                  0.5886654478976234,
                  0.003105590062111801,
                  322,
                  1
                ],
                [
                  0.5904936014625228,
                  0.0030959752321981426,
                  323,
                  1
                ],
                [
                  0.5923217550274223,
                  0.0030864197530864196,
                  324,
                  1
                ],
                [
                  0.5941499085923218,
                  0.003076923076923077,
                  325,
                  1
                ],
                [
                  0.5959780621572212,
                  0.003067484662576687,
                  326,
                  1
                ],
                [
                  0.5978062157221207,
                  0.0030581039755351682,
                  327,
                  1
                ],
                [
                  0.5996343692870201,
                  0.003048780487804878,
                  328,
                  1
                ],
                [
                  0.6014625228519196,
                  0.00303951367781155,
                  329,
                  1
                ],
                [
                  0.603290676416819,
                  0.0030303030303030303,
                  330,
                  1
                ],
                [
                  0.6051188299817185,
                  0.0030211480362537764,
                  331,
                  1
                ],
                [
                  0.6069469835466179,
                  0.0030120481927710845,
                  332,
                  1
                ],
                [
                  0.6087751371115173,
                  0.003003003003003003,
                  333,
                  1
                ],
                [
                  0.6106032906764168,
                  0.0029940119760479044,
                  334,
                  1
                ],
                [
                  0.6124314442413162,
                  0.0029850746268656717,
                  335,
                  1
                ],
                [
                  0.6142595978062158,
                  0.002976190476190476,
                  336,
                  1
                ],
                [
                  0.6160877513711152,
                  0.002967359050445104,
                  337,
                  1
                ],
                [
                  0.6179159049360147,
                  0.0029585798816568047,
                  338,
                  1
                ],
                [
                  0.6197440585009141,
                  0.0029498525073746312,
                  339,
                  1
                ],
                [
                  0.6215722120658135,
                  0.0029411764705882353,
                  340,
                  1
                ],
                [
                  0.623400365630713,
                  0.002932551319648094,
                  341,
                  1
                ],
                [
                  0.6252285191956124,
                  0.0029239766081871343,
                  342,
                  1
                ],
                [
                  0.6270566727605119,
                  0.0029154518950437317,
                  343,
                  1
                ],
                [
                  0.6288848263254113,
                  0.0029069767441860465,
                  344,
                  1
                ],
                [
                  0.6307129798903108,
                  0.005797101449275362,
                  345,
                  2
                ],
                [
                  0.6325411334552102,
                  0.005780346820809248,
                  346,
                  2
                ],
                [
                  0.6343692870201096,
                  0.005763688760806916,
                  347,
                  2
                ],
                [
                  0.6361974405850092,
                  0.005747126436781609,
                  348,
                  2
                ],
                [
                  0.6380255941499086,
                  0.0057306590257879654,
                  349,
                  2
                ],
                [
                  0.6398537477148081,
                  0.005714285714285714,
                  350,
                  2
                ],
                [
                  0.6416819012797075,
                  0.005698005698005698,
                  351,
                  2
                ],
                [
                  0.643510054844607,
                  0.005681818181818182,
                  352,
                  2
                ],
                [
                  0.6453382084095064,
                  0.0056657223796034,
                  353,
                  2
                ],
                [
                  0.6471663619744058,
                  0.005649717514124294,
                  354,
                  2
                ],
                [
                  0.6489945155393053,
                  0.005633802816901409,
                  355,
                  2
                ],
                [
                  0.6508226691042047,
                  0.0056179775280898875,
                  356,
                  2
                ],
                [
                  0.6526508226691042,
                  0.0056022408963585435,
                  357,
                  2
                ],
                [
                  0.6544789762340036,
                  0.00558659217877095,
                  358,
                  2
                ],
                [
                  0.6563071297989032,
                  0.005571030640668524,
                  359,
                  2
                ],
                [
                  0.6581352833638026,
                  0.005555555555555556,
                  360,
                  2
                ],
                [
                  0.659963436928702,
                  0.00554016620498615,
                  361,
                  2
                ],
                [
                  0.6617915904936015,
                  0.0055248618784530384,
                  362,
                  2
                ],
                [
                  0.6636197440585009,
                  0.005509641873278237,
                  363,
                  2
                ],
                [
                  0.6654478976234004,
                  0.005494505494505495,
                  364,
                  2
                ],
                [
                  0.6672760511882998,
                  0.005479452054794521,
                  365,
                  2
                ],
                [
                  0.6691042047531993,
                  0.00546448087431694,
                  366,
                  2
                ],
                [
                  0.6709323583180987,
                  0.005449591280653951,
                  367,
                  2
                ],
                [
                  0.6727605118829981,
                  0.005434782608695652,
                  368,
                  2
                ],
                [
                  0.6745886654478976,
                  0.005420054200542005,
                  369,
                  2
                ],
                [
                  0.676416819012797,
                  0.005405405405405406,
                  370,
                  2
                ],
                [
                  0.6782449725776966,
                  0.005390835579514825,
                  371,
                  2
                ],
                [
                  0.680073126142596,
                  0.005376344086021506,
                  372,
                  2
                ],
                [
                  0.6819012797074955,
                  0.005361930294906166,
                  373,
                  2
                ],
                [
                  0.6837294332723949,
                  0.0053475935828877,
                  374,
                  2
                ],
                [
                  0.6855575868372943,
                  0.005333333333333333,
                  375,
                  2
                ],
                [
                  0.6873857404021938,
                  0.005319148936170213,
                  376,
                  2
                ],
                [
                  0.6892138939670932,
                  0.005305039787798408,
                  377,
                  2
                ],
                [
                  0.6910420475319927,
                  0.005291005291005291,
                  378,
                  2
                ],
                [
                  0.6928702010968921,
                  0.005277044854881266,
                  379,
                  2
                ],
                [
                  0.6946983546617916,
                  0.005263157894736842,
                  380,
                  2
                ],
                [
                  0.696526508226691,
                  0.005249343832020997,
                  381,
                  2
                ],
                [
                  0.6983546617915904,
                  0.005235602094240838,
                  382,
                  2
                ],
                [
                  0.70018281535649,
                  0.007832898172323759,
                  383,
                  3
                ],
                [
                  0.7020109689213894,
                  0.0078125,
                  384,
                  3
                ],
                [
                  0.7038391224862889,
                  0.007792207792207792,
                  385,
                  3
                ],
                [
                  0.7056672760511883,
                  0.007772020725388601,
                  386,
                  3
                ],
                [
                  0.7074954296160878,
                  0.007751937984496124,
                  387,
                  3
                ],
                [
                  0.7093235831809872,
                  0.010309278350515464,
                  388,
                  4
                ],
                [
                  0.7111517367458866,
                  0.010282776349614395,
                  389,
                  4
                ],
                [
                  0.7129798903107861,
                  0.010256410256410256,
                  390,
                  4
                ],
                [
                  0.7148080438756855,
                  0.010230179028132993,
                  391,
                  4
                ],
                [
                  0.716636197440585,
                  0.01020408163265306,
                  392,
                  4
                ],
                [
                  0.7184643510054844,
                  0.010178117048346057,
                  393,
                  4
                ],
                [
                  0.720292504570384,
                  0.01015228426395939,
                  394,
                  4
                ],
                [
                  0.7221206581352834,
                  0.010126582278481013,
                  395,
                  4
                ],
                [
                  0.7239488117001828,
                  0.010101010101010102,
                  396,
                  4
                ],
                [
                  0.7257769652650823,
                  0.010075566750629723,
                  397,
                  4
                ],
                [
                  0.7276051188299817,
                  0.010050251256281407,
                  398,
                  4
                ],
                [
                  0.7294332723948812,
                  0.010025062656641603,
                  399,
                  4
                ],
                [
                  0.7312614259597806,
                  0.01,
                  400,
                  4
                ],
                [
                  0.7330895795246801,
                  0.00997506234413965,
                  401,
                  4
                ],
                [
                  0.7349177330895795,
                  0.009950248756218905,
                  402,
                  4
                ],
                [
                  0.7367458866544789,
                  0.009925558312655087,
                  403,
                  4
                ],
                [
                  0.7385740402193784,
                  0.009900990099009901,
                  404,
                  4
                ],
                [
                  0.7404021937842779,
                  0.009876543209876543,
                  405,
                  4
                ],
                [
                  0.7422303473491774,
                  0.009852216748768473,
                  406,
                  4
                ],
                [
                  0.7440585009140768,
                  0.009828009828009828,
                  407,
                  4
                ],
                [
                  0.7458866544789763,
                  0.00980392156862745,
                  408,
                  4
                ],
                [
                  0.7477148080438757,
                  0.009779951100244499,
                  409,
                  4
                ],
                [
                  0.7495429616087751,
                  0.00975609756097561,
                  410,
                  4
                ],
                [
                  0.7513711151736746,
                  0.009732360097323601,
                  411,
                  4
                ],
                [
                  0.753199268738574,
                  0.009708737864077669,
                  412,
                  4
                ],
                [
                  0.7550274223034735,
                  0.009685230024213076,
                  413,
                  4
                ],
                [
                  0.7568555758683729,
                  0.00966183574879227,
                  414,
                  4
                ],
                [
                  0.7586837294332724,
                  0.00963855421686747,
                  415,
                  4
                ],
                [
                  0.7605118829981719,
                  0.009615384615384616,
                  416,
                  4
                ],
                [
                  0.7623400365630713,
                  0.009592326139088728,
                  417,
                  4
                ],
                [
                  0.7641681901279708,
                  0.011961722488038277,
                  418,
                  5
                ],
                [
                  0.7659963436928702,
                  0.011933174224343675,
                  419,
                  5
                ],
                [
                  0.7678244972577697,
                  0.011904761904761904,
                  420,
                  5
                ],
                [
                  0.7696526508226691,
                  0.011876484560570071,
                  421,
                  5
                ],
                [
                  0.7714808043875686,
                  0.014218009478672985,
                  422,
                  6
                ],
                [
                  0.773308957952468,
                  0.014184397163120567,
                  423,
                  6
                ],
                [
                  0.7751371115173674,
                  0.014150943396226415,
                  424,
                  6
                ],
                [
                  0.7769652650822669,
                  0.01411764705882353,
                  425,
                  6
                ],
                [
                  0.7787934186471663,
                  0.014084507042253521,
                  426,
                  6
                ],
                [
                  0.7806215722120659,
                  0.01405152224824356,
                  427,
                  6
                ],
                [
                  0.7824497257769653,
                  0.014018691588785047,
                  428,
                  6
                ],
                [
                  0.7842778793418648,
                  0.013986013986013986,
                  429,
                  6
                ],
                [
                  0.7861060329067642,
                  0.013953488372093023,
                  430,
                  6
                ],
                [
                  0.7879341864716636,
                  0.013921113689095127,
                  431,
                  6
                ],
                [
                  0.7897623400365631,
                  0.013888888888888888,
                  432,
                  6
                ],
                [
                  0.7915904936014625,
                  0.013856812933025405,
                  433,
                  6
                ],
                [
                  0.793418647166362,
                  0.013824884792626729,
                  434,
                  6
                ],
                [
                  0.7952468007312614,
                  0.013793103448275862,
                  435,
                  6
                ],
                [
                  0.7970749542961609,
                  0.013761467889908258,
                  436,
                  6
                ],
                [
                  0.7989031078610603,
                  0.013729977116704805,
                  437,
                  6
                ],
                [
                  0.8007312614259597,
                  0.0136986301369863,
                  438,
                  6
                ],
                [
                  0.8025594149908593,
                  0.01366742596810934,
                  439,
                  6
                ],
                [
                  0.8043875685557587,
                  0.013636363636363636,
                  440,
                  6
                ],
                [
                  0.8062157221206582,
                  0.013605442176870748,
                  441,
                  6
                ],
                [
                  0.8080438756855576,
                  0.013574660633484163,
                  442,
                  6
                ],
                [
                  0.8098720292504571,
                  0.013544018058690745,
                  443,
                  6
                ],
                [
                  0.8117001828153565,
                  0.013513513513513514,
                  444,
                  6
                ],
                [
                  0.8135283363802559,
                  0.01348314606741573,
                  445,
                  6
                ],
                [
                  0.8153564899451554,
                  0.013452914798206279,
                  446,
                  6
                ],
                [
                  0.8171846435100548,
                  0.015659955257270694,
                  447,
                  7
                ],
                [
                  0.8190127970749543,
                  0.015625,
                  448,
                  7
                ],
                [
                  0.8208409506398537,
                  0.017817371937639197,
                  449,
                  8
                ],
                [
                  0.8226691042047533,
                  0.017777777777777778,
                  450,
                  8
                ],
                [
                  0.8244972577696527,
                  0.017738359201773836,
                  451,
                  8
                ],
                [
                  0.8263254113345521,
                  0.017699115044247787,
                  452,
                  8
                ],
                [
                  0.8281535648994516,
                  0.017660044150110375,
                  453,
                  8
                ],
                [
                  0.829981718464351,
                  0.019823788546255508,
                  454,
                  9
                ],
                [
                  0.8318098720292505,
                  0.01978021978021978,
                  455,
                  9
                ],
                [
                  0.8336380255941499,
                  0.019736842105263157,
                  456,
                  9
                ],
                [
                  0.8354661791590493,
                  0.019693654266958426,
                  457,
                  9
                ],
                [
                  0.8372943327239488,
                  0.019650655021834062,
                  458,
                  9
                ],
                [
                  0.8391224862888482,
                  0.02178649237472767,
                  459,
                  10
                ],
                [
                  0.8409506398537477,
                  0.02391304347826087,
                  460,
                  11
                ],
                [
                  0.8427787934186471,
                  0.02386117136659436,
                  461,
                  11
                ],
                [
                  0.8446069469835467,
                  0.025974025974025976,
                  462,
                  12
                ],
                [
                  0.8464351005484461,
                  0.028077753779697623,
                  463,
                  13
                ],
                [
                  0.8482632541133455,
                  0.03017241379310345,
                  464,
                  14
                ],
                [
                  0.850091407678245,
                  0.030107526881720432,
                  465,
                  14
                ],
                [
                  0.8519195612431444,
                  0.032188841201716736,
                  466,
                  15
                ],
                [
                  0.8537477148080439,
                  0.032119914346895075,
                  467,
                  15
                ],
                [
                  0.8555758683729433,
                  0.03418803418803419,
                  468,
                  16
                ],
                [
                  0.8574040219378428,
                  0.03411513859275053,
                  469,
                  16
                ],
                [
                  0.8592321755027422,
                  0.03404255319148936,
                  470,
                  16
                ],
                [
                  0.8610603290676416,
                  0.03397027600849257,
                  471,
                  16
                ],
                [
                  0.8628884826325411,
                  0.03389830508474576,
                  472,
                  16
                ],
                [
                  0.8647166361974405,
                  0.03382663847780127,
                  473,
                  16
                ],
                [
                  0.8665447897623401,
                  0.03375527426160337,
                  474,
                  16
                ],
                [
                  0.8683729433272395,
                  0.035789473684210524,
                  475,
                  17
                ],
                [
                  0.870201096892139,
                  0.037815126050420166,
                  476,
                  18
                ],
                [
                  0.8720292504570384,
                  0.03773584905660377,
                  477,
                  18
                ],
                [
                  0.8738574040219378,
                  0.03765690376569038,
                  478,
                  18
                ],
                [
                  0.8756855575868373,
                  0.03966597077244259,
                  479,
                  19
                ],
                [
                  0.8775137111517367,
                  0.03958333333333333,
                  480,
                  19
                ],
                [
                  0.8793418647166362,
                  0.04158004158004158,
                  481,
                  20
                ],
                [
                  0.8811700182815356,
                  0.04149377593360996,
                  482,
                  20
                ],
                [
                  0.8829981718464351,
                  0.041407867494824016,
                  483,
                  20
                ],
                [
                  0.8848263254113345,
                  0.04338842975206612,
                  484,
                  21
                ],
                [
                  0.886654478976234,
                  0.04329896907216495,
                  485,
                  21
                ],
                [
                  0.8884826325411335,
                  0.043209876543209874,
                  486,
                  21
                ],
                [
                  0.8903107861060329,
                  0.043121149897330596,
                  487,
                  21
                ],
                [
                  0.8921389396709324,
                  0.0430327868852459,
                  488,
                  21
                ],
                [
                  0.8939670932358318,
                  0.044989775051124746,
                  489,
                  22
                ],
                [
                  0.8957952468007313,
                  0.044897959183673466,
                  490,
                  22
                ],
                [
                  0.8976234003656307,
                  0.04480651731160896,
                  491,
                  22
                ],
                [
                  0.8994515539305301,
                  0.044715447154471545,
                  492,
                  22
                ],
                [
                  0.9012797074954296,
                  0.04665314401622718,
                  493,
                  23
                ],
                [
                  0.903107861060329,
                  0.0465587044534413,
                  494,
                  23
                ],
                [
                  0.9049360146252285,
                  0.048484848484848485,
                  495,
                  24
                ],
                [
                  0.906764168190128,
                  0.04838709677419355,
                  496,
                  24
                ],
                [
                  0.9085923217550275,
                  0.05030181086519115,
                  497,
                  25
                ],
                [
                  0.9104204753199269,
                  0.050200803212851405,
                  498,
                  25
                ],
                [
                  0.9122486288848263,
                  0.050100200400801605,
                  499,
                  25
                ],
                [
                  0.9140767824497258,
                  0.052,
                  500,
                  26
                ],
                [
                  0.9159049360146252,
                  0.05189620758483034,
                  501,
                  26
                ],
                [
                  0.9177330895795247,
                  0.05179282868525897,
                  502,
                  26
                ],
                [
                  0.9195612431444241,
                  0.05168986083499006,
                  503,
                  26
                ],
                [
                  0.9213893967093236,
                  0.051587301587301584,
                  504,
                  26
                ],
                [
                  0.923217550274223,
                  0.05148514851485148,
                  505,
                  26
                ],
                [
                  0.9250457038391224,
                  0.0533596837944664,
                  506,
                  27
                ],
                [
                  0.926873857404022,
                  0.055226824457593686,
                  507,
                  28
                ],
                [
                  0.9287020109689214,
                  0.05511811023622047,
                  508,
                  28
                ],
                [
                  0.9305301645338209,
                  0.0550098231827112,
                  509,
                  28
                ],
                [
                  0.9323583180987203,
                  0.056862745098039215,
                  510,
                  29
                ],
                [
                  0.9341864716636198,
                  0.05675146771037182,
                  511,
                  29
                ],
                [
                  0.9360146252285192,
                  0.056640625,
                  512,
                  29
                ],
                [
                  0.9378427787934186,
                  0.05847953216374269,
                  513,
                  30
                ],
                [
                  0.9396709323583181,
                  0.06031128404669261,
                  514,
                  31
                ],
                [
                  0.9414990859232175,
                  0.062135922330097085,
                  515,
                  32
                ],
                [
                  0.943327239488117,
                  0.06395348837209303,
                  516,
                  33
                ],
                [
                  0.9451553930530164,
                  0.06382978723404255,
                  517,
                  33
                ],
                [
                  0.946983546617916,
                  0.06563706563706563,
                  518,
                  34
                ],
                [
                  0.9488117001828154,
                  0.0674373795761079,
                  519,
                  35
                ],
                [
                  0.9506398537477148,
                  0.0673076923076923,
                  520,
                  35
                ],
                [
                  0.9524680073126143,
                  0.0671785028790787,
                  521,
                  35
                ],
                [
                  0.9542961608775137,
                  0.06896551724137931,
                  522,
                  36
                ],
                [
                  0.9561243144424132,
                  0.06883365200764818,
                  523,
                  36
                ],
                [
                  0.9579524680073126,
                  0.07061068702290077,
                  524,
                  37
                ],
                [
                  0.9597806215722121,
                  0.07047619047619047,
                  525,
                  37
                ],
                [
                  0.9616087751371115,
                  0.07034220532319392,
                  526,
                  37
                ],
                [
                  0.9634369287020109,
                  0.07020872865275142,
                  527,
                  37
                ],
                [
                  0.9652650822669104,
                  0.07196969696969698,
                  528,
                  38
                ],
                [
                  0.9670932358318098,
                  0.07183364839319471,
                  529,
                  38
                ],
                [
                  0.9689213893967094,
                  0.07358490566037736,
                  530,
                  39
                ],
                [
                  0.9707495429616088,
                  0.07344632768361582,
                  531,
                  39
                ],
                [
                  0.9725776965265083,
                  0.07518796992481203,
                  532,
                  40
                ],
                [
                  0.9744058500914077,
                  0.075046904315197,
                  533,
                  40
                ],
                [
                  0.9762340036563071,
                  0.07677902621722846,
                  534,
                  41
                ],
                [
                  0.9780621572212066,
                  0.07663551401869159,
                  535,
                  41
                ],
                [
                  0.979890310786106,
                  0.07649253731343283,
                  536,
                  41
                ],
                [
                  0.9817184643510055,
                  0.0782122905027933,
                  537,
                  42
                ],
                [
                  0.9835466179159049,
                  0.07806691449814127,
                  538,
                  42
                ],
                [
                  0.9853747714808044,
                  0.07792207792207792,
                  539,
                  42
                ],
                [
                  0.9872029250457038,
                  0.07777777777777778,
                  540,
                  42
                ],
                [
                  0.9890310786106032,
                  0.07948243992606285,
                  541,
                  43
                ],
                [
                  0.9908592321755028,
                  0.08118081180811808,
                  542,
                  44
                ],
                [
                  0.9926873857404022,
                  0.08287292817679558,
                  543,
                  45
                ],
                [
                  0.9945155393053017,
                  0.08455882352941177,
                  544,
                  46
                ],
                [
                  0.9963436928702011,
                  0.08623853211009175,
                  545,
                  47
                ],
                [
                  0.9981718464351006,
                  0.08608058608058608,
                  546,
                  47
                ],
                [
                  1.0,
                  0.08592321755027423,
                  547,
                  47
                ]
              ]
            },
            {
              "language": "English",
              "run": 550,
              "points": [
                [
                  0.0018281535648994515,
                  0.0,
                  1,
                  0
                ],
                [
                  0.003656307129798903,
                  0.0,
                  2,
                  0
                ],
                [
                  0.005484460694698354,
                  0.0,
                  3,
                  0
                ],
                [
                  0.007312614259597806,
                  0.0,
                  4,
                  0
                ],
                [
                  0.009140767824497258,
                  0.0,
                  5,
                  0
                ],
                [
                  0.010968921389396709,
                  0.0,
                  6,
                  0
                ],
                [
                  0.012797074954296161,
                  0.0,
                  7,
                  0
                ],
                [
                  0.014625228519195612,
                  0.0,
                  8,
                  0
                ],
                [
                  0.016453382084095063,
                  0.0,
                  9,
                  0
                ],
                [
                  0.018281535648994516,
                  0.0,
                  10,
                  0
                ],
                [
                  0.02010968921389397,
                  0.0,
                  11,
                  0
                ],
                [
                  0.021937842778793418,
                  0.0,
                  12,
                  0
                ],
                [
                  0.02376599634369287,
                  0.0,
                  13,
                  0
                ],
                [
                  0.025594149908592323,
                  0.0,
                  14,
                  0
                ],
                [
                  0.027422303473491772,
                  0.0,
                  15,
                  0
                ],
                [
                  0.029250457038391225,
                  0.0,
                  16,
                  0
                ],
                [
                  0.031078610603290677,
                  0.0,
                  17,
                  0
                ],
                [
                  0.03290676416819013,
                  0.0,
                  18,
                  0
                ],
                [
                  0.03473491773308958,
                  0.0,
                  19,
                  0
                ],
                [
                  0.03656307129798903,
                  0.0,
                  20,
                  0
                ],
                [
                  0.038391224862888484,
                  0.0,
                  21,
                  0
                ],
                [
                  0.04021937842778794,
                  0.0,
                  22,
                  0
                ],
                [
                  0.04204753199268738,
                  0.0,
                  23,
                  0
                ],
                [
                  0.043875685557586835,
                  0.0,
                  24,
                  0
                ],
                [
                  0.04570383912248629,
                  0.0,
                  25,
                  0
                ],
                [
                  0.04753199268738574,
                  0.0,
                  26,
                  0
                ],
                [
                  0.04936014625228519,
                  0.0,
                  27,
                  0
                ],
                [
                  0.051188299817184646,
                  0.0,
                  28,
                  0
                ],
                [
                  0.05301645338208409,
                  0.0,
                  29,
                  0
                ],
                [
                  0.054844606946983544,
                  0.0,
                  30,
                  0
                ],
                [
                  0.056672760511883,
                  0.0,
                  31,
                  0
                ],
                [
                  0.05850091407678245,
                  0.0,
                  32,
                  0
                ],
                [
                  0.0603290676416819,
                  0.0,
                  33,
                  0
                ],
                [
                  0.062157221206581355,
                  0.0,
                  34,
                  0
                ],
                [
                  0.06398537477148081,
                  0.0,
                  35,
                  0
                ],
                [
                  0.06581352833638025,
                  0.0,
                  36,
                  0
                ],
                [
                  0.06764168190127971,
                  0.0,
                  37,
                  0
                ],
                [
                  0.06946983546617916,
                  0.0,
                  38,
                  0
                ],
                [
                  0.0712979890310786,
                  0.0,
                  39,
                  0
                ],
                [
                  0.07312614259597806,
                  0.0,
                  40,
                  0
                ],
                [
                  0.07495429616087751,
                  0.0,
                  41,
                  0
                ],
                [
                  0.07678244972577697,
                  0.0,
                  42,
                  0
                ],
                [
                  0.07861060329067641,
                  0.0,
                  43,
                  0
                ],
                [
                  0.08043875685557587,
                  0.0,
                  44,
                  0
                ],
                [
                  0.08226691042047532,
                  0.0,
                  45,
                  0
                ],
                [
                  0.08409506398537477,
                  0.0,
                  46,
                  0
                ],
                [
                  0.08592321755027423,
                  0.0,
                  47,
                  0
                ],
                [
                  0.08775137111517367,
                  0.0,
                  48,
                  0
                ],
                [
                  0.08957952468007313,
                  0.0,
                  49,
                  0
                ],
                [
                  0.09140767824497258,
                  0.0,
                  50,
                  0
                ],
                [
                  0.09323583180987204,
                  0.0,
                  51,
                  0
                ],
                [
                  0.09506398537477148,
                  0.0,
                  52,
                  0
                ],
                [
                  0.09689213893967093,
                  0.0,
                  53,
                  0
                ],
                [
                  0.09872029250457039,
                  0.0,
                  54,
                  0
                ],
                [
                  0.10054844606946983,
                  0.0,
                  55,
                  0
                ],
                [
                  0.10237659963436929,
                  0.0,
                  56,
                  0
                ],
                [
                  0.10420475319926874,
                  0.0,
                  57,
                  0
                ],
                [
                  0.10603290676416818,
                  0.0,
                  58,
                  0
                ],
                [
                  0.10786106032906764,
                  0.0,
                  59,
                  0
                ],
                [
                  0.10968921389396709,
                  0.0,
                  60,
                  0
                ],
                [
                  0.11151736745886655,
                  0.0,
                  61,
                  0
                ],
                [
                  0.113345521023766,
                  0.0,
                  62,
                  0
                ],
                [
                  0.11517367458866545,
                  0.0,
                  63,
                  0
                ],
                [
                  0.1170018281535649,
                  0.0,
                  64,
                  0
                ],
                [
                  0.11882998171846434,
                  0.0,
                  65,
                  0
                ],
                [
                  0.1206581352833638,
                  0.0,
                  66,
                  0
                ],
                [
                  0.12248628884826325,
                  0.0,
                  67,
                  0
                ],
                [
                  0.12431444241316271,
                  0.0,
                  68,
                  0
                ],
                [
                  0.12614259597806216,
                  0.0,
                  69,
                  0
                ],
                [
                  0.12797074954296161,
                  0.0,
                  70,
                  0
                ],
                [
                  0.12979890310786105,
                  0.0,
                  71,
                  0
                ],
                [
                  0.1316270566727605,
                  0.0,
                  72,
                  0
                ],
                [
                  0.13345521023765997,
                  0.0,
                  73,
                  0
                ],
                [
                  0.13528336380255943,
                  0.013513513513513514,
                  74,
                  1
                ],
                [
                  0.13711151736745886,
                  0.013333333333333334,
                  75,
                  1
                ],
                [
                  0.13893967093235832,
                  0.013157894736842105,
                  76,
                  1
                ],
                [
                  0.14076782449725778,
                  0.012987012987012988,
                  77,
                  1
                ],
                [
                  0.1425959780621572,
                  0.01282051282051282,
                  78,
                  1
                ],
                [
                  0.14442413162705667,
                  0.012658227848101266,
                  79,
                  1
                ],
                [
                  0.14625228519195613,
                  0.0125,
                  80,
                  1
                ],
                [
                  0.1480804387568556,
                  0.012345679012345678,
                  81,
                  1
                ],
                [
                  0.14990859232175502,
                  0.012195121951219513,
                  82,
                  1
                ],
                [
                  0.15173674588665448,
                  0.012048192771084338,
                  83,
                  1
                ],
                [
                  0.15356489945155394,
                  0.011904761904761904,
                  84,
                  1
                ],
                [
                  0.15539305301645337,
                  0.011764705882352941,
                  85,
                  1
                ],
                [
                  0.15722120658135283,
                  0.011627906976744186,
                  86,
                  1
                ],
                [
                  0.1590493601462523,
                  0.011494252873563218,
                  87,
                  1
                ],
                [
                  0.16087751371115175,
                  0.011363636363636364,
                  88,
                  1
                ],
                [
                  0.16270566727605118,
                  0.011235955056179775,
                  89,
                  1
                ],
                [
                  0.16453382084095064,
                  0.011111111111111112,
                  90,
                  1
                ],
                [
                  0.1663619744058501,
                  0.01098901098901099,
                  91,
                  1
                ],
                [
                  0.16819012797074953,
                  0.010869565217391304,
                  92,
                  1
                ],
                [
                  0.170018281535649,
                  0.010752688172043012,
                  93,
                  1
                ],
                [
                  0.17184643510054845,
                  0.010638297872340425,
                  94,
                  1
                ],
                [
                  0.1736745886654479,
                  0.010526315789473684,
                  95,
                  1
                ],
                [
                  0.17550274223034734,
                  0.010416666666666666,
                  96,
                  1
                ],
                [
                  0.1773308957952468,
                  0.010309278350515464,
                  97,
                  1
                ],
                [
                  0.17915904936014626,
                  0.01020408163265306,
                  98,
                  1
                ],
                [
                  0.1809872029250457,
                  0.010101010101010102,
                  99,
                  1
                ],
                [
                  0.18281535648994515,
                  0.01,
                  100,
                  1
                ],
                [
                  0.1846435100548446,
                  0.009900990099009901,
                  101,
                  1
                ],
                [
                  0.18647166361974407,
                  0.00980392156862745,
                  102,
                  1
                ],
                [
                  0.1882998171846435,
                  0.009708737864077669,
                  103,
                  1
                ],
                [
                  0.19012797074954296,
                  0.009615384615384616,
                  104,
                  1
                ],
                [
                  0.19195612431444242,
                  0.009523809523809525,
                  105,
                  1
                ],
                [
                  0.19378427787934185,
                  0.009433962264150943,
                  106,
                  1
                ],
                [
                  0.1956124314442413,
                  0.009345794392523364,
                  107,
                  1
                ],
                [
                  0.19744058500914077,
                  0.009259259259259259,
                  108,
                  1
                ],
                [
                  0.19926873857404023,
                  0.009174311926605505,
                  109,
                  1
                ],
                [
                  0.20109689213893966,
                  0.00909090909090909,
                  110,
                  1
                ],
                [
                  0.20292504570383912,
                  0.009009009009009009,
                  111,
                  1
                ],
                [
                  0.20475319926873858,
                  0.008928571428571428,
                  112,
                  1
                ],
                [
                  0.20658135283363802,
                  0.008849557522123894,
                  113,
                  1
                ],
                [
                  0.20840950639853748,
                  0.008771929824561403,
                  114,
                  1
                ],
                [
                  0.21023765996343693,
                  0.008695652173913044,
                  115,
                  1
                ],
                [
                  0.21206581352833637,
                  0.008620689655172414,
                  116,
                  1
                ],
                [
                  0.21389396709323583,
                  0.008547008547008548,
                  117,
                  1
                ],
                [
                  0.21572212065813529,
                  0.00847457627118644,
                  118,
                  1
                ],
                [
                  0.21755027422303475,
                  0.008403361344537815,
                  119,
                  1
                ],
                [
                  0.21937842778793418,
                  0.008333333333333333,
                  120,
                  1
                ],
                [
                  0.22120658135283364,
                  0.008264462809917356,
                  121,
                  1
                ],
                [
                  0.2230347349177331,
                  0.00819672131147541,
                  122,
                  1
                ],
                [
                  0.22486288848263253,
                  0.008130081300813009,
                  123,
                  1
                ],
                [
                  0.226691042047532,
                  0.008064516129032258,
                  124,
                  1
                ],
                [
                  0.22851919561243145,
                  0.008,
                  125,
                  1
                ],
                [
                  0.2303473491773309,
                  0.007936507936507936,
                  126,
                  1
                ],
                [
                  0.23217550274223034,
                  0.007874015748031496,
                  127,
                  1
                ],
                [
                  0.2340036563071298,
                  0.0078125,
                  128,
                  1
                ],
                [
                  0.23583180987202926,
                  0.007751937984496124,
                  129,
                  1
                ],
                [
                  0.2376599634369287,
                  0.007692307692307693,
                  130,
                  1
                ],
                [
                  0.23948811700182815,
                  0.007633587786259542,
                  131,
                  1
                ],
                [
                  0.2413162705667276,
                  0.007575757575757576,
                  132,
                  1
                ],
                [
                  0.24314442413162707,
                  0.007518796992481203,
                  133,
                  1
                ],
                [
                  0.2449725776965265,
                  0.007462686567164179,
                  134,
                  1
                ],
                [
                  0.24680073126142596,
                  0.007407407407407408,
                  135,
                  1
                ],
                [
                  0.24862888482632542,
                  0.007352941176470588,
                  136,
                  1
                ],
                [
                  0.25045703839122485,
                  0.0072992700729927005,
                  137,
                  1
                ],
                [
                  0.2522851919561243,
                  0.007246376811594203,
                  138,
                  1
                ],
                [
                  0.25411334552102377,
                  0.007194244604316547,
                  139,
                  1
                ],
                [
                  0.25594149908592323,
                  0.007142857142857143,
                  140,
                  1
                ],
                [
                  0.2577696526508227,
                  0.0070921985815602835,
                  141,
                  1
                ],
                [
                  0.2595978062157221,
                  0.007042253521126761,
                  142,
                  1
                ],
                [
                  0.26142595978062155,
                  0.006993006993006993,
                  143,
                  1
                ],
                [
                  0.263254113345521,
                  0.006944444444444444,
                  144,
                  1
                ],
                [
                  0.26508226691042047,
                  0.006896551724137931,
                  145,
                  1
                ],
                [
                  0.26691042047531993,
                  0.00684931506849315,
                  146,
                  1
                ],
                [
                  0.2687385740402194,
                  0.006802721088435374,
                  147,
                  1
                ],
                [
                  0.27056672760511885,
                  0.006756756756756757,
                  148,
                  1
                ],
                [
                  0.27239488117001825,
                  0.006711409395973154,
                  149,
                  1
                ],
                [
                  0.2742230347349177,
                  0.006666666666666667,
                  150,
                  1
                ],
                [
                  0.2760511882998172,
                  0.006622516556291391,
                  151,
                  1
                ],
                [
                  0.27787934186471663,
                  0.006578947368421052,
                  152,
                  1
                ],
                [
                  0.2797074954296161,
                  0.006535947712418301,
                  153,
                  1
                ],
                [
                  0.28153564899451555,
                  0.006493506493506494,
                  154,
                  1
                ],
                [
                  0.283363802559415,
                  0.0064516129032258064,
                  155,
                  1
                ],
                [
                  0.2851919561243144,
                  0.00641025641025641,
                  156,
                  1
                ],
                [
                  0.2870201096892139,
                  0.006369426751592357,
                  157,
                  1
                ],
                [
                  0.28884826325411334,
                  0.006329113924050633,
                  158,
                  1
                ],
                [
                  0.2906764168190128,
                  0.006289308176100629,
                  159,
                  1
                ],
                [
                  0.29250457038391225,
                  0.00625,
                  160,
                  1
                ],
                [
                  0.2943327239488117,
                  0.006211180124223602,
                  161,
                  1
                ],
                [
                  0.2961608775137112,
                  0.006172839506172839,
                  162,
                  1
                ],
                [
                  0.2979890310786106,
                  0.006134969325153374,
                  163,
                  1
                ],
                [
                  0.29981718464351004,
                  0.006097560975609756,
                  164,
                  1
                ],
                [
                  0.3016453382084095,
                  0.006060606060606061,
                  165,
                  1
                ],
                [
                  0.30347349177330896,
                  0.006024096385542169,
                  166,
                  1
                ],
                [
                  0.3053016453382084,
                  0.005988023952095809,
                  167,
                  1
                ],
                [
                  0.3071297989031079,
                  0.005952380952380952,
                  168,
                  1
                ],
                [
                  0.30895795246800734,
                  0.005917159763313609,
                  169,
                  1
                ],
                [
                  0.31078610603290674,
                  0.0058823529411764705,
                  170,
                  1
                ],
                [
                  0.3126142595978062,
                  0.005847953216374269,
                  171,
                  1
                ],
                [
                  0.31444241316270566,
                  0.005813953488372093,
                  172,
                  1
                ],
                [
                  0.3162705667276051,
                  0.005780346820809248,
                  173,
                  1
                ],
                [
                  0.3180987202925046,
                  0.005747126436781609,
                  174,
                  1
                ],
                [
                  0.31992687385740404,
                  0.005714285714285714,
                  175,
                  1
                ],
                [
                  0.3217550274223035,
                  0.005681818181818182,
                  176,
                  1
                ],
                [
                  0.3235831809872029,
                  0.005649717514124294,
                  177,
                  1
                ],
                [
                  0.32541133455210236,
                  0.0056179775280898875,
                  178,
                  1
                ],
                [
                  0.3272394881170018,
                  0.00558659217877095,
                  179,
                  1
                ],
                [
                  0.3290676416819013,
                  0.005555555555555556,
                  180,
                  1
                ],
                [
                  0.33089579524680074,
                  0.0055248618784530384,
                  181,
                  1
                ],
                [
                  0.3327239488117002,
                  0.005494505494505495,
                  182,
                  1
                ],
                [
                  0.33455210237659966,
                  0.00546448087431694,
                  183,
                  1
                ],
                [
                  0.33638025594149906,
                  0.005434782608695652,
                  184,
                  1
                ],
                [
                  0.3382084095063985,
                  0.005405405405405406,
                  185,
                  1
                ],
                [
                  0.340036563071298,
                  0.005376344086021506,
                  186,
                  1
                ],
                [
                  0.34186471663619744,
                  0.0053475935828877,
                  187,
                  1
                ],
                [
                  0.3436928702010969,
                  0.010638297872340425,
                  188,
                  2
                ],
                [
                  0.34552102376599636,
                  0.010582010582010581,
                  189,
                  2
                ],
                [
                  0.3473491773308958,
                  0.010526315789473684,
                  190,
                  2
                ],
                [
                  0.3491773308957952,
                  0.010471204188481676,
                  191,
                  2
                ],
                [
                  0.3510054844606947,
                  0.010416666666666666,
                  192,
                  2
                ],
                [
                  0.35283363802559414,
                  0.010362694300518135,
                  193,
                  2
                ],
                [
                  0.3546617915904936,
                  0.010309278350515464,
                  194,
                  2
                ],
                [
                  0.35648994515539306,
                  0.010256410256410256,
                  195,
                  2
                ],
                [
                  0.3583180987202925,
                  0.01020408163265306,
                  196,
                  2
                ],
                [
                  0.360146252285192,
                  0.01015228426395939,
                  197,
                  2
                ],
                [
                  0.3619744058500914,
                  0.010101010101010102,
                  198,
                  2
                ],
                [
                  0.36380255941499084,
                  0.010050251256281407,
                  199,
                  2
                ],
                [
                  0.3656307129798903,
                  0.01,
                  200,
                  2
                ],
                [
                  0.36745886654478976,
                  0.009950248756218905,
                  201,
                  2
                ],
                [
                  0.3692870201096892,
                  0.009900990099009901,
                  202,
                  2
                ],
                [
                  0.3711151736745887,
                  0.009852216748768473,
                  203,
                  2
                ],
                [
                  0.37294332723948814,
                  0.00980392156862745,
                  204,
                  2
                ],
                [
                  0.37477148080438755,
                  0.00975609756097561,
                  205,
                  2
                ],
                [
                  0.376599634369287,
                  0.009708737864077669,
                  206,
                  2
                ],
                [
                  0.37842778793418647,
                  0.00966183574879227,
                  207,
                  2
                ],
                [
                  0.3802559414990859,
                  0.009615384615384616,
                  208,
                  2
                ],
                [
                  0.3820840950639854,
                  0.009569377990430622,
                  209,
                  2
                ],
                [
                  0.38391224862888484,
                  0.009523809523809525,
                  210,
                  2
                ],
                [
                  0.3857404021937843,
                  0.009478672985781991,
                  211,
                  2
                ],
                [
                  0.3875685557586837,
                  0.009433962264150943,
                  212,
                  2
                ],
                [
                  0.38939670932358317,
                  0.009389671361502348,
                  213,
                  2
                ],
                [
                  0.3912248628884826,
                  0.009345794392523364,
                  214,
                  2
                ],
                [
                  0.3930530164533821,
                  0.009302325581395349,
                  215,
                  2
                ],
                [
                  0.39488117001828155,
                  0.009259259259259259,
                  216,
                  2
                ],
                [
                  0.396709323583181,
                  0.009216589861751152,
                  217,
                  2
                ],
                [
                  0.39853747714808047,
                  0.009174311926605505,
                  218,
                  2
                ],
                [
                  0.40036563071297987,
                  0.0091324200913242,
                  219,
                  2
                ],
                [
                  0.40219378427787933,
                  0.00909090909090909,
                  220,
                  2
                ],
                [
                  0.4040219378427788,
                  0.00904977375565611,
                  221,
                  2
                ],
                [
                  0.40585009140767825,
                  0.009009009009009009,
                  222,
                  2
                ],
                [
                  0.4076782449725777,
                  0.008968609865470852,
                  223,
                  2
                ],
                [
                  0.40950639853747717,
                  0.008928571428571428,
                  224,
                  2
                ],
                [
                  0.4113345521023766,
                  0.008888888888888889,
                  225,
                  2
                ],
                [
                  0.41316270566727603,
                  0.008849557522123894,
                  226,
                  2
                ],
                [
                  0.4149908592321755,
                  0.00881057268722467,
                  227,
                  2
                ],
                [
                  0.41681901279707495,
                  0.008771929824561403,
                  228,
                  2
                ],
                [
                  0.4186471663619744,
                  0.008733624454148471,
                  229,
                  2
                ],
                [
                  0.42047531992687387,
                  0.008695652173913044,
                  230,
                  2
                ],
                [
                  0.42230347349177333,
                  0.008658008658008658,
                  231,
                  2
                ],
                [
                  0.42413162705667273,
                  0.008620689655172414,
                  232,
                  2
                ],
                [
                  0.4259597806215722,
                  0.008583690987124463,
                  233,
                  2
                ],
                [
                  0.42778793418647165,
                  0.008547008547008548,
                  234,
                  2
                ],
                [
                  0.4296160877513711,
                  0.00851063829787234,
                  235,
                  2
                ],
                [
                  0.43144424131627057,
                  0.00847457627118644,
                  236,
                  2
                ],
                [
                  0.43327239488117003,
                  0.008438818565400843,
                  237,
                  2
                ],
                [
                  0.4351005484460695,
                  0.008403361344537815,
                  238,
                  2
                ],
                [
                  0.4369287020109689,
                  0.008368200836820083,
                  239,
                  2
                ],
                [
                  0.43875685557586835,
                  0.008333333333333333,
                  240,
                  2
                ],
                [
                  0.4405850091407678,
                  0.008298755186721992,
                  241,
                  2
                ],
                [
                  0.4424131627056673,
                  0.008264462809917356,
                  242,
                  2
                ],
                [
                  0.44424131627056673,
                  0.00823045267489712,
                  243,
                  2
                ],
                [
                  0.4460694698354662,
                  0.00819672131147541,
                  244,
                  2
                ],
                [
                  0.44789762340036565,
                  0.00816326530612245,
                  245,
                  2
                ],
                [
                  0.44972577696526506,
                  0.008130081300813009,
                  246,
                  2
                ],
                [
                  0.4515539305301645,
                  0.008097165991902834,
                  247,
                  2
                ],
                [
                  0.453382084095064,
                  0.008064516129032258,
                  248,
                  2
                ],
                [
                  0.45521023765996343,
                  0.008032128514056224,
                  249,
                  2
                ],
                [
                  0.4570383912248629,
                  0.008,
                  250,
                  2
                ],
                [
                  0.45886654478976235,
                  0.00796812749003984,
                  251,
                  2
                ],
                [
                  0.4606946983546618,
                  0.007936507936507936,
                  252,
                  2
                ],
                [
                  0.4625228519195612,
                  0.007905138339920948,
                  253,
                  2
                ],
                [
                  0.4643510054844607,
                  0.007874015748031496,
                  254,
                  2
                ],
                [
                  0.46617915904936014,
                  0.00784313725490196,
                  255,
                  2
                ],
                [
                  0.4680073126142596,
                  0.0078125,
                  256,
                  2
                ],
                [
                  0.46983546617915906,
                  0.007782101167315175,
                  257,
                  2
                ],
                [
                  0.4716636197440585,
                  0.007751937984496124,
                  258,
                  2
                ],
                [
                  0.473491773308958,
                  0.007722007722007722,
                  259,
                  2
                ],
                [
                  0.4753199268738574,
                  0.007692307692307693,
                  260,
                  2
                ],
                [
                  0.47714808043875684,
                  0.007662835249042145,
                  261,
                  2
                ],
                [
                  0.4789762340036563,
                  0.007633587786259542,
                  262,
                  2
                ],
                [
                  0.48080438756855576,
                  0.0076045627376425855,
                  263,
                  2
                ],
                [
                  0.4826325411334552,
                  0.011363636363636364,
                  264,
                  3
                ],
                [
                  0.4844606946983547,
                  0.011320754716981131,
                  265,
                  3
                ],
                [
                  0.48628884826325414,
                  0.011278195488721804,
                  266,
                  3
                ],
                [
                  0.48811700182815354,
                  0.011235955056179775,
                  267,
                  3
                ],
                [
                  0.489945155393053,
                  0.011194029850746268,
                  268,
                  3
                ],
                [
                  0.49177330895795246,
                  0.011152416356877323,
                  269,
                  3
                ],
                [
                  0.4936014625228519,
                  0.011111111111111112,
                  270,
                  3
                ],
                [
                  0.4954296160877514,
                  0.01107011070110701,
                  271,
                  3
                ],
                [
                  0.49725776965265084,
                  0.011029411764705883,
                  272,
                  3
                ],
                [
                  0.4990859232175503,
                  0.01098901098901099,
                  273,
                  3
                ],
                [
                  0.5009140767824497,
                  0.010948905109489052,
                  274,
                  3
                ],
                [
                  0.5027422303473492,
                  0.01090909090909091,
                  275,
                  3
                ],
                [
                  0.5045703839122486,
                  0.010869565217391304,
                  276,
                  3
                ],
                [
                  0.506398537477148,
                  0.010830324909747292,
                  277,
                  3
                ],
                [
                  0.5082266910420475,
                  0.01079136690647482,
                  278,
                  3
                ],
                [
                  0.5100548446069469,
                  0.010752688172043012,
                  279,
                  3
                ],
                [
                  0.5118829981718465,
                  0.010714285714285714,
                  280,
                  3
                ],
                [
                  0.5137111517367459,
                  0.010676156583629894,
                  281,
                  3
                ],
                [
                  0.5155393053016454,
                  0.010638297872340425,
                  282,
                  3
                ],
                [
                  0.5173674588665448,
                  0.01060070671378092,
                  283,
                  3
                ],
                [
                  0.5191956124314442,
                  0.01056338028169014,
                  284,
                  3
                ],
                [
                  0.5210237659963437,
                  0.010526315789473684,
                  285,
                  3
                ],
                [
                  0.5228519195612431,
                  0.01048951048951049,
                  286,
                  3
                ],
                [
                  0.5246800731261426,
                  0.010452961672473868,
                  287,
                  3
                ],
                [
                  0.526508226691042,
                  0.010416666666666666,
                  288,
                  3
                ],
                [
                  0.5283363802559415,
                  0.010380622837370242,
                  289,
                  3
                ],
                [
                  0.5301645338208409,
                  0.010344827586206896,
                  290,
                  3
                ],
                [
                  0.5319926873857403,
                  0.010309278350515464,
                  291,
                  3
                ],
                [
                  0.5338208409506399,
                  0.010273972602739725,
                  292,
                  3
                ],
                [
                  0.5356489945155393,
                  0.010238907849829351,
                  293,
                  3
                ],
                [
                  0.5374771480804388,
                  0.01020408163265306,
                  294,
                  3
                ],
                [
                  0.5393053016453382,
                  0.010169491525423728,
                  295,
                  3
                ],
                [
                  0.5411334552102377,
                  0.010135135135135136,
                  296,
                  3
                ],
                [
                  0.5429616087751371,
                  0.010101010101010102,
                  297,
                  3
                ],
                [
                  0.5447897623400365,
                  0.010067114093959731,
                  298,
                  3
                ],
                [
                  0.546617915904936,
                  0.010033444816053512,
                  299,
                  3
                ],
                [
                  0.5484460694698354,
                  0.01,
                  300,
                  3
                ],
                [
                  0.5502742230347349,
                  0.009966777408637873,
                  301,
                  3
                ],
                [
                  0.5521023765996343,
                  0.009933774834437087,
                  302,
                  3
                ],
                [
                  0.5539305301645339,
                  0.013201320132013201,
                  303,
                  4
                ],
                [
                  0.5557586837294333,
                  0.013157894736842105,
                  304,
                  4
                ],
                [
                  0.5575868372943327,
                  0.013114754098360656,
                  305,
                  4
                ],
                [
                  0.5594149908592322,
                  0.013071895424836602,
                  306,
                  4
                ],
                [
                  0.5612431444241316,
                  0.013029315960912053,
                  307,
                  4
                ],
                [
                  0.5630712979890311,
                  0.012987012987012988,
                  308,
                  4
                ],
                [
                  0.5648994515539305,
                  0.012944983818770227,
                  309,
                  4
                ],
                [
                  0.56672760511883,
                  0.012903225806451613,
                  310,
                  4
                ],
                [
                  0.5685557586837294,
                  0.012861736334405145,
                  311,
                  4
                ],
                [
                  0.5703839122486288,
                  0.01282051282051282,
                  312,
                  4
                ],
                [
                  0.5722120658135283,
                  0.012779552715654952,
                  313,
                  4
                ],
                [
                  0.5740402193784278,
                  0.012738853503184714,
                  314,
                  4
                ],
                [
                  0.5758683729433273,
                  0.012698412698412698,
                  315,
                  4
                ],
                [
                  0.5776965265082267,
                  0.012658227848101266,
                  316,
                  4
                ],
                [
                  0.5795246800731262,
                  0.012618296529968454,
                  317,
                  4
                ],
                [
                  0.5813528336380256,
                  0.012578616352201259,
                  318,
                  4
                ],
                [
                  0.583180987202925,
                  0.012539184952978056,
                  319,
                  4
                ],
                [
                  0.5850091407678245,
                  0.0125,
                  320,
                  4
                ],
                [
                  0.5868372943327239,
                  0.012461059190031152,
                  321,
                  4
                ],
                [
                  0.5886654478976234,
                  0.012422360248447204,
                  322,
                  4
                ],
                [
                  0.5904936014625228,
                  0.01238390092879257,
                  323,
                  4
                ],
                [
                  0.5923217550274223,
                  0.012345679012345678,
                  324,
                  4
                ],
                [
                  0.5941499085923218,
                  0.015384615384615385,
                  325,
                  5
                ],
                [
                  0.5959780621572212,
                  0.015337423312883436,
                  326,
                  5
                ],
                [
                  0.5978062157221207,
                  0.01529051987767584,
                  327,
                  5
                ],
                [
                  0.5996343692870201,
                  0.01524390243902439,
                  328,
                  5
                ],
                [
                  0.6014625228519196,
                  0.015197568389057751,
                  329,
                  5
                ],
                [
                  0.603290676416819,
                  0.015151515151515152,
                  330,
                  5
                ],
                [
                  0.6051188299817185,
                  0.015105740181268883,
                  331,
                  5
                ],
                [
                  0.6069469835466179,
                  0.015060240963855422,
                  332,
                  5
                ],
                [
                  0.6087751371115173,
                  0.015015015015015015,
                  333,
                  5
                ],
                [
                  0.6106032906764168,
                  0.014970059880239521,
                  334,
                  5
                ],
                [
                  0.6124314442413162,
                  0.014925373134328358,
                  335,
                  5
                ],
                [
                  0.6142595978062158,
                  0.01488095238095238,
                  336,
                  5
                ],
                [
                  0.6160877513711152,
                  0.01483679525222552,
                  337,
                  5
                ],
                [
                  0.6179159049360147,
                  0.014792899408284023,
                  338,
                  5
                ],
                [
                  0.6197440585009141,
                  0.014749262536873156,
                  339,
                  5
                ],
                [
                  0.6215722120658135,
                  0.014705882352941176,
                  340,
                  5
                ],
                [
                  0.623400365630713,
                  0.01466275659824047,
                  341,
                  5
                ],
                [
                  0.6252285191956124,
                  0.014619883040935672,
                  342,
                  5
                ],
                [
                  0.6270566727605119,
                  0.014577259475218658,
                  343,
                  5
                ],
                [
                  0.6288848263254113,
                  0.014534883720930232,
                  344,
                  5
                ],
                [
                  0.6307129798903108,
                  0.014492753623188406,
                  345,
                  5
                ],
                [
                  0.6325411334552102,
                  0.014450867052023121,
                  346,
                  5
                ],
                [
                  0.6343692870201096,
                  0.01440922190201729,
                  347,
                  5
                ],
                [
                  0.6361974405850092,
                  0.014367816091954023,
                  348,
                  5
                ],
                [
                  0.6380255941499086,
                  0.014326647564469915,
                  349,
                  5
                ],
                [
                  0.6398537477148081,
                  0.014285714285714285,
                  350,
                  5
                ],
                [
                  0.6416819012797075,
                  0.014245014245014245,
                  351,
                  5
                ],
                [
                  0.643510054844607,
                  0.014204545454545454,
                  352,
                  5
                ],
                [
                  0.6453382084095064,
                  0.014164305949008499,
                  353,
                  5
                ],
                [
                  0.6471663619744058,
                  0.014124293785310734,
                  354,
                  5
                ],
                [
                  0.6489945155393053,
                  0.014084507042253521,
                  355,
                  5
                ],
                [
                  0.6508226691042047,
                  0.014044943820224719,
                  356,
                  5
                ],
                [
                  0.6526508226691042,
                  0.014005602240896359,
                  357,
                  5
                ],
                [
                  0.6544789762340036,
                  0.013966480446927373,
                  358,
                  5
                ],
                [
                  0.6563071297989032,
                  0.013927576601671309,
                  359,
                  5
                ],
                [
                  0.6581352833638026,
                  0.013888888888888888,
                  360,
                  5
                ],
                [
                  0.659963436928702,
                  0.013850415512465374,
                  361,
                  5
                ],
                [
                  0.6617915904936015,
                  0.013812154696132596,
                  362,
                  5
                ],
                [
                  0.6636197440585009,
                  0.013774104683195593,
                  363,
                  5
                ],
                [
                  0.6654478976234004,
                  0.013736263736263736,
                  364,
                  5
                ],
                [
                  0.6672760511882998,
                  0.01643835616438356,
                  365,
                  6
                ],
                [
                  0.6691042047531993,
                  0.01639344262295082,
                  366,
                  6
                ],
                [
                  0.6709323583180987,
                  0.01634877384196185,
                  367,
                  6
                ],
                [
                  0.6727605118829981,
                  0.016304347826086956,
                  368,
                  6
                ],
                [
                  0.6745886654478976,
                  0.016260162601626018,
                  369,
                  6
                ],
                [
                  0.676416819012797,
                  0.016216216216216217,
                  370,
                  6
                ],
                [
                  0.6782449725776966,
                  0.016172506738544475,
                  371,
                  6
                ],
                [
                  0.680073126142596,
                  0.016129032258064516,
                  372,
                  6
                ],
                [
                  0.6819012797074955,
                  0.0160857908847185,
                  373,
                  6
                ],
                [
                  0.6837294332723949,
                  0.016042780748663103,
                  374,
                  6
                ],
                [
                  0.6855575868372943,
                  0.016,
                  375,
                  6
                ],
                [
                  0.6873857404021938,
                  0.015957446808510637,
                  376,
                  6
                ],
                [
                  0.6892138939670932,
                  0.015915119363395226,
                  377,
                  6
                ],
                [
                  0.6910420475319927,
                  0.015873015873015872,
                  378,
                  6
                ],
                [
                  0.6928702010968921,
                  0.0158311345646438,
                  379,
                  6
                ],
                [
                  0.6946983546617916,
                  0.015789473684210527,
                  380,
                  6
                ],
                [
                  0.696526508226691,
                  0.015748031496062992,
                  381,
                  6
                ],
                [
                  0.6983546617915904,
                  0.015706806282722512,
                  382,
                  6
                ],
                [
                  0.70018281535649,
                  0.015665796344647518,
                  383,
                  6
                ],
                [
                  0.7020109689213894,
                  0.015625,
                  384,
                  6
                ],
                [
                  0.7038391224862889,
                  0.015584415584415584,
                  385,
                  6
                ],
                [
                  0.7056672760511883,
                  0.015544041450777202,
                  386,
                  6
                ],
                [
                  0.7074954296160878,
                  0.015503875968992248,
                  387,
                  6
                ],
                [
                  0.7093235831809872,
                  0.015463917525773196,
                  388,
                  6
                ],
                [
                  0.7111517367458866,
                  0.015424164524421594,
                  389,
                  6
                ],
                [
                  0.7129798903107861,
                  0.015384615384615385,
                  390,
                  6
                ],
                [
                  0.7148080438756855,
                  0.015345268542199489,
                  391,
                  6
                ],
                [
                  0.716636197440585,
                  0.015306122448979591,
                  392,
                  6
                ],
                [
                  0.7184643510054844,
                  0.015267175572519083,
                  393,
                  6
                ],
                [
                  0.720292504570384,
                  0.015228426395939087,
                  394,
                  6
                ],
                [
                  0.7221206581352834,
                  0.015189873417721518,
                  395,
                  6
                ],
                [
                  0.7239488117001828,
                  0.015151515151515152,
                  396,
                  6
                ],
                [
                  0.7257769652650823,
                  0.017632241813602016,
                  397,
                  7
                ],
                [
                  0.7276051188299817,
                  0.017587939698492462,
                  398,
                  7
                ],
                [
                  0.7294332723948812,
                  0.017543859649122806,
                  399,
                  7
                ],
                [
                  0.7312614259597806,
                  0.02,
                  400,
                  8
                ],
                [
                  0.7330895795246801,
                  0.022443890274314215,
                  401,
                  9
                ],
                [
                  0.7349177330895795,
                  0.022388059701492536,
                  402,
                  9
                ],
                [
                  0.7367458866544789,
                  0.02481389578163772,
                  403,
                  10
                ],
                [
                  0.7385740402193784,
                  0.024752475247524754,
                  404,
                  10
                ],
                [
                  0.7404021937842779,
                  0.024691358024691357,
                  405,
                  10
                ],
                [
                  0.7422303473491774,
                  0.024630541871921183,
                  406,
                  10
                ],
                [
                  0.7440585009140768,
                  0.02457002457002457,
                  407,
                  10
                ],
                [
                  0.7458866544789763,
                  0.024509803921568627,
                  408,
                  10
                ],
                [
                  0.7477148080438757,
                  0.02444987775061125,
                  409,
                  10
                ],
                [
                  0.7495429616087751,
                  0.024390243902439025,
                  410,
                  10
                ],
                [
                  0.7513711151736746,
                  0.024330900243309004,
                  411,
                  10
                ],
                [
                  0.753199268738574,
                  0.024271844660194174,
                  412,
                  10
                ],
                [
                  0.7550274223034735,
                  0.024213075060532687,
                  413,
                  10
                ],
                [
                  0.7568555758683729,
                  0.024154589371980676,
                  414,
                  10
                ],
                [
                  0.7586837294332724,
                  0.024096385542168676,
                  415,
                  10
                ],
                [
                  0.7605118829981719,
                  0.02403846153846154,
                  416,
                  10
                ],
                [
                  0.7623400365630713,
                  0.026378896882494004,
                  417,
                  11
                ],
                [
                  0.7641681901279708,
                  0.02631578947368421,
                  418,
                  11
                ],
                [
                  0.7659963436928702,
                  0.028639618138424822,
                  419,
                  12
                ],
                [
                  0.7678244972577697,
                  0.02857142857142857,
                  420,
                  12
                ],
                [
                  0.7696526508226691,
                  0.028503562945368172,
                  421,
                  12
                ],
                [
                  0.7714808043875686,
                  0.02843601895734597,
                  422,
                  12
                ],
                [
                  0.773308957952468,
                  0.028368794326241134,
                  423,
                  12
                ],
                [
                  0.7751371115173674,
                  0.030660377358490566,
                  424,
                  13
                ],
                [
                  0.7769652650822669,
                  0.03058823529411765,
                  425,
                  13
                ],
                [
                  0.7787934186471663,
                  0.03051643192488263,
                  426,
                  13
                ],
                [
                  0.7806215722120659,
                  0.03044496487119438,
                  427,
                  13
                ],
                [
                  0.7824497257769653,
                  0.030373831775700934,
                  428,
                  13
                ],
                [
                  0.7842778793418648,
                  0.03263403263403263,
                  429,
                  14
                ],
                [
                  0.7861060329067642,
                  0.03255813953488372,
                  430,
                  14
                ],
                [
                  0.7879341864716636,
                  0.03248259860788863,
                  431,
                  14
                ],
                [
                  0.7897623400365631,
                  0.032407407407407406,
                  432,
                  14
                ],
                [
                  0.7915904936014625,
                  0.03233256351039261,
                  433,
                  14
                ],
                [
                  0.793418647166362,
                  0.03225806451612903,
                  434,
                  14
                ],
                [
                  0.7952468007312614,
                  0.03218390804597701,
                  435,
                  14
                ],
                [
                  0.7970749542961609,
                  0.034403669724770644,
                  436,
                  15
                ],
                [
                  0.7989031078610603,
                  0.036613272311212815,
                  437,
                  16
                ],
                [
                  0.8007312614259597,
                  0.0365296803652968,
                  438,
                  16
                ],
                [
                  0.8025594149908593,
                  0.03644646924829157,
                  439,
                  16
                ],
                [
                  0.8043875685557587,
                  0.03636363636363636,
                  440,
                  16
                ],
                [
                  0.8062157221206582,
                  0.036281179138321996,
                  441,
                  16
                ],
                [
                  0.8080438756855576,
                  0.03619909502262444,
                  442,
                  16
                ],
                [
                  0.8098720292504571,
                  0.03611738148984198,
                  443,
                  16
                ],
                [
                  0.8117001828153565,
                  0.036036036036036036,
                  444,
                  16
                ],
                [
                  0.8135283363802559,
                  0.035955056179775284,
                  445,
                  16
                ],
                [
                  0.8153564899451554,
                  0.03587443946188341,
                  446,
                  16
                ],
                [
                  0.8171846435100548,
                  0.035794183445190156,
                  447,
                  16
                ],
                [
                  0.8190127970749543,
                  0.03794642857142857,
                  448,
                  17
                ],
                [
                  0.8208409506398537,
                  0.0378619153674833,
                  449,
                  17
                ],
                [
                  0.8226691042047533,
                  0.03777777777777778,
                  450,
                  17
                ],
                [
                  0.8244972577696527,
                  0.037694013303769404,
                  451,
                  17
                ],
                [
                  0.8263254113345521,
                  0.03761061946902655,
                  452,
                  17
                ],
                [
                  0.8281535648994516,
                  0.039735099337748346,
                  453,
                  18
                ],
                [
                  0.829981718464351,
                  0.039647577092511016,
                  454,
                  18
                ],
                [
                  0.8318098720292505,
                  0.041758241758241756,
                  455,
                  19
                ],
                [
                  0.8336380255941499,
                  0.041666666666666664,
                  456,
                  19
                ],
                [
                  0.8354661791590493,
                  0.0437636761487965,
                  457,
                  20
                ],
                [
                  0.8372943327239488,
                  0.043668122270742356,
                  458,
                  20
                ],
                [
                  0.8391224862888482,
                  0.0457516339869281,
                  459,
                  21
                ],
                [
                  0.8409506398537477,
                  0.04782608695652174,
                  460,
                  22
                ],
                [
                  0.8427787934186471,
                  0.049891540130151846,
                  461,
                  23
                ],
                [
                  0.8446069469835467,
                  0.05194805194805195,
                  462,
                  24
                ],
                [
                  0.8464351005484461,
                  0.05183585313174946,
                  463,
                  24
                ],
                [
                  0.8482632541133455,
                  0.05172413793103448,
                  464,
                  24
                ],
                [
                  0.850091407678245,
                  0.05161290322580645,
                  465,
                  24
                ],
                [
                  0.8519195612431444,
                  0.05150214592274678,
                  466,
                  24
                ],
                [
                  0.8537477148080439,
                  0.05139186295503212,
                  467,
                  24
                ],
                [
                  0.8555758683729433,
                  0.05128205128205128,
                  468,
                  24
                ],
                [
                  0.8574040219378428,
                  0.0511727078891258,
                  469,
                  24
                ],
                [
                  0.8592321755027422,
                  0.05106382978723404,
                  470,
                  24
                ],
                [
                  0.8610603290676416,
                  0.050955414012738856,
                  471,
                  24
                ],
                [
                  0.8628884826325411,
                  0.05296610169491525,
                  472,
                  25
                ],
                [
                  0.8647166361974405,
                  0.052854122621564484,
                  473,
                  25
                ],
                [
                  0.8665447897623401,
                  0.052742616033755275,
                  474,
                  25
                ],
                [
                  0.8683729433272395,
                  0.05263157894736842,
                  475,
                  25
                ],
                [
                  0.870201096892139,
                  0.0546218487394958,
                  476,
                  26
                ],
                [
                  0.8720292504570384,
                  0.05450733752620545,
                  477,
                  26
                ],
                [
                  0.8738574040219378,
                  0.05439330543933055,
                  478,
                  26
                ],
                [
                  0.8756855575868373,
                  0.05636743215031315,
                  479,
                  27
                ],
                [
                  0.8775137111517367,
                  0.058333333333333334,
                  480,
                  28
                ],
                [
                  0.8793418647166362,
                  0.058212058212058215,
                  481,
                  28
                ],
                [
                  0.8811700182815356,
                  0.058091286307053944,
                  482,
                  28
                ],
                [
                  0.8829981718464351,
                  0.057971014492753624,
                  483,
                  28
                ],
                [
                  0.8848263254113345,
                  0.05785123966942149,
                  484,
                  28
                ],
                [
                  0.886654478976234,
                  0.0577319587628866,
                  485,
                  28
                ],
                [
                  0.8884826325411335,
                  0.05761316872427984,
                  486,
                  28
                ],
                [
                  0.8903107861060329,
                  0.059548254620123205,
                  487,
                  29
                ],
                [
                  0.8921389396709324,
                  0.05942622950819672,
                  488,
                  29
                ],
                [
                  0.8939670932358318,
                  0.05930470347648262,
                  489,
                  29
                ],
                [
                  0.8957952468007313,
                  0.05918367346938776,
                  490,
                  29
                ],
                [
                  0.8976234003656307,
                  0.059063136456211814,
                  491,
                  29
                ],
                [
                  0.8994515539305301,
                  0.05894308943089431,
                  492,
                  29
                ],
                [
                  0.9012797074954296,
                  0.060851926977687626,
                  493,
                  30
                ],
                [
                  0.903107861060329,
                  0.06072874493927125,
                  494,
                  30
                ],
                [
                  0.9049360146252285,
                  0.06060606060606061,
                  495,
                  30
                ],
                [
                  0.906764168190128,
                  0.0625,
                  496,
                  31
                ],
                [
                  0.9085923217550275,
                  0.06237424547283702,
                  497,
                  31
                ],
                [
                  0.9104204753199269,
                  0.0642570281124498,
                  498,
                  32
                ],
                [
                  0.9122486288848263,
                  0.06412825651302605,
                  499,
                  32
                ],
                [
                  0.9140767824497258,
                  0.066,
                  500,
                  33
                ],
                [
                  0.9159049360146252,
                  0.0658682634730539,
                  501,
                  33
                ],
                [
                  0.9177330895795247,
                  0.06573705179282868,
                  502,
                  33
                ],
                [
                  0.9195612431444241,
                  0.06560636182902585,
                  503,
                  33
                ],
                [
                  0.9213893967093236,
                  0.06547619047619048,
                  504,
                  33
                ],
                [
                  0.923217550274223,
                  0.06534653465346535,
                  505,
                  33
                ],
                [
                  0.9250457038391224,
                  0.06719367588932806,
                  506,
                  34
                ],
                [
                  0.926873857404022,
                  0.0670611439842209,
                  507,
                  34
                ],
                [
                  0.9287020109689214,
                  0.0688976377952756,
                  508,
                  35
                ],
                [
                  0.9305301645338209,
                  0.07072691552062868,
                  509,
                  36
                ],
                [
                  0.9323583180987203,
                  0.07254901960784314,
                  510,
                  37
                ],
                [
                  0.9341864716636198,
                  0.07436399217221135,
                  511,
                  38
                ],
                [
                  0.9360146252285192,
                  0.07421875,
                  512,
                  38
                ],
                [
                  0.9378427787934186,
                  0.07407407407407407,
                  513,
                  38
                ],
                [
                  0.9396709323583181,
                  0.07587548638132295,
                  514,
                  39
                ],
                [
                  0.9414990859232175,
                  0.07766990291262135,
                  515,
                  40
                ],
                [
                  0.943327239488117,
                  0.07945736434108527,
                  516,
                  41
                ],
                [
                  0.9451553930530164,
                  0.08123791102514506,
                  517,
                  42
                ],
                [
                  0.946983546617916,
                  0.08108108108108109,
                  518,
                  42
                ],
                [
                  0.9488117001828154,
                  0.08092485549132948,
                  519,
                  42
                ],
                [
                  0.9506398537477148,
                  0.08269230769230769,
                  520,
                  43
                ],
                [
                  0.9524680073126143,
                  0.08253358925143954,
                  521,
                  43
                ],
                [
                  0.9542961608775137,
                  0.08237547892720307,
                  522,
                  43
                ],
                [
                  0.9561243144424132,
                  0.08221797323135756,
                  523,
                  43
                ],
                [
                  0.9579524680073126,
                  0.08396946564885496,
                  524,
                  44
                ],
                [
                  0.9597806215722121,
                  0.08571428571428572,
                  525,
                  45
                ],
                [
                  0.9616087751371115,
                  0.08555133079847908,
                  526,
                  45
                ],
                [
                  0.9634369287020109,
                  0.0872865275142315,
                  527,
                  46
                ],
                [
                  0.9652650822669104,
                  0.08712121212121213,
                  528,
                  46
                ],
                [
                  0.9670932358318098,
                  0.0888468809073724,
                  529,
                  47
                ],
                [
                  0.9689213893967094,
                  0.08867924528301886,
                  530,
                  47
                ],
                [
                  0.9707495429616088,
                  0.08851224105461393,
                  531,
                  47
                ],
                [
                  0.9725776965265083,
                  0.08834586466165413,
                  532,
                  47
                ],
                [
                  0.9744058500914077,
                  0.08818011257035648,
                  533,
                  47
                ],
                [
                  0.9762340036563071,
                  0.0898876404494382,
                  534,
                  48
                ],
                [
                  0.9780621572212066,
                  0.08971962616822429,
                  535,
                  48
                ],
                [
                  0.979890310786106,
                  0.0914179104477612,
                  536,
                  49
                ],
                [
                  0.9817184643510055,
                  0.0931098696461825,
                  537,
                  50
                ],
                [
                  0.9835466179159049,
                  0.09479553903345725,
                  538,
                  51
                ],
                [
                  0.9853747714808044,
                  0.09647495361781076,
                  539,
                  52
                ],
                [
                  0.9872029250457038,
                  0.09814814814814815,
                  540,
                  53
                ],
                [
                  0.9890310786106032,
                  0.09981515711645102,
                  541,
                  54
                ],
                [
                  0.9908592321755028,
                  0.0996309963099631,
                  542,
                  54
                ],
                [
                  0.9926873857404022,
                  0.10128913443830571,
                  543,
                  55
                ],
                [
                  0.9945155393053017,
                  0.10110294117647059,
                  544,
                  55
                ],
                [
                  0.9963436928702011,
                  0.10275229357798166,
                  545,
                  56
                ],
                [
                  0.9981718464351006,
                  0.1043956043956044,
                  546,
                  57
                ],
                [
                  1.0,
                  0.10603290676416818,
                  547,
                  58
                ]
              ]
            }
          ]
        },
        {
          "name": "Perplexity Decider 27B v1.2",
          "text_run": 217,
          "native": [
            {
              "language": "Turkish",
              "wording": "Bilmiyorum",
              "run": 372,
              "full_bank_restricted": true,
              "right": 497,
              "wrong": 50,
              "abstained": 0,
              "invalid": 0,
              "coverage": 1.0,
              "risk": 0.09140767824497258
            },
            {
              "language": "Turkish",
              "wording": "I don't know",
              "run": 531,
              "full_bank_restricted": true,
              "right": 495,
              "wrong": 51,
              "abstained": 1,
              "invalid": 0,
              "coverage": 0.9981718464351006,
              "risk": 0.09340659340659341
            },
            {
              "language": "English",
              "wording": "I don't know",
              "run": 529,
              "full_bank_restricted": false,
              "right": 481,
              "wrong": 64,
              "abstained": 2,
              "invalid": 0,
              "coverage": 0.9963436928702011,
              "risk": 0.11743119266055047
            }
          ],
          "curves": [
            {
              "language": "Turkish",
              "run": 217,
              "points": [
                [
                  0.0018281535648994515,
                  0.0,
                  1,
                  0
                ],
                [
                  0.003656307129798903,
                  0.0,
                  2,
                  0
                ],
                [
                  0.005484460694698354,
                  0.0,
                  3,
                  0
                ],
                [
                  0.007312614259597806,
                  0.0,
                  4,
                  0
                ],
                [
                  0.009140767824497258,
                  0.0,
                  5,
                  0
                ],
                [
                  0.010968921389396709,
                  0.0,
                  6,
                  0
                ],
                [
                  0.012797074954296161,
                  0.0,
                  7,
                  0
                ],
                [
                  0.014625228519195612,
                  0.0,
                  8,
                  0
                ],
                [
                  0.016453382084095063,
                  0.0,
                  9,
                  0
                ],
                [
                  0.018281535648994516,
                  0.0,
                  10,
                  0
                ],
                [
                  0.02010968921389397,
                  0.0,
                  11,
                  0
                ],
                [
                  0.021937842778793418,
                  0.0,
                  12,
                  0
                ],
                [
                  0.02376599634369287,
                  0.0,
                  13,
                  0
                ],
                [
                  0.025594149908592323,
                  0.0,
                  14,
                  0
                ],
                [
                  0.027422303473491772,
                  0.0,
                  15,
                  0
                ],
                [
                  0.029250457038391225,
                  0.0,
                  16,
                  0
                ],
                [
                  0.031078610603290677,
                  0.0,
                  17,
                  0
                ],
                [
                  0.03290676416819013,
                  0.0,
                  18,
                  0
                ],
                [
                  0.03473491773308958,
                  0.0,
                  19,
                  0
                ],
                [
                  0.03656307129798903,
                  0.0,
                  20,
                  0
                ],
                [
                  0.038391224862888484,
                  0.0,
                  21,
                  0
                ],
                [
                  0.04021937842778794,
                  0.0,
                  22,
                  0
                ],
                [
                  0.04204753199268738,
                  0.0,
                  23,
                  0
                ],
                [
                  0.043875685557586835,
                  0.0,
                  24,
                  0
                ],
                [
                  0.04570383912248629,
                  0.0,
                  25,
                  0
                ],
                [
                  0.04753199268738574,
                  0.0,
                  26,
                  0
                ],
                [
                  0.04936014625228519,
                  0.0,
                  27,
                  0
                ],
                [
                  0.051188299817184646,
                  0.0,
                  28,
                  0
                ],
                [
                  0.05301645338208409,
                  0.0,
                  29,
                  0
                ],
                [
                  0.054844606946983544,
                  0.0,
                  30,
                  0
                ],
                [
                  0.056672760511883,
                  0.0,
                  31,
                  0
                ],
                [
                  0.05850091407678245,
                  0.0,
                  32,
                  0
                ],
                [
                  0.0603290676416819,
                  0.0,
                  33,
                  0
                ],
                [
                  0.062157221206581355,
                  0.0,
                  34,
                  0
                ],
                [
                  0.06398537477148081,
                  0.0,
                  35,
                  0
                ],
                [
                  0.06581352833638025,
                  0.0,
                  36,
                  0
                ],
                [
                  0.06764168190127971,
                  0.0,
                  37,
                  0
                ],
                [
                  0.06946983546617916,
                  0.0,
                  38,
                  0
                ],
                [
                  0.0712979890310786,
                  0.0,
                  39,
                  0
                ],
                [
                  0.07312614259597806,
                  0.0,
                  40,
                  0
                ],
                [
                  0.07495429616087751,
                  0.0,
                  41,
                  0
                ],
                [
                  0.07678244972577697,
                  0.0,
                  42,
                  0
                ],
                [
                  0.07861060329067641,
                  0.0,
                  43,
                  0
                ],
                [
                  0.08043875685557587,
                  0.0,
                  44,
                  0
                ],
                [
                  0.08226691042047532,
                  0.0,
                  45,
                  0
                ],
                [
                  0.08409506398537477,
                  0.0,
                  46,
                  0
                ],
                [
                  0.08592321755027423,
                  0.0,
                  47,
                  0
                ],
                [
                  0.08775137111517367,
                  0.0,
                  48,
                  0
                ],
                [
                  0.08957952468007313,
                  0.0,
                  49,
                  0
                ],
                [
                  0.09140767824497258,
                  0.0,
                  50,
                  0
                ],
                [
                  0.09323583180987204,
                  0.0,
                  51,
                  0
                ],
                [
                  0.09506398537477148,
                  0.0,
                  52,
                  0
                ],
                [
                  0.09689213893967093,
                  0.0,
                  53,
                  0
                ],
                [
                  0.09872029250457039,
                  0.0,
                  54,
                  0
                ],
                [
                  0.10054844606946983,
                  0.0,
                  55,
                  0
                ],
                [
                  0.10237659963436929,
                  0.0,
                  56,
                  0
                ],
                [
                  0.10420475319926874,
                  0.0,
                  57,
                  0
                ],
                [
                  0.10603290676416818,
                  0.0,
                  58,
                  0
                ],
                [
                  0.10786106032906764,
                  0.0,
                  59,
                  0
                ],
                [
                  0.10968921389396709,
                  0.0,
                  60,
                  0
                ],
                [
                  0.11151736745886655,
                  0.0,
                  61,
                  0
                ],
                [
                  0.113345521023766,
                  0.0,
                  62,
                  0
                ],
                [
                  0.11517367458866545,
                  0.0,
                  63,
                  0
                ],
                [
                  0.1170018281535649,
                  0.0,
                  64,
                  0
                ],
                [
                  0.11882998171846434,
                  0.0,
                  65,
                  0
                ],
                [
                  0.1206581352833638,
                  0.0,
                  66,
                  0
                ],
                [
                  0.12248628884826325,
                  0.0,
                  67,
                  0
                ],
                [
                  0.12431444241316271,
                  0.0,
                  68,
                  0
                ],
                [
                  0.12614259597806216,
                  0.0,
                  69,
                  0
                ],
                [
                  0.12797074954296161,
                  0.0,
                  70,
                  0
                ],
                [
                  0.12979890310786105,
                  0.0,
                  71,
                  0
                ],
                [
                  0.1316270566727605,
                  0.0,
                  72,
                  0
                ],
                [
                  0.13345521023765997,
                  0.0,
                  73,
                  0
                ],
                [
                  0.13528336380255943,
                  0.0,
                  74,
                  0
                ],
                [
                  0.13711151736745886,
                  0.0,
                  75,
                  0
                ],
                [
                  0.13893967093235832,
                  0.0,
                  76,
                  0
                ],
                [
                  0.14076782449725778,
                  0.0,
                  77,
                  0
                ],
                [
                  0.1425959780621572,
                  0.0,
                  78,
                  0
                ],
                [
                  0.14442413162705667,
                  0.0,
                  79,
                  0
                ],
                [
                  0.14625228519195613,
                  0.0,
                  80,
                  0
                ],
                [
                  0.1480804387568556,
                  0.0,
                  81,
                  0
                ],
                [
                  0.14990859232175502,
                  0.0,
                  82,
                  0
                ],
                [
                  0.15173674588665448,
                  0.0,
                  83,
                  0
                ],
                [
                  0.15356489945155394,
                  0.0,
                  84,
                  0
                ],
                [
                  0.15539305301645337,
                  0.0,
                  85,
                  0
                ],
                [
                  0.15722120658135283,
                  0.0,
                  86,
                  0
                ],
                [
                  0.1590493601462523,
                  0.0,
                  87,
                  0
                ],
                [
                  0.16087751371115175,
                  0.0,
                  88,
                  0
                ],
                [
                  0.16270566727605118,
                  0.0,
                  89,
                  0
                ],
                [
                  0.16453382084095064,
                  0.0,
                  90,
                  0
                ],
                [
                  0.1663619744058501,
                  0.0,
                  91,
                  0
                ],
                [
                  0.16819012797074953,
                  0.0,
                  92,
                  0
                ],
                [
                  0.170018281535649,
                  0.0,
                  93,
                  0
                ],
                [
                  0.17184643510054845,
                  0.0,
                  94,
                  0
                ],
                [
                  0.1736745886654479,
                  0.0,
                  95,
                  0
                ],
                [
                  0.17550274223034734,
                  0.0,
                  96,
                  0
                ],
                [
                  0.1773308957952468,
                  0.0,
                  97,
                  0
                ],
                [
                  0.17915904936014626,
                  0.0,
                  98,
                  0
                ],
                [
                  0.1809872029250457,
                  0.0,
                  99,
                  0
                ],
                [
                  0.18281535648994515,
                  0.0,
                  100,
                  0
                ],
                [
                  0.1846435100548446,
                  0.0,
                  101,
                  0
                ],
                [
                  0.18647166361974407,
                  0.0,
                  102,
                  0
                ],
                [
                  0.1882998171846435,
                  0.0,
                  103,
                  0
                ],
                [
                  0.19012797074954296,
                  0.0,
                  104,
                  0
                ],
                [
                  0.19195612431444242,
                  0.0,
                  105,
                  0
                ],
                [
                  0.19378427787934185,
                  0.0,
                  106,
                  0
                ],
                [
                  0.1956124314442413,
                  0.0,
                  107,
                  0
                ],
                [
                  0.19744058500914077,
                  0.0,
                  108,
                  0
                ],
                [
                  0.19926873857404023,
                  0.0,
                  109,
                  0
                ],
                [
                  0.20109689213893966,
                  0.0,
                  110,
                  0
                ],
                [
                  0.20292504570383912,
                  0.0,
                  111,
                  0
                ],
                [
                  0.20475319926873858,
                  0.0,
                  112,
                  0
                ],
                [
                  0.20658135283363802,
                  0.0,
                  113,
                  0
                ],
                [
                  0.20840950639853748,
                  0.0,
                  114,
                  0
                ],
                [
                  0.21023765996343693,
                  0.0,
                  115,
                  0
                ],
                [
                  0.21206581352833637,
                  0.0,
                  116,
                  0
                ],
                [
                  0.21389396709323583,
                  0.0,
                  117,
                  0
                ],
                [
                  0.21572212065813529,
                  0.0,
                  118,
                  0
                ],
                [
                  0.21755027422303475,
                  0.0,
                  119,
                  0
                ],
                [
                  0.21937842778793418,
                  0.0,
                  120,
                  0
                ],
                [
                  0.22120658135283364,
                  0.0,
                  121,
                  0
                ],
                [
                  0.2230347349177331,
                  0.0,
                  122,
                  0
                ],
                [
                  0.22486288848263253,
                  0.0,
                  123,
                  0
                ],
                [
                  0.226691042047532,
                  0.0,
                  124,
                  0
                ],
                [
                  0.22851919561243145,
                  0.0,
                  125,
                  0
                ],
                [
                  0.2303473491773309,
                  0.0,
                  126,
                  0
                ],
                [
                  0.23217550274223034,
                  0.0,
                  127,
                  0
                ],
                [
                  0.2340036563071298,
                  0.0,
                  128,
                  0
                ],
                [
                  0.23583180987202926,
                  0.0,
                  129,
                  0
                ],
                [
                  0.2376599634369287,
                  0.0,
                  130,
                  0
                ],
                [
                  0.23948811700182815,
                  0.0,
                  131,
                  0
                ],
                [
                  0.2413162705667276,
                  0.0,
                  132,
                  0
                ],
                [
                  0.24314442413162707,
                  0.0,
                  133,
                  0
                ],
                [
                  0.2449725776965265,
                  0.0,
                  134,
                  0
                ],
                [
                  0.24680073126142596,
                  0.0,
                  135,
                  0
                ],
                [
                  0.24862888482632542,
                  0.0,
                  136,
                  0
                ],
                [
                  0.25045703839122485,
                  0.0,
                  137,
                  0
                ],
                [
                  0.2522851919561243,
                  0.0,
                  138,
                  0
                ],
                [
                  0.25411334552102377,
                  0.0,
                  139,
                  0
                ],
                [
                  0.25594149908592323,
                  0.0,
                  140,
                  0
                ],
                [
                  0.2577696526508227,
                  0.0,
                  141,
                  0
                ],
                [
                  0.2595978062157221,
                  0.0,
                  142,
                  0
                ],
                [
                  0.26142595978062155,
                  0.0,
                  143,
                  0
                ],
                [
                  0.263254113345521,
                  0.0,
                  144,
                  0
                ],
                [
                  0.26508226691042047,
                  0.0,
                  145,
                  0
                ],
                [
                  0.26691042047531993,
                  0.0,
                  146,
                  0
                ],
                [
                  0.2687385740402194,
                  0.0,
                  147,
                  0
                ],
                [
                  0.27056672760511885,
                  0.0,
                  148,
                  0
                ],
                [
                  0.27239488117001825,
                  0.0,
                  149,
                  0
                ],
                [
                  0.2742230347349177,
                  0.0,
                  150,
                  0
                ],
                [
                  0.2760511882998172,
                  0.0,
                  151,
                  0
                ],
                [
                  0.27787934186471663,
                  0.0,
                  152,
                  0
                ],
                [
                  0.2797074954296161,
                  0.0,
                  153,
                  0
                ],
                [
                  0.28153564899451555,
                  0.0,
                  154,
                  0
                ],
                [
                  0.283363802559415,
                  0.0,
                  155,
                  0
                ],
                [
                  0.2851919561243144,
                  0.0,
                  156,
                  0
                ],
                [
                  0.2870201096892139,
                  0.0,
                  157,
                  0
                ],
                [
                  0.28884826325411334,
                  0.0,
                  158,
                  0
                ],
                [
                  0.2906764168190128,
                  0.0,
                  159,
                  0
                ],
                [
                  0.29250457038391225,
                  0.0,
                  160,
                  0
                ],
                [
                  0.2943327239488117,
                  0.0,
                  161,
                  0
                ],
                [
                  0.2961608775137112,
                  0.0,
                  162,
                  0
                ],
                [
                  0.2979890310786106,
                  0.0,
                  163,
                  0
                ],
                [
                  0.29981718464351004,
                  0.0,
                  164,
                  0
                ],
                [
                  0.3016453382084095,
                  0.0,
                  165,
                  0
                ],
                [
                  0.30347349177330896,
                  0.0,
                  166,
                  0
                ],
                [
                  0.3053016453382084,
                  0.0,
                  167,
                  0
                ],
                [
                  0.3071297989031079,
                  0.0,
                  168,
                  0
                ],
                [
                  0.30895795246800734,
                  0.0,
                  169,
                  0
                ],
                [
                  0.31078610603290674,
                  0.0,
                  170,
                  0
                ],
                [
                  0.3126142595978062,
                  0.0,
                  171,
                  0
                ],
                [
                  0.31444241316270566,
                  0.0,
                  172,
                  0
                ],
                [
                  0.3162705667276051,
                  0.0,
                  173,
                  0
                ],
                [
                  0.3180987202925046,
                  0.0,
                  174,
                  0
                ],
                [
                  0.31992687385740404,
                  0.0,
                  175,
                  0
                ],
                [
                  0.3217550274223035,
                  0.0,
                  176,
                  0
                ],
                [
                  0.3235831809872029,
                  0.0,
                  177,
                  0
                ],
                [
                  0.32541133455210236,
                  0.0,
                  178,
                  0
                ],
                [
                  0.3272394881170018,
                  0.0,
                  179,
                  0
                ],
                [
                  0.3290676416819013,
                  0.0,
                  180,
                  0
                ],
                [
                  0.33089579524680074,
                  0.0,
                  181,
                  0
                ],
                [
                  0.3327239488117002,
                  0.0,
                  182,
                  0
                ],
                [
                  0.33455210237659966,
                  0.0,
                  183,
                  0
                ],
                [
                  0.33638025594149906,
                  0.0,
                  184,
                  0
                ],
                [
                  0.3382084095063985,
                  0.0,
                  185,
                  0
                ],
                [
                  0.340036563071298,
                  0.0,
                  186,
                  0
                ],
                [
                  0.34186471663619744,
                  0.0,
                  187,
                  0
                ],
                [
                  0.3436928702010969,
                  0.0,
                  188,
                  0
                ],
                [
                  0.34552102376599636,
                  0.0,
                  189,
                  0
                ],
                [
                  0.3473491773308958,
                  0.0,
                  190,
                  0
                ],
                [
                  0.3491773308957952,
                  0.0,
                  191,
                  0
                ],
                [
                  0.3510054844606947,
                  0.0,
                  192,
                  0
                ],
                [
                  0.35283363802559414,
                  0.0,
                  193,
                  0
                ],
                [
                  0.3546617915904936,
                  0.0,
                  194,
                  0
                ],
                [
                  0.35648994515539306,
                  0.0,
                  195,
                  0
                ],
                [
                  0.3583180987202925,
                  0.0,
                  196,
                  0
                ],
                [
                  0.360146252285192,
                  0.0,
                  197,
                  0
                ],
                [
                  0.3619744058500914,
                  0.0,
                  198,
                  0
                ],
                [
                  0.36380255941499084,
                  0.0,
                  199,
                  0
                ],
                [
                  0.3656307129798903,
                  0.0,
                  200,
                  0
                ],
                [
                  0.36745886654478976,
                  0.0,
                  201,
                  0
                ],
                [
                  0.3692870201096892,
                  0.0,
                  202,
                  0
                ],
                [
                  0.3711151736745887,
                  0.0,
                  203,
                  0
                ],
                [
                  0.37294332723948814,
                  0.0,
                  204,
                  0
                ],
                [
                  0.37477148080438755,
                  0.0,
                  205,
                  0
                ],
                [
                  0.376599634369287,
                  0.0,
                  206,
                  0
                ],
                [
                  0.37842778793418647,
                  0.0,
                  207,
                  0
                ],
                [
                  0.3802559414990859,
                  0.0,
                  208,
                  0
                ],
                [
                  0.3820840950639854,
                  0.0,
                  209,
                  0
                ],
                [
                  0.38391224862888484,
                  0.0,
                  210,
                  0
                ],
                [
                  0.3857404021937843,
                  0.0,
                  211,
                  0
                ],
                [
                  0.3875685557586837,
                  0.0,
                  212,
                  0
                ],
                [
                  0.38939670932358317,
                  0.0,
                  213,
                  0
                ],
                [
                  0.3912248628884826,
                  0.0,
                  214,
                  0
                ],
                [
                  0.3930530164533821,
                  0.0,
                  215,
                  0
                ],
                [
                  0.39488117001828155,
                  0.0,
                  216,
                  0
                ],
                [
                  0.396709323583181,
                  0.0,
                  217,
                  0
                ],
                [
                  0.39853747714808047,
                  0.0,
                  218,
                  0
                ],
                [
                  0.40036563071297987,
                  0.0,
                  219,
                  0
                ],
                [
                  0.40219378427787933,
                  0.0,
                  220,
                  0
                ],
                [
                  0.4040219378427788,
                  0.0,
                  221,
                  0
                ],
                [
                  0.40585009140767825,
                  0.0,
                  222,
                  0
                ],
                [
                  0.4076782449725777,
                  0.0,
                  223,
                  0
                ],
                [
                  0.40950639853747717,
                  0.0,
                  224,
                  0
                ],
                [
                  0.4113345521023766,
                  0.0,
                  225,
                  0
                ],
                [
                  0.41316270566727603,
                  0.0,
                  226,
                  0
                ],
                [
                  0.4149908592321755,
                  0.0,
                  227,
                  0
                ],
                [
                  0.41681901279707495,
                  0.0,
                  228,
                  0
                ],
                [
                  0.4186471663619744,
                  0.0,
                  229,
                  0
                ],
                [
                  0.42047531992687387,
                  0.0,
                  230,
                  0
                ],
                [
                  0.42230347349177333,
                  0.0,
                  231,
                  0
                ],
                [
                  0.42413162705667273,
                  0.0,
                  232,
                  0
                ],
                [
                  0.4259597806215722,
                  0.0,
                  233,
                  0
                ],
                [
                  0.42778793418647165,
                  0.0,
                  234,
                  0
                ],
                [
                  0.4296160877513711,
                  0.0,
                  235,
                  0
                ],
                [
                  0.43144424131627057,
                  0.0,
                  236,
                  0
                ],
                [
                  0.43327239488117003,
                  0.0,
                  237,
                  0
                ],
                [
                  0.4351005484460695,
                  0.0,
                  238,
                  0
                ],
                [
                  0.4369287020109689,
                  0.0,
                  239,
                  0
                ],
                [
                  0.43875685557586835,
                  0.0,
                  240,
                  0
                ],
                [
                  0.4405850091407678,
                  0.0,
                  241,
                  0
                ],
                [
                  0.4424131627056673,
                  0.0,
                  242,
                  0
                ],
                [
                  0.44424131627056673,
                  0.0,
                  243,
                  0
                ],
                [
                  0.4460694698354662,
                  0.0,
                  244,
                  0
                ],
                [
                  0.44789762340036565,
                  0.0,
                  245,
                  0
                ],
                [
                  0.44972577696526506,
                  0.0,
                  246,
                  0
                ],
                [
                  0.4515539305301645,
                  0.0,
                  247,
                  0
                ],
                [
                  0.453382084095064,
                  0.0,
                  248,
                  0
                ],
                [
                  0.45521023765996343,
                  0.0,
                  249,
                  0
                ],
                [
                  0.4570383912248629,
                  0.0,
                  250,
                  0
                ],
                [
                  0.45886654478976235,
                  0.0,
                  251,
                  0
                ],
                [
                  0.4606946983546618,
                  0.0,
                  252,
                  0
                ],
                [
                  0.4625228519195612,
                  0.0,
                  253,
                  0
                ],
                [
                  0.4643510054844607,
                  0.0,
                  254,
                  0
                ],
                [
                  0.46617915904936014,
                  0.0,
                  255,
                  0
                ],
                [
                  0.4680073126142596,
                  0.0,
                  256,
                  0
                ],
                [
                  0.46983546617915906,
                  0.0,
                  257,
                  0
                ],
                [
                  0.4716636197440585,
                  0.0,
                  258,
                  0
                ],
                [
                  0.473491773308958,
                  0.0,
                  259,
                  0
                ],
                [
                  0.4753199268738574,
                  0.0,
                  260,
                  0
                ],
                [
                  0.47714808043875684,
                  0.0,
                  261,
                  0
                ],
                [
                  0.4789762340036563,
                  0.0,
                  262,
                  0
                ],
                [
                  0.48080438756855576,
                  0.0,
                  263,
                  0
                ],
                [
                  0.4826325411334552,
                  0.0,
                  264,
                  0
                ],
                [
                  0.4844606946983547,
                  0.0,
                  265,
                  0
                ],
                [
                  0.48628884826325414,
                  0.0,
                  266,
                  0
                ],
                [
                  0.48811700182815354,
                  0.0,
                  267,
                  0
                ],
                [
                  0.489945155393053,
                  0.0,
                  268,
                  0
                ],
                [
                  0.49177330895795246,
                  0.0,
                  269,
                  0
                ],
                [
                  0.4936014625228519,
                  0.0,
                  270,
                  0
                ],
                [
                  0.4954296160877514,
                  0.0,
                  271,
                  0
                ],
                [
                  0.49725776965265084,
                  0.0,
                  272,
                  0
                ],
                [
                  0.4990859232175503,
                  0.0,
                  273,
                  0
                ],
                [
                  0.5009140767824497,
                  0.0,
                  274,
                  0
                ],
                [
                  0.5027422303473492,
                  0.0,
                  275,
                  0
                ],
                [
                  0.5045703839122486,
                  0.0,
                  276,
                  0
                ],
                [
                  0.506398537477148,
                  0.0,
                  277,
                  0
                ],
                [
                  0.5082266910420475,
                  0.0,
                  278,
                  0
                ],
                [
                  0.5100548446069469,
                  0.0,
                  279,
                  0
                ],
                [
                  0.5118829981718465,
                  0.0,
                  280,
                  0
                ],
                [
                  0.5137111517367459,
                  0.0,
                  281,
                  0
                ],
                [
                  0.5155393053016454,
                  0.0,
                  282,
                  0
                ],
                [
                  0.5173674588665448,
                  0.0,
                  283,
                  0
                ],
                [
                  0.5191956124314442,
                  0.0,
                  284,
                  0
                ],
                [
                  0.5210237659963437,
                  0.0,
                  285,
                  0
                ],
                [
                  0.5228519195612431,
                  0.0,
                  286,
                  0
                ],
                [
                  0.5246800731261426,
                  0.0,
                  287,
                  0
                ],
                [
                  0.526508226691042,
                  0.0,
                  288,
                  0
                ],
                [
                  0.5283363802559415,
                  0.0,
                  289,
                  0
                ],
                [
                  0.5301645338208409,
                  0.0,
                  290,
                  0
                ],
                [
                  0.5319926873857403,
                  0.0,
                  291,
                  0
                ],
                [
                  0.5338208409506399,
                  0.0,
                  292,
                  0
                ],
                [
                  0.5356489945155393,
                  0.0,
                  293,
                  0
                ],
                [
                  0.5374771480804388,
                  0.0,
                  294,
                  0
                ],
                [
                  0.5393053016453382,
                  0.0,
                  295,
                  0
                ],
                [
                  0.5411334552102377,
                  0.0,
                  296,
                  0
                ],
                [
                  0.5429616087751371,
                  0.0,
                  297,
                  0
                ],
                [
                  0.5447897623400365,
                  0.0,
                  298,
                  0
                ],
                [
                  0.546617915904936,
                  0.0,
                  299,
                  0
                ],
                [
                  0.5484460694698354,
                  0.0,
                  300,
                  0
                ],
                [
                  0.5502742230347349,
                  0.0,
                  301,
                  0
                ],
                [
                  0.5521023765996343,
                  0.0,
                  302,
                  0
                ],
                [
                  0.5539305301645339,
                  0.0,
                  303,
                  0
                ],
                [
                  0.5557586837294333,
                  0.0,
                  304,
                  0
                ],
                [
                  0.5575868372943327,
                  0.0,
                  305,
                  0
                ],
                [
                  0.5594149908592322,
                  0.0,
                  306,
                  0
                ],
                [
                  0.5612431444241316,
                  0.0,
                  307,
                  0
                ],
                [
                  0.5630712979890311,
                  0.0,
                  308,
                  0
                ],
                [
                  0.5648994515539305,
                  0.0,
                  309,
                  0
                ],
                [
                  0.56672760511883,
                  0.0,
                  310,
                  0
                ],
                [
                  0.5685557586837294,
                  0.0,
                  311,
                  0
                ],
                [
                  0.5703839122486288,
                  0.0,
                  312,
                  0
                ],
                [
                  0.5722120658135283,
                  0.0,
                  313,
                  0
                ],
                [
                  0.5740402193784278,
                  0.0,
                  314,
                  0
                ],
                [
                  0.5758683729433273,
                  0.0,
                  315,
                  0
                ],
                [
                  0.5776965265082267,
                  0.0,
                  316,
                  0
                ],
                [
                  0.5795246800731262,
                  0.0,
                  317,
                  0
                ],
                [
                  0.5813528336380256,
                  0.0,
                  318,
                  0
                ],
                [
                  0.583180987202925,
                  0.0,
                  319,
                  0
                ],
                [
                  0.5850091407678245,
                  0.0,
                  320,
                  0
                ],
                [
                  0.5868372943327239,
                  0.0,
                  321,
                  0
                ],
                [
                  0.5886654478976234,
                  0.0,
                  322,
                  0
                ],
                [
                  0.5904936014625228,
                  0.0,
                  323,
                  0
                ],
                [
                  0.5923217550274223,
                  0.0,
                  324,
                  0
                ],
                [
                  0.5941499085923218,
                  0.0,
                  325,
                  0
                ],
                [
                  0.5959780621572212,
                  0.0,
                  326,
                  0
                ],
                [
                  0.5978062157221207,
                  0.0,
                  327,
                  0
                ],
                [
                  0.5996343692870201,
                  0.0,
                  328,
                  0
                ],
                [
                  0.6014625228519196,
                  0.0,
                  329,
                  0
                ],
                [
                  0.603290676416819,
                  0.0,
                  330,
                  0
                ],
                [
                  0.6051188299817185,
                  0.0,
                  331,
                  0
                ],
                [
                  0.6069469835466179,
                  0.0,
                  332,
                  0
                ],
                [
                  0.6087751371115173,
                  0.0,
                  333,
                  0
                ],
                [
                  0.6106032906764168,
                  0.0,
                  334,
                  0
                ],
                [
                  0.6124314442413162,
                  0.0,
                  335,
                  0
                ],
                [
                  0.6142595978062158,
                  0.0,
                  336,
                  0
                ],
                [
                  0.6160877513711152,
                  0.0,
                  337,
                  0
                ],
                [
                  0.6179159049360147,
                  0.0,
                  338,
                  0
                ],
                [
                  0.6197440585009141,
                  0.0,
                  339,
                  0
                ],
                [
                  0.6215722120658135,
                  0.0,
                  340,
                  0
                ],
                [
                  0.623400365630713,
                  0.0,
                  341,
                  0
                ],
                [
                  0.6252285191956124,
                  0.0,
                  342,
                  0
                ],
                [
                  0.6270566727605119,
                  0.0,
                  343,
                  0
                ],
                [
                  0.6288848263254113,
                  0.0,
                  344,
                  0
                ],
                [
                  0.6307129798903108,
                  0.0,
                  345,
                  0
                ],
                [
                  0.6325411334552102,
                  0.0,
                  346,
                  0
                ],
                [
                  0.6343692870201096,
                  0.0,
                  347,
                  0
                ],
                [
                  0.6361974405850092,
                  0.0,
                  348,
                  0
                ],
                [
                  0.6380255941499086,
                  0.0,
                  349,
                  0
                ],
                [
                  0.6398537477148081,
                  0.0,
                  350,
                  0
                ],
                [
                  0.6416819012797075,
                  0.0,
                  351,
                  0
                ],
                [
                  0.643510054844607,
                  0.0,
                  352,
                  0
                ],
                [
                  0.6453382084095064,
                  0.0,
                  353,
                  0
                ],
                [
                  0.6471663619744058,
                  0.0,
                  354,
                  0
                ],
                [
                  0.6489945155393053,
                  0.0,
                  355,
                  0
                ],
                [
                  0.6508226691042047,
                  0.0,
                  356,
                  0
                ],
                [
                  0.6526508226691042,
                  0.0,
                  357,
                  0
                ],
                [
                  0.6544789762340036,
                  0.0,
                  358,
                  0
                ],
                [
                  0.6563071297989032,
                  0.0,
                  359,
                  0
                ],
                [
                  0.6581352833638026,
                  0.0,
                  360,
                  0
                ],
                [
                  0.659963436928702,
                  0.0,
                  361,
                  0
                ],
                [
                  0.6617915904936015,
                  0.0,
                  362,
                  0
                ],
                [
                  0.6636197440585009,
                  0.0,
                  363,
                  0
                ],
                [
                  0.6654478976234004,
                  0.0,
                  364,
                  0
                ],
                [
                  0.6672760511882998,
                  0.0,
                  365,
                  0
                ],
                [
                  0.6691042047531993,
                  0.0,
                  366,
                  0
                ],
                [
                  0.6709323583180987,
                  0.0,
                  367,
                  0
                ],
                [
                  0.6727605118829981,
                  0.0,
                  368,
                  0
                ],
                [
                  0.6745886654478976,
                  0.0,
                  369,
                  0
                ],
                [
                  0.676416819012797,
                  0.0,
                  370,
                  0
                ],
                [
                  0.6782449725776966,
                  0.0,
                  371,
                  0
                ],
                [
                  0.680073126142596,
                  0.0,
                  372,
                  0
                ],
                [
                  0.6819012797074955,
                  0.0,
                  373,
                  0
                ],
                [
                  0.6837294332723949,
                  0.0,
                  374,
                  0
                ],
                [
                  0.6855575868372943,
                  0.0,
                  375,
                  0
                ],
                [
                  0.6873857404021938,
                  0.0,
                  376,
                  0
                ],
                [
                  0.6892138939670932,
                  0.0,
                  377,
                  0
                ],
                [
                  0.6910420475319927,
                  0.0,
                  378,
                  0
                ],
                [
                  0.6928702010968921,
                  0.0,
                  379,
                  0
                ],
                [
                  0.6946983546617916,
                  0.0,
                  380,
                  0
                ],
                [
                  0.696526508226691,
                  0.0,
                  381,
                  0
                ],
                [
                  0.6983546617915904,
                  0.0,
                  382,
                  0
                ],
                [
                  0.70018281535649,
                  0.0,
                  383,
                  0
                ],
                [
                  0.7020109689213894,
                  0.0,
                  384,
                  0
                ],
                [
                  0.7038391224862889,
                  0.0,
                  385,
                  0
                ],
                [
                  0.7056672760511883,
                  0.0,
                  386,
                  0
                ],
                [
                  0.7074954296160878,
                  0.0,
                  387,
                  0
                ],
                [
                  0.7093235831809872,
                  0.0,
                  388,
                  0
                ],
                [
                  0.7111517367458866,
                  0.0,
                  389,
                  0
                ],
                [
                  0.7129798903107861,
                  0.0,
                  390,
                  0
                ],
                [
                  0.7148080438756855,
                  0.0,
                  391,
                  0
                ],
                [
                  0.716636197440585,
                  0.0,
                  392,
                  0
                ],
                [
                  0.7184643510054844,
                  0.0,
                  393,
                  0
                ],
                [
                  0.720292504570384,
                  0.0,
                  394,
                  0
                ],
                [
                  0.7221206581352834,
                  0.0,
                  395,
                  0
                ],
                [
                  0.7239488117001828,
                  0.0,
                  396,
                  0
                ],
                [
                  0.7257769652650823,
                  0.0,
                  397,
                  0
                ],
                [
                  0.7276051188299817,
                  0.0,
                  398,
                  0
                ],
                [
                  0.7294332723948812,
                  0.0,
                  399,
                  0
                ],
                [
                  0.7312614259597806,
                  0.0,
                  400,
                  0
                ],
                [
                  0.7330895795246801,
                  0.0,
                  401,
                  0
                ],
                [
                  0.7349177330895795,
                  0.0,
                  402,
                  0
                ],
                [
                  0.7367458866544789,
                  0.0,
                  403,
                  0
                ],
                [
                  0.7385740402193784,
                  0.0,
                  404,
                  0
                ],
                [
                  0.7404021937842779,
                  0.0,
                  405,
                  0
                ],
                [
                  0.7422303473491774,
                  0.0,
                  406,
                  0
                ],
                [
                  0.7440585009140768,
                  0.0,
                  407,
                  0
                ],
                [
                  0.7458866544789763,
                  0.0,
                  408,
                  0
                ],
                [
                  0.7477148080438757,
                  0.0024449877750611247,
                  409,
                  1
                ],
                [
                  0.7495429616087751,
                  0.0024390243902439024,
                  410,
                  1
                ],
                [
                  0.7513711151736746,
                  0.0024330900243309003,
                  411,
                  1
                ],
                [
                  0.753199268738574,
                  0.0048543689320388345,
                  412,
                  2
                ],
                [
                  0.7550274223034735,
                  0.004842615012106538,
                  413,
                  2
                ],
                [
                  0.7568555758683729,
                  0.004830917874396135,
                  414,
                  2
                ],
                [
                  0.7586837294332724,
                  0.004819277108433735,
                  415,
                  2
                ],
                [
                  0.7605118829981719,
                  0.004807692307692308,
                  416,
                  2
                ],
                [
                  0.7623400365630713,
                  0.004796163069544364,
                  417,
                  2
                ],
                [
                  0.7641681901279708,
                  0.004784688995215311,
                  418,
                  2
                ],
                [
                  0.7659963436928702,
                  0.007159904534606206,
                  419,
                  3
                ],
                [
                  0.7678244972577697,
                  0.007142857142857143,
                  420,
                  3
                ],
                [
                  0.7696526508226691,
                  0.009501187648456057,
                  421,
                  4
                ],
                [
                  0.7714808043875686,
                  0.011848341232227487,
                  422,
                  5
                ],
                [
                  0.773308957952468,
                  0.01182033096926714,
                  423,
                  5
                ],
                [
                  0.7751371115173674,
                  0.01179245283018868,
                  424,
                  5
                ],
                [
                  0.7769652650822669,
                  0.011764705882352941,
                  425,
                  5
                ],
                [
                  0.7787934186471663,
                  0.011737089201877934,
                  426,
                  5
                ],
                [
                  0.7806215722120659,
                  0.01405152224824356,
                  427,
                  6
                ],
                [
                  0.7824497257769653,
                  0.016355140186915886,
                  428,
                  7
                ],
                [
                  0.7842778793418648,
                  0.018648018648018648,
                  429,
                  8
                ],
                [
                  0.7861060329067642,
                  0.020930232558139535,
                  430,
                  9
                ],
                [
                  0.7879341864716636,
                  0.02088167053364269,
                  431,
                  9
                ],
                [
                  0.7897623400365631,
                  0.020833333333333332,
                  432,
                  9
                ],
                [
                  0.7915904936014625,
                  0.020785219399538105,
                  433,
                  9
                ],
                [
                  0.793418647166362,
                  0.020737327188940093,
                  434,
                  9
                ],
                [
                  0.7952468007312614,
                  0.020689655172413793,
                  435,
                  9
                ],
                [
                  0.7970749542961609,
                  0.020642201834862386,
                  436,
                  9
                ],
                [
                  0.7989031078610603,
                  0.020594965675057208,
                  437,
                  9
                ],
                [
                  0.8007312614259597,
                  0.02054794520547945,
                  438,
                  9
                ],
                [
                  0.8025594149908593,
                  0.022779043280182234,
                  439,
                  10
                ],
                [
                  0.8043875685557587,
                  0.022727272727272728,
                  440,
                  10
                ],
                [
                  0.8062157221206582,
                  0.024943310657596373,
                  441,
                  11
                ],
                [
                  0.8080438756855576,
                  0.024886877828054297,
                  442,
                  11
                ],
                [
                  0.8098720292504571,
                  0.024830699774266364,
                  443,
                  11
                ],
                [
                  0.8117001828153565,
                  0.024774774774774775,
                  444,
                  11
                ],
                [
                  0.8135283363802559,
                  0.024719101123595506,
                  445,
                  11
                ],
                [
                  0.8153564899451554,
                  0.02466367713004484,
                  446,
                  11
                ],
                [
                  0.8171846435100548,
                  0.024608501118568233,
                  447,
                  11
                ],
                [
                  0.8190127970749543,
                  0.024553571428571428,
                  448,
                  11
                ],
                [
                  0.8208409506398537,
                  0.024498886414253896,
                  449,
                  11
                ],
                [
                  0.8226691042047533,
                  0.02666666666666667,
                  450,
                  12
                ],
                [
                  0.8244972577696527,
                  0.028824833702882482,
                  451,
                  13
                ],
                [
                  0.8263254113345521,
                  0.028761061946902654,
                  452,
                  13
                ],
                [
                  0.8281535648994516,
                  0.02869757174392936,
                  453,
                  13
                ],
                [
                  0.829981718464351,
                  0.028634361233480177,
                  454,
                  13
                ],
                [
                  0.8318098720292505,
                  0.02857142857142857,
                  455,
                  13
                ],
                [
                  0.8336380255941499,
                  0.02850877192982456,
                  456,
                  13
                ],
                [
                  0.8354661791590493,
                  0.028446389496717725,
                  457,
                  13
                ],
                [
                  0.8372943327239488,
                  0.028384279475982533,
                  458,
                  13
                ],
                [
                  0.8391224862888482,
                  0.02832244008714597,
                  459,
                  13
                ],
                [
                  0.8409506398537477,
                  0.02826086956521739,
                  460,
                  13
                ],
                [
                  0.8427787934186471,
                  0.028199566160520606,
                  461,
                  13
                ],
                [
                  0.8446069469835467,
                  0.02813852813852814,
                  462,
                  13
                ],
                [
                  0.8464351005484461,
                  0.03023758099352052,
                  463,
                  14
                ],
                [
                  0.8482632541133455,
                  0.03017241379310345,
                  464,
                  14
                ],
                [
                  0.850091407678245,
                  0.030107526881720432,
                  465,
                  14
                ],
                [
                  0.8519195612431444,
                  0.030042918454935622,
                  466,
                  14
                ],
                [
                  0.8537477148080439,
                  0.029978586723768737,
                  467,
                  14
                ],
                [
                  0.8555758683729433,
                  0.03205128205128205,
                  468,
                  15
                ],
                [
                  0.8574040219378428,
                  0.031982942430703626,
                  469,
                  15
                ],
                [
                  0.8592321755027422,
                  0.031914893617021274,
                  470,
                  15
                ],
                [
                  0.8610603290676416,
                  0.03184713375796178,
                  471,
                  15
                ],
                [
                  0.8628884826325411,
                  0.03177966101694915,
                  472,
                  15
                ],
                [
                  0.8647166361974405,
                  0.03171247357293869,
                  473,
                  15
                ],
                [
                  0.8665447897623401,
                  0.03164556962025317,
                  474,
                  15
                ],
                [
                  0.8683729433272395,
                  0.031578947368421054,
                  475,
                  15
                ],
                [
                  0.870201096892139,
                  0.03361344537815126,
                  476,
                  16
                ],
                [
                  0.8720292504570384,
                  0.033542976939203356,
                  477,
                  16
                ],
                [
                  0.8738574040219378,
                  0.03556485355648536,
                  478,
                  17
                ],
                [
                  0.8756855575868373,
                  0.037578288100208766,
                  479,
                  18
                ],
                [
                  0.8775137111517367,
                  0.0375,
                  480,
                  18
                ],
                [
                  0.8793418647166362,
                  0.037422037422037424,
                  481,
                  18
                ],
                [
                  0.8811700182815356,
                  0.03734439834024896,
                  482,
                  18
                ],
                [
                  0.8829981718464351,
                  0.037267080745341616,
                  483,
                  18
                ],
                [
                  0.8848263254113345,
                  0.03925619834710744,
                  484,
                  19
                ],
                [
                  0.886654478976234,
                  0.03917525773195876,
                  485,
                  19
                ],
                [
                  0.8884826325411335,
                  0.0411522633744856,
                  486,
                  20
                ],
                [
                  0.8903107861060329,
                  0.04106776180698152,
                  487,
                  20
                ],
                [
                  0.8921389396709324,
                  0.0430327868852459,
                  488,
                  21
                ],
                [
                  0.8939670932358318,
                  0.04294478527607362,
                  489,
                  21
                ],
                [
                  0.8957952468007313,
                  0.044897959183673466,
                  490,
                  22
                ],
                [
                  0.8976234003656307,
                  0.04480651731160896,
                  491,
                  22
                ],
                [
                  0.8994515539305301,
                  0.044715447154471545,
                  492,
                  22
                ],
                [
                  0.9012797074954296,
                  0.04462474645030426,
                  493,
                  22
                ],
                [
                  0.903107861060329,
                  0.044534412955465584,
                  494,
                  22
                ],
                [
                  0.9049360146252285,
                  0.046464646464646465,
                  495,
                  23
                ],
                [
                  0.906764168190128,
                  0.04838709677419355,
                  496,
                  24
                ],
                [
                  0.9085923217550275,
                  0.0482897384305835,
                  497,
                  24
                ],
                [
                  0.9104204753199269,
                  0.050200803212851405,
                  498,
                  25
                ],
                [
                  0.9122486288848263,
                  0.052104208416833664,
                  499,
                  26
                ],
                [
                  0.9140767824497258,
                  0.054,
                  500,
                  27
                ],
                [
                  0.9159049360146252,
                  0.05389221556886228,
                  501,
                  27
                ],
                [
                  0.9177330895795247,
                  0.055776892430278883,
                  502,
                  28
                ],
                [
                  0.9195612431444241,
                  0.055666003976143144,
                  503,
                  28
                ],
                [
                  0.9213893967093236,
                  0.057539682539682536,
                  504,
                  29
                ],
                [
                  0.923217550274223,
                  0.05742574257425743,
                  505,
                  29
                ],
                [
                  0.9250457038391224,
                  0.05731225296442688,
                  506,
                  29
                ],
                [
                  0.926873857404022,
                  0.05719921104536489,
                  507,
                  29
                ],
                [
                  0.9287020109689214,
                  0.05708661417322835,
                  508,
                  29
                ],
                [
                  0.9305301645338209,
                  0.05697445972495088,
                  509,
                  29
                ],
                [
                  0.9323583180987203,
                  0.058823529411764705,
                  510,
                  30
                ],
                [
                  0.9341864716636198,
                  0.060665362035225046,
                  511,
                  31
                ],
                [
                  0.9360146252285192,
                  0.0625,
                  512,
                  32
                ],
                [
                  0.9378427787934186,
                  0.06237816764132553,
                  513,
                  32
                ],
                [
                  0.9396709323583181,
                  0.0622568093385214,
                  514,
                  32
                ],
                [
                  0.9414990859232175,
                  0.062135922330097085,
                  515,
                  32
                ],
                [
                  0.943327239488117,
                  0.06201550387596899,
                  516,
                  32
                ],
                [
                  0.9451553930530164,
                  0.061895551257253385,
                  517,
                  32
                ],
                [
                  0.946983546617916,
                  0.06177606177606178,
                  518,
                  32
                ],
                [
                  0.9488117001828154,
                  0.06165703275529865,
                  519,
                  32
                ],
                [
                  0.9506398537477148,
                  0.06346153846153846,
                  520,
                  33
                ],
                [
                  0.9524680073126143,
                  0.06333973128598848,
                  521,
                  33
                ],
                [
                  0.9542961608775137,
                  0.06513409961685823,
                  522,
                  34
                ],
                [
                  0.9561243144424132,
                  0.06692160611854685,
                  523,
                  35
                ],
                [
                  0.9579524680073126,
                  0.06679389312977099,
                  524,
                  35
                ],
                [
                  0.9597806215722121,
                  0.06666666666666667,
                  525,
                  35
                ],
                [
                  0.9616087751371115,
                  0.06653992395437262,
                  526,
                  35
                ],
                [
                  0.9634369287020109,
                  0.06641366223908918,
                  527,
                  35
                ],
                [
                  0.9652650822669104,
                  0.06818181818181818,
                  528,
                  36
                ],
                [
                  0.9670932358318098,
                  0.06994328922495274,
                  529,
                  37
                ],
                [
                  0.9689213893967094,
                  0.07169811320754717,
                  530,
                  38
                ],
                [
                  0.9707495429616088,
                  0.07156308851224105,
                  531,
                  38
                ],
                [
                  0.9725776965265083,
                  0.07330827067669173,
                  532,
                  39
                ],
                [
                  0.9744058500914077,
                  0.075046904315197,
                  533,
                  40
                ],
                [
                  0.9762340036563071,
                  0.0749063670411985,
                  534,
                  40
                ],
                [
                  0.9780621572212066,
                  0.07476635514018691,
                  535,
                  40
                ],
                [
                  0.979890310786106,
                  0.07462686567164178,
                  536,
                  40
                ],
                [
                  0.9817184643510055,
                  0.07635009310986965,
                  537,
                  41
                ],
                [
                  0.9835466179159049,
                  0.0762081784386617,
                  538,
                  41
                ],
                [
                  0.9853747714808044,
                  0.07606679035250463,
                  539,
                  41
                ],
                [
                  0.9872029250457038,
                  0.07777777777777778,
                  540,
                  42
                ],
                [
                  0.9890310786106032,
                  0.07948243992606285,
                  541,
                  43
                ],
                [
                  0.9908592321755028,
                  0.08118081180811808,
                  542,
                  44
                ],
                [
                  0.9926873857404022,
                  0.08287292817679558,
                  543,
                  45
                ],
                [
                  0.9945155393053017,
                  0.08455882352941177,
                  544,
                  46
                ],
                [
                  0.9963436928702011,
                  0.08440366972477065,
                  545,
                  46
                ],
                [
                  0.9981718464351006,
                  0.08424908424908426,
                  546,
                  46
                ],
                [
                  1.0,
                  0.08592321755027423,
                  547,
                  47
                ]
              ]
            },
            {
              "language": "English",
              "run": 527,
              "points": [
                [
                  0.0018281535648994515,
                  0.0,
                  1,
                  0
                ],
                [
                  0.003656307129798903,
                  0.0,
                  2,
                  0
                ],
                [
                  0.005484460694698354,
                  0.0,
                  3,
                  0
                ],
                [
                  0.007312614259597806,
                  0.0,
                  4,
                  0
                ],
                [
                  0.009140767824497258,
                  0.0,
                  5,
                  0
                ],
                [
                  0.010968921389396709,
                  0.0,
                  6,
                  0
                ],
                [
                  0.012797074954296161,
                  0.0,
                  7,
                  0
                ],
                [
                  0.014625228519195612,
                  0.0,
                  8,
                  0
                ],
                [
                  0.016453382084095063,
                  0.0,
                  9,
                  0
                ],
                [
                  0.018281535648994516,
                  0.0,
                  10,
                  0
                ],
                [
                  0.02010968921389397,
                  0.0,
                  11,
                  0
                ],
                [
                  0.021937842778793418,
                  0.0,
                  12,
                  0
                ],
                [
                  0.02376599634369287,
                  0.0,
                  13,
                  0
                ],
                [
                  0.025594149908592323,
                  0.0,
                  14,
                  0
                ],
                [
                  0.027422303473491772,
                  0.0,
                  15,
                  0
                ],
                [
                  0.029250457038391225,
                  0.0,
                  16,
                  0
                ],
                [
                  0.031078610603290677,
                  0.0,
                  17,
                  0
                ],
                [
                  0.03290676416819013,
                  0.0,
                  18,
                  0
                ],
                [
                  0.03473491773308958,
                  0.0,
                  19,
                  0
                ],
                [
                  0.03656307129798903,
                  0.0,
                  20,
                  0
                ],
                [
                  0.038391224862888484,
                  0.0,
                  21,
                  0
                ],
                [
                  0.04021937842778794,
                  0.0,
                  22,
                  0
                ],
                [
                  0.04204753199268738,
                  0.0,
                  23,
                  0
                ],
                [
                  0.043875685557586835,
                  0.0,
                  24,
                  0
                ],
                [
                  0.04570383912248629,
                  0.0,
                  25,
                  0
                ],
                [
                  0.04753199268738574,
                  0.0,
                  26,
                  0
                ],
                [
                  0.04936014625228519,
                  0.0,
                  27,
                  0
                ],
                [
                  0.051188299817184646,
                  0.0,
                  28,
                  0
                ],
                [
                  0.05301645338208409,
                  0.0,
                  29,
                  0
                ],
                [
                  0.054844606946983544,
                  0.0,
                  30,
                  0
                ],
                [
                  0.056672760511883,
                  0.0,
                  31,
                  0
                ],
                [
                  0.05850091407678245,
                  0.0,
                  32,
                  0
                ],
                [
                  0.0603290676416819,
                  0.0,
                  33,
                  0
                ],
                [
                  0.062157221206581355,
                  0.0,
                  34,
                  0
                ],
                [
                  0.06398537477148081,
                  0.0,
                  35,
                  0
                ],
                [
                  0.06581352833638025,
                  0.0,
                  36,
                  0
                ],
                [
                  0.06764168190127971,
                  0.0,
                  37,
                  0
                ],
                [
                  0.06946983546617916,
                  0.0,
                  38,
                  0
                ],
                [
                  0.0712979890310786,
                  0.0,
                  39,
                  0
                ],
                [
                  0.07312614259597806,
                  0.0,
                  40,
                  0
                ],
                [
                  0.07495429616087751,
                  0.0,
                  41,
                  0
                ],
                [
                  0.07678244972577697,
                  0.0,
                  42,
                  0
                ],
                [
                  0.07861060329067641,
                  0.0,
                  43,
                  0
                ],
                [
                  0.08043875685557587,
                  0.0,
                  44,
                  0
                ],
                [
                  0.08226691042047532,
                  0.0,
                  45,
                  0
                ],
                [
                  0.08409506398537477,
                  0.0,
                  46,
                  0
                ],
                [
                  0.08592321755027423,
                  0.0,
                  47,
                  0
                ],
                [
                  0.08775137111517367,
                  0.0,
                  48,
                  0
                ],
                [
                  0.08957952468007313,
                  0.0,
                  49,
                  0
                ],
                [
                  0.09140767824497258,
                  0.0,
                  50,
                  0
                ],
                [
                  0.09323583180987204,
                  0.0,
                  51,
                  0
                ],
                [
                  0.09506398537477148,
                  0.0,
                  52,
                  0
                ],
                [
                  0.09689213893967093,
                  0.0,
                  53,
                  0
                ],
                [
                  0.09872029250457039,
                  0.0,
                  54,
                  0
                ],
                [
                  0.10054844606946983,
                  0.0,
                  55,
                  0
                ],
                [
                  0.10237659963436929,
                  0.0,
                  56,
                  0
                ],
                [
                  0.10420475319926874,
                  0.0,
                  57,
                  0
                ],
                [
                  0.10603290676416818,
                  0.0,
                  58,
                  0
                ],
                [
                  0.10786106032906764,
                  0.0,
                  59,
                  0
                ],
                [
                  0.10968921389396709,
                  0.0,
                  60,
                  0
                ],
                [
                  0.11151736745886655,
                  0.0,
                  61,
                  0
                ],
                [
                  0.113345521023766,
                  0.0,
                  62,
                  0
                ],
                [
                  0.11517367458866545,
                  0.0,
                  63,
                  0
                ],
                [
                  0.1170018281535649,
                  0.0,
                  64,
                  0
                ],
                [
                  0.11882998171846434,
                  0.0,
                  65,
                  0
                ],
                [
                  0.1206581352833638,
                  0.0,
                  66,
                  0
                ],
                [
                  0.12248628884826325,
                  0.0,
                  67,
                  0
                ],
                [
                  0.12431444241316271,
                  0.0,
                  68,
                  0
                ],
                [
                  0.12614259597806216,
                  0.0,
                  69,
                  0
                ],
                [
                  0.12797074954296161,
                  0.0,
                  70,
                  0
                ],
                [
                  0.12979890310786105,
                  0.0,
                  71,
                  0
                ],
                [
                  0.1316270566727605,
                  0.0,
                  72,
                  0
                ],
                [
                  0.13345521023765997,
                  0.0,
                  73,
                  0
                ],
                [
                  0.13528336380255943,
                  0.0,
                  74,
                  0
                ],
                [
                  0.13711151736745886,
                  0.0,
                  75,
                  0
                ],
                [
                  0.13893967093235832,
                  0.0,
                  76,
                  0
                ],
                [
                  0.14076782449725778,
                  0.0,
                  77,
                  0
                ],
                [
                  0.1425959780621572,
                  0.0,
                  78,
                  0
                ],
                [
                  0.14442413162705667,
                  0.0,
                  79,
                  0
                ],
                [
                  0.14625228519195613,
                  0.0,
                  80,
                  0
                ],
                [
                  0.1480804387568556,
                  0.0,
                  81,
                  0
                ],
                [
                  0.14990859232175502,
                  0.0,
                  82,
                  0
                ],
                [
                  0.15173674588665448,
                  0.0,
                  83,
                  0
                ],
                [
                  0.15356489945155394,
                  0.0,
                  84,
                  0
                ],
                [
                  0.15539305301645337,
                  0.0,
                  85,
                  0
                ],
                [
                  0.15722120658135283,
                  0.0,
                  86,
                  0
                ],
                [
                  0.1590493601462523,
                  0.0,
                  87,
                  0
                ],
                [
                  0.16087751371115175,
                  0.0,
                  88,
                  0
                ],
                [
                  0.16270566727605118,
                  0.0,
                  89,
                  0
                ],
                [
                  0.16453382084095064,
                  0.0,
                  90,
                  0
                ],
                [
                  0.1663619744058501,
                  0.0,
                  91,
                  0
                ],
                [
                  0.16819012797074953,
                  0.0,
                  92,
                  0
                ],
                [
                  0.170018281535649,
                  0.0,
                  93,
                  0
                ],
                [
                  0.17184643510054845,
                  0.0,
                  94,
                  0
                ],
                [
                  0.1736745886654479,
                  0.0,
                  95,
                  0
                ],
                [
                  0.17550274223034734,
                  0.0,
                  96,
                  0
                ],
                [
                  0.1773308957952468,
                  0.0,
                  97,
                  0
                ],
                [
                  0.17915904936014626,
                  0.0,
                  98,
                  0
                ],
                [
                  0.1809872029250457,
                  0.0,
                  99,
                  0
                ],
                [
                  0.18281535648994515,
                  0.0,
                  100,
                  0
                ],
                [
                  0.1846435100548446,
                  0.0,
                  101,
                  0
                ],
                [
                  0.18647166361974407,
                  0.0,
                  102,
                  0
                ],
                [
                  0.1882998171846435,
                  0.0,
                  103,
                  0
                ],
                [
                  0.19012797074954296,
                  0.0,
                  104,
                  0
                ],
                [
                  0.19195612431444242,
                  0.0,
                  105,
                  0
                ],
                [
                  0.19378427787934185,
                  0.0,
                  106,
                  0
                ],
                [
                  0.1956124314442413,
                  0.0,
                  107,
                  0
                ],
                [
                  0.19744058500914077,
                  0.0,
                  108,
                  0
                ],
                [
                  0.19926873857404023,
                  0.0,
                  109,
                  0
                ],
                [
                  0.20109689213893966,
                  0.0,
                  110,
                  0
                ],
                [
                  0.20292504570383912,
                  0.0,
                  111,
                  0
                ],
                [
                  0.20475319926873858,
                  0.0,
                  112,
                  0
                ],
                [
                  0.20658135283363802,
                  0.0,
                  113,
                  0
                ],
                [
                  0.20840950639853748,
                  0.0,
                  114,
                  0
                ],
                [
                  0.21023765996343693,
                  0.0,
                  115,
                  0
                ],
                [
                  0.21206581352833637,
                  0.0,
                  116,
                  0
                ],
                [
                  0.21389396709323583,
                  0.0,
                  117,
                  0
                ],
                [
                  0.21572212065813529,
                  0.0,
                  118,
                  0
                ],
                [
                  0.21755027422303475,
                  0.0,
                  119,
                  0
                ],
                [
                  0.21937842778793418,
                  0.0,
                  120,
                  0
                ],
                [
                  0.22120658135283364,
                  0.0,
                  121,
                  0
                ],
                [
                  0.2230347349177331,
                  0.0,
                  122,
                  0
                ],
                [
                  0.22486288848263253,
                  0.0,
                  123,
                  0
                ],
                [
                  0.226691042047532,
                  0.0,
                  124,
                  0
                ],
                [
                  0.22851919561243145,
                  0.0,
                  125,
                  0
                ],
                [
                  0.2303473491773309,
                  0.0,
                  126,
                  0
                ],
                [
                  0.23217550274223034,
                  0.0,
                  127,
                  0
                ],
                [
                  0.2340036563071298,
                  0.0,
                  128,
                  0
                ],
                [
                  0.23583180987202926,
                  0.0,
                  129,
                  0
                ],
                [
                  0.2376599634369287,
                  0.0,
                  130,
                  0
                ],
                [
                  0.23948811700182815,
                  0.0,
                  131,
                  0
                ],
                [
                  0.2413162705667276,
                  0.0,
                  132,
                  0
                ],
                [
                  0.24314442413162707,
                  0.0,
                  133,
                  0
                ],
                [
                  0.2449725776965265,
                  0.0,
                  134,
                  0
                ],
                [
                  0.24680073126142596,
                  0.0,
                  135,
                  0
                ],
                [
                  0.24862888482632542,
                  0.0,
                  136,
                  0
                ],
                [
                  0.25045703839122485,
                  0.0,
                  137,
                  0
                ],
                [
                  0.2522851919561243,
                  0.0,
                  138,
                  0
                ],
                [
                  0.25411334552102377,
                  0.0,
                  139,
                  0
                ],
                [
                  0.25594149908592323,
                  0.0,
                  140,
                  0
                ],
                [
                  0.2577696526508227,
                  0.0,
                  141,
                  0
                ],
                [
                  0.2595978062157221,
                  0.0,
                  142,
                  0
                ],
                [
                  0.26142595978062155,
                  0.0,
                  143,
                  0
                ],
                [
                  0.263254113345521,
                  0.0,
                  144,
                  0
                ],
                [
                  0.26508226691042047,
                  0.0,
                  145,
                  0
                ],
                [
                  0.26691042047531993,
                  0.0,
                  146,
                  0
                ],
                [
                  0.2687385740402194,
                  0.0,
                  147,
                  0
                ],
                [
                  0.27056672760511885,
                  0.0,
                  148,
                  0
                ],
                [
                  0.27239488117001825,
                  0.0,
                  149,
                  0
                ],
                [
                  0.2742230347349177,
                  0.0,
                  150,
                  0
                ],
                [
                  0.2760511882998172,
                  0.0,
                  151,
                  0
                ],
                [
                  0.27787934186471663,
                  0.0,
                  152,
                  0
                ],
                [
                  0.2797074954296161,
                  0.0,
                  153,
                  0
                ],
                [
                  0.28153564899451555,
                  0.0,
                  154,
                  0
                ],
                [
                  0.283363802559415,
                  0.0,
                  155,
                  0
                ],
                [
                  0.2851919561243144,
                  0.0,
                  156,
                  0
                ],
                [
                  0.2870201096892139,
                  0.0,
                  157,
                  0
                ],
                [
                  0.28884826325411334,
                  0.0,
                  158,
                  0
                ],
                [
                  0.2906764168190128,
                  0.0,
                  159,
                  0
                ],
                [
                  0.29250457038391225,
                  0.0,
                  160,
                  0
                ],
                [
                  0.2943327239488117,
                  0.0,
                  161,
                  0
                ],
                [
                  0.2961608775137112,
                  0.0,
                  162,
                  0
                ],
                [
                  0.2979890310786106,
                  0.0,
                  163,
                  0
                ],
                [
                  0.29981718464351004,
                  0.0,
                  164,
                  0
                ],
                [
                  0.3016453382084095,
                  0.0,
                  165,
                  0
                ],
                [
                  0.30347349177330896,
                  0.0,
                  166,
                  0
                ],
                [
                  0.3053016453382084,
                  0.0,
                  167,
                  0
                ],
                [
                  0.3071297989031079,
                  0.0,
                  168,
                  0
                ],
                [
                  0.30895795246800734,
                  0.0,
                  169,
                  0
                ],
                [
                  0.31078610603290674,
                  0.0,
                  170,
                  0
                ],
                [
                  0.3126142595978062,
                  0.0,
                  171,
                  0
                ],
                [
                  0.31444241316270566,
                  0.0,
                  172,
                  0
                ],
                [
                  0.3162705667276051,
                  0.0,
                  173,
                  0
                ],
                [
                  0.3180987202925046,
                  0.0,
                  174,
                  0
                ],
                [
                  0.31992687385740404,
                  0.0,
                  175,
                  0
                ],
                [
                  0.3217550274223035,
                  0.0,
                  176,
                  0
                ],
                [
                  0.3235831809872029,
                  0.0,
                  177,
                  0
                ],
                [
                  0.32541133455210236,
                  0.0,
                  178,
                  0
                ],
                [
                  0.3272394881170018,
                  0.0,
                  179,
                  0
                ],
                [
                  0.3290676416819013,
                  0.0,
                  180,
                  0
                ],
                [
                  0.33089579524680074,
                  0.0,
                  181,
                  0
                ],
                [
                  0.3327239488117002,
                  0.0,
                  182,
                  0
                ],
                [
                  0.33455210237659966,
                  0.0,
                  183,
                  0
                ],
                [
                  0.33638025594149906,
                  0.0,
                  184,
                  0
                ],
                [
                  0.3382084095063985,
                  0.0,
                  185,
                  0
                ],
                [
                  0.340036563071298,
                  0.0,
                  186,
                  0
                ],
                [
                  0.34186471663619744,
                  0.0,
                  187,
                  0
                ],
                [
                  0.3436928702010969,
                  0.0,
                  188,
                  0
                ],
                [
                  0.34552102376599636,
                  0.0,
                  189,
                  0
                ],
                [
                  0.3473491773308958,
                  0.0,
                  190,
                  0
                ],
                [
                  0.3491773308957952,
                  0.0,
                  191,
                  0
                ],
                [
                  0.3510054844606947,
                  0.0,
                  192,
                  0
                ],
                [
                  0.35283363802559414,
                  0.0,
                  193,
                  0
                ],
                [
                  0.3546617915904936,
                  0.0,
                  194,
                  0
                ],
                [
                  0.35648994515539306,
                  0.0,
                  195,
                  0
                ],
                [
                  0.3583180987202925,
                  0.0,
                  196,
                  0
                ],
                [
                  0.360146252285192,
                  0.0,
                  197,
                  0
                ],
                [
                  0.3619744058500914,
                  0.0,
                  198,
                  0
                ],
                [
                  0.36380255941499084,
                  0.0,
                  199,
                  0
                ],
                [
                  0.3656307129798903,
                  0.0,
                  200,
                  0
                ],
                [
                  0.36745886654478976,
                  0.0,
                  201,
                  0
                ],
                [
                  0.3692870201096892,
                  0.0,
                  202,
                  0
                ],
                [
                  0.3711151736745887,
                  0.0,
                  203,
                  0
                ],
                [
                  0.37294332723948814,
                  0.0,
                  204,
                  0
                ],
                [
                  0.37477148080438755,
                  0.0,
                  205,
                  0
                ],
                [
                  0.376599634369287,
                  0.0,
                  206,
                  0
                ],
                [
                  0.37842778793418647,
                  0.0,
                  207,
                  0
                ],
                [
                  0.3802559414990859,
                  0.0,
                  208,
                  0
                ],
                [
                  0.3820840950639854,
                  0.0,
                  209,
                  0
                ],
                [
                  0.38391224862888484,
                  0.0,
                  210,
                  0
                ],
                [
                  0.3857404021937843,
                  0.0,
                  211,
                  0
                ],
                [
                  0.3875685557586837,
                  0.0,
                  212,
                  0
                ],
                [
                  0.38939670932358317,
                  0.0,
                  213,
                  0
                ],
                [
                  0.3912248628884826,
                  0.0,
                  214,
                  0
                ],
                [
                  0.3930530164533821,
                  0.0,
                  215,
                  0
                ],
                [
                  0.39488117001828155,
                  0.0,
                  216,
                  0
                ],
                [
                  0.396709323583181,
                  0.0,
                  217,
                  0
                ],
                [
                  0.39853747714808047,
                  0.0,
                  218,
                  0
                ],
                [
                  0.40036563071297987,
                  0.0,
                  219,
                  0
                ],
                [
                  0.40219378427787933,
                  0.0,
                  220,
                  0
                ],
                [
                  0.4040219378427788,
                  0.0,
                  221,
                  0
                ],
                [
                  0.40585009140767825,
                  0.0,
                  222,
                  0
                ],
                [
                  0.4076782449725777,
                  0.0,
                  223,
                  0
                ],
                [
                  0.40950639853747717,
                  0.0,
                  224,
                  0
                ],
                [
                  0.4113345521023766,
                  0.0,
                  225,
                  0
                ],
                [
                  0.41316270566727603,
                  0.0,
                  226,
                  0
                ],
                [
                  0.4149908592321755,
                  0.0,
                  227,
                  0
                ],
                [
                  0.41681901279707495,
                  0.0,
                  228,
                  0
                ],
                [
                  0.4186471663619744,
                  0.0,
                  229,
                  0
                ],
                [
                  0.42047531992687387,
                  0.0,
                  230,
                  0
                ],
                [
                  0.42230347349177333,
                  0.0,
                  231,
                  0
                ],
                [
                  0.42413162705667273,
                  0.0,
                  232,
                  0
                ],
                [
                  0.4259597806215722,
                  0.0,
                  233,
                  0
                ],
                [
                  0.42778793418647165,
                  0.0,
                  234,
                  0
                ],
                [
                  0.4296160877513711,
                  0.0,
                  235,
                  0
                ],
                [
                  0.43144424131627057,
                  0.0,
                  236,
                  0
                ],
                [
                  0.43327239488117003,
                  0.0,
                  237,
                  0
                ],
                [
                  0.4351005484460695,
                  0.0,
                  238,
                  0
                ],
                [
                  0.4369287020109689,
                  0.0,
                  239,
                  0
                ],
                [
                  0.43875685557586835,
                  0.0,
                  240,
                  0
                ],
                [
                  0.4405850091407678,
                  0.0,
                  241,
                  0
                ],
                [
                  0.4424131627056673,
                  0.0,
                  242,
                  0
                ],
                [
                  0.44424131627056673,
                  0.0,
                  243,
                  0
                ],
                [
                  0.4460694698354662,
                  0.0,
                  244,
                  0
                ],
                [
                  0.44789762340036565,
                  0.0,
                  245,
                  0
                ],
                [
                  0.44972577696526506,
                  0.0,
                  246,
                  0
                ],
                [
                  0.4515539305301645,
                  0.0,
                  247,
                  0
                ],
                [
                  0.453382084095064,
                  0.0,
                  248,
                  0
                ],
                [
                  0.45521023765996343,
                  0.0,
                  249,
                  0
                ],
                [
                  0.4570383912248629,
                  0.0,
                  250,
                  0
                ],
                [
                  0.45886654478976235,
                  0.0,
                  251,
                  0
                ],
                [
                  0.4606946983546618,
                  0.0,
                  252,
                  0
                ],
                [
                  0.4625228519195612,
                  0.0,
                  253,
                  0
                ],
                [
                  0.4643510054844607,
                  0.003937007874015748,
                  254,
                  1
                ],
                [
                  0.46617915904936014,
                  0.00784313725490196,
                  255,
                  2
                ],
                [
                  0.4680073126142596,
                  0.0078125,
                  256,
                  2
                ],
                [
                  0.46983546617915906,
                  0.007782101167315175,
                  257,
                  2
                ],
                [
                  0.4716636197440585,
                  0.007751937984496124,
                  258,
                  2
                ],
                [
                  0.473491773308958,
                  0.007722007722007722,
                  259,
                  2
                ],
                [
                  0.4753199268738574,
                  0.007692307692307693,
                  260,
                  2
                ],
                [
                  0.47714808043875684,
                  0.007662835249042145,
                  261,
                  2
                ],
                [
                  0.4789762340036563,
                  0.007633587786259542,
                  262,
                  2
                ],
                [
                  0.48080438756855576,
                  0.0076045627376425855,
                  263,
                  2
                ],
                [
                  0.4826325411334552,
                  0.007575757575757576,
                  264,
                  2
                ],
                [
                  0.4844606946983547,
                  0.007547169811320755,
                  265,
                  2
                ],
                [
                  0.48628884826325414,
                  0.007518796992481203,
                  266,
                  2
                ],
                [
                  0.48811700182815354,
                  0.00749063670411985,
                  267,
                  2
                ],
                [
                  0.489945155393053,
                  0.007462686567164179,
                  268,
                  2
                ],
                [
                  0.49177330895795246,
                  0.007434944237918215,
                  269,
                  2
                ],
                [
                  0.4936014625228519,
                  0.007407407407407408,
                  270,
                  2
                ],
                [
                  0.4954296160877514,
                  0.007380073800738007,
                  271,
                  2
                ],
                [
                  0.49725776965265084,
                  0.007352941176470588,
                  272,
                  2
                ],
                [
                  0.4990859232175503,
                  0.007326007326007326,
                  273,
                  2
                ],
                [
                  0.5009140767824497,
                  0.0072992700729927005,
                  274,
                  2
                ],
                [
                  0.5027422303473492,
                  0.007272727272727273,
                  275,
                  2
                ],
                [
                  0.5045703839122486,
                  0.007246376811594203,
                  276,
                  2
                ],
                [
                  0.506398537477148,
                  0.007220216606498195,
                  277,
                  2
                ],
                [
                  0.5082266910420475,
                  0.007194244604316547,
                  278,
                  2
                ],
                [
                  0.5100548446069469,
                  0.007168458781362007,
                  279,
                  2
                ],
                [
                  0.5118829981718465,
                  0.007142857142857143,
                  280,
                  2
                ],
                [
                  0.5137111517367459,
                  0.0071174377224199285,
                  281,
                  2
                ],
                [
                  0.5155393053016454,
                  0.0070921985815602835,
                  282,
                  2
                ],
                [
                  0.5173674588665448,
                  0.007067137809187279,
                  283,
                  2
                ],
                [
                  0.5191956124314442,
                  0.007042253521126761,
                  284,
                  2
                ],
                [
                  0.5210237659963437,
                  0.007017543859649123,
                  285,
                  2
                ],
                [
                  0.5228519195612431,
                  0.006993006993006993,
                  286,
                  2
                ],
                [
                  0.5246800731261426,
                  0.006968641114982578,
                  287,
                  2
                ],
                [
                  0.526508226691042,
                  0.006944444444444444,
                  288,
                  2
                ],
                [
                  0.5283363802559415,
                  0.006920415224913495,
                  289,
                  2
                ],
                [
                  0.5301645338208409,
                  0.006896551724137931,
                  290,
                  2
                ],
                [
                  0.5319926873857403,
                  0.006872852233676976,
                  291,
                  2
                ],
                [
                  0.5338208409506399,
                  0.00684931506849315,
                  292,
                  2
                ],
                [
                  0.5356489945155393,
                  0.006825938566552901,
                  293,
                  2
                ],
                [
                  0.5374771480804388,
                  0.006802721088435374,
                  294,
                  2
                ],
                [
                  0.5393053016453382,
                  0.006779661016949152,
                  295,
                  2
                ],
                [
                  0.5411334552102377,
                  0.006756756756756757,
                  296,
                  2
                ],
                [
                  0.5429616087751371,
                  0.006734006734006734,
                  297,
                  2
                ],
                [
                  0.5447897623400365,
                  0.006711409395973154,
                  298,
                  2
                ],
                [
                  0.546617915904936,
                  0.006688963210702341,
                  299,
                  2
                ],
                [
                  0.5484460694698354,
                  0.006666666666666667,
                  300,
                  2
                ],
                [
                  0.5502742230347349,
                  0.006644518272425249,
                  301,
                  2
                ],
                [
                  0.5521023765996343,
                  0.006622516556291391,
                  302,
                  2
                ],
                [
                  0.5539305301645339,
                  0.006600660066006601,
                  303,
                  2
                ],
                [
                  0.5557586837294333,
                  0.006578947368421052,
                  304,
                  2
                ],
                [
                  0.5575868372943327,
                  0.006557377049180328,
                  305,
                  2
                ],
                [
                  0.5594149908592322,
                  0.006535947712418301,
                  306,
                  2
                ],
                [
                  0.5612431444241316,
                  0.006514657980456026,
                  307,
                  2
                ],
                [
                  0.5630712979890311,
                  0.006493506493506494,
                  308,
                  2
                ],
                [
                  0.5648994515539305,
                  0.006472491909385114,
                  309,
                  2
                ],
                [
                  0.56672760511883,
                  0.0064516129032258064,
                  310,
                  2
                ],
                [
                  0.5685557586837294,
                  0.006430868167202572,
                  311,
                  2
                ],
                [
                  0.5703839122486288,
                  0.00641025641025641,
                  312,
                  2
                ],
                [
                  0.5722120658135283,
                  0.006389776357827476,
                  313,
                  2
                ],
                [
                  0.5740402193784278,
                  0.006369426751592357,
                  314,
                  2
                ],
                [
                  0.5758683729433273,
                  0.006349206349206349,
                  315,
                  2
                ],
                [
                  0.5776965265082267,
                  0.006329113924050633,
                  316,
                  2
                ],
                [
                  0.5795246800731262,
                  0.006309148264984227,
                  317,
                  2
                ],
                [
                  0.5813528336380256,
                  0.006289308176100629,
                  318,
                  2
                ],
                [
                  0.583180987202925,
                  0.006269592476489028,
                  319,
                  2
                ],
                [
                  0.5850091407678245,
                  0.00625,
                  320,
                  2
                ],
                [
                  0.5868372943327239,
                  0.006230529595015576,
                  321,
                  2
                ],
                [
                  0.5886654478976234,
                  0.006211180124223602,
                  322,
                  2
                ],
                [
                  0.5904936014625228,
                  0.006191950464396285,
                  323,
                  2
                ],
                [
                  0.5923217550274223,
                  0.006172839506172839,
                  324,
                  2
                ],
                [
                  0.5941499085923218,
                  0.006153846153846154,
                  325,
                  2
                ],
                [
                  0.5959780621572212,
                  0.006134969325153374,
                  326,
                  2
                ],
                [
                  0.5978062157221207,
                  0.0061162079510703364,
                  327,
                  2
                ],
                [
                  0.5996343692870201,
                  0.006097560975609756,
                  328,
                  2
                ],
                [
                  0.6014625228519196,
                  0.0060790273556231,
                  329,
                  2
                ],
                [
                  0.603290676416819,
                  0.00909090909090909,
                  330,
                  3
                ],
                [
                  0.6051188299817185,
                  0.00906344410876133,
                  331,
                  3
                ],
                [
                  0.6069469835466179,
                  0.009036144578313253,
                  332,
                  3
                ],
                [
                  0.6087751371115173,
                  0.009009009009009009,
                  333,
                  3
                ],
                [
                  0.6106032906764168,
                  0.008982035928143712,
                  334,
                  3
                ],
                [
                  0.6124314442413162,
                  0.008955223880597015,
                  335,
                  3
                ],
                [
                  0.6142595978062158,
                  0.008928571428571428,
                  336,
                  3
                ],
                [
                  0.6160877513711152,
                  0.008902077151335312,
                  337,
                  3
                ],
                [
                  0.6179159049360147,
                  0.008875739644970414,
                  338,
                  3
                ],
                [
                  0.6197440585009141,
                  0.008849557522123894,
                  339,
                  3
                ],
                [
                  0.6215722120658135,
                  0.008823529411764706,
                  340,
                  3
                ],
                [
                  0.623400365630713,
                  0.008797653958944282,
                  341,
                  3
                ],
                [
                  0.6252285191956124,
                  0.008771929824561403,
                  342,
                  3
                ],
                [
                  0.6270566727605119,
                  0.008746355685131196,
                  343,
                  3
                ],
                [
                  0.6288848263254113,
                  0.00872093023255814,
                  344,
                  3
                ],
                [
                  0.6307129798903108,
                  0.008695652173913044,
                  345,
                  3
                ],
                [
                  0.6325411334552102,
                  0.008670520231213872,
                  346,
                  3
                ],
                [
                  0.6343692870201096,
                  0.008645533141210375,
                  347,
                  3
                ],
                [
                  0.6361974405850092,
                  0.008620689655172414,
                  348,
                  3
                ],
                [
                  0.6380255941499086,
                  0.011461318051575931,
                  349,
                  4
                ],
                [
                  0.6398537477148081,
                  0.011428571428571429,
                  350,
                  4
                ],
                [
                  0.6416819012797075,
                  0.011396011396011397,
                  351,
                  4
                ],
                [
                  0.643510054844607,
                  0.011363636363636364,
                  352,
                  4
                ],
                [
                  0.6453382084095064,
                  0.0113314447592068,
                  353,
                  4
                ],
                [
                  0.6471663619744058,
                  0.011299435028248588,
                  354,
                  4
                ],
                [
                  0.6489945155393053,
                  0.011267605633802818,
                  355,
                  4
                ],
                [
                  0.6508226691042047,
                  0.011235955056179775,
                  356,
                  4
                ],
                [
                  0.6526508226691042,
                  0.011204481792717087,
                  357,
                  4
                ],
                [
                  0.6544789762340036,
                  0.0111731843575419,
                  358,
                  4
                ],
                [
                  0.6563071297989032,
                  0.011142061281337047,
                  359,
                  4
                ],
                [
                  0.6581352833638026,
                  0.011111111111111112,
                  360,
                  4
                ],
                [
                  0.659963436928702,
                  0.0110803324099723,
                  361,
                  4
                ],
                [
                  0.6617915904936015,
                  0.011049723756906077,
                  362,
                  4
                ],
                [
                  0.6636197440585009,
                  0.011019283746556474,
                  363,
                  4
                ],
                [
                  0.6654478976234004,
                  0.01098901098901099,
                  364,
                  4
                ],
                [
                  0.6672760511882998,
                  0.010958904109589041,
                  365,
                  4
                ],
                [
                  0.6691042047531993,
                  0.01092896174863388,
                  366,
                  4
                ],
                [
                  0.6709323583180987,
                  0.010899182561307902,
                  367,
                  4
                ],
                [
                  0.6727605118829981,
                  0.010869565217391304,
                  368,
                  4
                ],
                [
                  0.6745886654478976,
                  0.01084010840108401,
                  369,
                  4
                ],
                [
                  0.676416819012797,
                  0.010810810810810811,
                  370,
                  4
                ],
                [
                  0.6782449725776966,
                  0.01078167115902965,
                  371,
                  4
                ],
                [
                  0.680073126142596,
                  0.010752688172043012,
                  372,
                  4
                ],
                [
                  0.6819012797074955,
                  0.013404825737265416,
                  373,
                  5
                ],
                [
                  0.6837294332723949,
                  0.013368983957219251,
                  374,
                  5
                ],
                [
                  0.6855575868372943,
                  0.013333333333333334,
                  375,
                  5
                ],
                [
                  0.6873857404021938,
                  0.013297872340425532,
                  376,
                  5
                ],
                [
                  0.6892138939670932,
                  0.013262599469496022,
                  377,
                  5
                ],
                [
                  0.6910420475319927,
                  0.013227513227513227,
                  378,
                  5
                ],
                [
                  0.6928702010968921,
                  0.013192612137203167,
                  379,
                  5
                ],
                [
                  0.6946983546617916,
                  0.013157894736842105,
                  380,
                  5
                ],
                [
                  0.696526508226691,
                  0.013123359580052493,
                  381,
                  5
                ],
                [
                  0.6983546617915904,
                  0.013089005235602094,
                  382,
                  5
                ],
                [
                  0.70018281535649,
                  0.013054830287206266,
                  383,
                  5
                ],
                [
                  0.7020109689213894,
                  0.013020833333333334,
                  384,
                  5
                ],
                [
                  0.7038391224862889,
                  0.012987012987012988,
                  385,
                  5
                ],
                [
                  0.7056672760511883,
                  0.012953367875647668,
                  386,
                  5
                ],
                [
                  0.7074954296160878,
                  0.012919896640826873,
                  387,
                  5
                ],
                [
                  0.7093235831809872,
                  0.01288659793814433,
                  388,
                  5
                ],
                [
                  0.7111517367458866,
                  0.012853470437017995,
                  389,
                  5
                ],
                [
                  0.7129798903107861,
                  0.01282051282051282,
                  390,
                  5
                ],
                [
                  0.7148080438756855,
                  0.01278772378516624,
                  391,
                  5
                ],
                [
                  0.716636197440585,
                  0.012755102040816327,
                  392,
                  5
                ],
                [
                  0.7184643510054844,
                  0.01272264631043257,
                  393,
                  5
                ],
                [
                  0.720292504570384,
                  0.015228426395939087,
                  394,
                  6
                ],
                [
                  0.7221206581352834,
                  0.015189873417721518,
                  395,
                  6
                ],
                [
                  0.7239488117001828,
                  0.015151515151515152,
                  396,
                  6
                ],
                [
                  0.7257769652650823,
                  0.015113350125944584,
                  397,
                  6
                ],
                [
                  0.7276051188299817,
                  0.01507537688442211,
                  398,
                  6
                ],
                [
                  0.7294332723948812,
                  0.015037593984962405,
                  399,
                  6
                ],
                [
                  0.7312614259597806,
                  0.015,
                  400,
                  6
                ],
                [
                  0.7330895795246801,
                  0.014962593516209476,
                  401,
                  6
                ],
                [
                  0.7349177330895795,
                  0.014925373134328358,
                  402,
                  6
                ],
                [
                  0.7367458866544789,
                  0.01488833746898263,
                  403,
                  6
                ],
                [
                  0.7385740402193784,
                  0.01485148514851485,
                  404,
                  6
                ],
                [
                  0.7404021937842779,
                  0.014814814814814815,
                  405,
                  6
                ],
                [
                  0.7422303473491774,
                  0.014778325123152709,
                  406,
                  6
                ],
                [
                  0.7440585009140768,
                  0.014742014742014743,
                  407,
                  6
                ],
                [
                  0.7458866544789763,
                  0.014705882352941176,
                  408,
                  6
                ],
                [
                  0.7477148080438757,
                  0.017114914425427872,
                  409,
                  7
                ],
                [
                  0.7495429616087751,
                  0.01707317073170732,
                  410,
                  7
                ],
                [
                  0.7513711151736746,
                  0.019464720194647202,
                  411,
                  8
                ],
                [
                  0.753199268738574,
                  0.021844660194174758,
                  412,
                  9
                ],
                [
                  0.7550274223034735,
                  0.021791767554479417,
                  413,
                  9
                ],
                [
                  0.7568555758683729,
                  0.021739130434782608,
                  414,
                  9
                ],
                [
                  0.7586837294332724,
                  0.024096385542168676,
                  415,
                  10
                ],
                [
                  0.7605118829981719,
                  0.02403846153846154,
                  416,
                  10
                ],
                [
                  0.7623400365630713,
                  0.026378896882494004,
                  417,
                  11
                ],
                [
                  0.7641681901279708,
                  0.028708133971291867,
                  418,
                  12
                ],
                [
                  0.7659963436928702,
                  0.028639618138424822,
                  419,
                  12
                ],
                [
                  0.7678244972577697,
                  0.02857142857142857,
                  420,
                  12
                ],
                [
                  0.7696526508226691,
                  0.030878859857482184,
                  421,
                  13
                ],
                [
                  0.7714808043875686,
                  0.030805687203791468,
                  422,
                  13
                ],
                [
                  0.773308957952468,
                  0.030732860520094562,
                  423,
                  13
                ],
                [
                  0.7751371115173674,
                  0.030660377358490566,
                  424,
                  13
                ],
                [
                  0.7769652650822669,
                  0.03058823529411765,
                  425,
                  13
                ],
                [
                  0.7787934186471663,
                  0.03051643192488263,
                  426,
                  13
                ],
                [
                  0.7806215722120659,
                  0.03044496487119438,
                  427,
                  13
                ],
                [
                  0.7824497257769653,
                  0.030373831775700934,
                  428,
                  13
                ],
                [
                  0.7842778793418648,
                  0.030303030303030304,
                  429,
                  13
                ],
                [
                  0.7861060329067642,
                  0.030232558139534883,
                  430,
                  13
                ],
                [
                  0.7879341864716636,
                  0.03248259860788863,
                  431,
                  14
                ],
                [
                  0.7897623400365631,
                  0.032407407407407406,
                  432,
                  14
                ],
                [
                  0.7915904936014625,
                  0.03233256351039261,
                  433,
                  14
                ],
                [
                  0.793418647166362,
                  0.03456221198156682,
                  434,
                  15
                ],
                [
                  0.7952468007312614,
                  0.034482758620689655,
                  435,
                  15
                ],
                [
                  0.7970749542961609,
                  0.034403669724770644,
                  436,
                  15
                ],
                [
                  0.7989031078610603,
                  0.034324942791762014,
                  437,
                  15
                ],
                [
                  0.8007312614259597,
                  0.03424657534246575,
                  438,
                  15
                ],
                [
                  0.8025594149908593,
                  0.03644646924829157,
                  439,
                  16
                ],
                [
                  0.8043875685557587,
                  0.03636363636363636,
                  440,
                  16
                ],
                [
                  0.8062157221206582,
                  0.036281179138321996,
                  441,
                  16
                ],
                [
                  0.8080438756855576,
                  0.03619909502262444,
                  442,
                  16
                ],
                [
                  0.8098720292504571,
                  0.03837471783295711,
                  443,
                  17
                ],
                [
                  0.8117001828153565,
                  0.038288288288288286,
                  444,
                  17
                ],
                [
                  0.8135283363802559,
                  0.04044943820224719,
                  445,
                  18
                ],
                [
                  0.8153564899451554,
                  0.04035874439461883,
                  446,
                  18
                ],
                [
                  0.8171846435100548,
                  0.040268456375838924,
                  447,
                  18
                ],
                [
                  0.8190127970749543,
                  0.04017857142857143,
                  448,
                  18
                ],
                [
                  0.8208409506398537,
                  0.042316258351893093,
                  449,
                  19
                ],
                [
                  0.8226691042047533,
                  0.044444444444444446,
                  450,
                  20
                ],
                [
                  0.8244972577696527,
                  0.04656319290465632,
                  451,
                  21
                ],
                [
                  0.8263254113345521,
                  0.048672566371681415,
                  452,
                  22
                ],
                [
                  0.8281535648994516,
                  0.04856512141280353,
                  453,
                  22
                ],
                [
                  0.829981718464351,
                  0.048458149779735685,
                  454,
                  22
                ],
                [
                  0.8318098720292505,
                  0.05054945054945055,
                  455,
                  23
                ],
                [
                  0.8336380255941499,
                  0.05043859649122807,
                  456,
                  23
                ],
                [
                  0.8354661791590493,
                  0.05032822757111598,
                  457,
                  23
                ],
                [
                  0.8372943327239488,
                  0.05240174672489083,
                  458,
                  24
                ],
                [
                  0.8391224862888482,
                  0.05228758169934641,
                  459,
                  24
                ],
                [
                  0.8409506398537477,
                  0.05434782608695652,
                  460,
                  25
                ],
                [
                  0.8427787934186471,
                  0.05639913232104121,
                  461,
                  26
                ],
                [
                  0.8446069469835467,
                  0.05844155844155844,
                  462,
                  27
                ],
                [
                  0.8464351005484461,
                  0.058315334773218146,
                  463,
                  27
                ],
                [
                  0.8482632541133455,
                  0.05818965517241379,
                  464,
                  27
                ],
                [
                  0.850091407678245,
                  0.060215053763440864,
                  465,
                  28
                ],
                [
                  0.8519195612431444,
                  0.06223175965665236,
                  466,
                  29
                ],
                [
                  0.8537477148080439,
                  0.06209850107066381,
                  467,
                  29
                ],
                [
                  0.8555758683729433,
                  0.06196581196581197,
                  468,
                  29
                ],
                [
                  0.8574040219378428,
                  0.06183368869936034,
                  469,
                  29
                ],
                [
                  0.8592321755027422,
                  0.06170212765957447,
                  470,
                  29
                ],
                [
                  0.8610603290676416,
                  0.06157112526539278,
                  471,
                  29
                ],
                [
                  0.8628884826325411,
                  0.0635593220338983,
                  472,
                  30
                ],
                [
                  0.8647166361974405,
                  0.06342494714587738,
                  473,
                  30
                ],
                [
                  0.8665447897623401,
                  0.06329113924050633,
                  474,
                  30
                ],
                [
                  0.8683729433272395,
                  0.06526315789473684,
                  475,
                  31
                ],
                [
                  0.870201096892139,
                  0.06512605042016807,
                  476,
                  31
                ],
                [
                  0.8720292504570384,
                  0.0649895178197065,
                  477,
                  31
                ],
                [
                  0.8738574040219378,
                  0.06485355648535565,
                  478,
                  31
                ],
                [
                  0.8756855575868373,
                  0.06471816283924843,
                  479,
                  31
                ],
                [
                  0.8775137111517367,
                  0.06458333333333334,
                  480,
                  31
                ],
                [
                  0.8793418647166362,
                  0.06444906444906445,
                  481,
                  31
                ],
                [
                  0.8811700182815356,
                  0.06431535269709543,
                  482,
                  31
                ],
                [
                  0.8829981718464351,
                  0.06625258799171843,
                  483,
                  32
                ],
                [
                  0.8848263254113345,
                  0.06611570247933884,
                  484,
                  32
                ],
                [
                  0.886654478976234,
                  0.06597938144329897,
                  485,
                  32
                ],
                [
                  0.8884826325411335,
                  0.06584362139917696,
                  486,
                  32
                ],
                [
                  0.8903107861060329,
                  0.06570841889117043,
                  487,
                  32
                ],
                [
                  0.8921389396709324,
                  0.06557377049180328,
                  488,
                  32
                ],
                [
                  0.8939670932358318,
                  0.06748466257668712,
                  489,
                  33
                ],
                [
                  0.8957952468007313,
                  0.0673469387755102,
                  490,
                  33
                ],
                [
                  0.8976234003656307,
                  0.06720977596741344,
                  491,
                  33
                ],
                [
                  0.8994515539305301,
                  0.06707317073170732,
                  492,
                  33
                ],
                [
                  0.9012797074954296,
                  0.06693711967545639,
                  493,
                  33
                ],
                [
                  0.903107861060329,
                  0.06882591093117409,
                  494,
                  34
                ],
                [
                  0.9049360146252285,
                  0.06868686868686869,
                  495,
                  34
                ],
                [
                  0.906764168190128,
                  0.06854838709677419,
                  496,
                  34
                ],
                [
                  0.9085923217550275,
                  0.06841046277665996,
                  497,
                  34
                ],
                [
                  0.9104204753199269,
                  0.07028112449799197,
                  498,
                  35
                ],
                [
                  0.9122486288848263,
                  0.07014028056112225,
                  499,
                  35
                ],
                [
                  0.9140767824497258,
                  0.072,
                  500,
                  36
                ],
                [
                  0.9159049360146252,
                  0.0718562874251497,
                  501,
                  36
                ],
                [
                  0.9177330895795247,
                  0.07370517928286853,
                  502,
                  37
                ],
                [
                  0.9195612431444241,
                  0.07554671968190854,
                  503,
                  38
                ],
                [
                  0.9213893967093236,
                  0.07738095238095238,
                  504,
                  39
                ],
                [
                  0.923217550274223,
                  0.07722772277227723,
                  505,
                  39
                ],
                [
                  0.9250457038391224,
                  0.07707509881422925,
                  506,
                  39
                ],
                [
                  0.926873857404022,
                  0.07692307692307693,
                  507,
                  39
                ],
                [
                  0.9287020109689214,
                  0.07874015748031496,
                  508,
                  40
                ],
                [
                  0.9305301645338209,
                  0.07858546168958742,
                  509,
                  40
                ],
                [
                  0.9323583180987203,
                  0.0803921568627451,
                  510,
                  41
                ],
                [
                  0.9341864716636198,
                  0.08023483365949119,
                  511,
                  41
                ],
                [
                  0.9360146252285192,
                  0.080078125,
                  512,
                  41
                ],
                [
                  0.9378427787934186,
                  0.07992202729044834,
                  513,
                  41
                ],
                [
                  0.9396709323583181,
                  0.08171206225680934,
                  514,
                  42
                ],
                [
                  0.9414990859232175,
                  0.08349514563106795,
                  515,
                  43
                ],
                [
                  0.943327239488117,
                  0.08527131782945736,
                  516,
                  44
                ],
                [
                  0.9451553930530164,
                  0.08704061895551257,
                  517,
                  45
                ],
                [
                  0.946983546617916,
                  0.08687258687258688,
                  518,
                  45
                ],
                [
                  0.9488117001828154,
                  0.08863198458574181,
                  519,
                  46
                ],
                [
                  0.9506398537477148,
                  0.09038461538461538,
                  520,
                  47
                ],
                [
                  0.9524680073126143,
                  0.09213051823416507,
                  521,
                  48
                ],
                [
                  0.9542961608775137,
                  0.09195402298850575,
                  522,
                  48
                ],
                [
                  0.9561243144424132,
                  0.09369024856596558,
                  523,
                  49
                ],
                [
                  0.9579524680073126,
                  0.09351145038167939,
                  524,
                  49
                ],
                [
                  0.9597806215722121,
                  0.09333333333333334,
                  525,
                  49
                ],
                [
                  0.9616087751371115,
                  0.09315589353612168,
                  526,
                  49
                ],
                [
                  0.9634369287020109,
                  0.09487666034155598,
                  527,
                  50
                ],
                [
                  0.9652650822669104,
                  0.0946969696969697,
                  528,
                  50
                ],
                [
                  0.9670932358318098,
                  0.0945179584120983,
                  529,
                  50
                ],
                [
                  0.9689213893967094,
                  0.09433962264150944,
                  530,
                  50
                ],
                [
                  0.9707495429616088,
                  0.096045197740113,
                  531,
                  51
                ],
                [
                  0.9725776965265083,
                  0.09774436090225563,
                  532,
                  52
                ],
                [
                  0.9744058500914077,
                  0.09943714821763602,
                  533,
                  53
                ],
                [
                  0.9762340036563071,
                  0.10112359550561797,
                  534,
                  54
                ],
                [
                  0.9780621572212066,
                  0.102803738317757,
                  535,
                  55
                ],
                [
                  0.979890310786106,
                  0.10261194029850747,
                  536,
                  55
                ],
                [
                  0.9817184643510055,
                  0.1042830540037244,
                  537,
                  56
                ],
                [
                  0.9835466179159049,
                  0.10594795539033457,
                  538,
                  57
                ],
                [
                  0.9853747714808044,
                  0.10575139146567718,
                  539,
                  57
                ],
                [
                  0.9872029250457038,
                  0.10555555555555556,
                  540,
                  57
                ],
                [
                  0.9890310786106032,
                  0.10536044362292052,
                  541,
                  57
                ],
                [
                  0.9908592321755028,
                  0.1070110701107011,
                  542,
                  58
                ],
                [
                  0.9926873857404022,
                  0.10681399631675875,
                  543,
                  58
                ],
                [
                  0.9945155393053017,
                  0.10845588235294118,
                  544,
                  59
                ],
                [
                  0.9963436928702011,
                  0.11009174311926606,
                  545,
                  60
                ],
                [
                  0.9981718464351006,
                  0.11172161172161173,
                  546,
                  61
                ],
                [
                  1.0,
                  0.113345521023766,
                  547,
                  62
                ]
              ]
            }
          ]
        },
        {
          "name": "Drex Max 1.1",
          "text_run": 326,
          "native": [
            {
              "language": "Turkish",
              "wording": "Bilmiyorum",
              "run": 352,
              "full_bank_restricted": true,
              "right": 487,
              "wrong": 51,
              "abstained": 9,
              "invalid": 0,
              "coverage": 0.9835466179159049,
              "risk": 0.09479553903345725
            },
            {
              "language": "Turkish",
              "wording": "I don't know",
              "run": 513,
              "full_bank_restricted": true,
              "right": 481,
              "wrong": 48,
              "abstained": 18,
              "invalid": 0,
              "coverage": 0.9670932358318098,
              "risk": 0.09073724007561437
            },
            {
              "language": "English",
              "wording": "I don't know",
              "run": 447,
              "full_bank_restricted": false,
              "right": 463,
              "wrong": 41,
              "abstained": 43,
              "invalid": 0,
              "coverage": 0.9213893967093236,
              "risk": 0.08134920634920635
            }
          ],
          "curves": [
            {
              "language": "Turkish",
              "run": 326,
              "points": [
                [
                  0.0018281535648994515,
                  0.0,
                  1,
                  0
                ],
                [
                  0.003656307129798903,
                  0.0,
                  2,
                  0
                ],
                [
                  0.005484460694698354,
                  0.0,
                  3,
                  0
                ],
                [
                  0.007312614259597806,
                  0.0,
                  4,
                  0
                ],
                [
                  0.009140767824497258,
                  0.0,
                  5,
                  0
                ],
                [
                  0.010968921389396709,
                  0.0,
                  6,
                  0
                ],
                [
                  0.012797074954296161,
                  0.0,
                  7,
                  0
                ],
                [
                  0.014625228519195612,
                  0.0,
                  8,
                  0
                ],
                [
                  0.016453382084095063,
                  0.0,
                  9,
                  0
                ],
                [
                  0.018281535648994516,
                  0.0,
                  10,
                  0
                ],
                [
                  0.02010968921389397,
                  0.0,
                  11,
                  0
                ],
                [
                  0.021937842778793418,
                  0.0,
                  12,
                  0
                ],
                [
                  0.02376599634369287,
                  0.0,
                  13,
                  0
                ],
                [
                  0.025594149908592323,
                  0.0,
                  14,
                  0
                ],
                [
                  0.027422303473491772,
                  0.0,
                  15,
                  0
                ],
                [
                  0.029250457038391225,
                  0.0,
                  16,
                  0
                ],
                [
                  0.031078610603290677,
                  0.0,
                  17,
                  0
                ],
                [
                  0.03290676416819013,
                  0.0,
                  18,
                  0
                ],
                [
                  0.03473491773308958,
                  0.0,
                  19,
                  0
                ],
                [
                  0.03656307129798903,
                  0.0,
                  20,
                  0
                ],
                [
                  0.038391224862888484,
                  0.0,
                  21,
                  0
                ],
                [
                  0.04021937842778794,
                  0.0,
                  22,
                  0
                ],
                [
                  0.04204753199268738,
                  0.0,
                  23,
                  0
                ],
                [
                  0.043875685557586835,
                  0.0,
                  24,
                  0
                ],
                [
                  0.04570383912248629,
                  0.0,
                  25,
                  0
                ],
                [
                  0.04753199268738574,
                  0.0,
                  26,
                  0
                ],
                [
                  0.04936014625228519,
                  0.0,
                  27,
                  0
                ],
                [
                  0.051188299817184646,
                  0.0,
                  28,
                  0
                ],
                [
                  0.05301645338208409,
                  0.0,
                  29,
                  0
                ],
                [
                  0.054844606946983544,
                  0.0,
                  30,
                  0
                ],
                [
                  0.056672760511883,
                  0.0,
                  31,
                  0
                ],
                [
                  0.05850091407678245,
                  0.0,
                  32,
                  0
                ],
                [
                  0.0603290676416819,
                  0.0,
                  33,
                  0
                ],
                [
                  0.062157221206581355,
                  0.0,
                  34,
                  0
                ],
                [
                  0.06398537477148081,
                  0.0,
                  35,
                  0
                ],
                [
                  0.06581352833638025,
                  0.0,
                  36,
                  0
                ],
                [
                  0.06764168190127971,
                  0.0,
                  37,
                  0
                ],
                [
                  0.06946983546617916,
                  0.0,
                  38,
                  0
                ],
                [
                  0.0712979890310786,
                  0.0,
                  39,
                  0
                ],
                [
                  0.07312614259597806,
                  0.0,
                  40,
                  0
                ],
                [
                  0.07495429616087751,
                  0.0,
                  41,
                  0
                ],
                [
                  0.07678244972577697,
                  0.0,
                  42,
                  0
                ],
                [
                  0.07861060329067641,
                  0.0,
                  43,
                  0
                ],
                [
                  0.08043875685557587,
                  0.0,
                  44,
                  0
                ],
                [
                  0.08226691042047532,
                  0.0,
                  45,
                  0
                ],
                [
                  0.08409506398537477,
                  0.0,
                  46,
                  0
                ],
                [
                  0.08592321755027423,
                  0.0,
                  47,
                  0
                ],
                [
                  0.08775137111517367,
                  0.0,
                  48,
                  0
                ],
                [
                  0.08957952468007313,
                  0.0,
                  49,
                  0
                ],
                [
                  0.09140767824497258,
                  0.0,
                  50,
                  0
                ],
                [
                  0.09323583180987204,
                  0.0,
                  51,
                  0
                ],
                [
                  0.09506398537477148,
                  0.0,
                  52,
                  0
                ],
                [
                  0.09689213893967093,
                  0.0,
                  53,
                  0
                ],
                [
                  0.09872029250457039,
                  0.0,
                  54,
                  0
                ],
                [
                  0.10054844606946983,
                  0.0,
                  55,
                  0
                ],
                [
                  0.10237659963436929,
                  0.0,
                  56,
                  0
                ],
                [
                  0.10420475319926874,
                  0.0,
                  57,
                  0
                ],
                [
                  0.10603290676416818,
                  0.0,
                  58,
                  0
                ],
                [
                  0.10786106032906764,
                  0.0,
                  59,
                  0
                ],
                [
                  0.10968921389396709,
                  0.0,
                  60,
                  0
                ],
                [
                  0.11151736745886655,
                  0.0,
                  61,
                  0
                ],
                [
                  0.113345521023766,
                  0.0,
                  62,
                  0
                ],
                [
                  0.11517367458866545,
                  0.0,
                  63,
                  0
                ],
                [
                  0.1170018281535649,
                  0.0,
                  64,
                  0
                ],
                [
                  0.11882998171846434,
                  0.0,
                  65,
                  0
                ],
                [
                  0.1206581352833638,
                  0.0,
                  66,
                  0
                ],
                [
                  0.12248628884826325,
                  0.0,
                  67,
                  0
                ],
                [
                  0.12431444241316271,
                  0.0,
                  68,
                  0
                ],
                [
                  0.12614259597806216,
                  0.0,
                  69,
                  0
                ],
                [
                  0.12797074954296161,
                  0.0,
                  70,
                  0
                ],
                [
                  0.12979890310786105,
                  0.0,
                  71,
                  0
                ],
                [
                  0.1316270566727605,
                  0.0,
                  72,
                  0
                ],
                [
                  0.13345521023765997,
                  0.0,
                  73,
                  0
                ],
                [
                  0.13528336380255943,
                  0.0,
                  74,
                  0
                ],
                [
                  0.13711151736745886,
                  0.0,
                  75,
                  0
                ],
                [
                  0.13893967093235832,
                  0.0,
                  76,
                  0
                ],
                [
                  0.14076782449725778,
                  0.0,
                  77,
                  0
                ],
                [
                  0.1425959780621572,
                  0.0,
                  78,
                  0
                ],
                [
                  0.14442413162705667,
                  0.0,
                  79,
                  0
                ],
                [
                  0.14625228519195613,
                  0.0,
                  80,
                  0
                ],
                [
                  0.1480804387568556,
                  0.0,
                  81,
                  0
                ],
                [
                  0.14990859232175502,
                  0.0,
                  82,
                  0
                ],
                [
                  0.15173674588665448,
                  0.0,
                  83,
                  0
                ],
                [
                  0.15356489945155394,
                  0.0,
                  84,
                  0
                ],
                [
                  0.15539305301645337,
                  0.0,
                  85,
                  0
                ],
                [
                  0.15722120658135283,
                  0.0,
                  86,
                  0
                ],
                [
                  0.1590493601462523,
                  0.0,
                  87,
                  0
                ],
                [
                  0.16087751371115175,
                  0.0,
                  88,
                  0
                ],
                [
                  0.16270566727605118,
                  0.0,
                  89,
                  0
                ],
                [
                  0.16453382084095064,
                  0.0,
                  90,
                  0
                ],
                [
                  0.1663619744058501,
                  0.0,
                  91,
                  0
                ],
                [
                  0.16819012797074953,
                  0.0,
                  92,
                  0
                ],
                [
                  0.170018281535649,
                  0.0,
                  93,
                  0
                ],
                [
                  0.17184643510054845,
                  0.0,
                  94,
                  0
                ],
                [
                  0.1736745886654479,
                  0.0,
                  95,
                  0
                ],
                [
                  0.17550274223034734,
                  0.0,
                  96,
                  0
                ],
                [
                  0.1773308957952468,
                  0.0,
                  97,
                  0
                ],
                [
                  0.17915904936014626,
                  0.0,
                  98,
                  0
                ],
                [
                  0.1809872029250457,
                  0.0,
                  99,
                  0
                ],
                [
                  0.18281535648994515,
                  0.0,
                  100,
                  0
                ],
                [
                  0.1846435100548446,
                  0.0,
                  101,
                  0
                ],
                [
                  0.18647166361974407,
                  0.0,
                  102,
                  0
                ],
                [
                  0.1882998171846435,
                  0.0,
                  103,
                  0
                ],
                [
                  0.19012797074954296,
                  0.0,
                  104,
                  0
                ],
                [
                  0.19195612431444242,
                  0.0,
                  105,
                  0
                ],
                [
                  0.19378427787934185,
                  0.0,
                  106,
                  0
                ],
                [
                  0.1956124314442413,
                  0.0,
                  107,
                  0
                ],
                [
                  0.19744058500914077,
                  0.0,
                  108,
                  0
                ],
                [
                  0.19926873857404023,
                  0.0,
                  109,
                  0
                ],
                [
                  0.20109689213893966,
                  0.0,
                  110,
                  0
                ],
                [
                  0.20292504570383912,
                  0.0,
                  111,
                  0
                ],
                [
                  0.20475319926873858,
                  0.0,
                  112,
                  0
                ],
                [
                  0.20658135283363802,
                  0.0,
                  113,
                  0
                ],
                [
                  0.20840950639853748,
                  0.0,
                  114,
                  0
                ],
                [
                  0.21023765996343693,
                  0.0,
                  115,
                  0
                ],
                [
                  0.21206581352833637,
                  0.0,
                  116,
                  0
                ],
                [
                  0.21389396709323583,
                  0.0,
                  117,
                  0
                ],
                [
                  0.21572212065813529,
                  0.0,
                  118,
                  0
                ],
                [
                  0.21755027422303475,
                  0.0,
                  119,
                  0
                ],
                [
                  0.21937842778793418,
                  0.0,
                  120,
                  0
                ],
                [
                  0.22120658135283364,
                  0.0,
                  121,
                  0
                ],
                [
                  0.2230347349177331,
                  0.0,
                  122,
                  0
                ],
                [
                  0.22486288848263253,
                  0.0,
                  123,
                  0
                ],
                [
                  0.226691042047532,
                  0.0,
                  124,
                  0
                ],
                [
                  0.22851919561243145,
                  0.0,
                  125,
                  0
                ],
                [
                  0.2303473491773309,
                  0.0,
                  126,
                  0
                ],
                [
                  0.23217550274223034,
                  0.0,
                  127,
                  0
                ],
                [
                  0.2340036563071298,
                  0.0,
                  128,
                  0
                ],
                [
                  0.23583180987202926,
                  0.0,
                  129,
                  0
                ],
                [
                  0.2376599634369287,
                  0.0,
                  130,
                  0
                ],
                [
                  0.23948811700182815,
                  0.0,
                  131,
                  0
                ],
                [
                  0.2413162705667276,
                  0.0,
                  132,
                  0
                ],
                [
                  0.24314442413162707,
                  0.0,
                  133,
                  0
                ],
                [
                  0.2449725776965265,
                  0.0,
                  134,
                  0
                ],
                [
                  0.24680073126142596,
                  0.0,
                  135,
                  0
                ],
                [
                  0.24862888482632542,
                  0.0,
                  136,
                  0
                ],
                [
                  0.25045703839122485,
                  0.0,
                  137,
                  0
                ],
                [
                  0.2522851919561243,
                  0.0,
                  138,
                  0
                ],
                [
                  0.25411334552102377,
                  0.0,
                  139,
                  0
                ],
                [
                  0.25594149908592323,
                  0.0,
                  140,
                  0
                ],
                [
                  0.2577696526508227,
                  0.0,
                  141,
                  0
                ],
                [
                  0.2595978062157221,
                  0.0,
                  142,
                  0
                ],
                [
                  0.26142595978062155,
                  0.0,
                  143,
                  0
                ],
                [
                  0.263254113345521,
                  0.0,
                  144,
                  0
                ],
                [
                  0.26508226691042047,
                  0.0,
                  145,
                  0
                ],
                [
                  0.26691042047531993,
                  0.0,
                  146,
                  0
                ],
                [
                  0.2687385740402194,
                  0.0,
                  147,
                  0
                ],
                [
                  0.27056672760511885,
                  0.0,
                  148,
                  0
                ],
                [
                  0.27239488117001825,
                  0.0,
                  149,
                  0
                ],
                [
                  0.2742230347349177,
                  0.0,
                  150,
                  0
                ],
                [
                  0.2760511882998172,
                  0.0,
                  151,
                  0
                ],
                [
                  0.27787934186471663,
                  0.0,
                  152,
                  0
                ],
                [
                  0.2797074954296161,
                  0.0,
                  153,
                  0
                ],
                [
                  0.28153564899451555,
                  0.0,
                  154,
                  0
                ],
                [
                  0.283363802559415,
                  0.0,
                  155,
                  0
                ],
                [
                  0.2851919561243144,
                  0.0,
                  156,
                  0
                ],
                [
                  0.2870201096892139,
                  0.0,
                  157,
                  0
                ],
                [
                  0.28884826325411334,
                  0.0,
                  158,
                  0
                ],
                [
                  0.2906764168190128,
                  0.0,
                  159,
                  0
                ],
                [
                  0.29250457038391225,
                  0.0,
                  160,
                  0
                ],
                [
                  0.2943327239488117,
                  0.0,
                  161,
                  0
                ],
                [
                  0.2961608775137112,
                  0.0,
                  162,
                  0
                ],
                [
                  0.2979890310786106,
                  0.0,
                  163,
                  0
                ],
                [
                  0.29981718464351004,
                  0.0,
                  164,
                  0
                ],
                [
                  0.3016453382084095,
                  0.0,
                  165,
                  0
                ],
                [
                  0.30347349177330896,
                  0.0,
                  166,
                  0
                ],
                [
                  0.3053016453382084,
                  0.0,
                  167,
                  0
                ],
                [
                  0.3071297989031079,
                  0.0,
                  168,
                  0
                ],
                [
                  0.30895795246800734,
                  0.0,
                  169,
                  0
                ],
                [
                  0.31078610603290674,
                  0.0,
                  170,
                  0
                ],
                [
                  0.3126142595978062,
                  0.0,
                  171,
                  0
                ],
                [
                  0.31444241316270566,
                  0.0,
                  172,
                  0
                ],
                [
                  0.3162705667276051,
                  0.0,
                  173,
                  0
                ],
                [
                  0.3180987202925046,
                  0.0,
                  174,
                  0
                ],
                [
                  0.31992687385740404,
                  0.0,
                  175,
                  0
                ],
                [
                  0.3217550274223035,
                  0.0,
                  176,
                  0
                ],
                [
                  0.3235831809872029,
                  0.0,
                  177,
                  0
                ],
                [
                  0.32541133455210236,
                  0.0,
                  178,
                  0
                ],
                [
                  0.3272394881170018,
                  0.0,
                  179,
                  0
                ],
                [
                  0.3290676416819013,
                  0.0,
                  180,
                  0
                ],
                [
                  0.33089579524680074,
                  0.0,
                  181,
                  0
                ],
                [
                  0.3327239488117002,
                  0.0,
                  182,
                  0
                ],
                [
                  0.33455210237659966,
                  0.0,
                  183,
                  0
                ],
                [
                  0.33638025594149906,
                  0.0,
                  184,
                  0
                ],
                [
                  0.3382084095063985,
                  0.0,
                  185,
                  0
                ],
                [
                  0.340036563071298,
                  0.0,
                  186,
                  0
                ],
                [
                  0.34186471663619744,
                  0.0,
                  187,
                  0
                ],
                [
                  0.3436928702010969,
                  0.0,
                  188,
                  0
                ],
                [
                  0.34552102376599636,
                  0.0,
                  189,
                  0
                ],
                [
                  0.3473491773308958,
                  0.0,
                  190,
                  0
                ],
                [
                  0.3491773308957952,
                  0.0,
                  191,
                  0
                ],
                [
                  0.3510054844606947,
                  0.0,
                  192,
                  0
                ],
                [
                  0.35283363802559414,
                  0.0,
                  193,
                  0
                ],
                [
                  0.3546617915904936,
                  0.0,
                  194,
                  0
                ],
                [
                  0.35648994515539306,
                  0.0,
                  195,
                  0
                ],
                [
                  0.3583180987202925,
                  0.0,
                  196,
                  0
                ],
                [
                  0.360146252285192,
                  0.0,
                  197,
                  0
                ],
                [
                  0.3619744058500914,
                  0.0,
                  198,
                  0
                ],
                [
                  0.36380255941499084,
                  0.0,
                  199,
                  0
                ],
                [
                  0.3656307129798903,
                  0.0,
                  200,
                  0
                ],
                [
                  0.36745886654478976,
                  0.0,
                  201,
                  0
                ],
                [
                  0.3692870201096892,
                  0.0,
                  202,
                  0
                ],
                [
                  0.3711151736745887,
                  0.0,
                  203,
                  0
                ],
                [
                  0.37294332723948814,
                  0.0,
                  204,
                  0
                ],
                [
                  0.37477148080438755,
                  0.0,
                  205,
                  0
                ],
                [
                  0.376599634369287,
                  0.0,
                  206,
                  0
                ],
                [
                  0.37842778793418647,
                  0.0,
                  207,
                  0
                ],
                [
                  0.3802559414990859,
                  0.0,
                  208,
                  0
                ],
                [
                  0.3820840950639854,
                  0.0,
                  209,
                  0
                ],
                [
                  0.38391224862888484,
                  0.0,
                  210,
                  0
                ],
                [
                  0.3857404021937843,
                  0.0,
                  211,
                  0
                ],
                [
                  0.3875685557586837,
                  0.0,
                  212,
                  0
                ],
                [
                  0.38939670932358317,
                  0.0,
                  213,
                  0
                ],
                [
                  0.3912248628884826,
                  0.0,
                  214,
                  0
                ],
                [
                  0.3930530164533821,
                  0.0,
                  215,
                  0
                ],
                [
                  0.39488117001828155,
                  0.0,
                  216,
                  0
                ],
                [
                  0.396709323583181,
                  0.0,
                  217,
                  0
                ],
                [
                  0.39853747714808047,
                  0.0,
                  218,
                  0
                ],
                [
                  0.40036563071297987,
                  0.0,
                  219,
                  0
                ],
                [
                  0.40219378427787933,
                  0.0,
                  220,
                  0
                ],
                [
                  0.4040219378427788,
                  0.0,
                  221,
                  0
                ],
                [
                  0.40585009140767825,
                  0.0,
                  222,
                  0
                ],
                [
                  0.4076782449725777,
                  0.0,
                  223,
                  0
                ],
                [
                  0.40950639853747717,
                  0.0,
                  224,
                  0
                ],
                [
                  0.4113345521023766,
                  0.0,
                  225,
                  0
                ],
                [
                  0.41316270566727603,
                  0.0,
                  226,
                  0
                ],
                [
                  0.4149908592321755,
                  0.0,
                  227,
                  0
                ],
                [
                  0.41681901279707495,
                  0.0,
                  228,
                  0
                ],
                [
                  0.4186471663619744,
                  0.0,
                  229,
                  0
                ],
                [
                  0.42047531992687387,
                  0.0,
                  230,
                  0
                ],
                [
                  0.42230347349177333,
                  0.0,
                  231,
                  0
                ],
                [
                  0.42413162705667273,
                  0.0,
                  232,
                  0
                ],
                [
                  0.4259597806215722,
                  0.0,
                  233,
                  0
                ],
                [
                  0.42778793418647165,
                  0.0,
                  234,
                  0
                ],
                [
                  0.4296160877513711,
                  0.0,
                  235,
                  0
                ],
                [
                  0.43144424131627057,
                  0.0,
                  236,
                  0
                ],
                [
                  0.43327239488117003,
                  0.0,
                  237,
                  0
                ],
                [
                  0.4351005484460695,
                  0.0,
                  238,
                  0
                ],
                [
                  0.4369287020109689,
                  0.0,
                  239,
                  0
                ],
                [
                  0.43875685557586835,
                  0.0,
                  240,
                  0
                ],
                [
                  0.4405850091407678,
                  0.0,
                  241,
                  0
                ],
                [
                  0.4424131627056673,
                  0.0,
                  242,
                  0
                ],
                [
                  0.44424131627056673,
                  0.0,
                  243,
                  0
                ],
                [
                  0.4460694698354662,
                  0.0,
                  244,
                  0
                ],
                [
                  0.44789762340036565,
                  0.0,
                  245,
                  0
                ],
                [
                  0.44972577696526506,
                  0.0,
                  246,
                  0
                ],
                [
                  0.4515539305301645,
                  0.0,
                  247,
                  0
                ],
                [
                  0.453382084095064,
                  0.0,
                  248,
                  0
                ],
                [
                  0.45521023765996343,
                  0.0,
                  249,
                  0
                ],
                [
                  0.4570383912248629,
                  0.0,
                  250,
                  0
                ],
                [
                  0.45886654478976235,
                  0.0,
                  251,
                  0
                ],
                [
                  0.4606946983546618,
                  0.0,
                  252,
                  0
                ],
                [
                  0.4625228519195612,
                  0.0,
                  253,
                  0
                ],
                [
                  0.4643510054844607,
                  0.0,
                  254,
                  0
                ],
                [
                  0.46617915904936014,
                  0.0,
                  255,
                  0
                ],
                [
                  0.4680073126142596,
                  0.0,
                  256,
                  0
                ],
                [
                  0.46983546617915906,
                  0.0,
                  257,
                  0
                ],
                [
                  0.4716636197440585,
                  0.0,
                  258,
                  0
                ],
                [
                  0.473491773308958,
                  0.0,
                  259,
                  0
                ],
                [
                  0.4753199268738574,
                  0.0,
                  260,
                  0
                ],
                [
                  0.47714808043875684,
                  0.0,
                  261,
                  0
                ],
                [
                  0.4789762340036563,
                  0.0,
                  262,
                  0
                ],
                [
                  0.48080438756855576,
                  0.0,
                  263,
                  0
                ],
                [
                  0.4826325411334552,
                  0.0,
                  264,
                  0
                ],
                [
                  0.4844606946983547,
                  0.0,
                  265,
                  0
                ],
                [
                  0.48628884826325414,
                  0.0,
                  266,
                  0
                ],
                [
                  0.48811700182815354,
                  0.0,
                  267,
                  0
                ],
                [
                  0.489945155393053,
                  0.0,
                  268,
                  0
                ],
                [
                  0.49177330895795246,
                  0.0,
                  269,
                  0
                ],
                [
                  0.4936014625228519,
                  0.0,
                  270,
                  0
                ],
                [
                  0.4954296160877514,
                  0.0,
                  271,
                  0
                ],
                [
                  0.49725776965265084,
                  0.0,
                  272,
                  0
                ],
                [
                  0.4990859232175503,
                  0.0,
                  273,
                  0
                ],
                [
                  0.5009140767824497,
                  0.0,
                  274,
                  0
                ],
                [
                  0.5027422303473492,
                  0.0,
                  275,
                  0
                ],
                [
                  0.5045703839122486,
                  0.0,
                  276,
                  0
                ],
                [
                  0.506398537477148,
                  0.0,
                  277,
                  0
                ],
                [
                  0.5082266910420475,
                  0.0,
                  278,
                  0
                ],
                [
                  0.5100548446069469,
                  0.0,
                  279,
                  0
                ],
                [
                  0.5118829981718465,
                  0.0,
                  280,
                  0
                ],
                [
                  0.5137111517367459,
                  0.0,
                  281,
                  0
                ],
                [
                  0.5155393053016454,
                  0.0,
                  282,
                  0
                ],
                [
                  0.5173674588665448,
                  0.0,
                  283,
                  0
                ],
                [
                  0.5191956124314442,
                  0.0,
                  284,
                  0
                ],
                [
                  0.5210237659963437,
                  0.0,
                  285,
                  0
                ],
                [
                  0.5228519195612431,
                  0.0,
                  286,
                  0
                ],
                [
                  0.5246800731261426,
                  0.0,
                  287,
                  0
                ],
                [
                  0.526508226691042,
                  0.0,
                  288,
                  0
                ],
                [
                  0.5283363802559415,
                  0.0,
                  289,
                  0
                ],
                [
                  0.5301645338208409,
                  0.0,
                  290,
                  0
                ],
                [
                  0.5319926873857403,
                  0.0,
                  291,
                  0
                ],
                [
                  0.5338208409506399,
                  0.0,
                  292,
                  0
                ],
                [
                  0.5356489945155393,
                  0.0,
                  293,
                  0
                ],
                [
                  0.5374771480804388,
                  0.0,
                  294,
                  0
                ],
                [
                  0.5393053016453382,
                  0.0,
                  295,
                  0
                ],
                [
                  0.5411334552102377,
                  0.0,
                  296,
                  0
                ],
                [
                  0.5429616087751371,
                  0.0,
                  297,
                  0
                ],
                [
                  0.5447897623400365,
                  0.0,
                  298,
                  0
                ],
                [
                  0.546617915904936,
                  0.0,
                  299,
                  0
                ],
                [
                  0.5484460694698354,
                  0.0,
                  300,
                  0
                ],
                [
                  0.5502742230347349,
                  0.0,
                  301,
                  0
                ],
                [
                  0.5521023765996343,
                  0.0,
                  302,
                  0
                ],
                [
                  0.5539305301645339,
                  0.0,
                  303,
                  0
                ],
                [
                  0.5557586837294333,
                  0.0,
                  304,
                  0
                ],
                [
                  0.5575868372943327,
                  0.0,
                  305,
                  0
                ],
                [
                  0.5594149908592322,
                  0.0,
                  306,
                  0
                ],
                [
                  0.5612431444241316,
                  0.0,
                  307,
                  0
                ],
                [
                  0.5630712979890311,
                  0.0,
                  308,
                  0
                ],
                [
                  0.5648994515539305,
                  0.0,
                  309,
                  0
                ],
                [
                  0.56672760511883,
                  0.0,
                  310,
                  0
                ],
                [
                  0.5685557586837294,
                  0.0,
                  311,
                  0
                ],
                [
                  0.5703839122486288,
                  0.0,
                  312,
                  0
                ],
                [
                  0.5722120658135283,
                  0.0,
                  313,
                  0
                ],
                [
                  0.5740402193784278,
                  0.0,
                  314,
                  0
                ],
                [
                  0.5758683729433273,
                  0.0,
                  315,
                  0
                ],
                [
                  0.5776965265082267,
                  0.0,
                  316,
                  0
                ],
                [
                  0.5795246800731262,
                  0.0,
                  317,
                  0
                ],
                [
                  0.5813528336380256,
                  0.0,
                  318,
                  0
                ],
                [
                  0.583180987202925,
                  0.0,
                  319,
                  0
                ],
                [
                  0.5850091407678245,
                  0.0,
                  320,
                  0
                ],
                [
                  0.5868372943327239,
                  0.0,
                  321,
                  0
                ],
                [
                  0.5886654478976234,
                  0.0,
                  322,
                  0
                ],
                [
                  0.5904936014625228,
                  0.0,
                  323,
                  0
                ],
                [
                  0.5923217550274223,
                  0.0,
                  324,
                  0
                ],
                [
                  0.5941499085923218,
                  0.003076923076923077,
                  325,
                  1
                ],
                [
                  0.5959780621572212,
                  0.003067484662576687,
                  326,
                  1
                ],
                [
                  0.5978062157221207,
                  0.0030581039755351682,
                  327,
                  1
                ],
                [
                  0.5996343692870201,
                  0.003048780487804878,
                  328,
                  1
                ],
                [
                  0.6014625228519196,
                  0.00303951367781155,
                  329,
                  1
                ],
                [
                  0.603290676416819,
                  0.0030303030303030303,
                  330,
                  1
                ],
                [
                  0.6051188299817185,
                  0.0030211480362537764,
                  331,
                  1
                ],
                [
                  0.6069469835466179,
                  0.0030120481927710845,
                  332,
                  1
                ],
                [
                  0.6087751371115173,
                  0.003003003003003003,
                  333,
                  1
                ],
                [
                  0.6106032906764168,
                  0.0029940119760479044,
                  334,
                  1
                ],
                [
                  0.6124314442413162,
                  0.0029850746268656717,
                  335,
                  1
                ],
                [
                  0.6142595978062158,
                  0.002976190476190476,
                  336,
                  1
                ],
                [
                  0.6160877513711152,
                  0.002967359050445104,
                  337,
                  1
                ],
                [
                  0.6179159049360147,
                  0.0029585798816568047,
                  338,
                  1
                ],
                [
                  0.6197440585009141,
                  0.0029498525073746312,
                  339,
                  1
                ],
                [
                  0.6215722120658135,
                  0.0029411764705882353,
                  340,
                  1
                ],
                [
                  0.623400365630713,
                  0.002932551319648094,
                  341,
                  1
                ],
                [
                  0.6252285191956124,
                  0.0029239766081871343,
                  342,
                  1
                ],
                [
                  0.6270566727605119,
                  0.0029154518950437317,
                  343,
                  1
                ],
                [
                  0.6288848263254113,
                  0.0029069767441860465,
                  344,
                  1
                ],
                [
                  0.6307129798903108,
                  0.002898550724637681,
                  345,
                  1
                ],
                [
                  0.6325411334552102,
                  0.002890173410404624,
                  346,
                  1
                ],
                [
                  0.6343692870201096,
                  0.002881844380403458,
                  347,
                  1
                ],
                [
                  0.6361974405850092,
                  0.0028735632183908046,
                  348,
                  1
                ],
                [
                  0.6380255941499086,
                  0.0028653295128939827,
                  349,
                  1
                ],
                [
                  0.6398537477148081,
                  0.005714285714285714,
                  350,
                  2
                ],
                [
                  0.6416819012797075,
                  0.005698005698005698,
                  351,
                  2
                ],
                [
                  0.643510054844607,
                  0.005681818181818182,
                  352,
                  2
                ],
                [
                  0.6453382084095064,
                  0.0056657223796034,
                  353,
                  2
                ],
                [
                  0.6471663619744058,
                  0.005649717514124294,
                  354,
                  2
                ],
                [
                  0.6489945155393053,
                  0.005633802816901409,
                  355,
                  2
                ],
                [
                  0.6508226691042047,
                  0.0056179775280898875,
                  356,
                  2
                ],
                [
                  0.6526508226691042,
                  0.0056022408963585435,
                  357,
                  2
                ],
                [
                  0.6544789762340036,
                  0.00558659217877095,
                  358,
                  2
                ],
                [
                  0.6563071297989032,
                  0.005571030640668524,
                  359,
                  2
                ],
                [
                  0.6581352833638026,
                  0.005555555555555556,
                  360,
                  2
                ],
                [
                  0.659963436928702,
                  0.00554016620498615,
                  361,
                  2
                ],
                [
                  0.6617915904936015,
                  0.0055248618784530384,
                  362,
                  2
                ],
                [
                  0.6636197440585009,
                  0.005509641873278237,
                  363,
                  2
                ],
                [
                  0.6654478976234004,
                  0.005494505494505495,
                  364,
                  2
                ],
                [
                  0.6672760511882998,
                  0.005479452054794521,
                  365,
                  2
                ],
                [
                  0.6691042047531993,
                  0.00546448087431694,
                  366,
                  2
                ],
                [
                  0.6709323583180987,
                  0.005449591280653951,
                  367,
                  2
                ],
                [
                  0.6727605118829981,
                  0.005434782608695652,
                  368,
                  2
                ],
                [
                  0.6745886654478976,
                  0.005420054200542005,
                  369,
                  2
                ],
                [
                  0.676416819012797,
                  0.005405405405405406,
                  370,
                  2
                ],
                [
                  0.6782449725776966,
                  0.005390835579514825,
                  371,
                  2
                ],
                [
                  0.680073126142596,
                  0.005376344086021506,
                  372,
                  2
                ],
                [
                  0.6819012797074955,
                  0.005361930294906166,
                  373,
                  2
                ],
                [
                  0.6837294332723949,
                  0.0053475935828877,
                  374,
                  2
                ],
                [
                  0.6855575868372943,
                  0.005333333333333333,
                  375,
                  2
                ],
                [
                  0.6873857404021938,
                  0.005319148936170213,
                  376,
                  2
                ],
                [
                  0.6892138939670932,
                  0.005305039787798408,
                  377,
                  2
                ],
                [
                  0.6910420475319927,
                  0.005291005291005291,
                  378,
                  2
                ],
                [
                  0.6928702010968921,
                  0.005277044854881266,
                  379,
                  2
                ],
                [
                  0.6946983546617916,
                  0.005263157894736842,
                  380,
                  2
                ],
                [
                  0.696526508226691,
                  0.005249343832020997,
                  381,
                  2
                ],
                [
                  0.6983546617915904,
                  0.005235602094240838,
                  382,
                  2
                ],
                [
                  0.70018281535649,
                  0.005221932114882507,
                  383,
                  2
                ],
                [
                  0.7020109689213894,
                  0.005208333333333333,
                  384,
                  2
                ],
                [
                  0.7038391224862889,
                  0.005194805194805195,
                  385,
                  2
                ],
                [
                  0.7056672760511883,
                  0.0051813471502590676,
                  386,
                  2
                ],
                [
                  0.7074954296160878,
                  0.00516795865633075,
                  387,
                  2
                ],
                [
                  0.7093235831809872,
                  0.005154639175257732,
                  388,
                  2
                ],
                [
                  0.7111517367458866,
                  0.005141388174807198,
                  389,
                  2
                ],
                [
                  0.7129798903107861,
                  0.005128205128205128,
                  390,
                  2
                ],
                [
                  0.7148080438756855,
                  0.005115089514066497,
                  391,
                  2
                ],
                [
                  0.716636197440585,
                  0.00510204081632653,
                  392,
                  2
                ],
                [
                  0.7184643510054844,
                  0.005089058524173028,
                  393,
                  2
                ],
                [
                  0.720292504570384,
                  0.005076142131979695,
                  394,
                  2
                ],
                [
                  0.7221206581352834,
                  0.005063291139240506,
                  395,
                  2
                ],
                [
                  0.7239488117001828,
                  0.005050505050505051,
                  396,
                  2
                ],
                [
                  0.7257769652650823,
                  0.005037783375314861,
                  397,
                  2
                ],
                [
                  0.7276051188299817,
                  0.007537688442211055,
                  398,
                  3
                ],
                [
                  0.7294332723948812,
                  0.007518796992481203,
                  399,
                  3
                ],
                [
                  0.7312614259597806,
                  0.0075,
                  400,
                  3
                ],
                [
                  0.7330895795246801,
                  0.007481296758104738,
                  401,
                  3
                ],
                [
                  0.7349177330895795,
                  0.007462686567164179,
                  402,
                  3
                ],
                [
                  0.7367458866544789,
                  0.007444168734491315,
                  403,
                  3
                ],
                [
                  0.7385740402193784,
                  0.007425742574257425,
                  404,
                  3
                ],
                [
                  0.7404021937842779,
                  0.007407407407407408,
                  405,
                  3
                ],
                [
                  0.7422303473491774,
                  0.009852216748768473,
                  406,
                  4
                ],
                [
                  0.7440585009140768,
                  0.009828009828009828,
                  407,
                  4
                ],
                [
                  0.7458866544789763,
                  0.00980392156862745,
                  408,
                  4
                ],
                [
                  0.7477148080438757,
                  0.009779951100244499,
                  409,
                  4
                ],
                [
                  0.7495429616087751,
                  0.00975609756097561,
                  410,
                  4
                ],
                [
                  0.7513711151736746,
                  0.009732360097323601,
                  411,
                  4
                ],
                [
                  0.753199268738574,
                  0.009708737864077669,
                  412,
                  4
                ],
                [
                  0.7550274223034735,
                  0.009685230024213076,
                  413,
                  4
                ],
                [
                  0.7568555758683729,
                  0.00966183574879227,
                  414,
                  4
                ],
                [
                  0.7586837294332724,
                  0.012048192771084338,
                  415,
                  5
                ],
                [
                  0.7605118829981719,
                  0.01201923076923077,
                  416,
                  5
                ],
                [
                  0.7623400365630713,
                  0.011990407673860911,
                  417,
                  5
                ],
                [
                  0.7641681901279708,
                  0.011961722488038277,
                  418,
                  5
                ],
                [
                  0.7659963436928702,
                  0.011933174224343675,
                  419,
                  5
                ],
                [
                  0.7678244972577697,
                  0.011904761904761904,
                  420,
                  5
                ],
                [
                  0.7696526508226691,
                  0.011876484560570071,
                  421,
                  5
                ],
                [
                  0.7714808043875686,
                  0.011848341232227487,
                  422,
                  5
                ],
                [
                  0.773308957952468,
                  0.01182033096926714,
                  423,
                  5
                ],
                [
                  0.7751371115173674,
                  0.014150943396226415,
                  424,
                  6
                ],
                [
                  0.7769652650822669,
                  0.01647058823529412,
                  425,
                  7
                ],
                [
                  0.7787934186471663,
                  0.01643192488262911,
                  426,
                  7
                ],
                [
                  0.7806215722120659,
                  0.01639344262295082,
                  427,
                  7
                ],
                [
                  0.7824497257769653,
                  0.016355140186915886,
                  428,
                  7
                ],
                [
                  0.7842778793418648,
                  0.016317016317016316,
                  429,
                  7
                ],
                [
                  0.7861060329067642,
                  0.01627906976744186,
                  430,
                  7
                ],
                [
                  0.7879341864716636,
                  0.016241299303944315,
                  431,
                  7
                ],
                [
                  0.7897623400365631,
                  0.016203703703703703,
                  432,
                  7
                ],
                [
                  0.7915904936014625,
                  0.016166281755196306,
                  433,
                  7
                ],
                [
                  0.793418647166362,
                  0.016129032258064516,
                  434,
                  7
                ],
                [
                  0.7952468007312614,
                  0.016091954022988506,
                  435,
                  7
                ],
                [
                  0.7970749542961609,
                  0.016055045871559634,
                  436,
                  7
                ],
                [
                  0.7989031078610603,
                  0.016018306636155607,
                  437,
                  7
                ],
                [
                  0.8007312614259597,
                  0.01598173515981735,
                  438,
                  7
                ],
                [
                  0.8025594149908593,
                  0.015945330296127564,
                  439,
                  7
                ],
                [
                  0.8043875685557587,
                  0.015909090909090907,
                  440,
                  7
                ],
                [
                  0.8062157221206582,
                  0.015873015873015872,
                  441,
                  7
                ],
                [
                  0.8080438756855576,
                  0.01583710407239819,
                  442,
                  7
                ],
                [
                  0.8098720292504571,
                  0.01805869074492099,
                  443,
                  8
                ],
                [
                  0.8117001828153565,
                  0.02027027027027027,
                  444,
                  9
                ],
                [
                  0.8135283363802559,
                  0.02247191011235955,
                  445,
                  10
                ],
                [
                  0.8153564899451554,
                  0.02242152466367713,
                  446,
                  10
                ],
                [
                  0.8171846435100548,
                  0.02237136465324385,
                  447,
                  10
                ],
                [
                  0.8190127970749543,
                  0.022321428571428572,
                  448,
                  10
                ],
                [
                  0.8208409506398537,
                  0.022271714922048998,
                  449,
                  10
                ],
                [
                  0.8226691042047533,
                  0.022222222222222223,
                  450,
                  10
                ],
                [
                  0.8244972577696527,
                  0.022172949002217297,
                  451,
                  10
                ],
                [
                  0.8263254113345521,
                  0.022123893805309734,
                  452,
                  10
                ],
                [
                  0.8281535648994516,
                  0.02207505518763797,
                  453,
                  10
                ],
                [
                  0.829981718464351,
                  0.024229074889867842,
                  454,
                  11
                ],
                [
                  0.8318098720292505,
                  0.026373626373626374,
                  455,
                  12
                ],
                [
                  0.8336380255941499,
                  0.02631578947368421,
                  456,
                  12
                ],
                [
                  0.8354661791590493,
                  0.0262582056892779,
                  457,
                  12
                ],
                [
                  0.8372943327239488,
                  0.026200873362445413,
                  458,
                  12
                ],
                [
                  0.8391224862888482,
                  0.026143790849673203,
                  459,
                  12
                ],
                [
                  0.8409506398537477,
                  0.02608695652173913,
                  460,
                  12
                ],
                [
                  0.8427787934186471,
                  0.026030368763557483,
                  461,
                  12
                ],
                [
                  0.8446069469835467,
                  0.02813852813852814,
                  462,
                  13
                ],
                [
                  0.8464351005484461,
                  0.028077753779697623,
                  463,
                  13
                ],
                [
                  0.8482632541133455,
                  0.028017241379310345,
                  464,
                  13
                ],
                [
                  0.850091407678245,
                  0.030107526881720432,
                  465,
                  14
                ],
                [
                  0.8519195612431444,
                  0.030042918454935622,
                  466,
                  14
                ],
                [
                  0.8537477148080439,
                  0.029978586723768737,
                  467,
                  14
                ],
                [
                  0.8555758683729433,
                  0.03205128205128205,
                  468,
                  15
                ],
                [
                  0.8574040219378428,
                  0.031982942430703626,
                  469,
                  15
                ],
                [
                  0.8592321755027422,
                  0.031914893617021274,
                  470,
                  15
                ],
                [
                  0.8610603290676416,
                  0.03397027600849257,
                  471,
                  16
                ],
                [
                  0.8628884826325411,
                  0.036016949152542374,
                  472,
                  17
                ],
                [
                  0.8647166361974405,
                  0.03805496828752643,
                  473,
                  18
                ],
                [
                  0.8665447897623401,
                  0.04008438818565401,
                  474,
                  19
                ],
                [
                  0.8683729433272395,
                  0.042105263157894736,
                  475,
                  20
                ],
                [
                  0.870201096892139,
                  0.04411764705882353,
                  476,
                  21
                ],
                [
                  0.8720292504570384,
                  0.0440251572327044,
                  477,
                  21
                ],
                [
                  0.8738574040219378,
                  0.043933054393305436,
                  478,
                  21
                ],
                [
                  0.8756855575868373,
                  0.04384133611691023,
                  479,
                  21
                ],
                [
                  0.8775137111517367,
                  0.04583333333333333,
                  480,
                  22
                ],
                [
                  0.8793418647166362,
                  0.04781704781704782,
                  481,
                  23
                ],
                [
                  0.8811700182815356,
                  0.04771784232365145,
                  482,
                  23
                ],
                [
                  0.8829981718464351,
                  0.047619047619047616,
                  483,
                  23
                ],
                [
                  0.8848263254113345,
                  0.047520661157024795,
                  484,
                  23
                ],
                [
                  0.886654478976234,
                  0.04742268041237113,
                  485,
                  23
                ],
                [
                  0.8884826325411335,
                  0.047325102880658436,
                  486,
                  23
                ],
                [
                  0.8903107861060329,
                  0.04722792607802875,
                  487,
                  23
                ],
                [
                  0.8921389396709324,
                  0.0471311475409836,
                  488,
                  23
                ],
                [
                  0.8939670932358318,
                  0.04703476482617587,
                  489,
                  23
                ],
                [
                  0.8957952468007313,
                  0.04693877551020408,
                  490,
                  23
                ],
                [
                  0.8976234003656307,
                  0.04684317718940937,
                  491,
                  23
                ],
                [
                  0.8994515539305301,
                  0.046747967479674794,
                  492,
                  23
                ],
                [
                  0.9012797074954296,
                  0.0486815415821501,
                  493,
                  24
                ],
                [
                  0.903107861060329,
                  0.048582995951417005,
                  494,
                  24
                ],
                [
                  0.9049360146252285,
                  0.048484848484848485,
                  495,
                  24
                ],
                [
                  0.906764168190128,
                  0.04838709677419355,
                  496,
                  24
                ],
                [
                  0.9085923217550275,
                  0.05030181086519115,
                  497,
                  25
                ],
                [
                  0.9104204753199269,
                  0.050200803212851405,
                  498,
                  25
                ],
                [
                  0.9122486288848263,
                  0.052104208416833664,
                  499,
                  26
                ],
                [
                  0.9140767824497258,
                  0.052,
                  500,
                  26
                ],
                [
                  0.9159049360146252,
                  0.05189620758483034,
                  501,
                  26
                ],
                [
                  0.9177330895795247,
                  0.053784860557768925,
                  502,
                  27
                ],
                [
                  0.9195612431444241,
                  0.0536779324055666,
                  503,
                  27
                ],
                [
                  0.9213893967093236,
                  0.05555555555555555,
                  504,
                  28
                ],
                [
                  0.923217550274223,
                  0.05742574257425743,
                  505,
                  29
                ],
                [
                  0.9250457038391224,
                  0.05928853754940711,
                  506,
                  30
                ],
                [
                  0.926873857404022,
                  0.05917159763313609,
                  507,
                  30
                ],
                [
                  0.9287020109689214,
                  0.05905511811023622,
                  508,
                  30
                ],
                [
                  0.9305301645338209,
                  0.05893909626719057,
                  509,
                  30
                ],
                [
                  0.9323583180987203,
                  0.060784313725490195,
                  510,
                  31
                ],
                [
                  0.9341864716636198,
                  0.06262230919765166,
                  511,
                  32
                ],
                [
                  0.9360146252285192,
                  0.064453125,
                  512,
                  33
                ],
                [
                  0.9378427787934186,
                  0.06432748538011696,
                  513,
                  33
                ],
                [
                  0.9396709323583181,
                  0.06420233463035019,
                  514,
                  33
                ],
                [
                  0.9414990859232175,
                  0.06601941747572816,
                  515,
                  34
                ],
                [
                  0.943327239488117,
                  0.06782945736434108,
                  516,
                  35
                ],
                [
                  0.9451553930530164,
                  0.06963249516441006,
                  517,
                  36
                ],
                [
                  0.946983546617916,
                  0.0694980694980695,
                  518,
                  36
                ],
                [
                  0.9488117001828154,
                  0.06936416184971098,
                  519,
                  36
                ],
                [
                  0.9506398537477148,
                  0.07115384615384615,
                  520,
                  37
                ],
                [
                  0.9524680073126143,
                  0.0710172744721689,
                  521,
                  37
                ],
                [
                  0.9542961608775137,
                  0.07088122605363985,
                  522,
                  37
                ],
                [
                  0.9561243144424132,
                  0.07265774378585087,
                  523,
                  38
                ],
                [
                  0.9579524680073126,
                  0.07442748091603053,
                  524,
                  39
                ],
                [
                  0.9597806215722121,
                  0.0761904761904762,
                  525,
                  40
                ],
                [
                  0.9616087751371115,
                  0.07604562737642585,
                  526,
                  40
                ],
                [
                  0.9634369287020109,
                  0.0777988614800759,
                  527,
                  41
                ],
                [
                  0.9652650822669104,
                  0.07954545454545454,
                  528,
                  42
                ],
                [
                  0.9670932358318098,
                  0.07939508506616257,
                  529,
                  42
                ],
                [
                  0.9689213893967094,
                  0.08113207547169811,
                  530,
                  43
                ],
                [
                  0.9707495429616088,
                  0.08286252354048965,
                  531,
                  44
                ],
                [
                  0.9725776965265083,
                  0.08458646616541353,
                  532,
                  45
                ],
                [
                  0.9744058500914077,
                  0.08630393996247655,
                  533,
                  46
                ],
                [
                  0.9762340036563071,
                  0.08614232209737828,
                  534,
                  46
                ],
                [
                  0.9780621572212066,
                  0.08785046728971962,
                  535,
                  47
                ],
                [
                  0.979890310786106,
                  0.08955223880597014,
                  536,
                  48
                ],
                [
                  0.9817184643510055,
                  0.09124767225325885,
                  537,
                  49
                ],
                [
                  0.9835466179159049,
                  0.09293680297397769,
                  538,
                  50
                ],
                [
                  0.9853747714808044,
                  0.09276437847866419,
                  539,
                  50
                ],
                [
                  0.9872029250457038,
                  0.09259259259259259,
                  540,
                  50
                ],
                [
                  0.9890310786106032,
                  0.09242144177449169,
                  541,
                  50
                ],
                [
                  0.9908592321755028,
                  0.09225092250922509,
                  542,
                  50
                ],
                [
                  0.9926873857404022,
                  0.09392265193370165,
                  543,
                  51
                ],
                [
                  0.9945155393053017,
                  0.09558823529411764,
                  544,
                  52
                ],
                [
                  0.9963436928702011,
                  0.09724770642201835,
                  545,
                  53
                ],
                [
                  0.9981718464351006,
                  0.09706959706959707,
                  546,
                  53
                ],
                [
                  1.0,
                  0.09872029250457039,
                  547,
                  54
                ]
              ]
            },
            {
              "language": "English",
              "run": 399,
              "points": [
                [
                  0.0018281535648994515,
                  0.0,
                  1,
                  0
                ],
                [
                  0.003656307129798903,
                  0.0,
                  2,
                  0
                ],
                [
                  0.005484460694698354,
                  0.0,
                  3,
                  0
                ],
                [
                  0.007312614259597806,
                  0.0,
                  4,
                  0
                ],
                [
                  0.009140767824497258,
                  0.0,
                  5,
                  0
                ],
                [
                  0.010968921389396709,
                  0.0,
                  6,
                  0
                ],
                [
                  0.012797074954296161,
                  0.0,
                  7,
                  0
                ],
                [
                  0.014625228519195612,
                  0.0,
                  8,
                  0
                ],
                [
                  0.016453382084095063,
                  0.0,
                  9,
                  0
                ],
                [
                  0.018281535648994516,
                  0.0,
                  10,
                  0
                ],
                [
                  0.02010968921389397,
                  0.0,
                  11,
                  0
                ],
                [
                  0.021937842778793418,
                  0.0,
                  12,
                  0
                ],
                [
                  0.02376599634369287,
                  0.0,
                  13,
                  0
                ],
                [
                  0.025594149908592323,
                  0.0,
                  14,
                  0
                ],
                [
                  0.027422303473491772,
                  0.0,
                  15,
                  0
                ],
                [
                  0.029250457038391225,
                  0.0,
                  16,
                  0
                ],
                [
                  0.031078610603290677,
                  0.0,
                  17,
                  0
                ],
                [
                  0.03290676416819013,
                  0.0,
                  18,
                  0
                ],
                [
                  0.03473491773308958,
                  0.0,
                  19,
                  0
                ],
                [
                  0.03656307129798903,
                  0.0,
                  20,
                  0
                ],
                [
                  0.038391224862888484,
                  0.0,
                  21,
                  0
                ],
                [
                  0.04021937842778794,
                  0.0,
                  22,
                  0
                ],
                [
                  0.04204753199268738,
                  0.0,
                  23,
                  0
                ],
                [
                  0.043875685557586835,
                  0.0,
                  24,
                  0
                ],
                [
                  0.04570383912248629,
                  0.0,
                  25,
                  0
                ],
                [
                  0.04753199268738574,
                  0.0,
                  26,
                  0
                ],
                [
                  0.04936014625228519,
                  0.0,
                  27,
                  0
                ],
                [
                  0.051188299817184646,
                  0.0,
                  28,
                  0
                ],
                [
                  0.05301645338208409,
                  0.0,
                  29,
                  0
                ],
                [
                  0.054844606946983544,
                  0.0,
                  30,
                  0
                ],
                [
                  0.056672760511883,
                  0.0,
                  31,
                  0
                ],
                [
                  0.05850091407678245,
                  0.0,
                  32,
                  0
                ],
                [
                  0.0603290676416819,
                  0.0,
                  33,
                  0
                ],
                [
                  0.062157221206581355,
                  0.0,
                  34,
                  0
                ],
                [
                  0.06398537477148081,
                  0.0,
                  35,
                  0
                ],
                [
                  0.06581352833638025,
                  0.0,
                  36,
                  0
                ],
                [
                  0.06764168190127971,
                  0.0,
                  37,
                  0
                ],
                [
                  0.06946983546617916,
                  0.0,
                  38,
                  0
                ],
                [
                  0.0712979890310786,
                  0.0,
                  39,
                  0
                ],
                [
                  0.07312614259597806,
                  0.0,
                  40,
                  0
                ],
                [
                  0.07495429616087751,
                  0.0,
                  41,
                  0
                ],
                [
                  0.07678244972577697,
                  0.0,
                  42,
                  0
                ],
                [
                  0.07861060329067641,
                  0.0,
                  43,
                  0
                ],
                [
                  0.08043875685557587,
                  0.0,
                  44,
                  0
                ],
                [
                  0.08226691042047532,
                  0.0,
                  45,
                  0
                ],
                [
                  0.08409506398537477,
                  0.0,
                  46,
                  0
                ],
                [
                  0.08592321755027423,
                  0.0,
                  47,
                  0
                ],
                [
                  0.08775137111517367,
                  0.0,
                  48,
                  0
                ],
                [
                  0.08957952468007313,
                  0.0,
                  49,
                  0
                ],
                [
                  0.09140767824497258,
                  0.0,
                  50,
                  0
                ],
                [
                  0.09323583180987204,
                  0.0,
                  51,
                  0
                ],
                [
                  0.09506398537477148,
                  0.0,
                  52,
                  0
                ],
                [
                  0.09689213893967093,
                  0.0,
                  53,
                  0
                ],
                [
                  0.09872029250457039,
                  0.0,
                  54,
                  0
                ],
                [
                  0.10054844606946983,
                  0.0,
                  55,
                  0
                ],
                [
                  0.10237659963436929,
                  0.0,
                  56,
                  0
                ],
                [
                  0.10420475319926874,
                  0.0,
                  57,
                  0
                ],
                [
                  0.10603290676416818,
                  0.0,
                  58,
                  0
                ],
                [
                  0.10786106032906764,
                  0.0,
                  59,
                  0
                ],
                [
                  0.10968921389396709,
                  0.0,
                  60,
                  0
                ],
                [
                  0.11151736745886655,
                  0.0,
                  61,
                  0
                ],
                [
                  0.113345521023766,
                  0.0,
                  62,
                  0
                ],
                [
                  0.11517367458866545,
                  0.0,
                  63,
                  0
                ],
                [
                  0.1170018281535649,
                  0.0,
                  64,
                  0
                ],
                [
                  0.11882998171846434,
                  0.0,
                  65,
                  0
                ],
                [
                  0.1206581352833638,
                  0.0,
                  66,
                  0
                ],
                [
                  0.12248628884826325,
                  0.0,
                  67,
                  0
                ],
                [
                  0.12431444241316271,
                  0.0,
                  68,
                  0
                ],
                [
                  0.12614259597806216,
                  0.0,
                  69,
                  0
                ],
                [
                  0.12797074954296161,
                  0.0,
                  70,
                  0
                ],
                [
                  0.12979890310786105,
                  0.0,
                  71,
                  0
                ],
                [
                  0.1316270566727605,
                  0.0,
                  72,
                  0
                ],
                [
                  0.13345521023765997,
                  0.0,
                  73,
                  0
                ],
                [
                  0.13528336380255943,
                  0.0,
                  74,
                  0
                ],
                [
                  0.13711151736745886,
                  0.0,
                  75,
                  0
                ],
                [
                  0.13893967093235832,
                  0.0,
                  76,
                  0
                ],
                [
                  0.14076782449725778,
                  0.0,
                  77,
                  0
                ],
                [
                  0.1425959780621572,
                  0.0,
                  78,
                  0
                ],
                [
                  0.14442413162705667,
                  0.0,
                  79,
                  0
                ],
                [
                  0.14625228519195613,
                  0.0,
                  80,
                  0
                ],
                [
                  0.1480804387568556,
                  0.0,
                  81,
                  0
                ],
                [
                  0.14990859232175502,
                  0.0,
                  82,
                  0
                ],
                [
                  0.15173674588665448,
                  0.0,
                  83,
                  0
                ],
                [
                  0.15356489945155394,
                  0.0,
                  84,
                  0
                ],
                [
                  0.15539305301645337,
                  0.0,
                  85,
                  0
                ],
                [
                  0.15722120658135283,
                  0.0,
                  86,
                  0
                ],
                [
                  0.1590493601462523,
                  0.0,
                  87,
                  0
                ],
                [
                  0.16087751371115175,
                  0.0,
                  88,
                  0
                ],
                [
                  0.16270566727605118,
                  0.0,
                  89,
                  0
                ],
                [
                  0.16453382084095064,
                  0.0,
                  90,
                  0
                ],
                [
                  0.1663619744058501,
                  0.0,
                  91,
                  0
                ],
                [
                  0.16819012797074953,
                  0.0,
                  92,
                  0
                ],
                [
                  0.170018281535649,
                  0.0,
                  93,
                  0
                ],
                [
                  0.17184643510054845,
                  0.0,
                  94,
                  0
                ],
                [
                  0.1736745886654479,
                  0.0,
                  95,
                  0
                ],
                [
                  0.17550274223034734,
                  0.0,
                  96,
                  0
                ],
                [
                  0.1773308957952468,
                  0.0,
                  97,
                  0
                ],
                [
                  0.17915904936014626,
                  0.0,
                  98,
                  0
                ],
                [
                  0.1809872029250457,
                  0.0,
                  99,
                  0
                ],
                [
                  0.18281535648994515,
                  0.0,
                  100,
                  0
                ],
                [
                  0.1846435100548446,
                  0.0,
                  101,
                  0
                ],
                [
                  0.18647166361974407,
                  0.0,
                  102,
                  0
                ],
                [
                  0.1882998171846435,
                  0.0,
                  103,
                  0
                ],
                [
                  0.19012797074954296,
                  0.0,
                  104,
                  0
                ],
                [
                  0.19195612431444242,
                  0.0,
                  105,
                  0
                ],
                [
                  0.19378427787934185,
                  0.0,
                  106,
                  0
                ],
                [
                  0.1956124314442413,
                  0.0,
                  107,
                  0
                ],
                [
                  0.19744058500914077,
                  0.0,
                  108,
                  0
                ],
                [
                  0.19926873857404023,
                  0.0,
                  109,
                  0
                ],
                [
                  0.20109689213893966,
                  0.0,
                  110,
                  0
                ],
                [
                  0.20292504570383912,
                  0.0,
                  111,
                  0
                ],
                [
                  0.20475319926873858,
                  0.0,
                  112,
                  0
                ],
                [
                  0.20658135283363802,
                  0.0,
                  113,
                  0
                ],
                [
                  0.20840950639853748,
                  0.0,
                  114,
                  0
                ],
                [
                  0.21023765996343693,
                  0.0,
                  115,
                  0
                ],
                [
                  0.21206581352833637,
                  0.0,
                  116,
                  0
                ],
                [
                  0.21389396709323583,
                  0.0,
                  117,
                  0
                ],
                [
                  0.21572212065813529,
                  0.0,
                  118,
                  0
                ],
                [
                  0.21755027422303475,
                  0.0,
                  119,
                  0
                ],
                [
                  0.21937842778793418,
                  0.0,
                  120,
                  0
                ],
                [
                  0.22120658135283364,
                  0.0,
                  121,
                  0
                ],
                [
                  0.2230347349177331,
                  0.0,
                  122,
                  0
                ],
                [
                  0.22486288848263253,
                  0.0,
                  123,
                  0
                ],
                [
                  0.226691042047532,
                  0.0,
                  124,
                  0
                ],
                [
                  0.22851919561243145,
                  0.0,
                  125,
                  0
                ],
                [
                  0.2303473491773309,
                  0.0,
                  126,
                  0
                ],
                [
                  0.23217550274223034,
                  0.0,
                  127,
                  0
                ],
                [
                  0.2340036563071298,
                  0.0,
                  128,
                  0
                ],
                [
                  0.23583180987202926,
                  0.0,
                  129,
                  0
                ],
                [
                  0.2376599634369287,
                  0.0,
                  130,
                  0
                ],
                [
                  0.23948811700182815,
                  0.0,
                  131,
                  0
                ],
                [
                  0.2413162705667276,
                  0.0,
                  132,
                  0
                ],
                [
                  0.24314442413162707,
                  0.0,
                  133,
                  0
                ],
                [
                  0.2449725776965265,
                  0.0,
                  134,
                  0
                ],
                [
                  0.24680073126142596,
                  0.0,
                  135,
                  0
                ],
                [
                  0.24862888482632542,
                  0.0,
                  136,
                  0
                ],
                [
                  0.25045703839122485,
                  0.0,
                  137,
                  0
                ],
                [
                  0.2522851919561243,
                  0.0,
                  138,
                  0
                ],
                [
                  0.25411334552102377,
                  0.0,
                  139,
                  0
                ],
                [
                  0.25594149908592323,
                  0.0,
                  140,
                  0
                ],
                [
                  0.2577696526508227,
                  0.0,
                  141,
                  0
                ],
                [
                  0.2595978062157221,
                  0.0,
                  142,
                  0
                ],
                [
                  0.26142595978062155,
                  0.0,
                  143,
                  0
                ],
                [
                  0.263254113345521,
                  0.0,
                  144,
                  0
                ],
                [
                  0.26508226691042047,
                  0.0,
                  145,
                  0
                ],
                [
                  0.26691042047531993,
                  0.0,
                  146,
                  0
                ],
                [
                  0.2687385740402194,
                  0.0,
                  147,
                  0
                ],
                [
                  0.27056672760511885,
                  0.0,
                  148,
                  0
                ],
                [
                  0.27239488117001825,
                  0.0,
                  149,
                  0
                ],
                [
                  0.2742230347349177,
                  0.0,
                  150,
                  0
                ],
                [
                  0.2760511882998172,
                  0.0,
                  151,
                  0
                ],
                [
                  0.27787934186471663,
                  0.0,
                  152,
                  0
                ],
                [
                  0.2797074954296161,
                  0.0,
                  153,
                  0
                ],
                [
                  0.28153564899451555,
                  0.0,
                  154,
                  0
                ],
                [
                  0.283363802559415,
                  0.0,
                  155,
                  0
                ],
                [
                  0.2851919561243144,
                  0.0,
                  156,
                  0
                ],
                [
                  0.2870201096892139,
                  0.0,
                  157,
                  0
                ],
                [
                  0.28884826325411334,
                  0.0,
                  158,
                  0
                ],
                [
                  0.2906764168190128,
                  0.0,
                  159,
                  0
                ],
                [
                  0.29250457038391225,
                  0.0,
                  160,
                  0
                ],
                [
                  0.2943327239488117,
                  0.0,
                  161,
                  0
                ],
                [
                  0.2961608775137112,
                  0.0,
                  162,
                  0
                ],
                [
                  0.2979890310786106,
                  0.0,
                  163,
                  0
                ],
                [
                  0.29981718464351004,
                  0.0,
                  164,
                  0
                ],
                [
                  0.3016453382084095,
                  0.0,
                  165,
                  0
                ],
                [
                  0.30347349177330896,
                  0.0,
                  166,
                  0
                ],
                [
                  0.3053016453382084,
                  0.0,
                  167,
                  0
                ],
                [
                  0.3071297989031079,
                  0.0,
                  168,
                  0
                ],
                [
                  0.30895795246800734,
                  0.0,
                  169,
                  0
                ],
                [
                  0.31078610603290674,
                  0.0,
                  170,
                  0
                ],
                [
                  0.3126142595978062,
                  0.0,
                  171,
                  0
                ],
                [
                  0.31444241316270566,
                  0.0,
                  172,
                  0
                ],
                [
                  0.3162705667276051,
                  0.005780346820809248,
                  173,
                  1
                ],
                [
                  0.3180987202925046,
                  0.005747126436781609,
                  174,
                  1
                ],
                [
                  0.31992687385740404,
                  0.005714285714285714,
                  175,
                  1
                ],
                [
                  0.3217550274223035,
                  0.005681818181818182,
                  176,
                  1
                ],
                [
                  0.3235831809872029,
                  0.005649717514124294,
                  177,
                  1
                ],
                [
                  0.32541133455210236,
                  0.0056179775280898875,
                  178,
                  1
                ],
                [
                  0.3272394881170018,
                  0.00558659217877095,
                  179,
                  1
                ],
                [
                  0.3290676416819013,
                  0.005555555555555556,
                  180,
                  1
                ],
                [
                  0.33089579524680074,
                  0.0055248618784530384,
                  181,
                  1
                ],
                [
                  0.3327239488117002,
                  0.005494505494505495,
                  182,
                  1
                ],
                [
                  0.33455210237659966,
                  0.00546448087431694,
                  183,
                  1
                ],
                [
                  0.33638025594149906,
                  0.005434782608695652,
                  184,
                  1
                ],
                [
                  0.3382084095063985,
                  0.005405405405405406,
                  185,
                  1
                ],
                [
                  0.340036563071298,
                  0.005376344086021506,
                  186,
                  1
                ],
                [
                  0.34186471663619744,
                  0.0053475935828877,
                  187,
                  1
                ],
                [
                  0.3436928702010969,
                  0.005319148936170213,
                  188,
                  1
                ],
                [
                  0.34552102376599636,
                  0.005291005291005291,
                  189,
                  1
                ],
                [
                  0.3473491773308958,
                  0.005263157894736842,
                  190,
                  1
                ],
                [
                  0.3491773308957952,
                  0.005235602094240838,
                  191,
                  1
                ],
                [
                  0.3510054844606947,
                  0.005208333333333333,
                  192,
                  1
                ],
                [
                  0.35283363802559414,
                  0.0051813471502590676,
                  193,
                  1
                ],
                [
                  0.3546617915904936,
                  0.005154639175257732,
                  194,
                  1
                ],
                [
                  0.35648994515539306,
                  0.005128205128205128,
                  195,
                  1
                ],
                [
                  0.3583180987202925,
                  0.00510204081632653,
                  196,
                  1
                ],
                [
                  0.360146252285192,
                  0.005076142131979695,
                  197,
                  1
                ],
                [
                  0.3619744058500914,
                  0.005050505050505051,
                  198,
                  1
                ],
                [
                  0.36380255941499084,
                  0.005025125628140704,
                  199,
                  1
                ],
                [
                  0.3656307129798903,
                  0.005,
                  200,
                  1
                ],
                [
                  0.36745886654478976,
                  0.004975124378109453,
                  201,
                  1
                ],
                [
                  0.3692870201096892,
                  0.0049504950495049506,
                  202,
                  1
                ],
                [
                  0.3711151736745887,
                  0.0049261083743842365,
                  203,
                  1
                ],
                [
                  0.37294332723948814,
                  0.004901960784313725,
                  204,
                  1
                ],
                [
                  0.37477148080438755,
                  0.004878048780487805,
                  205,
                  1
                ],
                [
                  0.376599634369287,
                  0.0048543689320388345,
                  206,
                  1
                ],
                [
                  0.37842778793418647,
                  0.004830917874396135,
                  207,
                  1
                ],
                [
                  0.3802559414990859,
                  0.004807692307692308,
                  208,
                  1
                ],
                [
                  0.3820840950639854,
                  0.004784688995215311,
                  209,
                  1
                ],
                [
                  0.38391224862888484,
                  0.004761904761904762,
                  210,
                  1
                ],
                [
                  0.3857404021937843,
                  0.004739336492890996,
                  211,
                  1
                ],
                [
                  0.3875685557586837,
                  0.0047169811320754715,
                  212,
                  1
                ],
                [
                  0.38939670932358317,
                  0.004694835680751174,
                  213,
                  1
                ],
                [
                  0.3912248628884826,
                  0.004672897196261682,
                  214,
                  1
                ],
                [
                  0.3930530164533821,
                  0.004651162790697674,
                  215,
                  1
                ],
                [
                  0.39488117001828155,
                  0.004629629629629629,
                  216,
                  1
                ],
                [
                  0.396709323583181,
                  0.004608294930875576,
                  217,
                  1
                ],
                [
                  0.39853747714808047,
                  0.0045871559633027525,
                  218,
                  1
                ],
                [
                  0.40036563071297987,
                  0.0045662100456621,
                  219,
                  1
                ],
                [
                  0.40219378427787933,
                  0.004545454545454545,
                  220,
                  1
                ],
                [
                  0.4040219378427788,
                  0.004524886877828055,
                  221,
                  1
                ],
                [
                  0.40585009140767825,
                  0.0045045045045045045,
                  222,
                  1
                ],
                [
                  0.4076782449725777,
                  0.004484304932735426,
                  223,
                  1
                ],
                [
                  0.40950639853747717,
                  0.004464285714285714,
                  224,
                  1
                ],
                [
                  0.4113345521023766,
                  0.0044444444444444444,
                  225,
                  1
                ],
                [
                  0.41316270566727603,
                  0.004424778761061947,
                  226,
                  1
                ],
                [
                  0.4149908592321755,
                  0.004405286343612335,
                  227,
                  1
                ],
                [
                  0.41681901279707495,
                  0.0043859649122807015,
                  228,
                  1
                ],
                [
                  0.4186471663619744,
                  0.004366812227074236,
                  229,
                  1
                ],
                [
                  0.42047531992687387,
                  0.004347826086956522,
                  230,
                  1
                ],
                [
                  0.42230347349177333,
                  0.004329004329004329,
                  231,
                  1
                ],
                [
                  0.42413162705667273,
                  0.004310344827586207,
                  232,
                  1
                ],
                [
                  0.4259597806215722,
                  0.004291845493562232,
                  233,
                  1
                ],
                [
                  0.42778793418647165,
                  0.004273504273504274,
                  234,
                  1
                ],
                [
                  0.4296160877513711,
                  0.00425531914893617,
                  235,
                  1
                ],
                [
                  0.43144424131627057,
                  0.00423728813559322,
                  236,
                  1
                ],
                [
                  0.43327239488117003,
                  0.004219409282700422,
                  237,
                  1
                ],
                [
                  0.4351005484460695,
                  0.004201680672268907,
                  238,
                  1
                ],
                [
                  0.4369287020109689,
                  0.0041841004184100415,
                  239,
                  1
                ],
                [
                  0.43875685557586835,
                  0.004166666666666667,
                  240,
                  1
                ],
                [
                  0.4405850091407678,
                  0.004149377593360996,
                  241,
                  1
                ],
                [
                  0.4424131627056673,
                  0.004132231404958678,
                  242,
                  1
                ],
                [
                  0.44424131627056673,
                  0.00411522633744856,
                  243,
                  1
                ],
                [
                  0.4460694698354662,
                  0.004098360655737705,
                  244,
                  1
                ],
                [
                  0.44789762340036565,
                  0.004081632653061225,
                  245,
                  1
                ],
                [
                  0.44972577696526506,
                  0.0040650406504065045,
                  246,
                  1
                ],
                [
                  0.4515539305301645,
                  0.004048582995951417,
                  247,
                  1
                ],
                [
                  0.453382084095064,
                  0.004032258064516129,
                  248,
                  1
                ],
                [
                  0.45521023765996343,
                  0.004016064257028112,
                  249,
                  1
                ],
                [
                  0.4570383912248629,
                  0.004,
                  250,
                  1
                ],
                [
                  0.45886654478976235,
                  0.00398406374501992,
                  251,
                  1
                ],
                [
                  0.4606946983546618,
                  0.003968253968253968,
                  252,
                  1
                ],
                [
                  0.4625228519195612,
                  0.003952569169960474,
                  253,
                  1
                ],
                [
                  0.4643510054844607,
                  0.003937007874015748,
                  254,
                  1
                ],
                [
                  0.46617915904936014,
                  0.00392156862745098,
                  255,
                  1
                ],
                [
                  0.4680073126142596,
                  0.00390625,
                  256,
                  1
                ],
                [
                  0.46983546617915906,
                  0.0038910505836575876,
                  257,
                  1
                ],
                [
                  0.4716636197440585,
                  0.003875968992248062,
                  258,
                  1
                ],
                [
                  0.473491773308958,
                  0.003861003861003861,
                  259,
                  1
                ],
                [
                  0.4753199268738574,
                  0.0038461538461538464,
                  260,
                  1
                ],
                [
                  0.47714808043875684,
                  0.0038314176245210726,
                  261,
                  1
                ],
                [
                  0.4789762340036563,
                  0.003816793893129771,
                  262,
                  1
                ],
                [
                  0.48080438756855576,
                  0.0038022813688212928,
                  263,
                  1
                ],
                [
                  0.4826325411334552,
                  0.003787878787878788,
                  264,
                  1
                ],
                [
                  0.4844606946983547,
                  0.0037735849056603774,
                  265,
                  1
                ],
                [
                  0.48628884826325414,
                  0.0037593984962406013,
                  266,
                  1
                ],
                [
                  0.48811700182815354,
                  0.003745318352059925,
                  267,
                  1
                ],
                [
                  0.489945155393053,
                  0.0037313432835820895,
                  268,
                  1
                ],
                [
                  0.49177330895795246,
                  0.0037174721189591076,
                  269,
                  1
                ],
                [
                  0.4936014625228519,
                  0.003703703703703704,
                  270,
                  1
                ],
                [
                  0.4954296160877514,
                  0.0036900369003690036,
                  271,
                  1
                ],
                [
                  0.49725776965265084,
                  0.003676470588235294,
                  272,
                  1
                ],
                [
                  0.4990859232175503,
                  0.003663003663003663,
                  273,
                  1
                ],
                [
                  0.5009140767824497,
                  0.0036496350364963502,
                  274,
                  1
                ],
                [
                  0.5027422303473492,
                  0.0036363636363636364,
                  275,
                  1
                ],
                [
                  0.5045703839122486,
                  0.0036231884057971015,
                  276,
                  1
                ],
                [
                  0.506398537477148,
                  0.0036101083032490976,
                  277,
                  1
                ],
                [
                  0.5082266910420475,
                  0.0035971223021582736,
                  278,
                  1
                ],
                [
                  0.5100548446069469,
                  0.0035842293906810036,
                  279,
                  1
                ],
                [
                  0.5118829981718465,
                  0.0035714285714285713,
                  280,
                  1
                ],
                [
                  0.5137111517367459,
                  0.0035587188612099642,
                  281,
                  1
                ],
                [
                  0.5155393053016454,
                  0.0035460992907801418,
                  282,
                  1
                ],
                [
                  0.5173674588665448,
                  0.0035335689045936395,
                  283,
                  1
                ],
                [
                  0.5191956124314442,
                  0.0035211267605633804,
                  284,
                  1
                ],
                [
                  0.5210237659963437,
                  0.0035087719298245615,
                  285,
                  1
                ],
                [
                  0.5228519195612431,
                  0.0034965034965034965,
                  286,
                  1
                ],
                [
                  0.5246800731261426,
                  0.003484320557491289,
                  287,
                  1
                ],
                [
                  0.526508226691042,
                  0.003472222222222222,
                  288,
                  1
                ],
                [
                  0.5283363802559415,
                  0.0034602076124567475,
                  289,
                  1
                ],
                [
                  0.5301645338208409,
                  0.0034482758620689655,
                  290,
                  1
                ],
                [
                  0.5319926873857403,
                  0.003436426116838488,
                  291,
                  1
                ],
                [
                  0.5338208409506399,
                  0.003424657534246575,
                  292,
                  1
                ],
                [
                  0.5356489945155393,
                  0.0034129692832764505,
                  293,
                  1
                ],
                [
                  0.5374771480804388,
                  0.003401360544217687,
                  294,
                  1
                ],
                [
                  0.5393053016453382,
                  0.003389830508474576,
                  295,
                  1
                ],
                [
                  0.5411334552102377,
                  0.0033783783783783786,
                  296,
                  1
                ],
                [
                  0.5429616087751371,
                  0.003367003367003367,
                  297,
                  1
                ],
                [
                  0.5447897623400365,
                  0.003355704697986577,
                  298,
                  1
                ],
                [
                  0.546617915904936,
                  0.0033444816053511705,
                  299,
                  1
                ],
                [
                  0.5484460694698354,
                  0.0033333333333333335,
                  300,
                  1
                ],
                [
                  0.5502742230347349,
                  0.0033222591362126247,
                  301,
                  1
                ],
                [
                  0.5521023765996343,
                  0.0033112582781456954,
                  302,
                  1
                ],
                [
                  0.5539305301645339,
                  0.0033003300330033004,
                  303,
                  1
                ],
                [
                  0.5557586837294333,
                  0.003289473684210526,
                  304,
                  1
                ],
                [
                  0.5575868372943327,
                  0.003278688524590164,
                  305,
                  1
                ],
                [
                  0.5594149908592322,
                  0.0032679738562091504,
                  306,
                  1
                ],
                [
                  0.5612431444241316,
                  0.003257328990228013,
                  307,
                  1
                ],
                [
                  0.5630712979890311,
                  0.003246753246753247,
                  308,
                  1
                ],
                [
                  0.5648994515539305,
                  0.003236245954692557,
                  309,
                  1
                ],
                [
                  0.56672760511883,
                  0.0032258064516129032,
                  310,
                  1
                ],
                [
                  0.5685557586837294,
                  0.003215434083601286,
                  311,
                  1
                ],
                [
                  0.5703839122486288,
                  0.003205128205128205,
                  312,
                  1
                ],
                [
                  0.5722120658135283,
                  0.003194888178913738,
                  313,
                  1
                ],
                [
                  0.5740402193784278,
                  0.0031847133757961785,
                  314,
                  1
                ],
                [
                  0.5758683729433273,
                  0.0031746031746031746,
                  315,
                  1
                ],
                [
                  0.5776965265082267,
                  0.0031645569620253164,
                  316,
                  1
                ],
                [
                  0.5795246800731262,
                  0.0031545741324921135,
                  317,
                  1
                ],
                [
                  0.5813528336380256,
                  0.0031446540880503146,
                  318,
                  1
                ],
                [
                  0.583180987202925,
                  0.003134796238244514,
                  319,
                  1
                ],
                [
                  0.5850091407678245,
                  0.003125,
                  320,
                  1
                ],
                [
                  0.5868372943327239,
                  0.003115264797507788,
                  321,
                  1
                ],
                [
                  0.5886654478976234,
                  0.003105590062111801,
                  322,
                  1
                ],
                [
                  0.5904936014625228,
                  0.0030959752321981426,
                  323,
                  1
                ],
                [
                  0.5923217550274223,
                  0.0030864197530864196,
                  324,
                  1
                ],
                [
                  0.5941499085923218,
                  0.003076923076923077,
                  325,
                  1
                ],
                [
                  0.5959780621572212,
                  0.003067484662576687,
                  326,
                  1
                ],
                [
                  0.5978062157221207,
                  0.0030581039755351682,
                  327,
                  1
                ],
                [
                  0.5996343692870201,
                  0.003048780487804878,
                  328,
                  1
                ],
                [
                  0.6014625228519196,
                  0.00303951367781155,
                  329,
                  1
                ],
                [
                  0.603290676416819,
                  0.0030303030303030303,
                  330,
                  1
                ],
                [
                  0.6051188299817185,
                  0.0030211480362537764,
                  331,
                  1
                ],
                [
                  0.6069469835466179,
                  0.0030120481927710845,
                  332,
                  1
                ],
                [
                  0.6087751371115173,
                  0.003003003003003003,
                  333,
                  1
                ],
                [
                  0.6106032906764168,
                  0.0029940119760479044,
                  334,
                  1
                ],
                [
                  0.6124314442413162,
                  0.0029850746268656717,
                  335,
                  1
                ],
                [
                  0.6142595978062158,
                  0.002976190476190476,
                  336,
                  1
                ],
                [
                  0.6160877513711152,
                  0.002967359050445104,
                  337,
                  1
                ],
                [
                  0.6179159049360147,
                  0.0029585798816568047,
                  338,
                  1
                ],
                [
                  0.6197440585009141,
                  0.0029498525073746312,
                  339,
                  1
                ],
                [
                  0.6215722120658135,
                  0.0029411764705882353,
                  340,
                  1
                ],
                [
                  0.623400365630713,
                  0.002932551319648094,
                  341,
                  1
                ],
                [
                  0.6252285191956124,
                  0.005847953216374269,
                  342,
                  2
                ],
                [
                  0.6270566727605119,
                  0.0058309037900874635,
                  343,
                  2
                ],
                [
                  0.6288848263254113,
                  0.005813953488372093,
                  344,
                  2
                ],
                [
                  0.6307129798903108,
                  0.005797101449275362,
                  345,
                  2
                ],
                [
                  0.6325411334552102,
                  0.005780346820809248,
                  346,
                  2
                ],
                [
                  0.6343692870201096,
                  0.005763688760806916,
                  347,
                  2
                ],
                [
                  0.6361974405850092,
                  0.005747126436781609,
                  348,
                  2
                ],
                [
                  0.6380255941499086,
                  0.0057306590257879654,
                  349,
                  2
                ],
                [
                  0.6398537477148081,
                  0.005714285714285714,
                  350,
                  2
                ],
                [
                  0.6416819012797075,
                  0.005698005698005698,
                  351,
                  2
                ],
                [
                  0.643510054844607,
                  0.005681818181818182,
                  352,
                  2
                ],
                [
                  0.6453382084095064,
                  0.0056657223796034,
                  353,
                  2
                ],
                [
                  0.6471663619744058,
                  0.005649717514124294,
                  354,
                  2
                ],
                [
                  0.6489945155393053,
                  0.005633802816901409,
                  355,
                  2
                ],
                [
                  0.6508226691042047,
                  0.008426966292134831,
                  356,
                  3
                ],
                [
                  0.6526508226691042,
                  0.008403361344537815,
                  357,
                  3
                ],
                [
                  0.6544789762340036,
                  0.008379888268156424,
                  358,
                  3
                ],
                [
                  0.6563071297989032,
                  0.008356545961002786,
                  359,
                  3
                ],
                [
                  0.6581352833638026,
                  0.008333333333333333,
                  360,
                  3
                ],
                [
                  0.659963436928702,
                  0.008310249307479225,
                  361,
                  3
                ],
                [
                  0.6617915904936015,
                  0.008287292817679558,
                  362,
                  3
                ],
                [
                  0.6636197440585009,
                  0.008264462809917356,
                  363,
                  3
                ],
                [
                  0.6654478976234004,
                  0.008241758241758242,
                  364,
                  3
                ],
                [
                  0.6672760511882998,
                  0.00821917808219178,
                  365,
                  3
                ],
                [
                  0.6691042047531993,
                  0.00819672131147541,
                  366,
                  3
                ],
                [
                  0.6709323583180987,
                  0.008174386920980926,
                  367,
                  3
                ],
                [
                  0.6727605118829981,
                  0.008152173913043478,
                  368,
                  3
                ],
                [
                  0.6745886654478976,
                  0.008130081300813009,
                  369,
                  3
                ],
                [
                  0.676416819012797,
                  0.008108108108108109,
                  370,
                  3
                ],
                [
                  0.6782449725776966,
                  0.008086253369272238,
                  371,
                  3
                ],
                [
                  0.680073126142596,
                  0.008064516129032258,
                  372,
                  3
                ],
                [
                  0.6819012797074955,
                  0.00804289544235925,
                  373,
                  3
                ],
                [
                  0.6837294332723949,
                  0.008021390374331552,
                  374,
                  3
                ],
                [
                  0.6855575868372943,
                  0.008,
                  375,
                  3
                ],
                [
                  0.6873857404021938,
                  0.007978723404255319,
                  376,
                  3
                ],
                [
                  0.6892138939670932,
                  0.007957559681697613,
                  377,
                  3
                ],
                [
                  0.6910420475319927,
                  0.010582010582010581,
                  378,
                  4
                ],
                [
                  0.6928702010968921,
                  0.010554089709762533,
                  379,
                  4
                ],
                [
                  0.6946983546617916,
                  0.010526315789473684,
                  380,
                  4
                ],
                [
                  0.696526508226691,
                  0.010498687664041995,
                  381,
                  4
                ],
                [
                  0.6983546617915904,
                  0.010471204188481676,
                  382,
                  4
                ],
                [
                  0.70018281535649,
                  0.010443864229765013,
                  383,
                  4
                ],
                [
                  0.7020109689213894,
                  0.010416666666666666,
                  384,
                  4
                ],
                [
                  0.7038391224862889,
                  0.01038961038961039,
                  385,
                  4
                ],
                [
                  0.7056672760511883,
                  0.010362694300518135,
                  386,
                  4
                ],
                [
                  0.7074954296160878,
                  0.0103359173126615,
                  387,
                  4
                ],
                [
                  0.7093235831809872,
                  0.010309278350515464,
                  388,
                  4
                ],
                [
                  0.7111517367458866,
                  0.010282776349614395,
                  389,
                  4
                ],
                [
                  0.7129798903107861,
                  0.01282051282051282,
                  390,
                  5
                ],
                [
                  0.7148080438756855,
                  0.01278772378516624,
                  391,
                  5
                ],
                [
                  0.716636197440585,
                  0.012755102040816327,
                  392,
                  5
                ],
                [
                  0.7184643510054844,
                  0.015267175572519083,
                  393,
                  6
                ],
                [
                  0.720292504570384,
                  0.017766497461928935,
                  394,
                  7
                ],
                [
                  0.7221206581352834,
                  0.017721518987341773,
                  395,
                  7
                ],
                [
                  0.7239488117001828,
                  0.017676767676767676,
                  396,
                  7
                ],
                [
                  0.7257769652650823,
                  0.017632241813602016,
                  397,
                  7
                ],
                [
                  0.7276051188299817,
                  0.017587939698492462,
                  398,
                  7
                ],
                [
                  0.7294332723948812,
                  0.017543859649122806,
                  399,
                  7
                ],
                [
                  0.7312614259597806,
                  0.0175,
                  400,
                  7
                ],
                [
                  0.7330895795246801,
                  0.0199501246882793,
                  401,
                  8
                ],
                [
                  0.7349177330895795,
                  0.022388059701492536,
                  402,
                  9
                ],
                [
                  0.7367458866544789,
                  0.02481389578163772,
                  403,
                  10
                ],
                [
                  0.7385740402193784,
                  0.024752475247524754,
                  404,
                  10
                ],
                [
                  0.7404021937842779,
                  0.024691358024691357,
                  405,
                  10
                ],
                [
                  0.7422303473491774,
                  0.024630541871921183,
                  406,
                  10
                ],
                [
                  0.7440585009140768,
                  0.02457002457002457,
                  407,
                  10
                ],
                [
                  0.7458866544789763,
                  0.024509803921568627,
                  408,
                  10
                ],
                [
                  0.7477148080438757,
                  0.02444987775061125,
                  409,
                  10
                ],
                [
                  0.7495429616087751,
                  0.024390243902439025,
                  410,
                  10
                ],
                [
                  0.7513711151736746,
                  0.024330900243309004,
                  411,
                  10
                ],
                [
                  0.753199268738574,
                  0.024271844660194174,
                  412,
                  10
                ],
                [
                  0.7550274223034735,
                  0.026634382566585957,
                  413,
                  11
                ],
                [
                  0.7568555758683729,
                  0.026570048309178744,
                  414,
                  11
                ],
                [
                  0.7586837294332724,
                  0.02650602409638554,
                  415,
                  11
                ],
                [
                  0.7605118829981719,
                  0.026442307692307692,
                  416,
                  11
                ],
                [
                  0.7623400365630713,
                  0.02877697841726619,
                  417,
                  12
                ],
                [
                  0.7641681901279708,
                  0.028708133971291867,
                  418,
                  12
                ],
                [
                  0.7659963436928702,
                  0.028639618138424822,
                  419,
                  12
                ],
                [
                  0.7678244972577697,
                  0.02857142857142857,
                  420,
                  12
                ],
                [
                  0.7696526508226691,
                  0.028503562945368172,
                  421,
                  12
                ],
                [
                  0.7714808043875686,
                  0.02843601895734597,
                  422,
                  12
                ],
                [
                  0.773308957952468,
                  0.028368794326241134,
                  423,
                  12
                ],
                [
                  0.7751371115173674,
                  0.02830188679245283,
                  424,
                  12
                ],
                [
                  0.7769652650822669,
                  0.02823529411764706,
                  425,
                  12
                ],
                [
                  0.7787934186471663,
                  0.03051643192488263,
                  426,
                  13
                ],
                [
                  0.7806215722120659,
                  0.03044496487119438,
                  427,
                  13
                ],
                [
                  0.7824497257769653,
                  0.030373831775700934,
                  428,
                  13
                ],
                [
                  0.7842778793418648,
                  0.030303030303030304,
                  429,
                  13
                ],
                [
                  0.7861060329067642,
                  0.030232558139534883,
                  430,
                  13
                ],
                [
                  0.7879341864716636,
                  0.03248259860788863,
                  431,
                  14
                ],
                [
                  0.7897623400365631,
                  0.032407407407407406,
                  432,
                  14
                ],
                [
                  0.7915904936014625,
                  0.03464203233256351,
                  433,
                  15
                ],
                [
                  0.793418647166362,
                  0.03686635944700461,
                  434,
                  16
                ],
                [
                  0.7952468007312614,
                  0.0367816091954023,
                  435,
                  16
                ],
                [
                  0.7970749542961609,
                  0.03669724770642202,
                  436,
                  16
                ],
                [
                  0.7989031078610603,
                  0.036613272311212815,
                  437,
                  16
                ],
                [
                  0.8007312614259597,
                  0.0365296803652968,
                  438,
                  16
                ],
                [
                  0.8025594149908593,
                  0.03644646924829157,
                  439,
                  16
                ],
                [
                  0.8043875685557587,
                  0.03636363636363636,
                  440,
                  16
                ],
                [
                  0.8062157221206582,
                  0.036281179138321996,
                  441,
                  16
                ],
                [
                  0.8080438756855576,
                  0.038461538461538464,
                  442,
                  17
                ],
                [
                  0.8098720292504571,
                  0.03837471783295711,
                  443,
                  17
                ],
                [
                  0.8117001828153565,
                  0.04054054054054054,
                  444,
                  18
                ],
                [
                  0.8135283363802559,
                  0.04269662921348315,
                  445,
                  19
                ],
                [
                  0.8153564899451554,
                  0.042600896860986545,
                  446,
                  19
                ],
                [
                  0.8171846435100548,
                  0.042505592841163314,
                  447,
                  19
                ],
                [
                  0.8190127970749543,
                  0.044642857142857144,
                  448,
                  20
                ],
                [
                  0.8208409506398537,
                  0.0467706013363029,
                  449,
                  21
                ],
                [
                  0.8226691042047533,
                  0.04666666666666667,
                  450,
                  21
                ],
                [
                  0.8244972577696527,
                  0.04656319290465632,
                  451,
                  21
                ],
                [
                  0.8263254113345521,
                  0.046460176991150445,
                  452,
                  21
                ],
                [
                  0.8281535648994516,
                  0.046357615894039736,
                  453,
                  21
                ],
                [
                  0.829981718464351,
                  0.046255506607929514,
                  454,
                  21
                ],
                [
                  0.8318098720292505,
                  0.046153846153846156,
                  455,
                  21
                ],
                [
                  0.8336380255941499,
                  0.046052631578947366,
                  456,
                  21
                ],
                [
                  0.8354661791590493,
                  0.04814004376367615,
                  457,
                  22
                ],
                [
                  0.8372943327239488,
                  0.048034934497816595,
                  458,
                  22
                ],
                [
                  0.8391224862888482,
                  0.05010893246187364,
                  459,
                  23
                ],
                [
                  0.8409506398537477,
                  0.05,
                  460,
                  23
                ],
                [
                  0.8427787934186471,
                  0.049891540130151846,
                  461,
                  23
                ],
                [
                  0.8446069469835467,
                  0.049783549783549784,
                  462,
                  23
                ],
                [
                  0.8464351005484461,
                  0.04967602591792657,
                  463,
                  23
                ],
                [
                  0.8482632541133455,
                  0.04956896551724138,
                  464,
                  23
                ],
                [
                  0.850091407678245,
                  0.04946236559139785,
                  465,
                  23
                ],
                [
                  0.8519195612431444,
                  0.04935622317596566,
                  466,
                  23
                ],
                [
                  0.8537477148080439,
                  0.05139186295503212,
                  467,
                  24
                ],
                [
                  0.8555758683729433,
                  0.05128205128205128,
                  468,
                  24
                ],
                [
                  0.8574040219378428,
                  0.0511727078891258,
                  469,
                  24
                ],
                [
                  0.8592321755027422,
                  0.05106382978723404,
                  470,
                  24
                ],
                [
                  0.8610603290676416,
                  0.05307855626326964,
                  471,
                  25
                ],
                [
                  0.8628884826325411,
                  0.05296610169491525,
                  472,
                  25
                ],
                [
                  0.8647166361974405,
                  0.052854122621564484,
                  473,
                  25
                ],
                [
                  0.8665447897623401,
                  0.052742616033755275,
                  474,
                  25
                ],
                [
                  0.8683729433272395,
                  0.05263157894736842,
                  475,
                  25
                ],
                [
                  0.870201096892139,
                  0.052521008403361345,
                  476,
                  25
                ],
                [
                  0.8720292504570384,
                  0.05241090146750524,
                  477,
                  25
                ],
                [
                  0.8738574040219378,
                  0.05439330543933055,
                  478,
                  26
                ],
                [
                  0.8756855575868373,
                  0.054279749478079335,
                  479,
                  26
                ],
                [
                  0.8775137111517367,
                  0.05416666666666667,
                  480,
                  26
                ],
                [
                  0.8793418647166362,
                  0.05405405405405406,
                  481,
                  26
                ],
                [
                  0.8811700182815356,
                  0.056016597510373446,
                  482,
                  27
                ],
                [
                  0.8829981718464351,
                  0.055900621118012424,
                  483,
                  27
                ],
                [
                  0.8848263254113345,
                  0.05578512396694215,
                  484,
                  27
                ],
                [
                  0.886654478976234,
                  0.05567010309278351,
                  485,
                  27
                ],
                [
                  0.8884826325411335,
                  0.05761316872427984,
                  486,
                  28
                ],
                [
                  0.8903107861060329,
                  0.057494866529774126,
                  487,
                  28
                ],
                [
                  0.8921389396709324,
                  0.05737704918032787,
                  488,
                  28
                ],
                [
                  0.8939670932358318,
                  0.05725971370143149,
                  489,
                  28
                ],
                [
                  0.8957952468007313,
                  0.05714285714285714,
                  490,
                  28
                ],
                [
                  0.8976234003656307,
                  0.059063136456211814,
                  491,
                  29
                ],
                [
                  0.8994515539305301,
                  0.05894308943089431,
                  492,
                  29
                ],
                [
                  0.9012797074954296,
                  0.060851926977687626,
                  493,
                  30
                ],
                [
                  0.903107861060329,
                  0.06275303643724696,
                  494,
                  31
                ],
                [
                  0.9049360146252285,
                  0.06262626262626263,
                  495,
                  31
                ],
                [
                  0.906764168190128,
                  0.06451612903225806,
                  496,
                  32
                ],
                [
                  0.9085923217550275,
                  0.06639839034205232,
                  497,
                  33
                ],
                [
                  0.9104204753199269,
                  0.06827309236947791,
                  498,
                  34
                ],
                [
                  0.9122486288848263,
                  0.07014028056112225,
                  499,
                  35
                ],
                [
                  0.9140767824497258,
                  0.07,
                  500,
                  35
                ],
                [
                  0.9159049360146252,
                  0.06986027944111776,
                  501,
                  35
                ],
                [
                  0.9177330895795247,
                  0.0697211155378486,
                  502,
                  35
                ],
                [
                  0.9195612431444241,
                  0.07157057654075547,
                  503,
                  36
                ],
                [
                  0.9213893967093236,
                  0.07142857142857142,
                  504,
                  36
                ],
                [
                  0.923217550274223,
                  0.07128712871287128,
                  505,
                  36
                ],
                [
                  0.9250457038391224,
                  0.07114624505928854,
                  506,
                  36
                ],
                [
                  0.926873857404022,
                  0.07100591715976332,
                  507,
                  36
                ],
                [
                  0.9287020109689214,
                  0.07283464566929133,
                  508,
                  37
                ],
                [
                  0.9305301645338209,
                  0.07269155206286837,
                  509,
                  37
                ],
                [
                  0.9323583180987203,
                  0.07254901960784314,
                  510,
                  37
                ],
                [
                  0.9341864716636198,
                  0.07436399217221135,
                  511,
                  38
                ],
                [
                  0.9360146252285192,
                  0.076171875,
                  512,
                  39
                ],
                [
                  0.9378427787934186,
                  0.07797270955165692,
                  513,
                  40
                ],
                [
                  0.9396709323583181,
                  0.07976653696498054,
                  514,
                  41
                ],
                [
                  0.9414990859232175,
                  0.07961165048543689,
                  515,
                  41
                ],
                [
                  0.943327239488117,
                  0.08139534883720931,
                  516,
                  42
                ],
                [
                  0.9451553930530164,
                  0.08317214700193423,
                  517,
                  43
                ],
                [
                  0.946983546617916,
                  0.08301158301158301,
                  518,
                  43
                ],
                [
                  0.9488117001828154,
                  0.08285163776493257,
                  519,
                  43
                ],
                [
                  0.9506398537477148,
                  0.08461538461538462,
                  520,
                  44
                ],
                [
                  0.9524680073126143,
                  0.08637236084452975,
                  521,
                  45
                ],
                [
                  0.9542961608775137,
                  0.08812260536398467,
                  522,
                  46
                ],
                [
                  0.9561243144424132,
                  0.08795411089866156,
                  523,
                  46
                ],
                [
                  0.9579524680073126,
                  0.08969465648854962,
                  524,
                  47
                ],
                [
                  0.9597806215722121,
                  0.08952380952380952,
                  525,
                  47
                ],
                [
                  0.9616087751371115,
                  0.09125475285171103,
                  526,
                  48
                ],
                [
                  0.9634369287020109,
                  0.09297912713472485,
                  527,
                  49
                ],
                [
                  0.9652650822669104,
                  0.0946969696969697,
                  528,
                  50
                ],
                [
                  0.9670932358318098,
                  0.0945179584120983,
                  529,
                  50
                ],
                [
                  0.9689213893967094,
                  0.09622641509433963,
                  530,
                  51
                ],
                [
                  0.9707495429616088,
                  0.096045197740113,
                  531,
                  51
                ],
                [
                  0.9725776965265083,
                  0.09774436090225563,
                  532,
                  52
                ],
                [
                  0.9744058500914077,
                  0.09943714821763602,
                  533,
                  53
                ],
                [
                  0.9762340036563071,
                  0.10112359550561797,
                  534,
                  54
                ],
                [
                  0.9780621572212066,
                  0.102803738317757,
                  535,
                  55
                ],
                [
                  0.979890310786106,
                  0.1044776119402985,
                  536,
                  56
                ],
                [
                  0.9817184643510055,
                  0.1042830540037244,
                  537,
                  56
                ],
                [
                  0.9835466179159049,
                  0.10594795539033457,
                  538,
                  57
                ],
                [
                  0.9853747714808044,
                  0.10760667903525047,
                  539,
                  58
                ],
                [
                  0.9872029250457038,
                  0.10740740740740741,
                  540,
                  58
                ],
                [
                  0.9890310786106032,
                  0.10720887245841035,
                  541,
                  58
                ],
                [
                  0.9908592321755028,
                  0.1088560885608856,
                  542,
                  59
                ],
                [
                  0.9926873857404022,
                  0.11049723756906077,
                  543,
                  60
                ],
                [
                  0.9945155393053017,
                  0.11213235294117647,
                  544,
                  61
                ],
                [
                  0.9963436928702011,
                  0.11376146788990826,
                  545,
                  62
                ],
                [
                  0.9981718464351006,
                  0.11538461538461539,
                  546,
                  63
                ],
                [
                  1.0,
                  0.11517367458866545,
                  547,
                  63
                ]
              ]
            }
          ]
        },
        {
          "name": "Liquid d1",
          "text_run": 52,
          "native": [
            {
              "language": "Turkish",
              "wording": "Bilmiyorum",
              "run": 472,
              "full_bank_restricted": false,
              "right": 457,
              "wrong": 89,
              "abstained": 1,
              "invalid": 0,
              "coverage": 0.9981718464351006,
              "risk": 0.163003663003663
            },
            {
              "language": "Turkish",
              "wording": "I don't know",
              "run": null
            },
            {
              "language": "English",
              "wording": "I don't know",
              "run": 445,
              "full_bank_restricted": false,
              "right": 478,
              "wrong": 67,
              "abstained": 2,
              "invalid": 0,
              "coverage": 0.9963436928702011,
              "risk": 0.12293577981651377
            }
          ],
          "curves": [
            {
              "language": "Turkish",
              "run": 52,
              "points": [
                [
                  0.0018281535648994515,
                  0.0,
                  1,
                  0
                ],
                [
                  0.003656307129798903,
                  0.0,
                  2,
                  0
                ],
                [
                  0.005484460694698354,
                  0.0,
                  3,
                  0
                ],
                [
                  0.007312614259597806,
                  0.0,
                  4,
                  0
                ],
                [
                  0.009140767824497258,
                  0.0,
                  5,
                  0
                ],
                [
                  0.010968921389396709,
                  0.0,
                  6,
                  0
                ],
                [
                  0.012797074954296161,
                  0.0,
                  7,
                  0
                ],
                [
                  0.014625228519195612,
                  0.0,
                  8,
                  0
                ],
                [
                  0.016453382084095063,
                  0.0,
                  9,
                  0
                ],
                [
                  0.018281535648994516,
                  0.0,
                  10,
                  0
                ],
                [
                  0.02010968921389397,
                  0.0,
                  11,
                  0
                ],
                [
                  0.021937842778793418,
                  0.0,
                  12,
                  0
                ],
                [
                  0.02376599634369287,
                  0.0,
                  13,
                  0
                ],
                [
                  0.025594149908592323,
                  0.0,
                  14,
                  0
                ],
                [
                  0.027422303473491772,
                  0.0,
                  15,
                  0
                ],
                [
                  0.029250457038391225,
                  0.0,
                  16,
                  0
                ],
                [
                  0.031078610603290677,
                  0.0,
                  17,
                  0
                ],
                [
                  0.03290676416819013,
                  0.0,
                  18,
                  0
                ],
                [
                  0.03473491773308958,
                  0.0,
                  19,
                  0
                ],
                [
                  0.03656307129798903,
                  0.0,
                  20,
                  0
                ],
                [
                  0.038391224862888484,
                  0.0,
                  21,
                  0
                ],
                [
                  0.04021937842778794,
                  0.0,
                  22,
                  0
                ],
                [
                  0.04204753199268738,
                  0.0,
                  23,
                  0
                ],
                [
                  0.043875685557586835,
                  0.0,
                  24,
                  0
                ],
                [
                  0.04570383912248629,
                  0.0,
                  25,
                  0
                ],
                [
                  0.04753199268738574,
                  0.0,
                  26,
                  0
                ],
                [
                  0.04936014625228519,
                  0.0,
                  27,
                  0
                ],
                [
                  0.051188299817184646,
                  0.0,
                  28,
                  0
                ],
                [
                  0.05301645338208409,
                  0.0,
                  29,
                  0
                ],
                [
                  0.054844606946983544,
                  0.0,
                  30,
                  0
                ],
                [
                  0.056672760511883,
                  0.0,
                  31,
                  0
                ],
                [
                  0.05850091407678245,
                  0.0,
                  32,
                  0
                ],
                [
                  0.0603290676416819,
                  0.0,
                  33,
                  0
                ],
                [
                  0.062157221206581355,
                  0.0,
                  34,
                  0
                ],
                [
                  0.06398537477148081,
                  0.0,
                  35,
                  0
                ],
                [
                  0.06581352833638025,
                  0.0,
                  36,
                  0
                ],
                [
                  0.06764168190127971,
                  0.0,
                  37,
                  0
                ],
                [
                  0.06946983546617916,
                  0.0,
                  38,
                  0
                ],
                [
                  0.0712979890310786,
                  0.0,
                  39,
                  0
                ],
                [
                  0.07312614259597806,
                  0.0,
                  40,
                  0
                ],
                [
                  0.07495429616087751,
                  0.0,
                  41,
                  0
                ],
                [
                  0.07678244972577697,
                  0.0,
                  42,
                  0
                ],
                [
                  0.07861060329067641,
                  0.0,
                  43,
                  0
                ],
                [
                  0.08043875685557587,
                  0.0,
                  44,
                  0
                ],
                [
                  0.08226691042047532,
                  0.0,
                  45,
                  0
                ],
                [
                  0.08409506398537477,
                  0.0,
                  46,
                  0
                ],
                [
                  0.08592321755027423,
                  0.0,
                  47,
                  0
                ],
                [
                  0.08775137111517367,
                  0.0,
                  48,
                  0
                ],
                [
                  0.08957952468007313,
                  0.0,
                  49,
                  0
                ],
                [
                  0.09140767824497258,
                  0.0,
                  50,
                  0
                ],
                [
                  0.09323583180987204,
                  0.0,
                  51,
                  0
                ],
                [
                  0.09506398537477148,
                  0.0,
                  52,
                  0
                ],
                [
                  0.09689213893967093,
                  0.0,
                  53,
                  0
                ],
                [
                  0.09872029250457039,
                  0.0,
                  54,
                  0
                ],
                [
                  0.10054844606946983,
                  0.0,
                  55,
                  0
                ],
                [
                  0.10237659963436929,
                  0.0,
                  56,
                  0
                ],
                [
                  0.10420475319926874,
                  0.0,
                  57,
                  0
                ],
                [
                  0.10603290676416818,
                  0.0,
                  58,
                  0
                ],
                [
                  0.10786106032906764,
                  0.0,
                  59,
                  0
                ],
                [
                  0.10968921389396709,
                  0.0,
                  60,
                  0
                ],
                [
                  0.11151736745886655,
                  0.0,
                  61,
                  0
                ],
                [
                  0.113345521023766,
                  0.0,
                  62,
                  0
                ],
                [
                  0.11517367458866545,
                  0.0,
                  63,
                  0
                ],
                [
                  0.1170018281535649,
                  0.0,
                  64,
                  0
                ],
                [
                  0.11882998171846434,
                  0.0,
                  65,
                  0
                ],
                [
                  0.1206581352833638,
                  0.0,
                  66,
                  0
                ],
                [
                  0.12248628884826325,
                  0.0,
                  67,
                  0
                ],
                [
                  0.12431444241316271,
                  0.0,
                  68,
                  0
                ],
                [
                  0.12614259597806216,
                  0.0,
                  69,
                  0
                ],
                [
                  0.12797074954296161,
                  0.0,
                  70,
                  0
                ],
                [
                  0.12979890310786105,
                  0.0,
                  71,
                  0
                ],
                [
                  0.1316270566727605,
                  0.0,
                  72,
                  0
                ],
                [
                  0.13345521023765997,
                  0.0,
                  73,
                  0
                ],
                [
                  0.13528336380255943,
                  0.0,
                  74,
                  0
                ],
                [
                  0.13711151736745886,
                  0.0,
                  75,
                  0
                ],
                [
                  0.13893967093235832,
                  0.0,
                  76,
                  0
                ],
                [
                  0.14076782449725778,
                  0.0,
                  77,
                  0
                ],
                [
                  0.1425959780621572,
                  0.0,
                  78,
                  0
                ],
                [
                  0.14442413162705667,
                  0.0,
                  79,
                  0
                ],
                [
                  0.14625228519195613,
                  0.0,
                  80,
                  0
                ],
                [
                  0.1480804387568556,
                  0.0,
                  81,
                  0
                ],
                [
                  0.14990859232175502,
                  0.0,
                  82,
                  0
                ],
                [
                  0.15173674588665448,
                  0.0,
                  83,
                  0
                ],
                [
                  0.15356489945155394,
                  0.0,
                  84,
                  0
                ],
                [
                  0.15539305301645337,
                  0.0,
                  85,
                  0
                ],
                [
                  0.15722120658135283,
                  0.0,
                  86,
                  0
                ],
                [
                  0.1590493601462523,
                  0.0,
                  87,
                  0
                ],
                [
                  0.16087751371115175,
                  0.0,
                  88,
                  0
                ],
                [
                  0.16270566727605118,
                  0.0,
                  89,
                  0
                ],
                [
                  0.16453382084095064,
                  0.0,
                  90,
                  0
                ],
                [
                  0.1663619744058501,
                  0.0,
                  91,
                  0
                ],
                [
                  0.16819012797074953,
                  0.0,
                  92,
                  0
                ],
                [
                  0.170018281535649,
                  0.0,
                  93,
                  0
                ],
                [
                  0.17184643510054845,
                  0.0,
                  94,
                  0
                ],
                [
                  0.1736745886654479,
                  0.0,
                  95,
                  0
                ],
                [
                  0.17550274223034734,
                  0.0,
                  96,
                  0
                ],
                [
                  0.1773308957952468,
                  0.0,
                  97,
                  0
                ],
                [
                  0.17915904936014626,
                  0.0,
                  98,
                  0
                ],
                [
                  0.1809872029250457,
                  0.0,
                  99,
                  0
                ],
                [
                  0.18281535648994515,
                  0.0,
                  100,
                  0
                ],
                [
                  0.1846435100548446,
                  0.0,
                  101,
                  0
                ],
                [
                  0.18647166361974407,
                  0.0,
                  102,
                  0
                ],
                [
                  0.1882998171846435,
                  0.0,
                  103,
                  0
                ],
                [
                  0.19012797074954296,
                  0.0,
                  104,
                  0
                ],
                [
                  0.19195612431444242,
                  0.0,
                  105,
                  0
                ],
                [
                  0.19378427787934185,
                  0.0,
                  106,
                  0
                ],
                [
                  0.1956124314442413,
                  0.0,
                  107,
                  0
                ],
                [
                  0.19744058500914077,
                  0.0,
                  108,
                  0
                ],
                [
                  0.19926873857404023,
                  0.0,
                  109,
                  0
                ],
                [
                  0.20109689213893966,
                  0.0,
                  110,
                  0
                ],
                [
                  0.20292504570383912,
                  0.0,
                  111,
                  0
                ],
                [
                  0.20475319926873858,
                  0.0,
                  112,
                  0
                ],
                [
                  0.20658135283363802,
                  0.0,
                  113,
                  0
                ],
                [
                  0.20840950639853748,
                  0.0,
                  114,
                  0
                ],
                [
                  0.21023765996343693,
                  0.0,
                  115,
                  0
                ],
                [
                  0.21206581352833637,
                  0.0,
                  116,
                  0
                ],
                [
                  0.21389396709323583,
                  0.0,
                  117,
                  0
                ],
                [
                  0.21572212065813529,
                  0.0,
                  118,
                  0
                ],
                [
                  0.21755027422303475,
                  0.0,
                  119,
                  0
                ],
                [
                  0.21937842778793418,
                  0.0,
                  120,
                  0
                ],
                [
                  0.22120658135283364,
                  0.0,
                  121,
                  0
                ],
                [
                  0.2230347349177331,
                  0.0,
                  122,
                  0
                ],
                [
                  0.22486288848263253,
                  0.0,
                  123,
                  0
                ],
                [
                  0.226691042047532,
                  0.0,
                  124,
                  0
                ],
                [
                  0.22851919561243145,
                  0.0,
                  125,
                  0
                ],
                [
                  0.2303473491773309,
                  0.0,
                  126,
                  0
                ],
                [
                  0.23217550274223034,
                  0.0,
                  127,
                  0
                ],
                [
                  0.2340036563071298,
                  0.0,
                  128,
                  0
                ],
                [
                  0.23583180987202926,
                  0.0,
                  129,
                  0
                ],
                [
                  0.2376599634369287,
                  0.0,
                  130,
                  0
                ],
                [
                  0.23948811700182815,
                  0.0,
                  131,
                  0
                ],
                [
                  0.2413162705667276,
                  0.0,
                  132,
                  0
                ],
                [
                  0.24314442413162707,
                  0.0,
                  133,
                  0
                ],
                [
                  0.2449725776965265,
                  0.0,
                  134,
                  0
                ],
                [
                  0.24680073126142596,
                  0.0,
                  135,
                  0
                ],
                [
                  0.24862888482632542,
                  0.0,
                  136,
                  0
                ],
                [
                  0.25045703839122485,
                  0.0,
                  137,
                  0
                ],
                [
                  0.2522851919561243,
                  0.0,
                  138,
                  0
                ],
                [
                  0.25411334552102377,
                  0.0,
                  139,
                  0
                ],
                [
                  0.25594149908592323,
                  0.0,
                  140,
                  0
                ],
                [
                  0.2577696526508227,
                  0.0,
                  141,
                  0
                ],
                [
                  0.2595978062157221,
                  0.0,
                  142,
                  0
                ],
                [
                  0.26142595978062155,
                  0.0,
                  143,
                  0
                ],
                [
                  0.263254113345521,
                  0.0,
                  144,
                  0
                ],
                [
                  0.26508226691042047,
                  0.0,
                  145,
                  0
                ],
                [
                  0.26691042047531993,
                  0.0,
                  146,
                  0
                ],
                [
                  0.2687385740402194,
                  0.0,
                  147,
                  0
                ],
                [
                  0.27056672760511885,
                  0.0,
                  148,
                  0
                ],
                [
                  0.27239488117001825,
                  0.0,
                  149,
                  0
                ],
                [
                  0.2742230347349177,
                  0.0,
                  150,
                  0
                ],
                [
                  0.2760511882998172,
                  0.0,
                  151,
                  0
                ],
                [
                  0.27787934186471663,
                  0.0,
                  152,
                  0
                ],
                [
                  0.2797074954296161,
                  0.0,
                  153,
                  0
                ],
                [
                  0.28153564899451555,
                  0.0,
                  154,
                  0
                ],
                [
                  0.283363802559415,
                  0.0,
                  155,
                  0
                ],
                [
                  0.2851919561243144,
                  0.0,
                  156,
                  0
                ],
                [
                  0.2870201096892139,
                  0.0,
                  157,
                  0
                ],
                [
                  0.28884826325411334,
                  0.0,
                  158,
                  0
                ],
                [
                  0.2906764168190128,
                  0.0,
                  159,
                  0
                ],
                [
                  0.29250457038391225,
                  0.0,
                  160,
                  0
                ],
                [
                  0.2943327239488117,
                  0.0,
                  161,
                  0
                ],
                [
                  0.2961608775137112,
                  0.0,
                  162,
                  0
                ],
                [
                  0.2979890310786106,
                  0.0,
                  163,
                  0
                ],
                [
                  0.29981718464351004,
                  0.0,
                  164,
                  0
                ],
                [
                  0.3016453382084095,
                  0.0,
                  165,
                  0
                ],
                [
                  0.30347349177330896,
                  0.0,
                  166,
                  0
                ],
                [
                  0.3053016453382084,
                  0.0,
                  167,
                  0
                ],
                [
                  0.3071297989031079,
                  0.0,
                  168,
                  0
                ],
                [
                  0.30895795246800734,
                  0.0,
                  169,
                  0
                ],
                [
                  0.31078610603290674,
                  0.0,
                  170,
                  0
                ],
                [
                  0.3126142595978062,
                  0.0,
                  171,
                  0
                ],
                [
                  0.31444241316270566,
                  0.0,
                  172,
                  0
                ],
                [
                  0.3162705667276051,
                  0.0,
                  173,
                  0
                ],
                [
                  0.3180987202925046,
                  0.0,
                  174,
                  0
                ],
                [
                  0.31992687385740404,
                  0.0,
                  175,
                  0
                ],
                [
                  0.3217550274223035,
                  0.0,
                  176,
                  0
                ],
                [
                  0.3235831809872029,
                  0.0,
                  177,
                  0
                ],
                [
                  0.32541133455210236,
                  0.0,
                  178,
                  0
                ],
                [
                  0.3272394881170018,
                  0.0,
                  179,
                  0
                ],
                [
                  0.3290676416819013,
                  0.0,
                  180,
                  0
                ],
                [
                  0.33089579524680074,
                  0.0,
                  181,
                  0
                ],
                [
                  0.3327239488117002,
                  0.0,
                  182,
                  0
                ],
                [
                  0.33455210237659966,
                  0.0,
                  183,
                  0
                ],
                [
                  0.33638025594149906,
                  0.0,
                  184,
                  0
                ],
                [
                  0.3382084095063985,
                  0.0,
                  185,
                  0
                ],
                [
                  0.340036563071298,
                  0.0,
                  186,
                  0
                ],
                [
                  0.34186471663619744,
                  0.0,
                  187,
                  0
                ],
                [
                  0.3436928702010969,
                  0.0,
                  188,
                  0
                ],
                [
                  0.34552102376599636,
                  0.0,
                  189,
                  0
                ],
                [
                  0.3473491773308958,
                  0.0,
                  190,
                  0
                ],
                [
                  0.3491773308957952,
                  0.0,
                  191,
                  0
                ],
                [
                  0.3510054844606947,
                  0.0,
                  192,
                  0
                ],
                [
                  0.35283363802559414,
                  0.0,
                  193,
                  0
                ],
                [
                  0.3546617915904936,
                  0.0,
                  194,
                  0
                ],
                [
                  0.35648994515539306,
                  0.0,
                  195,
                  0
                ],
                [
                  0.3583180987202925,
                  0.0,
                  196,
                  0
                ],
                [
                  0.360146252285192,
                  0.0,
                  197,
                  0
                ],
                [
                  0.3619744058500914,
                  0.0,
                  198,
                  0
                ],
                [
                  0.36380255941499084,
                  0.0,
                  199,
                  0
                ],
                [
                  0.3656307129798903,
                  0.0,
                  200,
                  0
                ],
                [
                  0.36745886654478976,
                  0.0,
                  201,
                  0
                ],
                [
                  0.3692870201096892,
                  0.0,
                  202,
                  0
                ],
                [
                  0.3711151736745887,
                  0.0,
                  203,
                  0
                ],
                [
                  0.37294332723948814,
                  0.0,
                  204,
                  0
                ],
                [
                  0.37477148080438755,
                  0.0,
                  205,
                  0
                ],
                [
                  0.376599634369287,
                  0.0,
                  206,
                  0
                ],
                [
                  0.37842778793418647,
                  0.0,
                  207,
                  0
                ],
                [
                  0.3802559414990859,
                  0.0,
                  208,
                  0
                ],
                [
                  0.3820840950639854,
                  0.0,
                  209,
                  0
                ],
                [
                  0.38391224862888484,
                  0.0,
                  210,
                  0
                ],
                [
                  0.3857404021937843,
                  0.0,
                  211,
                  0
                ],
                [
                  0.3875685557586837,
                  0.0,
                  212,
                  0
                ],
                [
                  0.38939670932358317,
                  0.0,
                  213,
                  0
                ],
                [
                  0.3912248628884826,
                  0.0,
                  214,
                  0
                ],
                [
                  0.3930530164533821,
                  0.0,
                  215,
                  0
                ],
                [
                  0.39488117001828155,
                  0.0,
                  216,
                  0
                ],
                [
                  0.396709323583181,
                  0.0,
                  217,
                  0
                ],
                [
                  0.39853747714808047,
                  0.0,
                  218,
                  0
                ],
                [
                  0.40036563071297987,
                  0.0,
                  219,
                  0
                ],
                [
                  0.40219378427787933,
                  0.0,
                  220,
                  0
                ],
                [
                  0.4040219378427788,
                  0.0,
                  221,
                  0
                ],
                [
                  0.40585009140767825,
                  0.0045045045045045045,
                  222,
                  1
                ],
                [
                  0.4076782449725777,
                  0.004484304932735426,
                  223,
                  1
                ],
                [
                  0.40950639853747717,
                  0.004464285714285714,
                  224,
                  1
                ],
                [
                  0.4113345521023766,
                  0.0044444444444444444,
                  225,
                  1
                ],
                [
                  0.41316270566727603,
                  0.004424778761061947,
                  226,
                  1
                ],
                [
                  0.4149908592321755,
                  0.004405286343612335,
                  227,
                  1
                ],
                [
                  0.41681901279707495,
                  0.0043859649122807015,
                  228,
                  1
                ],
                [
                  0.4186471663619744,
                  0.004366812227074236,
                  229,
                  1
                ],
                [
                  0.42047531992687387,
                  0.004347826086956522,
                  230,
                  1
                ],
                [
                  0.42230347349177333,
                  0.004329004329004329,
                  231,
                  1
                ],
                [
                  0.42413162705667273,
                  0.004310344827586207,
                  232,
                  1
                ],
                [
                  0.4259597806215722,
                  0.004291845493562232,
                  233,
                  1
                ],
                [
                  0.42778793418647165,
                  0.004273504273504274,
                  234,
                  1
                ],
                [
                  0.4296160877513711,
                  0.00425531914893617,
                  235,
                  1
                ],
                [
                  0.43144424131627057,
                  0.00423728813559322,
                  236,
                  1
                ],
                [
                  0.43327239488117003,
                  0.004219409282700422,
                  237,
                  1
                ],
                [
                  0.4351005484460695,
                  0.004201680672268907,
                  238,
                  1
                ],
                [
                  0.4369287020109689,
                  0.0041841004184100415,
                  239,
                  1
                ],
                [
                  0.43875685557586835,
                  0.004166666666666667,
                  240,
                  1
                ],
                [
                  0.4405850091407678,
                  0.004149377593360996,
                  241,
                  1
                ],
                [
                  0.4424131627056673,
                  0.004132231404958678,
                  242,
                  1
                ],
                [
                  0.44424131627056673,
                  0.00411522633744856,
                  243,
                  1
                ],
                [
                  0.4460694698354662,
                  0.00819672131147541,
                  244,
                  2
                ],
                [
                  0.44789762340036565,
                  0.00816326530612245,
                  245,
                  2
                ],
                [
                  0.44972577696526506,
                  0.008130081300813009,
                  246,
                  2
                ],
                [
                  0.4515539305301645,
                  0.008097165991902834,
                  247,
                  2
                ],
                [
                  0.453382084095064,
                  0.008064516129032258,
                  248,
                  2
                ],
                [
                  0.45521023765996343,
                  0.008032128514056224,
                  249,
                  2
                ],
                [
                  0.4570383912248629,
                  0.008,
                  250,
                  2
                ],
                [
                  0.45886654478976235,
                  0.00796812749003984,
                  251,
                  2
                ],
                [
                  0.4606946983546618,
                  0.007936507936507936,
                  252,
                  2
                ],
                [
                  0.4625228519195612,
                  0.007905138339920948,
                  253,
                  2
                ],
                [
                  0.4643510054844607,
                  0.007874015748031496,
                  254,
                  2
                ],
                [
                  0.46617915904936014,
                  0.00784313725490196,
                  255,
                  2
                ],
                [
                  0.4680073126142596,
                  0.0078125,
                  256,
                  2
                ],
                [
                  0.46983546617915906,
                  0.007782101167315175,
                  257,
                  2
                ],
                [
                  0.4716636197440585,
                  0.007751937984496124,
                  258,
                  2
                ],
                [
                  0.473491773308958,
                  0.007722007722007722,
                  259,
                  2
                ],
                [
                  0.4753199268738574,
                  0.007692307692307693,
                  260,
                  2
                ],
                [
                  0.47714808043875684,
                  0.007662835249042145,
                  261,
                  2
                ],
                [
                  0.4789762340036563,
                  0.007633587786259542,
                  262,
                  2
                ],
                [
                  0.48080438756855576,
                  0.0076045627376425855,
                  263,
                  2
                ],
                [
                  0.4826325411334552,
                  0.007575757575757576,
                  264,
                  2
                ],
                [
                  0.4844606946983547,
                  0.007547169811320755,
                  265,
                  2
                ],
                [
                  0.48628884826325414,
                  0.011278195488721804,
                  266,
                  3
                ],
                [
                  0.48811700182815354,
                  0.011235955056179775,
                  267,
                  3
                ],
                [
                  0.489945155393053,
                  0.011194029850746268,
                  268,
                  3
                ],
                [
                  0.49177330895795246,
                  0.011152416356877323,
                  269,
                  3
                ],
                [
                  0.4936014625228519,
                  0.011111111111111112,
                  270,
                  3
                ],
                [
                  0.4954296160877514,
                  0.01107011070110701,
                  271,
                  3
                ],
                [
                  0.49725776965265084,
                  0.011029411764705883,
                  272,
                  3
                ],
                [
                  0.4990859232175503,
                  0.01098901098901099,
                  273,
                  3
                ],
                [
                  0.5009140767824497,
                  0.010948905109489052,
                  274,
                  3
                ],
                [
                  0.5027422303473492,
                  0.01090909090909091,
                  275,
                  3
                ],
                [
                  0.5045703839122486,
                  0.010869565217391304,
                  276,
                  3
                ],
                [
                  0.506398537477148,
                  0.010830324909747292,
                  277,
                  3
                ],
                [
                  0.5082266910420475,
                  0.01079136690647482,
                  278,
                  3
                ],
                [
                  0.5100548446069469,
                  0.010752688172043012,
                  279,
                  3
                ],
                [
                  0.5118829981718465,
                  0.010714285714285714,
                  280,
                  3
                ],
                [
                  0.5137111517367459,
                  0.010676156583629894,
                  281,
                  3
                ],
                [
                  0.5155393053016454,
                  0.010638297872340425,
                  282,
                  3
                ],
                [
                  0.5173674588665448,
                  0.01060070671378092,
                  283,
                  3
                ],
                [
                  0.5191956124314442,
                  0.01056338028169014,
                  284,
                  3
                ],
                [
                  0.5210237659963437,
                  0.010526315789473684,
                  285,
                  3
                ],
                [
                  0.5228519195612431,
                  0.01048951048951049,
                  286,
                  3
                ],
                [
                  0.5246800731261426,
                  0.010452961672473868,
                  287,
                  3
                ],
                [
                  0.526508226691042,
                  0.010416666666666666,
                  288,
                  3
                ],
                [
                  0.5283363802559415,
                  0.010380622837370242,
                  289,
                  3
                ],
                [
                  0.5301645338208409,
                  0.010344827586206896,
                  290,
                  3
                ],
                [
                  0.5319926873857403,
                  0.010309278350515464,
                  291,
                  3
                ],
                [
                  0.5338208409506399,
                  0.010273972602739725,
                  292,
                  3
                ],
                [
                  0.5356489945155393,
                  0.010238907849829351,
                  293,
                  3
                ],
                [
                  0.5374771480804388,
                  0.01020408163265306,
                  294,
                  3
                ],
                [
                  0.5393053016453382,
                  0.010169491525423728,
                  295,
                  3
                ],
                [
                  0.5411334552102377,
                  0.010135135135135136,
                  296,
                  3
                ],
                [
                  0.5429616087751371,
                  0.010101010101010102,
                  297,
                  3
                ],
                [
                  0.5447897623400365,
                  0.010067114093959731,
                  298,
                  3
                ],
                [
                  0.546617915904936,
                  0.010033444816053512,
                  299,
                  3
                ],
                [
                  0.5484460694698354,
                  0.01,
                  300,
                  3
                ],
                [
                  0.5502742230347349,
                  0.009966777408637873,
                  301,
                  3
                ],
                [
                  0.5521023765996343,
                  0.009933774834437087,
                  302,
                  3
                ],
                [
                  0.5539305301645339,
                  0.009900990099009901,
                  303,
                  3
                ],
                [
                  0.5557586837294333,
                  0.009868421052631578,
                  304,
                  3
                ],
                [
                  0.5575868372943327,
                  0.009836065573770493,
                  305,
                  3
                ],
                [
                  0.5594149908592322,
                  0.00980392156862745,
                  306,
                  3
                ],
                [
                  0.5612431444241316,
                  0.009771986970684038,
                  307,
                  3
                ],
                [
                  0.5630712979890311,
                  0.00974025974025974,
                  308,
                  3
                ],
                [
                  0.5648994515539305,
                  0.009708737864077669,
                  309,
                  3
                ],
                [
                  0.56672760511883,
                  0.00967741935483871,
                  310,
                  3
                ],
                [
                  0.5685557586837294,
                  0.00964630225080386,
                  311,
                  3
                ],
                [
                  0.5703839122486288,
                  0.009615384615384616,
                  312,
                  3
                ],
                [
                  0.5722120658135283,
                  0.009584664536741214,
                  313,
                  3
                ],
                [
                  0.5740402193784278,
                  0.009554140127388535,
                  314,
                  3
                ],
                [
                  0.5758683729433273,
                  0.009523809523809525,
                  315,
                  3
                ],
                [
                  0.5776965265082267,
                  0.00949367088607595,
                  316,
                  3
                ],
                [
                  0.5795246800731262,
                  0.00946372239747634,
                  317,
                  3
                ],
                [
                  0.5813528336380256,
                  0.009433962264150943,
                  318,
                  3
                ],
                [
                  0.583180987202925,
                  0.009404388714733543,
                  319,
                  3
                ],
                [
                  0.5850091407678245,
                  0.009375,
                  320,
                  3
                ],
                [
                  0.5868372943327239,
                  0.009345794392523364,
                  321,
                  3
                ],
                [
                  0.5886654478976234,
                  0.009316770186335404,
                  322,
                  3
                ],
                [
                  0.5904936014625228,
                  0.009287925696594427,
                  323,
                  3
                ],
                [
                  0.5923217550274223,
                  0.009259259259259259,
                  324,
                  3
                ],
                [
                  0.5941499085923218,
                  0.009230769230769232,
                  325,
                  3
                ],
                [
                  0.5959780621572212,
                  0.009202453987730062,
                  326,
                  3
                ],
                [
                  0.5978062157221207,
                  0.009174311926605505,
                  327,
                  3
                ],
                [
                  0.5996343692870201,
                  0.009146341463414634,
                  328,
                  3
                ],
                [
                  0.6014625228519196,
                  0.00911854103343465,
                  329,
                  3
                ],
                [
                  0.603290676416819,
                  0.00909090909090909,
                  330,
                  3
                ],
                [
                  0.6051188299817185,
                  0.00906344410876133,
                  331,
                  3
                ],
                [
                  0.6069469835466179,
                  0.009036144578313253,
                  332,
                  3
                ],
                [
                  0.6087751371115173,
                  0.009009009009009009,
                  333,
                  3
                ],
                [
                  0.6106032906764168,
                  0.008982035928143712,
                  334,
                  3
                ],
                [
                  0.6124314442413162,
                  0.008955223880597015,
                  335,
                  3
                ],
                [
                  0.6142595978062158,
                  0.008928571428571428,
                  336,
                  3
                ],
                [
                  0.6160877513711152,
                  0.008902077151335312,
                  337,
                  3
                ],
                [
                  0.6179159049360147,
                  0.008875739644970414,
                  338,
                  3
                ],
                [
                  0.6197440585009141,
                  0.008849557522123894,
                  339,
                  3
                ],
                [
                  0.6215722120658135,
                  0.008823529411764706,
                  340,
                  3
                ],
                [
                  0.623400365630713,
                  0.008797653958944282,
                  341,
                  3
                ],
                [
                  0.6252285191956124,
                  0.008771929824561403,
                  342,
                  3
                ],
                [
                  0.6270566727605119,
                  0.008746355685131196,
                  343,
                  3
                ],
                [
                  0.6288848263254113,
                  0.00872093023255814,
                  344,
                  3
                ],
                [
                  0.6307129798903108,
                  0.008695652173913044,
                  345,
                  3
                ],
                [
                  0.6325411334552102,
                  0.011560693641618497,
                  346,
                  4
                ],
                [
                  0.6343692870201096,
                  0.011527377521613832,
                  347,
                  4
                ],
                [
                  0.6361974405850092,
                  0.011494252873563218,
                  348,
                  4
                ],
                [
                  0.6380255941499086,
                  0.011461318051575931,
                  349,
                  4
                ],
                [
                  0.6398537477148081,
                  0.011428571428571429,
                  350,
                  4
                ],
                [
                  0.6416819012797075,
                  0.011396011396011397,
                  351,
                  4
                ],
                [
                  0.643510054844607,
                  0.011363636363636364,
                  352,
                  4
                ],
                [
                  0.6453382084095064,
                  0.014164305949008499,
                  353,
                  5
                ],
                [
                  0.6471663619744058,
                  0.014124293785310734,
                  354,
                  5
                ],
                [
                  0.6489945155393053,
                  0.016901408450704224,
                  355,
                  6
                ],
                [
                  0.6508226691042047,
                  0.019662921348314606,
                  356,
                  7
                ],
                [
                  0.6526508226691042,
                  0.0196078431372549,
                  357,
                  7
                ],
                [
                  0.6544789762340036,
                  0.019553072625698324,
                  358,
                  7
                ],
                [
                  0.6563071297989032,
                  0.022284122562674095,
                  359,
                  8
                ],
                [
                  0.6581352833638026,
                  0.022222222222222223,
                  360,
                  8
                ],
                [
                  0.659963436928702,
                  0.0221606648199446,
                  361,
                  8
                ],
                [
                  0.6617915904936015,
                  0.022099447513812154,
                  362,
                  8
                ],
                [
                  0.6636197440585009,
                  0.02203856749311295,
                  363,
                  8
                ],
                [
                  0.6654478976234004,
                  0.02197802197802198,
                  364,
                  8
                ],
                [
                  0.6672760511882998,
                  0.024657534246575342,
                  365,
                  9
                ],
                [
                  0.6691042047531993,
                  0.02459016393442623,
                  366,
                  9
                ],
                [
                  0.6709323583180987,
                  0.02452316076294278,
                  367,
                  9
                ],
                [
                  0.6727605118829981,
                  0.024456521739130436,
                  368,
                  9
                ],
                [
                  0.6745886654478976,
                  0.024390243902439025,
                  369,
                  9
                ],
                [
                  0.676416819012797,
                  0.024324324324324326,
                  370,
                  9
                ],
                [
                  0.6782449725776966,
                  0.026954177897574125,
                  371,
                  10
                ],
                [
                  0.680073126142596,
                  0.026881720430107527,
                  372,
                  10
                ],
                [
                  0.6819012797074955,
                  0.02680965147453083,
                  373,
                  10
                ],
                [
                  0.6837294332723949,
                  0.026737967914438502,
                  374,
                  10
                ],
                [
                  0.6855575868372943,
                  0.02666666666666667,
                  375,
                  10
                ],
                [
                  0.6873857404021938,
                  0.026595744680851064,
                  376,
                  10
                ],
                [
                  0.6892138939670932,
                  0.026525198938992044,
                  377,
                  10
                ],
                [
                  0.6910420475319927,
                  0.026455026455026454,
                  378,
                  10
                ],
                [
                  0.6928702010968921,
                  0.026385224274406333,
                  379,
                  10
                ],
                [
                  0.6946983546617916,
                  0.02631578947368421,
                  380,
                  10
                ],
                [
                  0.696526508226691,
                  0.026246719160104987,
                  381,
                  10
                ],
                [
                  0.6983546617915904,
                  0.02617801047120419,
                  382,
                  10
                ],
                [
                  0.70018281535649,
                  0.02610966057441253,
                  383,
                  10
                ],
                [
                  0.7020109689213894,
                  0.026041666666666668,
                  384,
                  10
                ],
                [
                  0.7038391224862889,
                  0.025974025974025976,
                  385,
                  10
                ],
                [
                  0.7056672760511883,
                  0.025906735751295335,
                  386,
                  10
                ],
                [
                  0.7074954296160878,
                  0.025839793281653745,
                  387,
                  10
                ],
                [
                  0.7093235831809872,
                  0.028350515463917526,
                  388,
                  11
                ],
                [
                  0.7111517367458866,
                  0.030848329048843187,
                  389,
                  12
                ],
                [
                  0.7129798903107861,
                  0.03333333333333333,
                  390,
                  13
                ],
                [
                  0.7148080438756855,
                  0.03324808184143223,
                  391,
                  13
                ],
                [
                  0.716636197440585,
                  0.03316326530612245,
                  392,
                  13
                ],
                [
                  0.7184643510054844,
                  0.03307888040712468,
                  393,
                  13
                ],
                [
                  0.720292504570384,
                  0.03299492385786802,
                  394,
                  13
                ],
                [
                  0.7221206581352834,
                  0.03291139240506329,
                  395,
                  13
                ],
                [
                  0.7239488117001828,
                  0.03282828282828283,
                  396,
                  13
                ],
                [
                  0.7257769652650823,
                  0.03526448362720403,
                  397,
                  14
                ],
                [
                  0.7276051188299817,
                  0.035175879396984924,
                  398,
                  14
                ],
                [
                  0.7294332723948812,
                  0.03508771929824561,
                  399,
                  14
                ],
                [
                  0.7312614259597806,
                  0.0375,
                  400,
                  15
                ],
                [
                  0.7330895795246801,
                  0.03740648379052369,
                  401,
                  15
                ],
                [
                  0.7349177330895795,
                  0.03980099502487562,
                  402,
                  16
                ],
                [
                  0.7367458866544789,
                  0.03970223325062035,
                  403,
                  16
                ],
                [
                  0.7385740402193784,
                  0.039603960396039604,
                  404,
                  16
                ],
                [
                  0.7404021937842779,
                  0.03950617283950617,
                  405,
                  16
                ],
                [
                  0.7422303473491774,
                  0.03940886699507389,
                  406,
                  16
                ],
                [
                  0.7440585009140768,
                  0.04176904176904177,
                  407,
                  17
                ],
                [
                  0.7458866544789763,
                  0.041666666666666664,
                  408,
                  17
                ],
                [
                  0.7477148080438757,
                  0.044009779951100246,
                  409,
                  18
                ],
                [
                  0.7495429616087751,
                  0.046341463414634146,
                  410,
                  19
                ],
                [
                  0.7513711151736746,
                  0.046228710462287104,
                  411,
                  19
                ],
                [
                  0.753199268738574,
                  0.04611650485436893,
                  412,
                  19
                ],
                [
                  0.7550274223034735,
                  0.04600484261501211,
                  413,
                  19
                ],
                [
                  0.7568555758683729,
                  0.04830917874396135,
                  414,
                  20
                ],
                [
                  0.7586837294332724,
                  0.04819277108433735,
                  415,
                  20
                ],
                [
                  0.7605118829981719,
                  0.04807692307692308,
                  416,
                  20
                ],
                [
                  0.7623400365630713,
                  0.047961630695443645,
                  417,
                  20
                ],
                [
                  0.7641681901279708,
                  0.04784688995215311,
                  418,
                  20
                ],
                [
                  0.7659963436928702,
                  0.0477326968973747,
                  419,
                  20
                ],
                [
                  0.7678244972577697,
                  0.05,
                  420,
                  21
                ],
                [
                  0.7696526508226691,
                  0.0498812351543943,
                  421,
                  21
                ],
                [
                  0.7714808043875686,
                  0.052132701421800945,
                  422,
                  22
                ],
                [
                  0.773308957952468,
                  0.05200945626477541,
                  423,
                  22
                ],
                [
                  0.7751371115173674,
                  0.054245283018867926,
                  424,
                  23
                ],
                [
                  0.7769652650822669,
                  0.05411764705882353,
                  425,
                  23
                ],
                [
                  0.7787934186471663,
                  0.0539906103286385,
                  426,
                  23
                ],
                [
                  0.7806215722120659,
                  0.05620608899297424,
                  427,
                  24
                ],
                [
                  0.7824497257769653,
                  0.056074766355140186,
                  428,
                  24
                ],
                [
                  0.7842778793418648,
                  0.055944055944055944,
                  429,
                  24
                ],
                [
                  0.7861060329067642,
                  0.05581395348837209,
                  430,
                  24
                ],
                [
                  0.7879341864716636,
                  0.05568445475638051,
                  431,
                  24
                ],
                [
                  0.7897623400365631,
                  0.05555555555555555,
                  432,
                  24
                ],
                [
                  0.7915904936014625,
                  0.05542725173210162,
                  433,
                  24
                ],
                [
                  0.793418647166362,
                  0.055299539170506916,
                  434,
                  24
                ],
                [
                  0.7952468007312614,
                  0.05517241379310345,
                  435,
                  24
                ],
                [
                  0.7970749542961609,
                  0.05504587155963303,
                  436,
                  24
                ],
                [
                  0.7989031078610603,
                  0.05491990846681922,
                  437,
                  24
                ],
                [
                  0.8007312614259597,
                  0.05707762557077625,
                  438,
                  25
                ],
                [
                  0.8025594149908593,
                  0.05922551252847381,
                  439,
                  26
                ],
                [
                  0.8043875685557587,
                  0.05909090909090909,
                  440,
                  26
                ],
                [
                  0.8062157221206582,
                  0.061224489795918366,
                  441,
                  27
                ],
                [
                  0.8080438756855576,
                  0.06108597285067873,
                  442,
                  27
                ],
                [
                  0.8098720292504571,
                  0.06320541760722348,
                  443,
                  28
                ],
                [
                  0.8117001828153565,
                  0.06306306306306306,
                  444,
                  28
                ],
                [
                  0.8135283363802559,
                  0.06292134831460675,
                  445,
                  28
                ],
                [
                  0.8153564899451554,
                  0.06502242152466367,
                  446,
                  29
                ],
                [
                  0.8171846435100548,
                  0.06487695749440715,
                  447,
                  29
                ],
                [
                  0.8190127970749543,
                  0.06696428571428571,
                  448,
                  30
                ],
                [
                  0.8208409506398537,
                  0.066815144766147,
                  449,
                  30
                ],
                [
                  0.8226691042047533,
                  0.06888888888888889,
                  450,
                  31
                ],
                [
                  0.8244972577696527,
                  0.06873614190687362,
                  451,
                  31
                ],
                [
                  0.8263254113345521,
                  0.07079646017699115,
                  452,
                  32
                ],
                [
                  0.8281535648994516,
                  0.0706401766004415,
                  453,
                  32
                ],
                [
                  0.829981718464351,
                  0.07048458149779736,
                  454,
                  32
                ],
                [
                  0.8318098720292505,
                  0.07032967032967033,
                  455,
                  32
                ],
                [
                  0.8336380255941499,
                  0.07236842105263158,
                  456,
                  33
                ],
                [
                  0.8354661791590493,
                  0.07221006564551423,
                  457,
                  33
                ],
                [
                  0.8372943327239488,
                  0.07205240174672489,
                  458,
                  33
                ],
                [
                  0.8391224862888482,
                  0.07407407407407407,
                  459,
                  34
                ],
                [
                  0.8409506398537477,
                  0.07608695652173914,
                  460,
                  35
                ],
                [
                  0.8427787934186471,
                  0.07592190889370933,
                  461,
                  35
                ],
                [
                  0.8446069469835467,
                  0.07575757575757576,
                  462,
                  35
                ],
                [
                  0.8464351005484461,
                  0.0755939524838013,
                  463,
                  35
                ],
                [
                  0.8482632541133455,
                  0.07543103448275862,
                  464,
                  35
                ],
                [
                  0.850091407678245,
                  0.07526881720430108,
                  465,
                  35
                ],
                [
                  0.8519195612431444,
                  0.07510729613733906,
                  466,
                  35
                ],
                [
                  0.8537477148080439,
                  0.07708779443254818,
                  467,
                  36
                ],
                [
                  0.8555758683729433,
                  0.07905982905982906,
                  468,
                  37
                ],
                [
                  0.8574040219378428,
                  0.07889125799573561,
                  469,
                  37
                ],
                [
                  0.8592321755027422,
                  0.07872340425531915,
                  470,
                  37
                ],
                [
                  0.8610603290676416,
                  0.08067940552016985,
                  471,
                  38
                ],
                [
                  0.8628884826325411,
                  0.0826271186440678,
                  472,
                  39
                ],
                [
                  0.8647166361974405,
                  0.08456659619450317,
                  473,
                  40
                ],
                [
                  0.8665447897623401,
                  0.08438818565400844,
                  474,
                  40
                ],
                [
                  0.8683729433272395,
                  0.08421052631578947,
                  475,
                  40
                ],
                [
                  0.870201096892139,
                  0.08403361344537816,
                  476,
                  40
                ],
                [
                  0.8720292504570384,
                  0.0859538784067086,
                  477,
                  41
                ],
                [
                  0.8738574040219378,
                  0.08577405857740586,
                  478,
                  41
                ],
                [
                  0.8756855575868373,
                  0.08559498956158663,
                  479,
                  41
                ],
                [
                  0.8775137111517367,
                  0.08541666666666667,
                  480,
                  41
                ],
                [
                  0.8793418647166362,
                  0.08731808731808732,
                  481,
                  42
                ],
                [
                  0.8811700182815356,
                  0.08921161825726141,
                  482,
                  43
                ],
                [
                  0.8829981718464351,
                  0.09109730848861283,
                  483,
                  44
                ],
                [
                  0.8848263254113345,
                  0.09297520661157024,
                  484,
                  45
                ],
                [
                  0.886654478976234,
                  0.09278350515463918,
                  485,
                  45
                ],
                [
                  0.8884826325411335,
                  0.09259259259259259,
                  486,
                  45
                ],
                [
                  0.8903107861060329,
                  0.09240246406570841,
                  487,
                  45
                ],
                [
                  0.8921389396709324,
                  0.09221311475409837,
                  488,
                  45
                ],
                [
                  0.8939670932358318,
                  0.09406952965235174,
                  489,
                  46
                ],
                [
                  0.8957952468007313,
                  0.09387755102040816,
                  490,
                  46
                ],
                [
                  0.8976234003656307,
                  0.09368635437881874,
                  491,
                  46
                ],
                [
                  0.8994515539305301,
                  0.09349593495934959,
                  492,
                  46
                ],
                [
                  0.9012797074954296,
                  0.09330628803245436,
                  493,
                  46
                ],
                [
                  0.903107861060329,
                  0.0951417004048583,
                  494,
                  47
                ],
                [
                  0.9049360146252285,
                  0.09696969696969697,
                  495,
                  48
                ],
                [
                  0.906764168190128,
                  0.09879032258064516,
                  496,
                  49
                ],
                [
                  0.9085923217550275,
                  0.1006036217303823,
                  497,
                  50
                ],
                [
                  0.9104204753199269,
                  0.10240963855421686,
                  498,
                  51
                ],
                [
                  0.9122486288848263,
                  0.10220440881763528,
                  499,
                  51
                ],
                [
                  0.9140767824497258,
                  0.104,
                  500,
                  52
                ],
                [
                  0.9159049360146252,
                  0.10379241516966067,
                  501,
                  52
                ],
                [
                  0.9177330895795247,
                  0.10557768924302789,
                  502,
                  53
                ],
                [
                  0.9195612431444241,
                  0.1073558648111332,
                  503,
                  54
                ],
                [
                  0.9213893967093236,
                  0.10912698412698413,
                  504,
                  55
                ],
                [
                  0.923217550274223,
                  0.11089108910891089,
                  505,
                  56
                ],
                [
                  0.9250457038391224,
                  0.11067193675889328,
                  506,
                  56
                ],
                [
                  0.926873857404022,
                  0.11045364891518737,
                  507,
                  56
                ],
                [
                  0.9287020109689214,
                  0.11023622047244094,
                  508,
                  56
                ],
                [
                  0.9305301645338209,
                  0.1100196463654224,
                  509,
                  56
                ],
                [
                  0.9323583180987203,
                  0.10980392156862745,
                  510,
                  56
                ],
                [
                  0.9341864716636198,
                  0.11154598825831702,
                  511,
                  57
                ],
                [
                  0.9360146252285192,
                  0.11328125,
                  512,
                  58
                ],
                [
                  0.9378427787934186,
                  0.11500974658869395,
                  513,
                  59
                ],
                [
                  0.9396709323583181,
                  0.11673151750972763,
                  514,
                  60
                ],
                [
                  0.9414990859232175,
                  0.11650485436893204,
                  515,
                  60
                ],
                [
                  0.943327239488117,
                  0.1182170542635659,
                  516,
                  61
                ],
                [
                  0.9451553930530164,
                  0.11992263056092843,
                  517,
                  62
                ],
                [
                  0.946983546617916,
                  0.12162162162162163,
                  518,
                  63
                ],
                [
                  0.9488117001828154,
                  0.1233140655105973,
                  519,
                  64
                ],
                [
                  0.9506398537477148,
                  0.125,
                  520,
                  65
                ],
                [
                  0.9524680073126143,
                  0.12667946257197696,
                  521,
                  66
                ],
                [
                  0.9542961608775137,
                  0.12643678160919541,
                  522,
                  66
                ],
                [
                  0.9561243144424132,
                  0.12810707456978968,
                  523,
                  67
                ],
                [
                  0.9579524680073126,
                  0.12786259541984732,
                  524,
                  67
                ],
                [
                  0.9597806215722121,
                  0.1295238095238095,
                  525,
                  68
                ],
                [
                  0.9616087751371115,
                  0.12927756653992395,
                  526,
                  68
                ],
                [
                  0.9634369287020109,
                  0.12903225806451613,
                  527,
                  68
                ],
                [
                  0.9652650822669104,
                  0.13068181818181818,
                  528,
                  69
                ],
                [
                  0.9670932358318098,
                  0.13043478260869565,
                  529,
                  69
                ],
                [
                  0.9689213893967094,
                  0.13018867924528302,
                  530,
                  69
                ],
                [
                  0.9707495429616088,
                  0.1318267419962335,
                  531,
                  70
                ],
                [
                  0.9725776965265083,
                  0.13345864661654136,
                  532,
                  71
                ],
                [
                  0.9744058500914077,
                  0.1350844277673546,
                  533,
                  72
                ],
                [
                  0.9762340036563071,
                  0.1348314606741573,
                  534,
                  72
                ],
                [
                  0.9780621572212066,
                  0.13644859813084112,
                  535,
                  73
                ],
                [
                  0.979890310786106,
                  0.13619402985074627,
                  536,
                  73
                ],
                [
                  0.9817184643510055,
                  0.1378026070763501,
                  537,
                  74
                ],
                [
                  0.9835466179159049,
                  0.137546468401487,
                  538,
                  74
                ],
                [
                  0.9853747714808044,
                  0.1391465677179963,
                  539,
                  75
                ],
                [
                  0.9872029250457038,
                  0.14074074074074075,
                  540,
                  76
                ],
                [
                  0.9890310786106032,
                  0.14048059149722736,
                  541,
                  76
                ],
                [
                  0.9908592321755028,
                  0.14206642066420663,
                  542,
                  77
                ],
                [
                  0.9926873857404022,
                  0.143646408839779,
                  543,
                  78
                ],
                [
                  0.9945155393053017,
                  0.14522058823529413,
                  544,
                  79
                ],
                [
                  0.9963436928702011,
                  0.14678899082568808,
                  545,
                  80
                ],
                [
                  0.9981718464351006,
                  0.14835164835164835,
                  546,
                  81
                ],
                [
                  1.0,
                  0.14990859232175502,
                  547,
                  82
                ]
              ]
            },
            {
              "language": "English",
              "run": 430,
              "points": [
                [
                  0.0018281535648994515,
                  0.0,
                  1,
                  0
                ],
                [
                  0.003656307129798903,
                  0.0,
                  2,
                  0
                ],
                [
                  0.005484460694698354,
                  0.0,
                  3,
                  0
                ],
                [
                  0.007312614259597806,
                  0.0,
                  4,
                  0
                ],
                [
                  0.009140767824497258,
                  0.0,
                  5,
                  0
                ],
                [
                  0.010968921389396709,
                  0.0,
                  6,
                  0
                ],
                [
                  0.012797074954296161,
                  0.0,
                  7,
                  0
                ],
                [
                  0.014625228519195612,
                  0.0,
                  8,
                  0
                ],
                [
                  0.016453382084095063,
                  0.0,
                  9,
                  0
                ],
                [
                  0.018281535648994516,
                  0.0,
                  10,
                  0
                ],
                [
                  0.02010968921389397,
                  0.0,
                  11,
                  0
                ],
                [
                  0.021937842778793418,
                  0.0,
                  12,
                  0
                ],
                [
                  0.02376599634369287,
                  0.0,
                  13,
                  0
                ],
                [
                  0.025594149908592323,
                  0.0,
                  14,
                  0
                ],
                [
                  0.027422303473491772,
                  0.0,
                  15,
                  0
                ],
                [
                  0.029250457038391225,
                  0.0,
                  16,
                  0
                ],
                [
                  0.031078610603290677,
                  0.0,
                  17,
                  0
                ],
                [
                  0.03290676416819013,
                  0.0,
                  18,
                  0
                ],
                [
                  0.03473491773308958,
                  0.0,
                  19,
                  0
                ],
                [
                  0.03656307129798903,
                  0.0,
                  20,
                  0
                ],
                [
                  0.038391224862888484,
                  0.0,
                  21,
                  0
                ],
                [
                  0.04021937842778794,
                  0.0,
                  22,
                  0
                ],
                [
                  0.04204753199268738,
                  0.0,
                  23,
                  0
                ],
                [
                  0.043875685557586835,
                  0.0,
                  24,
                  0
                ],
                [
                  0.04570383912248629,
                  0.0,
                  25,
                  0
                ],
                [
                  0.04753199268738574,
                  0.0,
                  26,
                  0
                ],
                [
                  0.04936014625228519,
                  0.0,
                  27,
                  0
                ],
                [
                  0.051188299817184646,
                  0.0,
                  28,
                  0
                ],
                [
                  0.05301645338208409,
                  0.0,
                  29,
                  0
                ],
                [
                  0.054844606946983544,
                  0.0,
                  30,
                  0
                ],
                [
                  0.056672760511883,
                  0.0,
                  31,
                  0
                ],
                [
                  0.05850091407678245,
                  0.0,
                  32,
                  0
                ],
                [
                  0.0603290676416819,
                  0.0,
                  33,
                  0
                ],
                [
                  0.062157221206581355,
                  0.0,
                  34,
                  0
                ],
                [
                  0.06398537477148081,
                  0.0,
                  35,
                  0
                ],
                [
                  0.06581352833638025,
                  0.0,
                  36,
                  0
                ],
                [
                  0.06764168190127971,
                  0.0,
                  37,
                  0
                ],
                [
                  0.06946983546617916,
                  0.0,
                  38,
                  0
                ],
                [
                  0.0712979890310786,
                  0.0,
                  39,
                  0
                ],
                [
                  0.07312614259597806,
                  0.0,
                  40,
                  0
                ],
                [
                  0.07495429616087751,
                  0.0,
                  41,
                  0
                ],
                [
                  0.07678244972577697,
                  0.0,
                  42,
                  0
                ],
                [
                  0.07861060329067641,
                  0.023255813953488372,
                  43,
                  1
                ],
                [
                  0.08043875685557587,
                  0.022727272727272728,
                  44,
                  1
                ],
                [
                  0.08226691042047532,
                  0.022222222222222223,
                  45,
                  1
                ],
                [
                  0.08409506398537477,
                  0.021739130434782608,
                  46,
                  1
                ],
                [
                  0.08592321755027423,
                  0.02127659574468085,
                  47,
                  1
                ],
                [
                  0.08775137111517367,
                  0.020833333333333332,
                  48,
                  1
                ],
                [
                  0.08957952468007313,
                  0.02040816326530612,
                  49,
                  1
                ],
                [
                  0.09140767824497258,
                  0.02,
                  50,
                  1
                ],
                [
                  0.09323583180987204,
                  0.0196078431372549,
                  51,
                  1
                ],
                [
                  0.09506398537477148,
                  0.019230769230769232,
                  52,
                  1
                ],
                [
                  0.09689213893967093,
                  0.018867924528301886,
                  53,
                  1
                ],
                [
                  0.09872029250457039,
                  0.018518518518518517,
                  54,
                  1
                ],
                [
                  0.10054844606946983,
                  0.01818181818181818,
                  55,
                  1
                ],
                [
                  0.10237659963436929,
                  0.017857142857142856,
                  56,
                  1
                ],
                [
                  0.10420475319926874,
                  0.017543859649122806,
                  57,
                  1
                ],
                [
                  0.10603290676416818,
                  0.017241379310344827,
                  58,
                  1
                ],
                [
                  0.10786106032906764,
                  0.01694915254237288,
                  59,
                  1
                ],
                [
                  0.10968921389396709,
                  0.016666666666666666,
                  60,
                  1
                ],
                [
                  0.11151736745886655,
                  0.01639344262295082,
                  61,
                  1
                ],
                [
                  0.113345521023766,
                  0.016129032258064516,
                  62,
                  1
                ],
                [
                  0.11517367458866545,
                  0.015873015873015872,
                  63,
                  1
                ],
                [
                  0.1170018281535649,
                  0.015625,
                  64,
                  1
                ],
                [
                  0.11882998171846434,
                  0.015384615384615385,
                  65,
                  1
                ],
                [
                  0.1206581352833638,
                  0.015151515151515152,
                  66,
                  1
                ],
                [
                  0.12248628884826325,
                  0.014925373134328358,
                  67,
                  1
                ],
                [
                  0.12431444241316271,
                  0.014705882352941176,
                  68,
                  1
                ],
                [
                  0.12614259597806216,
                  0.014492753623188406,
                  69,
                  1
                ],
                [
                  0.12797074954296161,
                  0.014285714285714285,
                  70,
                  1
                ],
                [
                  0.12979890310786105,
                  0.014084507042253521,
                  71,
                  1
                ],
                [
                  0.1316270566727605,
                  0.013888888888888888,
                  72,
                  1
                ],
                [
                  0.13345521023765997,
                  0.0136986301369863,
                  73,
                  1
                ],
                [
                  0.13528336380255943,
                  0.013513513513513514,
                  74,
                  1
                ],
                [
                  0.13711151736745886,
                  0.013333333333333334,
                  75,
                  1
                ],
                [
                  0.13893967093235832,
                  0.013157894736842105,
                  76,
                  1
                ],
                [
                  0.14076782449725778,
                  0.012987012987012988,
                  77,
                  1
                ],
                [
                  0.1425959780621572,
                  0.01282051282051282,
                  78,
                  1
                ],
                [
                  0.14442413162705667,
                  0.012658227848101266,
                  79,
                  1
                ],
                [
                  0.14625228519195613,
                  0.0125,
                  80,
                  1
                ],
                [
                  0.1480804387568556,
                  0.012345679012345678,
                  81,
                  1
                ],
                [
                  0.14990859232175502,
                  0.012195121951219513,
                  82,
                  1
                ],
                [
                  0.15173674588665448,
                  0.012048192771084338,
                  83,
                  1
                ],
                [
                  0.15356489945155394,
                  0.011904761904761904,
                  84,
                  1
                ],
                [
                  0.15539305301645337,
                  0.011764705882352941,
                  85,
                  1
                ],
                [
                  0.15722120658135283,
                  0.011627906976744186,
                  86,
                  1
                ],
                [
                  0.1590493601462523,
                  0.011494252873563218,
                  87,
                  1
                ],
                [
                  0.16087751371115175,
                  0.011363636363636364,
                  88,
                  1
                ],
                [
                  0.16270566727605118,
                  0.011235955056179775,
                  89,
                  1
                ],
                [
                  0.16453382084095064,
                  0.011111111111111112,
                  90,
                  1
                ],
                [
                  0.1663619744058501,
                  0.01098901098901099,
                  91,
                  1
                ],
                [
                  0.16819012797074953,
                  0.010869565217391304,
                  92,
                  1
                ],
                [
                  0.170018281535649,
                  0.010752688172043012,
                  93,
                  1
                ],
                [
                  0.17184643510054845,
                  0.010638297872340425,
                  94,
                  1
                ],
                [
                  0.1736745886654479,
                  0.010526315789473684,
                  95,
                  1
                ],
                [
                  0.17550274223034734,
                  0.010416666666666666,
                  96,
                  1
                ],
                [
                  0.1773308957952468,
                  0.010309278350515464,
                  97,
                  1
                ],
                [
                  0.17915904936014626,
                  0.01020408163265306,
                  98,
                  1
                ],
                [
                  0.1809872029250457,
                  0.010101010101010102,
                  99,
                  1
                ],
                [
                  0.18281535648994515,
                  0.01,
                  100,
                  1
                ],
                [
                  0.1846435100548446,
                  0.009900990099009901,
                  101,
                  1
                ],
                [
                  0.18647166361974407,
                  0.00980392156862745,
                  102,
                  1
                ],
                [
                  0.1882998171846435,
                  0.009708737864077669,
                  103,
                  1
                ],
                [
                  0.19012797074954296,
                  0.009615384615384616,
                  104,
                  1
                ],
                [
                  0.19195612431444242,
                  0.009523809523809525,
                  105,
                  1
                ],
                [
                  0.19378427787934185,
                  0.009433962264150943,
                  106,
                  1
                ],
                [
                  0.1956124314442413,
                  0.009345794392523364,
                  107,
                  1
                ],
                [
                  0.19744058500914077,
                  0.009259259259259259,
                  108,
                  1
                ],
                [
                  0.19926873857404023,
                  0.009174311926605505,
                  109,
                  1
                ],
                [
                  0.20109689213893966,
                  0.00909090909090909,
                  110,
                  1
                ],
                [
                  0.20292504570383912,
                  0.009009009009009009,
                  111,
                  1
                ],
                [
                  0.20475319926873858,
                  0.008928571428571428,
                  112,
                  1
                ],
                [
                  0.20658135283363802,
                  0.008849557522123894,
                  113,
                  1
                ],
                [
                  0.20840950639853748,
                  0.008771929824561403,
                  114,
                  1
                ],
                [
                  0.21023765996343693,
                  0.008695652173913044,
                  115,
                  1
                ],
                [
                  0.21206581352833637,
                  0.008620689655172414,
                  116,
                  1
                ],
                [
                  0.21389396709323583,
                  0.008547008547008548,
                  117,
                  1
                ],
                [
                  0.21572212065813529,
                  0.00847457627118644,
                  118,
                  1
                ],
                [
                  0.21755027422303475,
                  0.008403361344537815,
                  119,
                  1
                ],
                [
                  0.21937842778793418,
                  0.008333333333333333,
                  120,
                  1
                ],
                [
                  0.22120658135283364,
                  0.008264462809917356,
                  121,
                  1
                ],
                [
                  0.2230347349177331,
                  0.00819672131147541,
                  122,
                  1
                ],
                [
                  0.22486288848263253,
                  0.008130081300813009,
                  123,
                  1
                ],
                [
                  0.226691042047532,
                  0.008064516129032258,
                  124,
                  1
                ],
                [
                  0.22851919561243145,
                  0.008,
                  125,
                  1
                ],
                [
                  0.2303473491773309,
                  0.007936507936507936,
                  126,
                  1
                ],
                [
                  0.23217550274223034,
                  0.007874015748031496,
                  127,
                  1
                ],
                [
                  0.2340036563071298,
                  0.0078125,
                  128,
                  1
                ],
                [
                  0.23583180987202926,
                  0.007751937984496124,
                  129,
                  1
                ],
                [
                  0.2376599634369287,
                  0.007692307692307693,
                  130,
                  1
                ],
                [
                  0.23948811700182815,
                  0.007633587786259542,
                  131,
                  1
                ],
                [
                  0.2413162705667276,
                  0.007575757575757576,
                  132,
                  1
                ],
                [
                  0.24314442413162707,
                  0.007518796992481203,
                  133,
                  1
                ],
                [
                  0.2449725776965265,
                  0.007462686567164179,
                  134,
                  1
                ],
                [
                  0.24680073126142596,
                  0.007407407407407408,
                  135,
                  1
                ],
                [
                  0.24862888482632542,
                  0.007352941176470588,
                  136,
                  1
                ],
                [
                  0.25045703839122485,
                  0.0072992700729927005,
                  137,
                  1
                ],
                [
                  0.2522851919561243,
                  0.007246376811594203,
                  138,
                  1
                ],
                [
                  0.25411334552102377,
                  0.007194244604316547,
                  139,
                  1
                ],
                [
                  0.25594149908592323,
                  0.007142857142857143,
                  140,
                  1
                ],
                [
                  0.2577696526508227,
                  0.0070921985815602835,
                  141,
                  1
                ],
                [
                  0.2595978062157221,
                  0.007042253521126761,
                  142,
                  1
                ],
                [
                  0.26142595978062155,
                  0.006993006993006993,
                  143,
                  1
                ],
                [
                  0.263254113345521,
                  0.006944444444444444,
                  144,
                  1
                ],
                [
                  0.26508226691042047,
                  0.006896551724137931,
                  145,
                  1
                ],
                [
                  0.26691042047531993,
                  0.00684931506849315,
                  146,
                  1
                ],
                [
                  0.2687385740402194,
                  0.006802721088435374,
                  147,
                  1
                ],
                [
                  0.27056672760511885,
                  0.006756756756756757,
                  148,
                  1
                ],
                [
                  0.27239488117001825,
                  0.006711409395973154,
                  149,
                  1
                ],
                [
                  0.2742230347349177,
                  0.006666666666666667,
                  150,
                  1
                ],
                [
                  0.2760511882998172,
                  0.006622516556291391,
                  151,
                  1
                ],
                [
                  0.27787934186471663,
                  0.006578947368421052,
                  152,
                  1
                ],
                [
                  0.2797074954296161,
                  0.006535947712418301,
                  153,
                  1
                ],
                [
                  0.28153564899451555,
                  0.006493506493506494,
                  154,
                  1
                ],
                [
                  0.283363802559415,
                  0.0064516129032258064,
                  155,
                  1
                ],
                [
                  0.2851919561243144,
                  0.00641025641025641,
                  156,
                  1
                ],
                [
                  0.2870201096892139,
                  0.006369426751592357,
                  157,
                  1
                ],
                [
                  0.28884826325411334,
                  0.006329113924050633,
                  158,
                  1
                ],
                [
                  0.2906764168190128,
                  0.006289308176100629,
                  159,
                  1
                ],
                [
                  0.29250457038391225,
                  0.00625,
                  160,
                  1
                ],
                [
                  0.2943327239488117,
                  0.006211180124223602,
                  161,
                  1
                ],
                [
                  0.2961608775137112,
                  0.006172839506172839,
                  162,
                  1
                ],
                [
                  0.2979890310786106,
                  0.006134969325153374,
                  163,
                  1
                ],
                [
                  0.29981718464351004,
                  0.006097560975609756,
                  164,
                  1
                ],
                [
                  0.3016453382084095,
                  0.006060606060606061,
                  165,
                  1
                ],
                [
                  0.30347349177330896,
                  0.006024096385542169,
                  166,
                  1
                ],
                [
                  0.3053016453382084,
                  0.005988023952095809,
                  167,
                  1
                ],
                [
                  0.3071297989031079,
                  0.005952380952380952,
                  168,
                  1
                ],
                [
                  0.30895795246800734,
                  0.005917159763313609,
                  169,
                  1
                ],
                [
                  0.31078610603290674,
                  0.0058823529411764705,
                  170,
                  1
                ],
                [
                  0.3126142595978062,
                  0.005847953216374269,
                  171,
                  1
                ],
                [
                  0.31444241316270566,
                  0.005813953488372093,
                  172,
                  1
                ],
                [
                  0.3162705667276051,
                  0.005780346820809248,
                  173,
                  1
                ],
                [
                  0.3180987202925046,
                  0.005747126436781609,
                  174,
                  1
                ],
                [
                  0.31992687385740404,
                  0.005714285714285714,
                  175,
                  1
                ],
                [
                  0.3217550274223035,
                  0.005681818181818182,
                  176,
                  1
                ],
                [
                  0.3235831809872029,
                  0.005649717514124294,
                  177,
                  1
                ],
                [
                  0.32541133455210236,
                  0.0056179775280898875,
                  178,
                  1
                ],
                [
                  0.3272394881170018,
                  0.00558659217877095,
                  179,
                  1
                ],
                [
                  0.3290676416819013,
                  0.005555555555555556,
                  180,
                  1
                ],
                [
                  0.33089579524680074,
                  0.0055248618784530384,
                  181,
                  1
                ],
                [
                  0.3327239488117002,
                  0.005494505494505495,
                  182,
                  1
                ],
                [
                  0.33455210237659966,
                  0.00546448087431694,
                  183,
                  1
                ],
                [
                  0.33638025594149906,
                  0.005434782608695652,
                  184,
                  1
                ],
                [
                  0.3382084095063985,
                  0.005405405405405406,
                  185,
                  1
                ],
                [
                  0.340036563071298,
                  0.005376344086021506,
                  186,
                  1
                ],
                [
                  0.34186471663619744,
                  0.0053475935828877,
                  187,
                  1
                ],
                [
                  0.3436928702010969,
                  0.005319148936170213,
                  188,
                  1
                ],
                [
                  0.34552102376599636,
                  0.005291005291005291,
                  189,
                  1
                ],
                [
                  0.3473491773308958,
                  0.005263157894736842,
                  190,
                  1
                ],
                [
                  0.3491773308957952,
                  0.005235602094240838,
                  191,
                  1
                ],
                [
                  0.3510054844606947,
                  0.005208333333333333,
                  192,
                  1
                ],
                [
                  0.35283363802559414,
                  0.0051813471502590676,
                  193,
                  1
                ],
                [
                  0.3546617915904936,
                  0.005154639175257732,
                  194,
                  1
                ],
                [
                  0.35648994515539306,
                  0.005128205128205128,
                  195,
                  1
                ],
                [
                  0.3583180987202925,
                  0.00510204081632653,
                  196,
                  1
                ],
                [
                  0.360146252285192,
                  0.005076142131979695,
                  197,
                  1
                ],
                [
                  0.3619744058500914,
                  0.005050505050505051,
                  198,
                  1
                ],
                [
                  0.36380255941499084,
                  0.005025125628140704,
                  199,
                  1
                ],
                [
                  0.3656307129798903,
                  0.005,
                  200,
                  1
                ],
                [
                  0.36745886654478976,
                  0.004975124378109453,
                  201,
                  1
                ],
                [
                  0.3692870201096892,
                  0.0049504950495049506,
                  202,
                  1
                ],
                [
                  0.3711151736745887,
                  0.0049261083743842365,
                  203,
                  1
                ],
                [
                  0.37294332723948814,
                  0.004901960784313725,
                  204,
                  1
                ],
                [
                  0.37477148080438755,
                  0.004878048780487805,
                  205,
                  1
                ],
                [
                  0.376599634369287,
                  0.0048543689320388345,
                  206,
                  1
                ],
                [
                  0.37842778793418647,
                  0.004830917874396135,
                  207,
                  1
                ],
                [
                  0.3802559414990859,
                  0.004807692307692308,
                  208,
                  1
                ],
                [
                  0.3820840950639854,
                  0.004784688995215311,
                  209,
                  1
                ],
                [
                  0.38391224862888484,
                  0.004761904761904762,
                  210,
                  1
                ],
                [
                  0.3857404021937843,
                  0.009478672985781991,
                  211,
                  2
                ],
                [
                  0.3875685557586837,
                  0.009433962264150943,
                  212,
                  2
                ],
                [
                  0.38939670932358317,
                  0.009389671361502348,
                  213,
                  2
                ],
                [
                  0.3912248628884826,
                  0.009345794392523364,
                  214,
                  2
                ],
                [
                  0.3930530164533821,
                  0.009302325581395349,
                  215,
                  2
                ],
                [
                  0.39488117001828155,
                  0.009259259259259259,
                  216,
                  2
                ],
                [
                  0.396709323583181,
                  0.009216589861751152,
                  217,
                  2
                ],
                [
                  0.39853747714808047,
                  0.009174311926605505,
                  218,
                  2
                ],
                [
                  0.40036563071297987,
                  0.0091324200913242,
                  219,
                  2
                ],
                [
                  0.40219378427787933,
                  0.00909090909090909,
                  220,
                  2
                ],
                [
                  0.4040219378427788,
                  0.00904977375565611,
                  221,
                  2
                ],
                [
                  0.40585009140767825,
                  0.009009009009009009,
                  222,
                  2
                ],
                [
                  0.4076782449725777,
                  0.008968609865470852,
                  223,
                  2
                ],
                [
                  0.40950639853747717,
                  0.008928571428571428,
                  224,
                  2
                ],
                [
                  0.4113345521023766,
                  0.008888888888888889,
                  225,
                  2
                ],
                [
                  0.41316270566727603,
                  0.008849557522123894,
                  226,
                  2
                ],
                [
                  0.4149908592321755,
                  0.00881057268722467,
                  227,
                  2
                ],
                [
                  0.41681901279707495,
                  0.008771929824561403,
                  228,
                  2
                ],
                [
                  0.4186471663619744,
                  0.008733624454148471,
                  229,
                  2
                ],
                [
                  0.42047531992687387,
                  0.008695652173913044,
                  230,
                  2
                ],
                [
                  0.42230347349177333,
                  0.008658008658008658,
                  231,
                  2
                ],
                [
                  0.42413162705667273,
                  0.008620689655172414,
                  232,
                  2
                ],
                [
                  0.4259597806215722,
                  0.008583690987124463,
                  233,
                  2
                ],
                [
                  0.42778793418647165,
                  0.008547008547008548,
                  234,
                  2
                ],
                [
                  0.4296160877513711,
                  0.00851063829787234,
                  235,
                  2
                ],
                [
                  0.43144424131627057,
                  0.00847457627118644,
                  236,
                  2
                ],
                [
                  0.43327239488117003,
                  0.008438818565400843,
                  237,
                  2
                ],
                [
                  0.4351005484460695,
                  0.008403361344537815,
                  238,
                  2
                ],
                [
                  0.4369287020109689,
                  0.008368200836820083,
                  239,
                  2
                ],
                [
                  0.43875685557586835,
                  0.008333333333333333,
                  240,
                  2
                ],
                [
                  0.4405850091407678,
                  0.008298755186721992,
                  241,
                  2
                ],
                [
                  0.4424131627056673,
                  0.008264462809917356,
                  242,
                  2
                ],
                [
                  0.44424131627056673,
                  0.00823045267489712,
                  243,
                  2
                ],
                [
                  0.4460694698354662,
                  0.00819672131147541,
                  244,
                  2
                ],
                [
                  0.44789762340036565,
                  0.00816326530612245,
                  245,
                  2
                ],
                [
                  0.44972577696526506,
                  0.008130081300813009,
                  246,
                  2
                ],
                [
                  0.4515539305301645,
                  0.008097165991902834,
                  247,
                  2
                ],
                [
                  0.453382084095064,
                  0.008064516129032258,
                  248,
                  2
                ],
                [
                  0.45521023765996343,
                  0.008032128514056224,
                  249,
                  2
                ],
                [
                  0.4570383912248629,
                  0.008,
                  250,
                  2
                ],
                [
                  0.45886654478976235,
                  0.00796812749003984,
                  251,
                  2
                ],
                [
                  0.4606946983546618,
                  0.007936507936507936,
                  252,
                  2
                ],
                [
                  0.4625228519195612,
                  0.007905138339920948,
                  253,
                  2
                ],
                [
                  0.4643510054844607,
                  0.007874015748031496,
                  254,
                  2
                ],
                [
                  0.46617915904936014,
                  0.00784313725490196,
                  255,
                  2
                ],
                [
                  0.4680073126142596,
                  0.0078125,
                  256,
                  2
                ],
                [
                  0.46983546617915906,
                  0.007782101167315175,
                  257,
                  2
                ],
                [
                  0.4716636197440585,
                  0.007751937984496124,
                  258,
                  2
                ],
                [
                  0.473491773308958,
                  0.007722007722007722,
                  259,
                  2
                ],
                [
                  0.4753199268738574,
                  0.007692307692307693,
                  260,
                  2
                ],
                [
                  0.47714808043875684,
                  0.007662835249042145,
                  261,
                  2
                ],
                [
                  0.4789762340036563,
                  0.007633587786259542,
                  262,
                  2
                ],
                [
                  0.48080438756855576,
                  0.0076045627376425855,
                  263,
                  2
                ],
                [
                  0.4826325411334552,
                  0.007575757575757576,
                  264,
                  2
                ],
                [
                  0.4844606946983547,
                  0.007547169811320755,
                  265,
                  2
                ],
                [
                  0.48628884826325414,
                  0.007518796992481203,
                  266,
                  2
                ],
                [
                  0.48811700182815354,
                  0.00749063670411985,
                  267,
                  2
                ],
                [
                  0.489945155393053,
                  0.007462686567164179,
                  268,
                  2
                ],
                [
                  0.49177330895795246,
                  0.007434944237918215,
                  269,
                  2
                ],
                [
                  0.4936014625228519,
                  0.007407407407407408,
                  270,
                  2
                ],
                [
                  0.4954296160877514,
                  0.007380073800738007,
                  271,
                  2
                ],
                [
                  0.49725776965265084,
                  0.007352941176470588,
                  272,
                  2
                ],
                [
                  0.4990859232175503,
                  0.007326007326007326,
                  273,
                  2
                ],
                [
                  0.5009140767824497,
                  0.0072992700729927005,
                  274,
                  2
                ],
                [
                  0.5027422303473492,
                  0.007272727272727273,
                  275,
                  2
                ],
                [
                  0.5045703839122486,
                  0.007246376811594203,
                  276,
                  2
                ],
                [
                  0.506398537477148,
                  0.007220216606498195,
                  277,
                  2
                ],
                [
                  0.5082266910420475,
                  0.007194244604316547,
                  278,
                  2
                ],
                [
                  0.5100548446069469,
                  0.007168458781362007,
                  279,
                  2
                ],
                [
                  0.5118829981718465,
                  0.007142857142857143,
                  280,
                  2
                ],
                [
                  0.5137111517367459,
                  0.0071174377224199285,
                  281,
                  2
                ],
                [
                  0.5155393053016454,
                  0.0070921985815602835,
                  282,
                  2
                ],
                [
                  0.5173674588665448,
                  0.007067137809187279,
                  283,
                  2
                ],
                [
                  0.5191956124314442,
                  0.007042253521126761,
                  284,
                  2
                ],
                [
                  0.5210237659963437,
                  0.007017543859649123,
                  285,
                  2
                ],
                [
                  0.5228519195612431,
                  0.006993006993006993,
                  286,
                  2
                ],
                [
                  0.5246800731261426,
                  0.006968641114982578,
                  287,
                  2
                ],
                [
                  0.526508226691042,
                  0.006944444444444444,
                  288,
                  2
                ],
                [
                  0.5283363802559415,
                  0.006920415224913495,
                  289,
                  2
                ],
                [
                  0.5301645338208409,
                  0.006896551724137931,
                  290,
                  2
                ],
                [
                  0.5319926873857403,
                  0.006872852233676976,
                  291,
                  2
                ],
                [
                  0.5338208409506399,
                  0.00684931506849315,
                  292,
                  2
                ],
                [
                  0.5356489945155393,
                  0.006825938566552901,
                  293,
                  2
                ],
                [
                  0.5374771480804388,
                  0.006802721088435374,
                  294,
                  2
                ],
                [
                  0.5393053016453382,
                  0.006779661016949152,
                  295,
                  2
                ],
                [
                  0.5411334552102377,
                  0.006756756756756757,
                  296,
                  2
                ],
                [
                  0.5429616087751371,
                  0.006734006734006734,
                  297,
                  2
                ],
                [
                  0.5447897623400365,
                  0.006711409395973154,
                  298,
                  2
                ],
                [
                  0.546617915904936,
                  0.006688963210702341,
                  299,
                  2
                ],
                [
                  0.5484460694698354,
                  0.006666666666666667,
                  300,
                  2
                ],
                [
                  0.5502742230347349,
                  0.006644518272425249,
                  301,
                  2
                ],
                [
                  0.5521023765996343,
                  0.006622516556291391,
                  302,
                  2
                ],
                [
                  0.5539305301645339,
                  0.006600660066006601,
                  303,
                  2
                ],
                [
                  0.5557586837294333,
                  0.006578947368421052,
                  304,
                  2
                ],
                [
                  0.5575868372943327,
                  0.006557377049180328,
                  305,
                  2
                ],
                [
                  0.5594149908592322,
                  0.006535947712418301,
                  306,
                  2
                ],
                [
                  0.5612431444241316,
                  0.006514657980456026,
                  307,
                  2
                ],
                [
                  0.5630712979890311,
                  0.006493506493506494,
                  308,
                  2
                ],
                [
                  0.5648994515539305,
                  0.006472491909385114,
                  309,
                  2
                ],
                [
                  0.56672760511883,
                  0.0064516129032258064,
                  310,
                  2
                ],
                [
                  0.5685557586837294,
                  0.006430868167202572,
                  311,
                  2
                ],
                [
                  0.5703839122486288,
                  0.009615384615384616,
                  312,
                  3
                ],
                [
                  0.5722120658135283,
                  0.009584664536741214,
                  313,
                  3
                ],
                [
                  0.5740402193784278,
                  0.009554140127388535,
                  314,
                  3
                ],
                [
                  0.5758683729433273,
                  0.009523809523809525,
                  315,
                  3
                ],
                [
                  0.5776965265082267,
                  0.012658227848101266,
                  316,
                  4
                ],
                [
                  0.5795246800731262,
                  0.012618296529968454,
                  317,
                  4
                ],
                [
                  0.5813528336380256,
                  0.012578616352201259,
                  318,
                  4
                ],
                [
                  0.583180987202925,
                  0.012539184952978056,
                  319,
                  4
                ],
                [
                  0.5850091407678245,
                  0.0125,
                  320,
                  4
                ],
                [
                  0.5868372943327239,
                  0.012461059190031152,
                  321,
                  4
                ],
                [
                  0.5886654478976234,
                  0.012422360248447204,
                  322,
                  4
                ],
                [
                  0.5904936014625228,
                  0.01238390092879257,
                  323,
                  4
                ],
                [
                  0.5923217550274223,
                  0.015432098765432098,
                  324,
                  5
                ],
                [
                  0.5941499085923218,
                  0.015384615384615385,
                  325,
                  5
                ],
                [
                  0.5959780621572212,
                  0.015337423312883436,
                  326,
                  5
                ],
                [
                  0.5978062157221207,
                  0.01529051987767584,
                  327,
                  5
                ],
                [
                  0.5996343692870201,
                  0.01524390243902439,
                  328,
                  5
                ],
                [
                  0.6014625228519196,
                  0.015197568389057751,
                  329,
                  5
                ],
                [
                  0.603290676416819,
                  0.015151515151515152,
                  330,
                  5
                ],
                [
                  0.6051188299817185,
                  0.015105740181268883,
                  331,
                  5
                ],
                [
                  0.6069469835466179,
                  0.015060240963855422,
                  332,
                  5
                ],
                [
                  0.6087751371115173,
                  0.015015015015015015,
                  333,
                  5
                ],
                [
                  0.6106032906764168,
                  0.017964071856287425,
                  334,
                  6
                ],
                [
                  0.6124314442413162,
                  0.01791044776119403,
                  335,
                  6
                ],
                [
                  0.6142595978062158,
                  0.017857142857142856,
                  336,
                  6
                ],
                [
                  0.6160877513711152,
                  0.017804154302670624,
                  337,
                  6
                ],
                [
                  0.6179159049360147,
                  0.01775147928994083,
                  338,
                  6
                ],
                [
                  0.6197440585009141,
                  0.017699115044247787,
                  339,
                  6
                ],
                [
                  0.6215722120658135,
                  0.01764705882352941,
                  340,
                  6
                ],
                [
                  0.623400365630713,
                  0.017595307917888565,
                  341,
                  6
                ],
                [
                  0.6252285191956124,
                  0.017543859649122806,
                  342,
                  6
                ],
                [
                  0.6270566727605119,
                  0.01749271137026239,
                  343,
                  6
                ],
                [
                  0.6288848263254113,
                  0.01744186046511628,
                  344,
                  6
                ],
                [
                  0.6307129798903108,
                  0.017391304347826087,
                  345,
                  6
                ],
                [
                  0.6325411334552102,
                  0.017341040462427744,
                  346,
                  6
                ],
                [
                  0.6343692870201096,
                  0.01729106628242075,
                  347,
                  6
                ],
                [
                  0.6361974405850092,
                  0.017241379310344827,
                  348,
                  6
                ],
                [
                  0.6380255941499086,
                  0.017191977077363897,
                  349,
                  6
                ],
                [
                  0.6398537477148081,
                  0.017142857142857144,
                  350,
                  6
                ],
                [
                  0.6416819012797075,
                  0.017094017094017096,
                  351,
                  6
                ],
                [
                  0.643510054844607,
                  0.017045454545454544,
                  352,
                  6
                ],
                [
                  0.6453382084095064,
                  0.0169971671388102,
                  353,
                  6
                ],
                [
                  0.6471663619744058,
                  0.01694915254237288,
                  354,
                  6
                ],
                [
                  0.6489945155393053,
                  0.016901408450704224,
                  355,
                  6
                ],
                [
                  0.6508226691042047,
                  0.016853932584269662,
                  356,
                  6
                ],
                [
                  0.6526508226691042,
                  0.01680672268907563,
                  357,
                  6
                ],
                [
                  0.6544789762340036,
                  0.019553072625698324,
                  358,
                  7
                ],
                [
                  0.6563071297989032,
                  0.019498607242339833,
                  359,
                  7
                ],
                [
                  0.6581352833638026,
                  0.019444444444444445,
                  360,
                  7
                ],
                [
                  0.659963436928702,
                  0.019390581717451522,
                  361,
                  7
                ],
                [
                  0.6617915904936015,
                  0.022099447513812154,
                  362,
                  8
                ],
                [
                  0.6636197440585009,
                  0.02203856749311295,
                  363,
                  8
                ],
                [
                  0.6654478976234004,
                  0.02197802197802198,
                  364,
                  8
                ],
                [
                  0.6672760511882998,
                  0.021917808219178082,
                  365,
                  8
                ],
                [
                  0.6691042047531993,
                  0.02185792349726776,
                  366,
                  8
                ],
                [
                  0.6709323583180987,
                  0.021798365122615803,
                  367,
                  8
                ],
                [
                  0.6727605118829981,
                  0.024456521739130436,
                  368,
                  9
                ],
                [
                  0.6745886654478976,
                  0.024390243902439025,
                  369,
                  9
                ],
                [
                  0.676416819012797,
                  0.024324324324324326,
                  370,
                  9
                ],
                [
                  0.6782449725776966,
                  0.02425876010781671,
                  371,
                  9
                ],
                [
                  0.680073126142596,
                  0.024193548387096774,
                  372,
                  9
                ],
                [
                  0.6819012797074955,
                  0.024128686327077747,
                  373,
                  9
                ],
                [
                  0.6837294332723949,
                  0.02406417112299465,
                  374,
                  9
                ],
                [
                  0.6855575868372943,
                  0.024,
                  375,
                  9
                ],
                [
                  0.6873857404021938,
                  0.023936170212765957,
                  376,
                  9
                ],
                [
                  0.6892138939670932,
                  0.026525198938992044,
                  377,
                  10
                ],
                [
                  0.6910420475319927,
                  0.026455026455026454,
                  378,
                  10
                ],
                [
                  0.6928702010968921,
                  0.026385224274406333,
                  379,
                  10
                ],
                [
                  0.6946983546617916,
                  0.02631578947368421,
                  380,
                  10
                ],
                [
                  0.696526508226691,
                  0.026246719160104987,
                  381,
                  10
                ],
                [
                  0.6983546617915904,
                  0.02617801047120419,
                  382,
                  10
                ],
                [
                  0.70018281535649,
                  0.028720626631853787,
                  383,
                  11
                ],
                [
                  0.7020109689213894,
                  0.028645833333333332,
                  384,
                  11
                ],
                [
                  0.7038391224862889,
                  0.02857142857142857,
                  385,
                  11
                ],
                [
                  0.7056672760511883,
                  0.02849740932642487,
                  386,
                  11
                ],
                [
                  0.7074954296160878,
                  0.028423772609819122,
                  387,
                  11
                ],
                [
                  0.7093235831809872,
                  0.028350515463917526,
                  388,
                  11
                ],
                [
                  0.7111517367458866,
                  0.028277634961439587,
                  389,
                  11
                ],
                [
                  0.7129798903107861,
                  0.028205128205128206,
                  390,
                  11
                ],
                [
                  0.7148080438756855,
                  0.028132992327365727,
                  391,
                  11
                ],
                [
                  0.716636197440585,
                  0.02806122448979592,
                  392,
                  11
                ],
                [
                  0.7184643510054844,
                  0.030534351145038167,
                  393,
                  12
                ],
                [
                  0.720292504570384,
                  0.030456852791878174,
                  394,
                  12
                ],
                [
                  0.7221206581352834,
                  0.030379746835443037,
                  395,
                  12
                ],
                [
                  0.7239488117001828,
                  0.030303030303030304,
                  396,
                  12
                ],
                [
                  0.7257769652650823,
                  0.030226700251889168,
                  397,
                  12
                ],
                [
                  0.7276051188299817,
                  0.032663316582914576,
                  398,
                  13
                ],
                [
                  0.7294332723948812,
                  0.03508771929824561,
                  399,
                  14
                ],
                [
                  0.7312614259597806,
                  0.035,
                  400,
                  14
                ],
                [
                  0.7330895795246801,
                  0.034912718204488775,
                  401,
                  14
                ],
                [
                  0.7349177330895795,
                  0.03482587064676617,
                  402,
                  14
                ],
                [
                  0.7367458866544789,
                  0.034739454094292806,
                  403,
                  14
                ],
                [
                  0.7385740402193784,
                  0.03712871287128713,
                  404,
                  15
                ],
                [
                  0.7404021937842779,
                  0.037037037037037035,
                  405,
                  15
                ],
                [
                  0.7422303473491774,
                  0.03694581280788178,
                  406,
                  15
                ],
                [
                  0.7440585009140768,
                  0.03931203931203931,
                  407,
                  16
                ],
                [
                  0.7458866544789763,
                  0.041666666666666664,
                  408,
                  17
                ],
                [
                  0.7477148080438757,
                  0.04156479217603912,
                  409,
                  17
                ],
                [
                  0.7495429616087751,
                  0.041463414634146344,
                  410,
                  17
                ],
                [
                  0.7513711151736746,
                  0.0413625304136253,
                  411,
                  17
                ],
                [
                  0.753199268738574,
                  0.043689320388349516,
                  412,
                  18
                ],
                [
                  0.7550274223034735,
                  0.043583535108958835,
                  413,
                  18
                ],
                [
                  0.7568555758683729,
                  0.043478260869565216,
                  414,
                  18
                ],
                [
                  0.7586837294332724,
                  0.04578313253012048,
                  415,
                  19
                ],
                [
                  0.7605118829981719,
                  0.04567307692307692,
                  416,
                  19
                ],
                [
                  0.7623400365630713,
                  0.045563549160671464,
                  417,
                  19
                ],
                [
                  0.7641681901279708,
                  0.045454545454545456,
                  418,
                  19
                ],
                [
                  0.7659963436928702,
                  0.045346062052505964,
                  419,
                  19
                ],
                [
                  0.7678244972577697,
                  0.047619047619047616,
                  420,
                  20
                ],
                [
                  0.7696526508226691,
                  0.047505938242280284,
                  421,
                  20
                ],
                [
                  0.7714808043875686,
                  0.04739336492890995,
                  422,
                  20
                ],
                [
                  0.773308957952468,
                  0.04728132387706856,
                  423,
                  20
                ],
                [
                  0.7751371115173674,
                  0.04716981132075472,
                  424,
                  20
                ],
                [
                  0.7769652650822669,
                  0.047058823529411764,
                  425,
                  20
                ],
                [
                  0.7787934186471663,
                  0.04929577464788732,
                  426,
                  21
                ],
                [
                  0.7806215722120659,
                  0.04918032786885246,
                  427,
                  21
                ],
                [
                  0.7824497257769653,
                  0.04906542056074766,
                  428,
                  21
                ],
                [
                  0.7842778793418648,
                  0.04895104895104895,
                  429,
                  21
                ],
                [
                  0.7861060329067642,
                  0.05116279069767442,
                  430,
                  22
                ],
                [
                  0.7879341864716636,
                  0.05104408352668213,
                  431,
                  22
                ],
                [
                  0.7897623400365631,
                  0.05324074074074074,
                  432,
                  23
                ],
                [
                  0.7915904936014625,
                  0.053117782909930716,
                  433,
                  23
                ],
                [
                  0.793418647166362,
                  0.052995391705069124,
                  434,
                  23
                ],
                [
                  0.7952468007312614,
                  0.052873563218390804,
                  435,
                  23
                ],
                [
                  0.7970749542961609,
                  0.052752293577981654,
                  436,
                  23
                ],
                [
                  0.7989031078610603,
                  0.05491990846681922,
                  437,
                  24
                ],
                [
                  0.8007312614259597,
                  0.05707762557077625,
                  438,
                  25
                ],
                [
                  0.8025594149908593,
                  0.05922551252847381,
                  439,
                  26
                ],
                [
                  0.8043875685557587,
                  0.05909090909090909,
                  440,
                  26
                ],
                [
                  0.8062157221206582,
                  0.05895691609977324,
                  441,
                  26
                ],
                [
                  0.8080438756855576,
                  0.058823529411764705,
                  442,
                  26
                ],
                [
                  0.8098720292504571,
                  0.05869074492099323,
                  443,
                  26
                ],
                [
                  0.8117001828153565,
                  0.05855855855855856,
                  444,
                  26
                ],
                [
                  0.8135283363802559,
                  0.058426966292134834,
                  445,
                  26
                ],
                [
                  0.8153564899451554,
                  0.06053811659192825,
                  446,
                  27
                ],
                [
                  0.8171846435100548,
                  0.06263982102908278,
                  447,
                  28
                ],
                [
                  0.8190127970749543,
                  0.06473214285714286,
                  448,
                  29
                ],
                [
                  0.8208409506398537,
                  0.066815144766147,
                  449,
                  30
                ],
                [
                  0.8226691042047533,
                  0.06666666666666667,
                  450,
                  30
                ],
                [
                  0.8244972577696527,
                  0.06651884700665188,
                  451,
                  30
                ],
                [
                  0.8263254113345521,
                  0.06637168141592921,
                  452,
                  30
                ],
                [
                  0.8281535648994516,
                  0.0684326710816777,
                  453,
                  31
                ],
                [
                  0.829981718464351,
                  0.06828193832599119,
                  454,
                  31
                ],
                [
                  0.8318098720292505,
                  0.06813186813186813,
                  455,
                  31
                ],
                [
                  0.8336380255941499,
                  0.06798245614035088,
                  456,
                  31
                ],
                [
                  0.8354661791590493,
                  0.06783369803063458,
                  457,
                  31
                ],
                [
                  0.8372943327239488,
                  0.06986899563318777,
                  458,
                  32
                ],
                [
                  0.8391224862888482,
                  0.06971677559912855,
                  459,
                  32
                ],
                [
                  0.8409506398537477,
                  0.06956521739130435,
                  460,
                  32
                ],
                [
                  0.8427787934186471,
                  0.06941431670281996,
                  461,
                  32
                ],
                [
                  0.8446069469835467,
                  0.07142857142857142,
                  462,
                  33
                ],
                [
                  0.8464351005484461,
                  0.07127429805615551,
                  463,
                  33
                ],
                [
                  0.8482632541133455,
                  0.07112068965517242,
                  464,
                  33
                ],
                [
                  0.850091407678245,
                  0.07096774193548387,
                  465,
                  33
                ],
                [
                  0.8519195612431444,
                  0.07296137339055794,
                  466,
                  34
                ],
                [
                  0.8537477148080439,
                  0.07494646680942184,
                  467,
                  35
                ],
                [
                  0.8555758683729433,
                  0.07692307692307693,
                  468,
                  36
                ],
                [
                  0.8574040219378428,
                  0.0767590618336887,
                  469,
                  36
                ],
                [
                  0.8592321755027422,
                  0.07659574468085106,
                  470,
                  36
                ],
                [
                  0.8610603290676416,
                  0.07643312101910828,
                  471,
                  36
                ],
                [
                  0.8628884826325411,
                  0.07627118644067797,
                  472,
                  36
                ],
                [
                  0.8647166361974405,
                  0.07822410147991543,
                  473,
                  37
                ],
                [
                  0.8665447897623401,
                  0.07805907172995781,
                  474,
                  37
                ],
                [
                  0.8683729433272395,
                  0.07789473684210527,
                  475,
                  37
                ],
                [
                  0.870201096892139,
                  0.07773109243697479,
                  476,
                  37
                ],
                [
                  0.8720292504570384,
                  0.07756813417190776,
                  477,
                  37
                ],
                [
                  0.8738574040219378,
                  0.07740585774058577,
                  478,
                  37
                ],
                [
                  0.8756855575868373,
                  0.07724425887265135,
                  479,
                  37
                ],
                [
                  0.8775137111517367,
                  0.07708333333333334,
                  480,
                  37
                ],
                [
                  0.8793418647166362,
                  0.07692307692307693,
                  481,
                  37
                ],
                [
                  0.8811700182815356,
                  0.07676348547717843,
                  482,
                  37
                ],
                [
                  0.8829981718464351,
                  0.07867494824016563,
                  483,
                  38
                ],
                [
                  0.8848263254113345,
                  0.08057851239669421,
                  484,
                  39
                ],
                [
                  0.886654478976234,
                  0.08247422680412371,
                  485,
                  40
                ],
                [
                  0.8884826325411335,
                  0.0823045267489712,
                  486,
                  40
                ],
                [
                  0.8903107861060329,
                  0.08418891170431211,
                  487,
                  41
                ],
                [
                  0.8921389396709324,
                  0.0860655737704918,
                  488,
                  42
                ],
                [
                  0.8939670932358318,
                  0.08793456032719836,
                  489,
                  43
                ],
                [
                  0.8957952468007313,
                  0.08775510204081632,
                  490,
                  43
                ],
                [
                  0.8976234003656307,
                  0.08757637474541752,
                  491,
                  43
                ],
                [
                  0.8994515539305301,
                  0.08739837398373984,
                  492,
                  43
                ],
                [
                  0.9012797074954296,
                  0.0872210953346856,
                  493,
                  43
                ],
                [
                  0.903107861060329,
                  0.08704453441295547,
                  494,
                  43
                ],
                [
                  0.9049360146252285,
                  0.08686868686868687,
                  495,
                  43
                ],
                [
                  0.906764168190128,
                  0.08870967741935484,
                  496,
                  44
                ],
                [
                  0.9085923217550275,
                  0.08853118712273642,
                  497,
                  44
                ],
                [
                  0.9104204753199269,
                  0.09036144578313253,
                  498,
                  45
                ],
                [
                  0.9122486288848263,
                  0.09218436873747494,
                  499,
                  46
                ],
                [
                  0.9140767824497258,
                  0.092,
                  500,
                  46
                ],
                [
                  0.9159049360146252,
                  0.09381237524950099,
                  501,
                  47
                ],
                [
                  0.9177330895795247,
                  0.09362549800796813,
                  502,
                  47
                ],
                [
                  0.9195612431444241,
                  0.09343936381709742,
                  503,
                  47
                ],
                [
                  0.9213893967093236,
                  0.09523809523809523,
                  504,
                  48
                ],
                [
                  0.923217550274223,
                  0.09504950495049505,
                  505,
                  48
                ],
                [
                  0.9250457038391224,
                  0.09683794466403162,
                  506,
                  49
                ],
                [
                  0.926873857404022,
                  0.09664694280078895,
                  507,
                  49
                ],
                [
                  0.9287020109689214,
                  0.0984251968503937,
                  508,
                  50
                ],
                [
                  0.9305301645338209,
                  0.10019646365422397,
                  509,
                  51
                ],
                [
                  0.9323583180987203,
                  0.1,
                  510,
                  51
                ],
                [
                  0.9341864716636198,
                  0.10176125244618395,
                  511,
                  52
                ],
                [
                  0.9360146252285192,
                  0.1015625,
                  512,
                  52
                ],
                [
                  0.9378427787934186,
                  0.10136452241715399,
                  513,
                  52
                ],
                [
                  0.9396709323583181,
                  0.10116731517509728,
                  514,
                  52
                ],
                [
                  0.9414990859232175,
                  0.1029126213592233,
                  515,
                  53
                ],
                [
                  0.943327239488117,
                  0.10465116279069768,
                  516,
                  54
                ],
                [
                  0.9451553930530164,
                  0.10638297872340426,
                  517,
                  55
                ],
                [
                  0.946983546617916,
                  0.10810810810810811,
                  518,
                  56
                ],
                [
                  0.9488117001828154,
                  0.10789980732177264,
                  519,
                  56
                ],
                [
                  0.9506398537477148,
                  0.1076923076923077,
                  520,
                  56
                ],
                [
                  0.9524680073126143,
                  0.10940499040307101,
                  521,
                  57
                ],
                [
                  0.9542961608775137,
                  0.10919540229885058,
                  522,
                  57
                ],
                [
                  0.9561243144424132,
                  0.1089866156787763,
                  523,
                  57
                ],
                [
                  0.9579524680073126,
                  0.11068702290076336,
                  524,
                  58
                ],
                [
                  0.9597806215722121,
                  0.11047619047619048,
                  525,
                  58
                ],
                [
                  0.9616087751371115,
                  0.11026615969581749,
                  526,
                  58
                ],
                [
                  0.9634369287020109,
                  0.11195445920303605,
                  527,
                  59
                ],
                [
                  0.9652650822669104,
                  0.11363636363636363,
                  528,
                  60
                ],
                [
                  0.9670932358318098,
                  0.11342155009451796,
                  529,
                  60
                ],
                [
                  0.9689213893967094,
                  0.11509433962264151,
                  530,
                  61
                ],
                [
                  0.9707495429616088,
                  0.1167608286252354,
                  531,
                  62
                ],
                [
                  0.9725776965265083,
                  0.11654135338345864,
                  532,
                  62
                ],
                [
                  0.9744058500914077,
                  0.11819887429643527,
                  533,
                  63
                ],
                [
                  0.9762340036563071,
                  0.11797752808988764,
                  534,
                  63
                ],
                [
                  0.9780621572212066,
                  0.11775700934579439,
                  535,
                  63
                ],
                [
                  0.979890310786106,
                  0.11753731343283583,
                  536,
                  63
                ],
                [
                  0.9817184643510055,
                  0.1191806331471136,
                  537,
                  64
                ],
                [
                  0.9835466179159049,
                  0.11895910780669144,
                  538,
                  64
                ],
                [
                  0.9853747714808044,
                  0.12059369202226346,
                  539,
                  65
                ],
                [
                  0.9872029250457038,
                  0.12222222222222222,
                  540,
                  66
                ],
                [
                  0.9890310786106032,
                  0.12384473197781885,
                  541,
                  67
                ],
                [
                  0.9908592321755028,
                  0.12546125461254612,
                  542,
                  68
                ],
                [
                  0.9926873857404022,
                  0.1270718232044199,
                  543,
                  69
                ],
                [
                  0.9945155393053017,
                  0.12867647058823528,
                  544,
                  70
                ],
                [
                  0.9963436928702011,
                  0.13027522935779817,
                  545,
                  71
                ],
                [
                  0.9981718464351006,
                  0.13186813186813187,
                  546,
                  72
                ],
                [
                  1.0,
                  0.1316270566727605,
                  547,
                  72
                ]
              ]
            }
          ]
        },
        {
          "name": "Jev 1.14.0",
          "text_run": 194,
          "native": [
            {
              "language": "Turkish",
              "wording": "Bilmiyorum",
              "run": 470,
              "full_bank_restricted": false,
              "right": 475,
              "wrong": 69,
              "abstained": 3,
              "invalid": 0,
              "coverage": 0.9945155393053017,
              "risk": 0.12683823529411764
            },
            {
              "language": "Turkish",
              "wording": "I don't know",
              "run": null
            },
            {
              "language": "English",
              "wording": "I don't know",
              "run": 446,
              "full_bank_restricted": false,
              "right": 472,
              "wrong": 72,
              "abstained": 3,
              "invalid": 0,
              "coverage": 0.9945155393053017,
              "risk": 0.1323529411764706
            }
          ],
          "curves": [
            {
              "language": "Turkish",
              "run": 194,
              "points": [
                [
                  0.3016453382084095,
                  0.0,
                  165,
                  0
                ],
                [
                  0.3802559414990859,
                  0.0,
                  208,
                  0
                ],
                [
                  0.4460694698354662,
                  0.004098360655737705,
                  244,
                  1
                ],
                [
                  0.48080438756855576,
                  0.0038022813688212928,
                  263,
                  1
                ],
                [
                  0.5082266910420475,
                  0.0035971223021582736,
                  278,
                  1
                ],
                [
                  0.5411334552102377,
                  0.0033783783783783786,
                  296,
                  1
                ],
                [
                  0.5484460694698354,
                  0.0033333333333333335,
                  300,
                  1
                ],
                [
                  0.5539305301645339,
                  0.0033003300330033004,
                  303,
                  1
                ],
                [
                  0.583180987202925,
                  0.006269592476489028,
                  319,
                  2
                ],
                [
                  0.5850091407678245,
                  0.00625,
                  320,
                  2
                ],
                [
                  0.5996343692870201,
                  0.009146341463414634,
                  328,
                  3
                ],
                [
                  0.6142595978062158,
                  0.011904761904761904,
                  336,
                  4
                ],
                [
                  0.6343692870201096,
                  0.011527377521613832,
                  347,
                  4
                ],
                [
                  0.6581352833638026,
                  0.019444444444444445,
                  360,
                  7
                ],
                [
                  0.6691042047531993,
                  0.01912568306010929,
                  366,
                  7
                ],
                [
                  0.6782449725776966,
                  0.02425876010781671,
                  371,
                  9
                ],
                [
                  0.6873857404021938,
                  0.023936170212765957,
                  376,
                  9
                ],
                [
                  0.696526508226691,
                  0.023622047244094488,
                  381,
                  9
                ],
                [
                  0.7056672760511883,
                  0.023316062176165803,
                  386,
                  9
                ],
                [
                  0.720292504570384,
                  0.02284263959390863,
                  394,
                  9
                ],
                [
                  0.7294332723948812,
                  0.02506265664160401,
                  399,
                  10
                ],
                [
                  0.7422303473491774,
                  0.029556650246305417,
                  406,
                  12
                ],
                [
                  0.7495429616087751,
                  0.03414634146341464,
                  410,
                  14
                ],
                [
                  0.7623400365630713,
                  0.0407673860911271,
                  417,
                  17
                ],
                [
                  0.7696526508226691,
                  0.04275534441805225,
                  421,
                  18
                ],
                [
                  0.7787934186471663,
                  0.04460093896713615,
                  426,
                  19
                ],
                [
                  0.7952468007312614,
                  0.05057471264367816,
                  435,
                  22
                ],
                [
                  0.8025594149908593,
                  0.05011389521640091,
                  439,
                  22
                ],
                [
                  0.8080438756855576,
                  0.05203619909502263,
                  442,
                  23
                ],
                [
                  0.8135283363802559,
                  0.05393258426966292,
                  445,
                  24
                ],
                [
                  0.8208409506398537,
                  0.05790645879732739,
                  449,
                  26
                ],
                [
                  0.8281535648994516,
                  0.059602649006622516,
                  453,
                  27
                ],
                [
                  0.829981718464351,
                  0.05947136563876652,
                  454,
                  27
                ],
                [
                  0.8336380255941499,
                  0.05921052631578947,
                  456,
                  27
                ],
                [
                  0.8409506398537477,
                  0.06521739130434782,
                  460,
                  30
                ],
                [
                  0.8427787934186471,
                  0.0650759219088937,
                  461,
                  30
                ],
                [
                  0.8464351005484461,
                  0.06695464362850972,
                  463,
                  31
                ],
                [
                  0.8555758683729433,
                  0.07264957264957266,
                  468,
                  34
                ],
                [
                  0.8592321755027422,
                  0.07446808510638298,
                  470,
                  35
                ],
                [
                  0.8610603290676416,
                  0.07430997876857749,
                  471,
                  35
                ],
                [
                  0.8720292504570384,
                  0.07756813417190776,
                  477,
                  37
                ],
                [
                  0.8738574040219378,
                  0.0794979079497908,
                  478,
                  38
                ],
                [
                  0.8793418647166362,
                  0.08316008316008316,
                  481,
                  40
                ],
                [
                  0.8829981718464351,
                  0.08488612836438923,
                  483,
                  41
                ],
                [
                  0.8884826325411335,
                  0.08641975308641975,
                  486,
                  42
                ],
                [
                  0.8903107861060329,
                  0.08829568788501027,
                  487,
                  43
                ],
                [
                  0.8939670932358318,
                  0.08793456032719836,
                  489,
                  43
                ],
                [
                  0.8976234003656307,
                  0.08757637474541752,
                  491,
                  43
                ],
                [
                  0.8994515539305301,
                  0.08739837398373984,
                  492,
                  43
                ],
                [
                  0.906764168190128,
                  0.08870967741935484,
                  496,
                  44
                ],
                [
                  0.9122486288848263,
                  0.09218436873747494,
                  499,
                  46
                ],
                [
                  0.9140767824497258,
                  0.094,
                  500,
                  47
                ],
                [
                  0.923217550274223,
                  0.09504950495049505,
                  505,
                  48
                ],
                [
                  0.9287020109689214,
                  0.0984251968503937,
                  508,
                  50
                ],
                [
                  0.9305301645338209,
                  0.09823182711198428,
                  509,
                  50
                ],
                [
                  0.9360146252285192,
                  0.1015625,
                  512,
                  52
                ],
                [
                  0.943327239488117,
                  0.10271317829457365,
                  516,
                  53
                ],
                [
                  0.9451553930530164,
                  0.10251450676982592,
                  517,
                  53
                ],
                [
                  0.946983546617916,
                  0.10424710424710425,
                  518,
                  54
                ],
                [
                  0.9506398537477148,
                  0.10576923076923077,
                  520,
                  55
                ],
                [
                  0.9542961608775137,
                  0.10727969348659004,
                  522,
                  56
                ],
                [
                  0.9597806215722121,
                  0.11238095238095239,
                  525,
                  59
                ],
                [
                  0.9634369287020109,
                  0.11195445920303605,
                  527,
                  59
                ],
                [
                  0.9689213893967094,
                  0.11509433962264151,
                  530,
                  61
                ],
                [
                  0.9725776965265083,
                  0.11654135338345864,
                  532,
                  62
                ],
                [
                  0.9762340036563071,
                  0.11797752808988764,
                  534,
                  63
                ],
                [
                  0.9817184643510055,
                  0.12104283054003724,
                  537,
                  65
                ],
                [
                  0.9890310786106032,
                  0.12384473197781885,
                  541,
                  67
                ],
                [
                  0.9908592321755028,
                  0.12546125461254612,
                  542,
                  68
                ],
                [
                  0.9963436928702011,
                  0.12660550458715597,
                  545,
                  69
                ],
                [
                  0.9981718464351006,
                  0.12637362637362637,
                  546,
                  69
                ],
                [
                  1.0,
                  0.12797074954296161,
                  547,
                  70
                ]
              ]
            },
            {
              "language": "English",
              "run": 410,
              "points": [
                [
                  0.3546617915904936,
                  0.010309278350515464,
                  194,
                  2
                ],
                [
                  0.4625228519195612,
                  0.011857707509881422,
                  253,
                  3
                ],
                [
                  0.5210237659963437,
                  0.014035087719298246,
                  285,
                  4
                ],
                [
                  0.5594149908592322,
                  0.016339869281045753,
                  306,
                  5
                ],
                [
                  0.5886654478976234,
                  0.015527950310559006,
                  322,
                  5
                ],
                [
                  0.6087751371115173,
                  0.015015015015015015,
                  333,
                  5
                ],
                [
                  0.6325411334552102,
                  0.02023121387283237,
                  346,
                  7
                ],
                [
                  0.6453382084095064,
                  0.0226628895184136,
                  353,
                  8
                ],
                [
                  0.6563071297989032,
                  0.022284122562674095,
                  359,
                  8
                ],
                [
                  0.6709323583180987,
                  0.021798365122615803,
                  367,
                  8
                ],
                [
                  0.6782449725776966,
                  0.0215633423180593,
                  371,
                  8
                ],
                [
                  0.6837294332723949,
                  0.0213903743315508,
                  374,
                  8
                ],
                [
                  0.6928702010968921,
                  0.021108179419525065,
                  379,
                  8
                ],
                [
                  0.7020109689213894,
                  0.0234375,
                  384,
                  9
                ],
                [
                  0.7056672760511883,
                  0.023316062176165803,
                  386,
                  9
                ],
                [
                  0.716636197440585,
                  0.025510204081632654,
                  392,
                  10
                ],
                [
                  0.7276051188299817,
                  0.02512562814070352,
                  398,
                  10
                ],
                [
                  0.7330895795246801,
                  0.02743142144638404,
                  401,
                  11
                ],
                [
                  0.7440585009140768,
                  0.029484029484029485,
                  407,
                  12
                ],
                [
                  0.7568555758683729,
                  0.033816425120772944,
                  414,
                  14
                ],
                [
                  0.7586837294332724,
                  0.033734939759036145,
                  415,
                  14
                ],
                [
                  0.7605118829981719,
                  0.03365384615384615,
                  416,
                  14
                ],
                [
                  0.7659963436928702,
                  0.03818615751789976,
                  419,
                  16
                ],
                [
                  0.7769652650822669,
                  0.04,
                  425,
                  17
                ],
                [
                  0.7824497257769653,
                  0.04205607476635514,
                  428,
                  18
                ],
                [
                  0.7861060329067642,
                  0.044186046511627906,
                  430,
                  19
                ],
                [
                  0.7970749542961609,
                  0.045871559633027525,
                  436,
                  20
                ],
                [
                  0.8080438756855576,
                  0.049773755656108594,
                  442,
                  22
                ],
                [
                  0.8153564899451554,
                  0.04932735426008968,
                  446,
                  22
                ],
                [
                  0.8190127970749543,
                  0.05133928571428571,
                  448,
                  23
                ],
                [
                  0.8263254113345521,
                  0.05088495575221239,
                  452,
                  23
                ],
                [
                  0.829981718464351,
                  0.05286343612334802,
                  454,
                  24
                ],
                [
                  0.8372943327239488,
                  0.05458515283842795,
                  458,
                  25
                ],
                [
                  0.8391224862888482,
                  0.054466230936819175,
                  459,
                  25
                ],
                [
                  0.8482632541133455,
                  0.05818965517241379,
                  464,
                  27
                ],
                [
                  0.8537477148080439,
                  0.059957173447537475,
                  467,
                  28
                ],
                [
                  0.8592321755027422,
                  0.06382978723404255,
                  470,
                  30
                ],
                [
                  0.8665447897623401,
                  0.06329113924050633,
                  474,
                  30
                ],
                [
                  0.8756855575868373,
                  0.06680584551148225,
                  479,
                  32
                ],
                [
                  0.8829981718464351,
                  0.07246376811594203,
                  483,
                  35
                ],
                [
                  0.886654478976234,
                  0.07422680412371134,
                  485,
                  36
                ],
                [
                  0.8994515539305301,
                  0.08333333333333333,
                  492,
                  41
                ],
                [
                  0.903107861060329,
                  0.08502024291497975,
                  494,
                  42
                ],
                [
                  0.9049360146252285,
                  0.08686868686868687,
                  495,
                  43
                ],
                [
                  0.9085923217550275,
                  0.08853118712273642,
                  497,
                  44
                ],
                [
                  0.9195612431444241,
                  0.08946322067594434,
                  503,
                  45
                ],
                [
                  0.923217550274223,
                  0.09306930693069307,
                  505,
                  47
                ],
                [
                  0.9305301645338209,
                  0.0962671905697446,
                  509,
                  49
                ],
                [
                  0.9360146252285192,
                  0.09765625,
                  512,
                  50
                ],
                [
                  0.943327239488117,
                  0.10271317829457365,
                  516,
                  53
                ],
                [
                  0.946983546617916,
                  0.10424710424710425,
                  518,
                  54
                ],
                [
                  0.9561243144424132,
                  0.11089866156787763,
                  523,
                  58
                ],
                [
                  0.9597806215722121,
                  0.11428571428571428,
                  525,
                  60
                ],
                [
                  0.9616087751371115,
                  0.11596958174904944,
                  526,
                  61
                ],
                [
                  0.9634369287020109,
                  0.11764705882352941,
                  527,
                  62
                ],
                [
                  0.9689213893967094,
                  0.11886792452830189,
                  530,
                  63
                ],
                [
                  0.9725776965265083,
                  0.12218045112781954,
                  532,
                  65
                ],
                [
                  0.9780621572212066,
                  0.12710280373831775,
                  535,
                  68
                ],
                [
                  0.9817184643510055,
                  0.12849162011173185,
                  537,
                  69
                ],
                [
                  0.9853747714808044,
                  0.12987012987012986,
                  539,
                  70
                ],
                [
                  0.9872029250457038,
                  0.13148148148148148,
                  540,
                  71
                ],
                [
                  0.9890310786106032,
                  0.13123844731977818,
                  541,
                  71
                ],
                [
                  0.9926873857404022,
                  0.13443830570902393,
                  543,
                  73
                ],
                [
                  0.9945155393053017,
                  0.13602941176470587,
                  544,
                  74
                ],
                [
                  0.9981718464351006,
                  0.13736263736263737,
                  546,
                  75
                ],
                [
                  1.0,
                  0.13893967093235832,
                  547,
                  76
                ]
              ]
            }
          ]
        },
        {
          "name": "SubQ Snap 1.1",
          "text_run": 346,
          "native": [
            {
              "language": "Turkish",
              "wording": "Bilmiyorum",
              "run": 392,
              "full_bank_restricted": true,
              "right": 477,
              "wrong": 51,
              "abstained": 19,
              "invalid": 0,
              "coverage": 0.9652650822669104,
              "risk": 0.09659090909090909
            },
            {
              "language": "Turkish",
              "wording": "I don't know",
              "run": 512,
              "full_bank_restricted": true,
              "right": 483,
              "wrong": 45,
              "abstained": 19,
              "invalid": 0,
              "coverage": 0.9652650822669104,
              "risk": 0.08522727272727272
            },
            {
              "language": "English",
              "wording": "I don't know",
              "run": 448,
              "full_bank_restricted": false,
              "right": 463,
              "wrong": 63,
              "abstained": 21,
              "invalid": 0,
              "coverage": 0.9616087751371115,
              "risk": 0.11977186311787072
            }
          ],
          "curves": [
            {
              "language": "Turkish",
              "run": 346,
              "points": [
                [
                  0.0018281535648994515,
                  0.0,
                  1,
                  0
                ],
                [
                  0.003656307129798903,
                  0.0,
                  2,
                  0
                ],
                [
                  0.005484460694698354,
                  0.0,
                  3,
                  0
                ],
                [
                  0.007312614259597806,
                  0.0,
                  4,
                  0
                ],
                [
                  0.009140767824497258,
                  0.0,
                  5,
                  0
                ],
                [
                  0.010968921389396709,
                  0.0,
                  6,
                  0
                ],
                [
                  0.012797074954296161,
                  0.0,
                  7,
                  0
                ],
                [
                  0.014625228519195612,
                  0.0,
                  8,
                  0
                ],
                [
                  0.016453382084095063,
                  0.0,
                  9,
                  0
                ],
                [
                  0.018281535648994516,
                  0.0,
                  10,
                  0
                ],
                [
                  0.02010968921389397,
                  0.0,
                  11,
                  0
                ],
                [
                  0.021937842778793418,
                  0.0,
                  12,
                  0
                ],
                [
                  0.02376599634369287,
                  0.0,
                  13,
                  0
                ],
                [
                  0.025594149908592323,
                  0.0,
                  14,
                  0
                ],
                [
                  0.027422303473491772,
                  0.0,
                  15,
                  0
                ],
                [
                  0.029250457038391225,
                  0.0,
                  16,
                  0
                ],
                [
                  0.031078610603290677,
                  0.0,
                  17,
                  0
                ],
                [
                  0.03290676416819013,
                  0.0,
                  18,
                  0
                ],
                [
                  0.03473491773308958,
                  0.0,
                  19,
                  0
                ],
                [
                  0.03656307129798903,
                  0.0,
                  20,
                  0
                ],
                [
                  0.038391224862888484,
                  0.0,
                  21,
                  0
                ],
                [
                  0.04021937842778794,
                  0.0,
                  22,
                  0
                ],
                [
                  0.04204753199268738,
                  0.0,
                  23,
                  0
                ],
                [
                  0.043875685557586835,
                  0.0,
                  24,
                  0
                ],
                [
                  0.04570383912248629,
                  0.0,
                  25,
                  0
                ],
                [
                  0.04753199268738574,
                  0.0,
                  26,
                  0
                ],
                [
                  0.04936014625228519,
                  0.0,
                  27,
                  0
                ],
                [
                  0.051188299817184646,
                  0.0,
                  28,
                  0
                ],
                [
                  0.05301645338208409,
                  0.0,
                  29,
                  0
                ],
                [
                  0.054844606946983544,
                  0.0,
                  30,
                  0
                ],
                [
                  0.056672760511883,
                  0.0,
                  31,
                  0
                ],
                [
                  0.05850091407678245,
                  0.0,
                  32,
                  0
                ],
                [
                  0.0603290676416819,
                  0.0,
                  33,
                  0
                ],
                [
                  0.062157221206581355,
                  0.0,
                  34,
                  0
                ],
                [
                  0.06398537477148081,
                  0.0,
                  35,
                  0
                ],
                [
                  0.06581352833638025,
                  0.0,
                  36,
                  0
                ],
                [
                  0.06764168190127971,
                  0.0,
                  37,
                  0
                ],
                [
                  0.06946983546617916,
                  0.0,
                  38,
                  0
                ],
                [
                  0.0712979890310786,
                  0.0,
                  39,
                  0
                ],
                [
                  0.07312614259597806,
                  0.0,
                  40,
                  0
                ],
                [
                  0.07495429616087751,
                  0.0,
                  41,
                  0
                ],
                [
                  0.07861060329067641,
                  0.0,
                  43,
                  0
                ],
                [
                  0.08043875685557587,
                  0.0,
                  44,
                  0
                ],
                [
                  0.08226691042047532,
                  0.0,
                  45,
                  0
                ],
                [
                  0.08409506398537477,
                  0.0,
                  46,
                  0
                ],
                [
                  0.08592321755027423,
                  0.0,
                  47,
                  0
                ],
                [
                  0.08775137111517367,
                  0.0,
                  48,
                  0
                ],
                [
                  0.08957952468007313,
                  0.0,
                  49,
                  0
                ],
                [
                  0.09140767824497258,
                  0.0,
                  50,
                  0
                ],
                [
                  0.09323583180987204,
                  0.0,
                  51,
                  0
                ],
                [
                  0.09506398537477148,
                  0.0,
                  52,
                  0
                ],
                [
                  0.09689213893967093,
                  0.0,
                  53,
                  0
                ],
                [
                  0.09872029250457039,
                  0.0,
                  54,
                  0
                ],
                [
                  0.10054844606946983,
                  0.0,
                  55,
                  0
                ],
                [
                  0.10237659963436929,
                  0.0,
                  56,
                  0
                ],
                [
                  0.10420475319926874,
                  0.0,
                  57,
                  0
                ],
                [
                  0.10603290676416818,
                  0.0,
                  58,
                  0
                ],
                [
                  0.10786106032906764,
                  0.0,
                  59,
                  0
                ],
                [
                  0.10968921389396709,
                  0.0,
                  60,
                  0
                ],
                [
                  0.11151736745886655,
                  0.0,
                  61,
                  0
                ],
                [
                  0.113345521023766,
                  0.0,
                  62,
                  0
                ],
                [
                  0.11517367458866545,
                  0.0,
                  63,
                  0
                ],
                [
                  0.1170018281535649,
                  0.0,
                  64,
                  0
                ],
                [
                  0.11882998171846434,
                  0.0,
                  65,
                  0
                ],
                [
                  0.1206581352833638,
                  0.0,
                  66,
                  0
                ],
                [
                  0.12248628884826325,
                  0.0,
                  67,
                  0
                ],
                [
                  0.12431444241316271,
                  0.0,
                  68,
                  0
                ],
                [
                  0.12614259597806216,
                  0.0,
                  69,
                  0
                ],
                [
                  0.12797074954296161,
                  0.0,
                  70,
                  0
                ],
                [
                  0.12979890310786105,
                  0.0,
                  71,
                  0
                ],
                [
                  0.1316270566727605,
                  0.0,
                  72,
                  0
                ],
                [
                  0.13345521023765997,
                  0.0,
                  73,
                  0
                ],
                [
                  0.13528336380255943,
                  0.0,
                  74,
                  0
                ],
                [
                  0.13711151736745886,
                  0.0,
                  75,
                  0
                ],
                [
                  0.13893967093235832,
                  0.0,
                  76,
                  0
                ],
                [
                  0.14076782449725778,
                  0.0,
                  77,
                  0
                ],
                [
                  0.1425959780621572,
                  0.0,
                  78,
                  0
                ],
                [
                  0.14442413162705667,
                  0.0,
                  79,
                  0
                ],
                [
                  0.14625228519195613,
                  0.0,
                  80,
                  0
                ],
                [
                  0.1480804387568556,
                  0.0,
                  81,
                  0
                ],
                [
                  0.14990859232175502,
                  0.0,
                  82,
                  0
                ],
                [
                  0.15173674588665448,
                  0.0,
                  83,
                  0
                ],
                [
                  0.15356489945155394,
                  0.0,
                  84,
                  0
                ],
                [
                  0.15539305301645337,
                  0.0,
                  85,
                  0
                ],
                [
                  0.15722120658135283,
                  0.0,
                  86,
                  0
                ],
                [
                  0.1590493601462523,
                  0.0,
                  87,
                  0
                ],
                [
                  0.16087751371115175,
                  0.0,
                  88,
                  0
                ],
                [
                  0.16270566727605118,
                  0.0,
                  89,
                  0
                ],
                [
                  0.16453382084095064,
                  0.0,
                  90,
                  0
                ],
                [
                  0.1663619744058501,
                  0.0,
                  91,
                  0
                ],
                [
                  0.16819012797074953,
                  0.0,
                  92,
                  0
                ],
                [
                  0.170018281535649,
                  0.0,
                  93,
                  0
                ],
                [
                  0.17184643510054845,
                  0.0,
                  94,
                  0
                ],
                [
                  0.1736745886654479,
                  0.0,
                  95,
                  0
                ],
                [
                  0.17550274223034734,
                  0.0,
                  96,
                  0
                ],
                [
                  0.1773308957952468,
                  0.0,
                  97,
                  0
                ],
                [
                  0.17915904936014626,
                  0.0,
                  98,
                  0
                ],
                [
                  0.1809872029250457,
                  0.0,
                  99,
                  0
                ],
                [
                  0.18281535648994515,
                  0.0,
                  100,
                  0
                ],
                [
                  0.1846435100548446,
                  0.0,
                  101,
                  0
                ],
                [
                  0.18647166361974407,
                  0.0,
                  102,
                  0
                ],
                [
                  0.1882998171846435,
                  0.0,
                  103,
                  0
                ],
                [
                  0.19012797074954296,
                  0.0,
                  104,
                  0
                ],
                [
                  0.19195612431444242,
                  0.0,
                  105,
                  0
                ],
                [
                  0.19378427787934185,
                  0.0,
                  106,
                  0
                ],
                [
                  0.1956124314442413,
                  0.0,
                  107,
                  0
                ],
                [
                  0.19744058500914077,
                  0.0,
                  108,
                  0
                ],
                [
                  0.19926873857404023,
                  0.0,
                  109,
                  0
                ],
                [
                  0.20109689213893966,
                  0.0,
                  110,
                  0
                ],
                [
                  0.20292504570383912,
                  0.0,
                  111,
                  0
                ],
                [
                  0.20475319926873858,
                  0.0,
                  112,
                  0
                ],
                [
                  0.20658135283363802,
                  0.0,
                  113,
                  0
                ],
                [
                  0.20840950639853748,
                  0.0,
                  114,
                  0
                ],
                [
                  0.21023765996343693,
                  0.0,
                  115,
                  0
                ],
                [
                  0.21206581352833637,
                  0.0,
                  116,
                  0
                ],
                [
                  0.21389396709323583,
                  0.0,
                  117,
                  0
                ],
                [
                  0.21572212065813529,
                  0.0,
                  118,
                  0
                ],
                [
                  0.21755027422303475,
                  0.0,
                  119,
                  0
                ],
                [
                  0.21937842778793418,
                  0.0,
                  120,
                  0
                ],
                [
                  0.22120658135283364,
                  0.0,
                  121,
                  0
                ],
                [
                  0.2230347349177331,
                  0.0,
                  122,
                  0
                ],
                [
                  0.22486288848263253,
                  0.0,
                  123,
                  0
                ],
                [
                  0.226691042047532,
                  0.0,
                  124,
                  0
                ],
                [
                  0.22851919561243145,
                  0.0,
                  125,
                  0
                ],
                [
                  0.2303473491773309,
                  0.0,
                  126,
                  0
                ],
                [
                  0.23217550274223034,
                  0.0,
                  127,
                  0
                ],
                [
                  0.2340036563071298,
                  0.0,
                  128,
                  0
                ],
                [
                  0.23583180987202926,
                  0.0,
                  129,
                  0
                ],
                [
                  0.2376599634369287,
                  0.0,
                  130,
                  0
                ],
                [
                  0.23948811700182815,
                  0.0,
                  131,
                  0
                ],
                [
                  0.2413162705667276,
                  0.0,
                  132,
                  0
                ],
                [
                  0.24314442413162707,
                  0.0,
                  133,
                  0
                ],
                [
                  0.2449725776965265,
                  0.0,
                  134,
                  0
                ],
                [
                  0.24680073126142596,
                  0.0,
                  135,
                  0
                ],
                [
                  0.24862888482632542,
                  0.0,
                  136,
                  0
                ],
                [
                  0.25045703839122485,
                  0.0,
                  137,
                  0
                ],
                [
                  0.2522851919561243,
                  0.0,
                  138,
                  0
                ],
                [
                  0.25411334552102377,
                  0.0,
                  139,
                  0
                ],
                [
                  0.25594149908592323,
                  0.0,
                  140,
                  0
                ],
                [
                  0.2577696526508227,
                  0.0,
                  141,
                  0
                ],
                [
                  0.2595978062157221,
                  0.0,
                  142,
                  0
                ],
                [
                  0.26142595978062155,
                  0.0,
                  143,
                  0
                ],
                [
                  0.263254113345521,
                  0.0,
                  144,
                  0
                ],
                [
                  0.26508226691042047,
                  0.0,
                  145,
                  0
                ],
                [
                  0.26691042047531993,
                  0.0,
                  146,
                  0
                ],
                [
                  0.2687385740402194,
                  0.0,
                  147,
                  0
                ],
                [
                  0.27056672760511885,
                  0.0,
                  148,
                  0
                ],
                [
                  0.27239488117001825,
                  0.0,
                  149,
                  0
                ],
                [
                  0.2742230347349177,
                  0.0,
                  150,
                  0
                ],
                [
                  0.2760511882998172,
                  0.0,
                  151,
                  0
                ],
                [
                  0.27787934186471663,
                  0.0,
                  152,
                  0
                ],
                [
                  0.2797074954296161,
                  0.0,
                  153,
                  0
                ],
                [
                  0.28153564899451555,
                  0.0,
                  154,
                  0
                ],
                [
                  0.283363802559415,
                  0.0,
                  155,
                  0
                ],
                [
                  0.2851919561243144,
                  0.0,
                  156,
                  0
                ],
                [
                  0.2870201096892139,
                  0.0,
                  157,
                  0
                ],
                [
                  0.28884826325411334,
                  0.0,
                  158,
                  0
                ],
                [
                  0.2906764168190128,
                  0.0,
                  159,
                  0
                ],
                [
                  0.29250457038391225,
                  0.0,
                  160,
                  0
                ],
                [
                  0.2943327239488117,
                  0.0,
                  161,
                  0
                ],
                [
                  0.2961608775137112,
                  0.0,
                  162,
                  0
                ],
                [
                  0.2979890310786106,
                  0.0,
                  163,
                  0
                ],
                [
                  0.29981718464351004,
                  0.0,
                  164,
                  0
                ],
                [
                  0.3016453382084095,
                  0.0,
                  165,
                  0
                ],
                [
                  0.30347349177330896,
                  0.0,
                  166,
                  0
                ],
                [
                  0.3053016453382084,
                  0.0,
                  167,
                  0
                ],
                [
                  0.3071297989031079,
                  0.0,
                  168,
                  0
                ],
                [
                  0.30895795246800734,
                  0.0,
                  169,
                  0
                ],
                [
                  0.31078610603290674,
                  0.0,
                  170,
                  0
                ],
                [
                  0.3126142595978062,
                  0.0,
                  171,
                  0
                ],
                [
                  0.31444241316270566,
                  0.0,
                  172,
                  0
                ],
                [
                  0.3162705667276051,
                  0.0,
                  173,
                  0
                ],
                [
                  0.3180987202925046,
                  0.0,
                  174,
                  0
                ],
                [
                  0.31992687385740404,
                  0.0,
                  175,
                  0
                ],
                [
                  0.3217550274223035,
                  0.0,
                  176,
                  0
                ],
                [
                  0.3235831809872029,
                  0.0,
                  177,
                  0
                ],
                [
                  0.32541133455210236,
                  0.0,
                  178,
                  0
                ],
                [
                  0.3272394881170018,
                  0.0,
                  179,
                  0
                ],
                [
                  0.3290676416819013,
                  0.0,
                  180,
                  0
                ],
                [
                  0.33089579524680074,
                  0.0,
                  181,
                  0
                ],
                [
                  0.3327239488117002,
                  0.0,
                  182,
                  0
                ],
                [
                  0.33455210237659966,
                  0.0,
                  183,
                  0
                ],
                [
                  0.33638025594149906,
                  0.0,
                  184,
                  0
                ],
                [
                  0.3382084095063985,
                  0.0,
                  185,
                  0
                ],
                [
                  0.340036563071298,
                  0.0,
                  186,
                  0
                ],
                [
                  0.34186471663619744,
                  0.0,
                  187,
                  0
                ],
                [
                  0.3436928702010969,
                  0.0,
                  188,
                  0
                ],
                [
                  0.34552102376599636,
                  0.0,
                  189,
                  0
                ],
                [
                  0.3473491773308958,
                  0.0,
                  190,
                  0
                ],
                [
                  0.3491773308957952,
                  0.0,
                  191,
                  0
                ],
                [
                  0.3510054844606947,
                  0.0,
                  192,
                  0
                ],
                [
                  0.35283363802559414,
                  0.0,
                  193,
                  0
                ],
                [
                  0.3546617915904936,
                  0.0,
                  194,
                  0
                ],
                [
                  0.35648994515539306,
                  0.0,
                  195,
                  0
                ],
                [
                  0.3583180987202925,
                  0.0,
                  196,
                  0
                ],
                [
                  0.360146252285192,
                  0.0,
                  197,
                  0
                ],
                [
                  0.3619744058500914,
                  0.0,
                  198,
                  0
                ],
                [
                  0.36380255941499084,
                  0.0,
                  199,
                  0
                ],
                [
                  0.3656307129798903,
                  0.0,
                  200,
                  0
                ],
                [
                  0.36745886654478976,
                  0.0,
                  201,
                  0
                ],
                [
                  0.3692870201096892,
                  0.0,
                  202,
                  0
                ],
                [
                  0.3711151736745887,
                  0.0,
                  203,
                  0
                ],
                [
                  0.37294332723948814,
                  0.0,
                  204,
                  0
                ],
                [
                  0.37477148080438755,
                  0.0,
                  205,
                  0
                ],
                [
                  0.376599634369287,
                  0.0,
                  206,
                  0
                ],
                [
                  0.37842778793418647,
                  0.0,
                  207,
                  0
                ],
                [
                  0.3802559414990859,
                  0.0,
                  208,
                  0
                ],
                [
                  0.3820840950639854,
                  0.0,
                  209,
                  0
                ],
                [
                  0.38391224862888484,
                  0.0,
                  210,
                  0
                ],
                [
                  0.3857404021937843,
                  0.0,
                  211,
                  0
                ],
                [
                  0.3875685557586837,
                  0.0,
                  212,
                  0
                ],
                [
                  0.38939670932358317,
                  0.0,
                  213,
                  0
                ],
                [
                  0.3912248628884826,
                  0.0,
                  214,
                  0
                ],
                [
                  0.3930530164533821,
                  0.0,
                  215,
                  0
                ],
                [
                  0.39488117001828155,
                  0.0,
                  216,
                  0
                ],
                [
                  0.396709323583181,
                  0.0,
                  217,
                  0
                ],
                [
                  0.39853747714808047,
                  0.0,
                  218,
                  0
                ],
                [
                  0.40036563071297987,
                  0.0,
                  219,
                  0
                ],
                [
                  0.40219378427787933,
                  0.0,
                  220,
                  0
                ],
                [
                  0.4040219378427788,
                  0.0,
                  221,
                  0
                ],
                [
                  0.40585009140767825,
                  0.0,
                  222,
                  0
                ],
                [
                  0.4076782449725777,
                  0.0,
                  223,
                  0
                ],
                [
                  0.40950639853747717,
                  0.0,
                  224,
                  0
                ],
                [
                  0.4113345521023766,
                  0.0,
                  225,
                  0
                ],
                [
                  0.41316270566727603,
                  0.0,
                  226,
                  0
                ],
                [
                  0.4149908592321755,
                  0.0,
                  227,
                  0
                ],
                [
                  0.41681901279707495,
                  0.0,
                  228,
                  0
                ],
                [
                  0.4186471663619744,
                  0.0,
                  229,
                  0
                ],
                [
                  0.42047531992687387,
                  0.0,
                  230,
                  0
                ],
                [
                  0.42230347349177333,
                  0.0,
                  231,
                  0
                ],
                [
                  0.42413162705667273,
                  0.0,
                  232,
                  0
                ],
                [
                  0.4259597806215722,
                  0.0,
                  233,
                  0
                ],
                [
                  0.42778793418647165,
                  0.0,
                  234,
                  0
                ],
                [
                  0.4296160877513711,
                  0.0,
                  235,
                  0
                ],
                [
                  0.43144424131627057,
                  0.0,
                  236,
                  0
                ],
                [
                  0.43327239488117003,
                  0.0,
                  237,
                  0
                ],
                [
                  0.4351005484460695,
                  0.0,
                  238,
                  0
                ],
                [
                  0.4369287020109689,
                  0.0,
                  239,
                  0
                ],
                [
                  0.43875685557586835,
                  0.0,
                  240,
                  0
                ],
                [
                  0.4405850091407678,
                  0.0,
                  241,
                  0
                ],
                [
                  0.4424131627056673,
                  0.0,
                  242,
                  0
                ],
                [
                  0.44424131627056673,
                  0.0,
                  243,
                  0
                ],
                [
                  0.4460694698354662,
                  0.0,
                  244,
                  0
                ],
                [
                  0.44789762340036565,
                  0.0,
                  245,
                  0
                ],
                [
                  0.44972577696526506,
                  0.0,
                  246,
                  0
                ],
                [
                  0.4515539305301645,
                  0.0,
                  247,
                  0
                ],
                [
                  0.453382084095064,
                  0.0,
                  248,
                  0
                ],
                [
                  0.45521023765996343,
                  0.0,
                  249,
                  0
                ],
                [
                  0.4570383912248629,
                  0.0,
                  250,
                  0
                ],
                [
                  0.45886654478976235,
                  0.0,
                  251,
                  0
                ],
                [
                  0.4606946983546618,
                  0.0,
                  252,
                  0
                ],
                [
                  0.4625228519195612,
                  0.0,
                  253,
                  0
                ],
                [
                  0.4643510054844607,
                  0.0,
                  254,
                  0
                ],
                [
                  0.46617915904936014,
                  0.0,
                  255,
                  0
                ],
                [
                  0.4680073126142596,
                  0.0,
                  256,
                  0
                ],
                [
                  0.46983546617915906,
                  0.0,
                  257,
                  0
                ],
                [
                  0.4716636197440585,
                  0.0,
                  258,
                  0
                ],
                [
                  0.473491773308958,
                  0.0,
                  259,
                  0
                ],
                [
                  0.4753199268738574,
                  0.0,
                  260,
                  0
                ],
                [
                  0.47714808043875684,
                  0.0,
                  261,
                  0
                ],
                [
                  0.4789762340036563,
                  0.0,
                  262,
                  0
                ],
                [
                  0.48080438756855576,
                  0.0,
                  263,
                  0
                ],
                [
                  0.4826325411334552,
                  0.0,
                  264,
                  0
                ],
                [
                  0.4844606946983547,
                  0.0,
                  265,
                  0
                ],
                [
                  0.48628884826325414,
                  0.0,
                  266,
                  0
                ],
                [
                  0.48811700182815354,
                  0.0,
                  267,
                  0
                ],
                [
                  0.489945155393053,
                  0.0,
                  268,
                  0
                ],
                [
                  0.49177330895795246,
                  0.0,
                  269,
                  0
                ],
                [
                  0.4936014625228519,
                  0.0,
                  270,
                  0
                ],
                [
                  0.4954296160877514,
                  0.0,
                  271,
                  0
                ],
                [
                  0.49725776965265084,
                  0.0,
                  272,
                  0
                ],
                [
                  0.4990859232175503,
                  0.0,
                  273,
                  0
                ],
                [
                  0.5009140767824497,
                  0.0,
                  274,
                  0
                ],
                [
                  0.5027422303473492,
                  0.0,
                  275,
                  0
                ],
                [
                  0.5045703839122486,
                  0.0,
                  276,
                  0
                ],
                [
                  0.506398537477148,
                  0.0,
                  277,
                  0
                ],
                [
                  0.5082266910420475,
                  0.0,
                  278,
                  0
                ],
                [
                  0.5100548446069469,
                  0.0,
                  279,
                  0
                ],
                [
                  0.5118829981718465,
                  0.0,
                  280,
                  0
                ],
                [
                  0.5137111517367459,
                  0.0,
                  281,
                  0
                ],
                [
                  0.5155393053016454,
                  0.0,
                  282,
                  0
                ],
                [
                  0.5173674588665448,
                  0.0,
                  283,
                  0
                ],
                [
                  0.5191956124314442,
                  0.0,
                  284,
                  0
                ],
                [
                  0.5210237659963437,
                  0.0,
                  285,
                  0
                ],
                [
                  0.5228519195612431,
                  0.0,
                  286,
                  0
                ],
                [
                  0.5246800731261426,
                  0.0,
                  287,
                  0
                ],
                [
                  0.526508226691042,
                  0.0,
                  288,
                  0
                ],
                [
                  0.5283363802559415,
                  0.0,
                  289,
                  0
                ],
                [
                  0.5301645338208409,
                  0.0,
                  290,
                  0
                ],
                [
                  0.5319926873857403,
                  0.0,
                  291,
                  0
                ],
                [
                  0.5338208409506399,
                  0.0,
                  292,
                  0
                ],
                [
                  0.5356489945155393,
                  0.0,
                  293,
                  0
                ],
                [
                  0.5374771480804388,
                  0.0,
                  294,
                  0
                ],
                [
                  0.5393053016453382,
                  0.0,
                  295,
                  0
                ],
                [
                  0.5411334552102377,
                  0.0,
                  296,
                  0
                ],
                [
                  0.5429616087751371,
                  0.0,
                  297,
                  0
                ],
                [
                  0.5447897623400365,
                  0.0,
                  298,
                  0
                ],
                [
                  0.546617915904936,
                  0.0,
                  299,
                  0
                ],
                [
                  0.5484460694698354,
                  0.0,
                  300,
                  0
                ],
                [
                  0.5502742230347349,
                  0.0,
                  301,
                  0
                ],
                [
                  0.5521023765996343,
                  0.0,
                  302,
                  0
                ],
                [
                  0.5539305301645339,
                  0.0,
                  303,
                  0
                ],
                [
                  0.5557586837294333,
                  0.0,
                  304,
                  0
                ],
                [
                  0.5575868372943327,
                  0.003278688524590164,
                  305,
                  1
                ],
                [
                  0.5594149908592322,
                  0.0032679738562091504,
                  306,
                  1
                ],
                [
                  0.5612431444241316,
                  0.003257328990228013,
                  307,
                  1
                ],
                [
                  0.5630712979890311,
                  0.003246753246753247,
                  308,
                  1
                ],
                [
                  0.5648994515539305,
                  0.003236245954692557,
                  309,
                  1
                ],
                [
                  0.56672760511883,
                  0.0032258064516129032,
                  310,
                  1
                ],
                [
                  0.5685557586837294,
                  0.003215434083601286,
                  311,
                  1
                ],
                [
                  0.5703839122486288,
                  0.003205128205128205,
                  312,
                  1
                ],
                [
                  0.5722120658135283,
                  0.003194888178913738,
                  313,
                  1
                ],
                [
                  0.5740402193784278,
                  0.0031847133757961785,
                  314,
                  1
                ],
                [
                  0.5758683729433273,
                  0.0031746031746031746,
                  315,
                  1
                ],
                [
                  0.5776965265082267,
                  0.0031645569620253164,
                  316,
                  1
                ],
                [
                  0.5795246800731262,
                  0.0031545741324921135,
                  317,
                  1
                ],
                [
                  0.5813528336380256,
                  0.0031446540880503146,
                  318,
                  1
                ],
                [
                  0.583180987202925,
                  0.003134796238244514,
                  319,
                  1
                ],
                [
                  0.5850091407678245,
                  0.003125,
                  320,
                  1
                ],
                [
                  0.5868372943327239,
                  0.003115264797507788,
                  321,
                  1
                ],
                [
                  0.5886654478976234,
                  0.003105590062111801,
                  322,
                  1
                ],
                [
                  0.5904936014625228,
                  0.0030959752321981426,
                  323,
                  1
                ],
                [
                  0.5923217550274223,
                  0.0030864197530864196,
                  324,
                  1
                ],
                [
                  0.5941499085923218,
                  0.003076923076923077,
                  325,
                  1
                ],
                [
                  0.5959780621572212,
                  0.003067484662576687,
                  326,
                  1
                ],
                [
                  0.5978062157221207,
                  0.0030581039755351682,
                  327,
                  1
                ],
                [
                  0.5996343692870201,
                  0.003048780487804878,
                  328,
                  1
                ],
                [
                  0.6014625228519196,
                  0.00303951367781155,
                  329,
                  1
                ],
                [
                  0.603290676416819,
                  0.0030303030303030303,
                  330,
                  1
                ],
                [
                  0.6051188299817185,
                  0.0030211480362537764,
                  331,
                  1
                ],
                [
                  0.6069469835466179,
                  0.0030120481927710845,
                  332,
                  1
                ],
                [
                  0.6087751371115173,
                  0.003003003003003003,
                  333,
                  1
                ],
                [
                  0.6106032906764168,
                  0.005988023952095809,
                  334,
                  2
                ],
                [
                  0.6124314442413162,
                  0.005970149253731343,
                  335,
                  2
                ],
                [
                  0.6142595978062158,
                  0.005952380952380952,
                  336,
                  2
                ],
                [
                  0.6160877513711152,
                  0.005934718100890208,
                  337,
                  2
                ],
                [
                  0.6179159049360147,
                  0.005917159763313609,
                  338,
                  2
                ],
                [
                  0.6197440585009141,
                  0.0058997050147492625,
                  339,
                  2
                ],
                [
                  0.6215722120658135,
                  0.0058823529411764705,
                  340,
                  2
                ],
                [
                  0.623400365630713,
                  0.005865102639296188,
                  341,
                  2
                ],
                [
                  0.6252285191956124,
                  0.005847953216374269,
                  342,
                  2
                ],
                [
                  0.6270566727605119,
                  0.0058309037900874635,
                  343,
                  2
                ],
                [
                  0.6288848263254113,
                  0.005813953488372093,
                  344,
                  2
                ],
                [
                  0.6307129798903108,
                  0.005797101449275362,
                  345,
                  2
                ],
                [
                  0.6325411334552102,
                  0.005780346820809248,
                  346,
                  2
                ],
                [
                  0.6343692870201096,
                  0.005763688760806916,
                  347,
                  2
                ],
                [
                  0.6361974405850092,
                  0.005747126436781609,
                  348,
                  2
                ],
                [
                  0.6380255941499086,
                  0.008595988538681949,
                  349,
                  3
                ],
                [
                  0.6398537477148081,
                  0.008571428571428572,
                  350,
                  3
                ],
                [
                  0.6416819012797075,
                  0.008547008547008548,
                  351,
                  3
                ],
                [
                  0.643510054844607,
                  0.008522727272727272,
                  352,
                  3
                ],
                [
                  0.6453382084095064,
                  0.0084985835694051,
                  353,
                  3
                ],
                [
                  0.6471663619744058,
                  0.00847457627118644,
                  354,
                  3
                ],
                [
                  0.6489945155393053,
                  0.008450704225352112,
                  355,
                  3
                ],
                [
                  0.6508226691042047,
                  0.008426966292134831,
                  356,
                  3
                ],
                [
                  0.6526508226691042,
                  0.011204481792717087,
                  357,
                  4
                ],
                [
                  0.6544789762340036,
                  0.0111731843575419,
                  358,
                  4
                ],
                [
                  0.6563071297989032,
                  0.011142061281337047,
                  359,
                  4
                ],
                [
                  0.6581352833638026,
                  0.011111111111111112,
                  360,
                  4
                ],
                [
                  0.659963436928702,
                  0.0110803324099723,
                  361,
                  4
                ],
                [
                  0.6617915904936015,
                  0.011049723756906077,
                  362,
                  4
                ],
                [
                  0.6636197440585009,
                  0.011019283746556474,
                  363,
                  4
                ],
                [
                  0.6654478976234004,
                  0.01098901098901099,
                  364,
                  4
                ],
                [
                  0.6672760511882998,
                  0.010958904109589041,
                  365,
                  4
                ],
                [
                  0.6691042047531993,
                  0.01092896174863388,
                  366,
                  4
                ],
                [
                  0.6709323583180987,
                  0.010899182561307902,
                  367,
                  4
                ],
                [
                  0.6727605118829981,
                  0.01358695652173913,
                  368,
                  5
                ],
                [
                  0.6745886654478976,
                  0.013550135501355014,
                  369,
                  5
                ],
                [
                  0.676416819012797,
                  0.013513513513513514,
                  370,
                  5
                ],
                [
                  0.6782449725776966,
                  0.013477088948787063,
                  371,
                  5
                ],
                [
                  0.680073126142596,
                  0.013440860215053764,
                  372,
                  5
                ],
                [
                  0.6819012797074955,
                  0.013404825737265416,
                  373,
                  5
                ],
                [
                  0.6837294332723949,
                  0.016042780748663103,
                  374,
                  6
                ],
                [
                  0.6855575868372943,
                  0.016,
                  375,
                  6
                ],
                [
                  0.6873857404021938,
                  0.015957446808510637,
                  376,
                  6
                ],
                [
                  0.6892138939670932,
                  0.015915119363395226,
                  377,
                  6
                ],
                [
                  0.6910420475319927,
                  0.015873015873015872,
                  378,
                  6
                ],
                [
                  0.6928702010968921,
                  0.018469656992084433,
                  379,
                  7
                ],
                [
                  0.6946983546617916,
                  0.018421052631578946,
                  380,
                  7
                ],
                [
                  0.696526508226691,
                  0.01837270341207349,
                  381,
                  7
                ],
                [
                  0.6983546617915904,
                  0.01832460732984293,
                  382,
                  7
                ],
                [
                  0.70018281535649,
                  0.018276762402088774,
                  383,
                  7
                ],
                [
                  0.7020109689213894,
                  0.018229166666666668,
                  384,
                  7
                ],
                [
                  0.7038391224862889,
                  0.01818181818181818,
                  385,
                  7
                ],
                [
                  0.7056672760511883,
                  0.018134715025906734,
                  386,
                  7
                ],
                [
                  0.7074954296160878,
                  0.01808785529715762,
                  387,
                  7
                ],
                [
                  0.7093235831809872,
                  0.01804123711340206,
                  388,
                  7
                ],
                [
                  0.7111517367458866,
                  0.017994858611825194,
                  389,
                  7
                ],
                [
                  0.7129798903107861,
                  0.017948717948717947,
                  390,
                  7
                ],
                [
                  0.7148080438756855,
                  0.017902813299232736,
                  391,
                  7
                ],
                [
                  0.716636197440585,
                  0.017857142857142856,
                  392,
                  7
                ],
                [
                  0.7184643510054844,
                  0.017811704834605598,
                  393,
                  7
                ],
                [
                  0.720292504570384,
                  0.017766497461928935,
                  394,
                  7
                ],
                [
                  0.7221206581352834,
                  0.017721518987341773,
                  395,
                  7
                ],
                [
                  0.7239488117001828,
                  0.017676767676767676,
                  396,
                  7
                ],
                [
                  0.7257769652650823,
                  0.017632241813602016,
                  397,
                  7
                ],
                [
                  0.7276051188299817,
                  0.017587939698492462,
                  398,
                  7
                ],
                [
                  0.7294332723948812,
                  0.017543859649122806,
                  399,
                  7
                ],
                [
                  0.7312614259597806,
                  0.0175,
                  400,
                  7
                ],
                [
                  0.7330895795246801,
                  0.017456359102244388,
                  401,
                  7
                ],
                [
                  0.7349177330895795,
                  0.017412935323383085,
                  402,
                  7
                ],
                [
                  0.7367458866544789,
                  0.017369727047146403,
                  403,
                  7
                ],
                [
                  0.7385740402193784,
                  0.017326732673267328,
                  404,
                  7
                ],
                [
                  0.7404021937842779,
                  0.01728395061728395,
                  405,
                  7
                ],
                [
                  0.7422303473491774,
                  0.017241379310344827,
                  406,
                  7
                ],
                [
                  0.7440585009140768,
                  0.019656019656019656,
                  407,
                  8
                ],
                [
                  0.7458866544789763,
                  0.0196078431372549,
                  408,
                  8
                ],
                [
                  0.7477148080438757,
                  0.019559902200488997,
                  409,
                  8
                ],
                [
                  0.7495429616087751,
                  0.01951219512195122,
                  410,
                  8
                ],
                [
                  0.7513711151736746,
                  0.019464720194647202,
                  411,
                  8
                ],
                [
                  0.753199268738574,
                  0.019417475728155338,
                  412,
                  8
                ],
                [
                  0.7550274223034735,
                  0.01937046004842615,
                  413,
                  8
                ],
                [
                  0.7568555758683729,
                  0.01932367149758454,
                  414,
                  8
                ],
                [
                  0.7586837294332724,
                  0.01927710843373494,
                  415,
                  8
                ],
                [
                  0.7605118829981719,
                  0.019230769230769232,
                  416,
                  8
                ],
                [
                  0.7623400365630713,
                  0.019184652278177457,
                  417,
                  8
                ],
                [
                  0.7641681901279708,
                  0.019138755980861243,
                  418,
                  8
                ],
                [
                  0.7659963436928702,
                  0.01909307875894988,
                  419,
                  8
                ],
                [
                  0.7678244972577697,
                  0.01904761904761905,
                  420,
                  8
                ],
                [
                  0.7696526508226691,
                  0.019002375296912115,
                  421,
                  8
                ],
                [
                  0.7714808043875686,
                  0.018957345971563982,
                  422,
                  8
                ],
                [
                  0.773308957952468,
                  0.018912529550827423,
                  423,
                  8
                ],
                [
                  0.7751371115173674,
                  0.018867924528301886,
                  424,
                  8
                ],
                [
                  0.7769652650822669,
                  0.018823529411764704,
                  425,
                  8
                ],
                [
                  0.7787934186471663,
                  0.018779342723004695,
                  426,
                  8
                ],
                [
                  0.7806215722120659,
                  0.01873536299765808,
                  427,
                  8
                ],
                [
                  0.7824497257769653,
                  0.018691588785046728,
                  428,
                  8
                ],
                [
                  0.7842778793418648,
                  0.018648018648018648,
                  429,
                  8
                ],
                [
                  0.7861060329067642,
                  0.018604651162790697,
                  430,
                  8
                ],
                [
                  0.7879341864716636,
                  0.018561484918793503,
                  431,
                  8
                ],
                [
                  0.7897623400365631,
                  0.018518518518518517,
                  432,
                  8
                ],
                [
                  0.7915904936014625,
                  0.018475750577367205,
                  433,
                  8
                ],
                [
                  0.793418647166362,
                  0.018433179723502304,
                  434,
                  8
                ],
                [
                  0.7952468007312614,
                  0.01839080459770115,
                  435,
                  8
                ],
                [
                  0.7970749542961609,
                  0.020642201834862386,
                  436,
                  9
                ],
                [
                  0.7989031078610603,
                  0.020594965675057208,
                  437,
                  9
                ],
                [
                  0.8007312614259597,
                  0.0228310502283105,
                  438,
                  10
                ],
                [
                  0.8025594149908593,
                  0.022779043280182234,
                  439,
                  10
                ],
                [
                  0.8043875685557587,
                  0.025,
                  440,
                  11
                ],
                [
                  0.8062157221206582,
                  0.024943310657596373,
                  441,
                  11
                ],
                [
                  0.8080438756855576,
                  0.024886877828054297,
                  442,
                  11
                ],
                [
                  0.8098720292504571,
                  0.02708803611738149,
                  443,
                  12
                ],
                [
                  0.8117001828153565,
                  0.02702702702702703,
                  444,
                  12
                ],
                [
                  0.8135283363802559,
                  0.029213483146067417,
                  445,
                  13
                ],
                [
                  0.8153564899451554,
                  0.02914798206278027,
                  446,
                  13
                ],
                [
                  0.8171846435100548,
                  0.029082774049217,
                  447,
                  13
                ],
                [
                  0.8190127970749543,
                  0.029017857142857144,
                  448,
                  13
                ],
                [
                  0.8208409506398537,
                  0.028953229398663696,
                  449,
                  13
                ],
                [
                  0.8226691042047533,
                  0.028888888888888888,
                  450,
                  13
                ],
                [
                  0.8244972577696527,
                  0.028824833702882482,
                  451,
                  13
                ],
                [
                  0.8263254113345521,
                  0.028761061946902654,
                  452,
                  13
                ],
                [
                  0.8281535648994516,
                  0.02869757174392936,
                  453,
                  13
                ],
                [
                  0.829981718464351,
                  0.030837004405286344,
                  454,
                  14
                ],
                [
                  0.8318098720292505,
                  0.03296703296703297,
                  455,
                  15
                ],
                [
                  0.8336380255941499,
                  0.03289473684210526,
                  456,
                  15
                ],
                [
                  0.8354661791590493,
                  0.03282275711159737,
                  457,
                  15
                ],
                [
                  0.8372943327239488,
                  0.03275109170305677,
                  458,
                  15
                ],
                [
                  0.8391224862888482,
                  0.034858387799564274,
                  459,
                  16
                ],
                [
                  0.8409506398537477,
                  0.034782608695652174,
                  460,
                  16
                ],
                [
                  0.8427787934186471,
                  0.03470715835140998,
                  461,
                  16
                ],
                [
                  0.8446069469835467,
                  0.03463203463203463,
                  462,
                  16
                ],
                [
                  0.8464351005484461,
                  0.03455723542116631,
                  463,
                  16
                ],
                [
                  0.8482632541133455,
                  0.034482758620689655,
                  464,
                  16
                ],
                [
                  0.850091407678245,
                  0.034408602150537634,
                  465,
                  16
                ],
                [
                  0.8519195612431444,
                  0.034334763948497854,
                  466,
                  16
                ],
                [
                  0.8537477148080439,
                  0.034261241970021415,
                  467,
                  16
                ],
                [
                  0.8555758683729433,
                  0.03632478632478633,
                  468,
                  17
                ],
                [
                  0.8574040219378428,
                  0.03624733475479744,
                  469,
                  17
                ],
                [
                  0.8592321755027422,
                  0.036170212765957444,
                  470,
                  17
                ],
                [
                  0.8610603290676416,
                  0.036093418259023353,
                  471,
                  17
                ],
                [
                  0.8628884826325411,
                  0.036016949152542374,
                  472,
                  17
                ],
                [
                  0.8647166361974405,
                  0.03805496828752643,
                  473,
                  18
                ],
                [
                  0.8665447897623401,
                  0.0379746835443038,
                  474,
                  18
                ],
                [
                  0.8683729433272395,
                  0.04,
                  475,
                  19
                ],
                [
                  0.870201096892139,
                  0.03991596638655462,
                  476,
                  19
                ],
                [
                  0.8720292504570384,
                  0.039832285115303984,
                  477,
                  19
                ],
                [
                  0.8738574040219378,
                  0.04184100418410042,
                  478,
                  20
                ],
                [
                  0.8756855575868373,
                  0.04384133611691023,
                  479,
                  21
                ],
                [
                  0.8775137111517367,
                  0.04375,
                  480,
                  21
                ],
                [
                  0.8793418647166362,
                  0.04365904365904366,
                  481,
                  21
                ],
                [
                  0.8811700182815356,
                  0.04564315352697095,
                  482,
                  22
                ],
                [
                  0.8829981718464351,
                  0.047619047619047616,
                  483,
                  23
                ],
                [
                  0.8848263254113345,
                  0.047520661157024795,
                  484,
                  23
                ],
                [
                  0.886654478976234,
                  0.04742268041237113,
                  485,
                  23
                ],
                [
                  0.8884826325411335,
                  0.047325102880658436,
                  486,
                  23
                ],
                [
                  0.8903107861060329,
                  0.049281314168377825,
                  487,
                  24
                ],
                [
                  0.8921389396709324,
                  0.04918032786885246,
                  488,
                  24
                ],
                [
                  0.8939670932358318,
                  0.05112474437627812,
                  489,
                  25
                ],
                [
                  0.8957952468007313,
                  0.05102040816326531,
                  490,
                  25
                ],
                [
                  0.8976234003656307,
                  0.05091649694501019,
                  491,
                  25
                ],
                [
                  0.8994515539305301,
                  0.0508130081300813,
                  492,
                  25
                ],
                [
                  0.9012797074954296,
                  0.05070993914807302,
                  493,
                  25
                ],
                [
                  0.903107861060329,
                  0.05263157894736842,
                  494,
                  26
                ],
                [
                  0.9049360146252285,
                  0.052525252525252523,
                  495,
                  26
                ],
                [
                  0.906764168190128,
                  0.05443548387096774,
                  496,
                  27
                ],
                [
                  0.9085923217550275,
                  0.056338028169014086,
                  497,
                  28
                ],
                [
                  0.9104204753199269,
                  0.05622489959839357,
                  498,
                  28
                ],
                [
                  0.9122486288848263,
                  0.05811623246492986,
                  499,
                  29
                ],
                [
                  0.9140767824497258,
                  0.06,
                  500,
                  30
                ],
                [
                  0.9159049360146252,
                  0.059880239520958084,
                  501,
                  30
                ],
                [
                  0.9177330895795247,
                  0.061752988047808766,
                  502,
                  31
                ],
                [
                  0.9195612431444241,
                  0.0636182902584493,
                  503,
                  32
                ],
                [
                  0.9213893967093236,
                  0.06349206349206349,
                  504,
                  32
                ],
                [
                  0.923217550274223,
                  0.06336633663366337,
                  505,
                  32
                ],
                [
                  0.9250457038391224,
                  0.06324110671936758,
                  506,
                  32
                ],
                [
                  0.926873857404022,
                  0.0631163708086785,
                  507,
                  32
                ],
                [
                  0.9287020109689214,
                  0.06299212598425197,
                  508,
                  32
                ],
                [
                  0.9305301645338209,
                  0.06483300589390963,
                  509,
                  33
                ],
                [
                  0.9323583180987203,
                  0.06666666666666667,
                  510,
                  34
                ],
                [
                  0.9341864716636198,
                  0.0684931506849315,
                  511,
                  35
                ],
                [
                  0.9360146252285192,
                  0.068359375,
                  512,
                  35
                ],
                [
                  0.9378427787934186,
                  0.07017543859649122,
                  513,
                  36
                ],
                [
                  0.9396709323583181,
                  0.07198443579766536,
                  514,
                  37
                ],
                [
                  0.9414990859232175,
                  0.07378640776699029,
                  515,
                  38
                ],
                [
                  0.943327239488117,
                  0.0755813953488372,
                  516,
                  39
                ],
                [
                  0.9451553930530164,
                  0.07736943907156674,
                  517,
                  40
                ],
                [
                  0.946983546617916,
                  0.07915057915057915,
                  518,
                  41
                ],
                [
                  0.9488117001828154,
                  0.08092485549132948,
                  519,
                  42
                ],
                [
                  0.9506398537477148,
                  0.08076923076923077,
                  520,
                  42
                ],
                [
                  0.9524680073126143,
                  0.08253358925143954,
                  521,
                  43
                ],
                [
                  0.9542961608775137,
                  0.08237547892720307,
                  522,
                  43
                ],
                [
                  0.9561243144424132,
                  0.0841300191204589,
                  523,
                  44
                ],
                [
                  0.9579524680073126,
                  0.08587786259541985,
                  524,
                  45
                ],
                [
                  0.9597806215722121,
                  0.08761904761904762,
                  525,
                  46
                ],
                [
                  0.9616087751371115,
                  0.08935361216730038,
                  526,
                  47
                ],
                [
                  0.9634369287020109,
                  0.08918406072106262,
                  527,
                  47
                ],
                [
                  0.9652650822669104,
                  0.08901515151515152,
                  528,
                  47
                ],
                [
                  0.9670932358318098,
                  0.09073724007561437,
                  529,
                  48
                ],
                [
                  0.9689213893967094,
                  0.09245283018867924,
                  530,
                  49
                ],
                [
                  0.9707495429616088,
                  0.09416195856873823,
                  531,
                  50
                ],
                [
                  0.9725776965265083,
                  0.09398496240601503,
                  532,
                  50
                ],
                [
                  0.9744058500914077,
                  0.09568480300187618,
                  533,
                  51
                ],
                [
                  0.9762340036563071,
                  0.09737827715355805,
                  534,
                  52
                ],
                [
                  0.9780621572212066,
                  0.09719626168224299,
                  535,
                  52
                ],
                [
                  0.979890310786106,
                  0.09701492537313433,
                  536,
                  52
                ],
                [
                  0.9817184643510055,
                  0.09869646182495345,
                  537,
                  53
                ],
                [
                  0.9835466179159049,
                  0.10037174721189591,
                  538,
                  54
                ],
                [
                  0.9853747714808044,
                  0.10204081632653061,
                  539,
                  55
                ],
                [
                  0.9872029250457038,
                  0.1037037037037037,
                  540,
                  56
                ],
                [
                  0.9890310786106032,
                  0.10351201478743069,
                  541,
                  56
                ],
                [
                  0.9908592321755028,
                  0.10516605166051661,
                  542,
                  57
                ],
                [
                  0.9926873857404022,
                  0.10497237569060773,
                  543,
                  57
                ],
                [
                  0.9945155393053017,
                  0.10661764705882353,
                  544,
                  58
                ],
                [
                  0.9963436928702011,
                  0.10642201834862386,
                  545,
                  58
                ],
                [
                  0.9981718464351006,
                  0.10622710622710622,
                  546,
                  58
                ],
                [
                  1.0,
                  0.10786106032906764,
                  547,
                  59
                ]
              ]
            },
            {
              "language": "English",
              "run": 413,
              "points": [
                [
                  0.0018281535648994515,
                  0.0,
                  1,
                  0
                ],
                [
                  0.003656307129798903,
                  0.0,
                  2,
                  0
                ],
                [
                  0.005484460694698354,
                  0.0,
                  3,
                  0
                ],
                [
                  0.007312614259597806,
                  0.0,
                  4,
                  0
                ],
                [
                  0.009140767824497258,
                  0.0,
                  5,
                  0
                ],
                [
                  0.010968921389396709,
                  0.0,
                  6,
                  0
                ],
                [
                  0.012797074954296161,
                  0.0,
                  7,
                  0
                ],
                [
                  0.014625228519195612,
                  0.0,
                  8,
                  0
                ],
                [
                  0.016453382084095063,
                  0.0,
                  9,
                  0
                ],
                [
                  0.018281535648994516,
                  0.0,
                  10,
                  0
                ],
                [
                  0.02010968921389397,
                  0.0,
                  11,
                  0
                ],
                [
                  0.021937842778793418,
                  0.0,
                  12,
                  0
                ],
                [
                  0.02376599634369287,
                  0.0,
                  13,
                  0
                ],
                [
                  0.025594149908592323,
                  0.0,
                  14,
                  0
                ],
                [
                  0.027422303473491772,
                  0.0,
                  15,
                  0
                ],
                [
                  0.029250457038391225,
                  0.0,
                  16,
                  0
                ],
                [
                  0.031078610603290677,
                  0.0,
                  17,
                  0
                ],
                [
                  0.03290676416819013,
                  0.0,
                  18,
                  0
                ],
                [
                  0.03473491773308958,
                  0.0,
                  19,
                  0
                ],
                [
                  0.03656307129798903,
                  0.0,
                  20,
                  0
                ],
                [
                  0.038391224862888484,
                  0.0,
                  21,
                  0
                ],
                [
                  0.04021937842778794,
                  0.0,
                  22,
                  0
                ],
                [
                  0.04204753199268738,
                  0.0,
                  23,
                  0
                ],
                [
                  0.043875685557586835,
                  0.0,
                  24,
                  0
                ],
                [
                  0.04570383912248629,
                  0.0,
                  25,
                  0
                ],
                [
                  0.04753199268738574,
                  0.0,
                  26,
                  0
                ],
                [
                  0.04936014625228519,
                  0.0,
                  27,
                  0
                ],
                [
                  0.051188299817184646,
                  0.0,
                  28,
                  0
                ],
                [
                  0.05301645338208409,
                  0.0,
                  29,
                  0
                ],
                [
                  0.054844606946983544,
                  0.0,
                  30,
                  0
                ],
                [
                  0.056672760511883,
                  0.0,
                  31,
                  0
                ],
                [
                  0.05850091407678245,
                  0.0,
                  32,
                  0
                ],
                [
                  0.0603290676416819,
                  0.0,
                  33,
                  0
                ],
                [
                  0.062157221206581355,
                  0.0,
                  34,
                  0
                ],
                [
                  0.06398537477148081,
                  0.0,
                  35,
                  0
                ],
                [
                  0.06581352833638025,
                  0.0,
                  36,
                  0
                ],
                [
                  0.06764168190127971,
                  0.0,
                  37,
                  0
                ],
                [
                  0.06946983546617916,
                  0.0,
                  38,
                  0
                ],
                [
                  0.0712979890310786,
                  0.0,
                  39,
                  0
                ],
                [
                  0.07312614259597806,
                  0.0,
                  40,
                  0
                ],
                [
                  0.07495429616087751,
                  0.0,
                  41,
                  0
                ],
                [
                  0.07678244972577697,
                  0.0,
                  42,
                  0
                ],
                [
                  0.07861060329067641,
                  0.0,
                  43,
                  0
                ],
                [
                  0.08043875685557587,
                  0.0,
                  44,
                  0
                ],
                [
                  0.08226691042047532,
                  0.0,
                  45,
                  0
                ],
                [
                  0.08409506398537477,
                  0.0,
                  46,
                  0
                ],
                [
                  0.08592321755027423,
                  0.0,
                  47,
                  0
                ],
                [
                  0.08775137111517367,
                  0.0,
                  48,
                  0
                ],
                [
                  0.08957952468007313,
                  0.0,
                  49,
                  0
                ],
                [
                  0.09140767824497258,
                  0.0,
                  50,
                  0
                ],
                [
                  0.09323583180987204,
                  0.0,
                  51,
                  0
                ],
                [
                  0.09506398537477148,
                  0.0,
                  52,
                  0
                ],
                [
                  0.09689213893967093,
                  0.0,
                  53,
                  0
                ],
                [
                  0.09872029250457039,
                  0.0,
                  54,
                  0
                ],
                [
                  0.10054844606946983,
                  0.0,
                  55,
                  0
                ],
                [
                  0.10237659963436929,
                  0.0,
                  56,
                  0
                ],
                [
                  0.10420475319926874,
                  0.0,
                  57,
                  0
                ],
                [
                  0.10603290676416818,
                  0.0,
                  58,
                  0
                ],
                [
                  0.10786106032906764,
                  0.0,
                  59,
                  0
                ],
                [
                  0.10968921389396709,
                  0.0,
                  60,
                  0
                ],
                [
                  0.11151736745886655,
                  0.0,
                  61,
                  0
                ],
                [
                  0.113345521023766,
                  0.0,
                  62,
                  0
                ],
                [
                  0.11517367458866545,
                  0.0,
                  63,
                  0
                ],
                [
                  0.1170018281535649,
                  0.0,
                  64,
                  0
                ],
                [
                  0.11882998171846434,
                  0.0,
                  65,
                  0
                ],
                [
                  0.1206581352833638,
                  0.0,
                  66,
                  0
                ],
                [
                  0.12248628884826325,
                  0.0,
                  67,
                  0
                ],
                [
                  0.12431444241316271,
                  0.0,
                  68,
                  0
                ],
                [
                  0.12614259597806216,
                  0.0,
                  69,
                  0
                ],
                [
                  0.12797074954296161,
                  0.0,
                  70,
                  0
                ],
                [
                  0.12979890310786105,
                  0.0,
                  71,
                  0
                ],
                [
                  0.1316270566727605,
                  0.0,
                  72,
                  0
                ],
                [
                  0.13345521023765997,
                  0.0,
                  73,
                  0
                ],
                [
                  0.13528336380255943,
                  0.0,
                  74,
                  0
                ],
                [
                  0.13711151736745886,
                  0.0,
                  75,
                  0
                ],
                [
                  0.13893967093235832,
                  0.0,
                  76,
                  0
                ],
                [
                  0.14076782449725778,
                  0.0,
                  77,
                  0
                ],
                [
                  0.1425959780621572,
                  0.0,
                  78,
                  0
                ],
                [
                  0.14442413162705667,
                  0.0,
                  79,
                  0
                ],
                [
                  0.14625228519195613,
                  0.0,
                  80,
                  0
                ],
                [
                  0.1480804387568556,
                  0.0,
                  81,
                  0
                ],
                [
                  0.14990859232175502,
                  0.0,
                  82,
                  0
                ],
                [
                  0.15173674588665448,
                  0.0,
                  83,
                  0
                ],
                [
                  0.15356489945155394,
                  0.0,
                  84,
                  0
                ],
                [
                  0.15539305301645337,
                  0.0,
                  85,
                  0
                ],
                [
                  0.15722120658135283,
                  0.0,
                  86,
                  0
                ],
                [
                  0.1590493601462523,
                  0.0,
                  87,
                  0
                ],
                [
                  0.16087751371115175,
                  0.0,
                  88,
                  0
                ],
                [
                  0.16270566727605118,
                  0.0,
                  89,
                  0
                ],
                [
                  0.16453382084095064,
                  0.0,
                  90,
                  0
                ],
                [
                  0.1663619744058501,
                  0.0,
                  91,
                  0
                ],
                [
                  0.16819012797074953,
                  0.0,
                  92,
                  0
                ],
                [
                  0.170018281535649,
                  0.0,
                  93,
                  0
                ],
                [
                  0.17184643510054845,
                  0.0,
                  94,
                  0
                ],
                [
                  0.1736745886654479,
                  0.0,
                  95,
                  0
                ],
                [
                  0.17550274223034734,
                  0.0,
                  96,
                  0
                ],
                [
                  0.1773308957952468,
                  0.0,
                  97,
                  0
                ],
                [
                  0.17915904936014626,
                  0.0,
                  98,
                  0
                ],
                [
                  0.1809872029250457,
                  0.0,
                  99,
                  0
                ],
                [
                  0.18281535648994515,
                  0.0,
                  100,
                  0
                ],
                [
                  0.1846435100548446,
                  0.0,
                  101,
                  0
                ],
                [
                  0.18647166361974407,
                  0.0,
                  102,
                  0
                ],
                [
                  0.1882998171846435,
                  0.0,
                  103,
                  0
                ],
                [
                  0.19012797074954296,
                  0.0,
                  104,
                  0
                ],
                [
                  0.19195612431444242,
                  0.0,
                  105,
                  0
                ],
                [
                  0.19378427787934185,
                  0.0,
                  106,
                  0
                ],
                [
                  0.1956124314442413,
                  0.0,
                  107,
                  0
                ],
                [
                  0.19744058500914077,
                  0.0,
                  108,
                  0
                ],
                [
                  0.19926873857404023,
                  0.0,
                  109,
                  0
                ],
                [
                  0.20109689213893966,
                  0.0,
                  110,
                  0
                ],
                [
                  0.20292504570383912,
                  0.0,
                  111,
                  0
                ],
                [
                  0.20475319926873858,
                  0.0,
                  112,
                  0
                ],
                [
                  0.20658135283363802,
                  0.0,
                  113,
                  0
                ],
                [
                  0.20840950639853748,
                  0.0,
                  114,
                  0
                ],
                [
                  0.21023765996343693,
                  0.0,
                  115,
                  0
                ],
                [
                  0.21206581352833637,
                  0.0,
                  116,
                  0
                ],
                [
                  0.21389396709323583,
                  0.0,
                  117,
                  0
                ],
                [
                  0.21572212065813529,
                  0.0,
                  118,
                  0
                ],
                [
                  0.21755027422303475,
                  0.0,
                  119,
                  0
                ],
                [
                  0.21937842778793418,
                  0.0,
                  120,
                  0
                ],
                [
                  0.22120658135283364,
                  0.0,
                  121,
                  0
                ],
                [
                  0.2230347349177331,
                  0.0,
                  122,
                  0
                ],
                [
                  0.22486288848263253,
                  0.0,
                  123,
                  0
                ],
                [
                  0.226691042047532,
                  0.0,
                  124,
                  0
                ],
                [
                  0.22851919561243145,
                  0.0,
                  125,
                  0
                ],
                [
                  0.2303473491773309,
                  0.0,
                  126,
                  0
                ],
                [
                  0.23217550274223034,
                  0.0,
                  127,
                  0
                ],
                [
                  0.2340036563071298,
                  0.0,
                  128,
                  0
                ],
                [
                  0.23583180987202926,
                  0.0,
                  129,
                  0
                ],
                [
                  0.2376599634369287,
                  0.0,
                  130,
                  0
                ],
                [
                  0.23948811700182815,
                  0.0,
                  131,
                  0
                ],
                [
                  0.2413162705667276,
                  0.0,
                  132,
                  0
                ],
                [
                  0.24314442413162707,
                  0.0,
                  133,
                  0
                ],
                [
                  0.2449725776965265,
                  0.0,
                  134,
                  0
                ],
                [
                  0.24680073126142596,
                  0.0,
                  135,
                  0
                ],
                [
                  0.24862888482632542,
                  0.0,
                  136,
                  0
                ],
                [
                  0.25045703839122485,
                  0.0,
                  137,
                  0
                ],
                [
                  0.2522851919561243,
                  0.0,
                  138,
                  0
                ],
                [
                  0.25411334552102377,
                  0.0,
                  139,
                  0
                ],
                [
                  0.25594149908592323,
                  0.0,
                  140,
                  0
                ],
                [
                  0.2577696526508227,
                  0.0,
                  141,
                  0
                ],
                [
                  0.2595978062157221,
                  0.0,
                  142,
                  0
                ],
                [
                  0.26142595978062155,
                  0.0,
                  143,
                  0
                ],
                [
                  0.263254113345521,
                  0.0,
                  144,
                  0
                ],
                [
                  0.26508226691042047,
                  0.0,
                  145,
                  0
                ],
                [
                  0.26691042047531993,
                  0.0,
                  146,
                  0
                ],
                [
                  0.2687385740402194,
                  0.0,
                  147,
                  0
                ],
                [
                  0.27056672760511885,
                  0.0,
                  148,
                  0
                ],
                [
                  0.27239488117001825,
                  0.0,
                  149,
                  0
                ],
                [
                  0.2742230347349177,
                  0.0,
                  150,
                  0
                ],
                [
                  0.2760511882998172,
                  0.0,
                  151,
                  0
                ],
                [
                  0.27787934186471663,
                  0.0,
                  152,
                  0
                ],
                [
                  0.2797074954296161,
                  0.0,
                  153,
                  0
                ],
                [
                  0.28153564899451555,
                  0.0,
                  154,
                  0
                ],
                [
                  0.283363802559415,
                  0.0,
                  155,
                  0
                ],
                [
                  0.2851919561243144,
                  0.0,
                  156,
                  0
                ],
                [
                  0.2870201096892139,
                  0.0,
                  157,
                  0
                ],
                [
                  0.28884826325411334,
                  0.0,
                  158,
                  0
                ],
                [
                  0.2906764168190128,
                  0.0,
                  159,
                  0
                ],
                [
                  0.29250457038391225,
                  0.0,
                  160,
                  0
                ],
                [
                  0.2943327239488117,
                  0.0,
                  161,
                  0
                ],
                [
                  0.2961608775137112,
                  0.0,
                  162,
                  0
                ],
                [
                  0.2979890310786106,
                  0.0,
                  163,
                  0
                ],
                [
                  0.29981718464351004,
                  0.0,
                  164,
                  0
                ],
                [
                  0.3016453382084095,
                  0.0,
                  165,
                  0
                ],
                [
                  0.30347349177330896,
                  0.0,
                  166,
                  0
                ],
                [
                  0.3053016453382084,
                  0.0,
                  167,
                  0
                ],
                [
                  0.3071297989031079,
                  0.0,
                  168,
                  0
                ],
                [
                  0.30895795246800734,
                  0.0,
                  169,
                  0
                ],
                [
                  0.31078610603290674,
                  0.0,
                  170,
                  0
                ],
                [
                  0.3126142595978062,
                  0.0,
                  171,
                  0
                ],
                [
                  0.31444241316270566,
                  0.0,
                  172,
                  0
                ],
                [
                  0.3162705667276051,
                  0.0,
                  173,
                  0
                ],
                [
                  0.3180987202925046,
                  0.0,
                  174,
                  0
                ],
                [
                  0.31992687385740404,
                  0.0,
                  175,
                  0
                ],
                [
                  0.3217550274223035,
                  0.0,
                  176,
                  0
                ],
                [
                  0.3235831809872029,
                  0.0,
                  177,
                  0
                ],
                [
                  0.32541133455210236,
                  0.0,
                  178,
                  0
                ],
                [
                  0.3272394881170018,
                  0.0,
                  179,
                  0
                ],
                [
                  0.3290676416819013,
                  0.0,
                  180,
                  0
                ],
                [
                  0.33089579524680074,
                  0.0,
                  181,
                  0
                ],
                [
                  0.3327239488117002,
                  0.0,
                  182,
                  0
                ],
                [
                  0.33455210237659966,
                  0.0,
                  183,
                  0
                ],
                [
                  0.33638025594149906,
                  0.0,
                  184,
                  0
                ],
                [
                  0.3382084095063985,
                  0.0,
                  185,
                  0
                ],
                [
                  0.340036563071298,
                  0.0,
                  186,
                  0
                ],
                [
                  0.34186471663619744,
                  0.0,
                  187,
                  0
                ],
                [
                  0.3436928702010969,
                  0.0,
                  188,
                  0
                ],
                [
                  0.34552102376599636,
                  0.0,
                  189,
                  0
                ],
                [
                  0.3473491773308958,
                  0.0,
                  190,
                  0
                ],
                [
                  0.3491773308957952,
                  0.0,
                  191,
                  0
                ],
                [
                  0.3510054844606947,
                  0.0,
                  192,
                  0
                ],
                [
                  0.35283363802559414,
                  0.0,
                  193,
                  0
                ],
                [
                  0.3546617915904936,
                  0.0,
                  194,
                  0
                ],
                [
                  0.35648994515539306,
                  0.0,
                  195,
                  0
                ],
                [
                  0.3583180987202925,
                  0.0,
                  196,
                  0
                ],
                [
                  0.360146252285192,
                  0.0,
                  197,
                  0
                ],
                [
                  0.3619744058500914,
                  0.0,
                  198,
                  0
                ],
                [
                  0.36380255941499084,
                  0.0,
                  199,
                  0
                ],
                [
                  0.3656307129798903,
                  0.0,
                  200,
                  0
                ],
                [
                  0.36745886654478976,
                  0.0,
                  201,
                  0
                ],
                [
                  0.3692870201096892,
                  0.0,
                  202,
                  0
                ],
                [
                  0.3711151736745887,
                  0.0,
                  203,
                  0
                ],
                [
                  0.37294332723948814,
                  0.0,
                  204,
                  0
                ],
                [
                  0.37477148080438755,
                  0.0,
                  205,
                  0
                ],
                [
                  0.376599634369287,
                  0.0,
                  206,
                  0
                ],
                [
                  0.37842778793418647,
                  0.0,
                  207,
                  0
                ],
                [
                  0.3802559414990859,
                  0.0,
                  208,
                  0
                ],
                [
                  0.3820840950639854,
                  0.0,
                  209,
                  0
                ],
                [
                  0.38391224862888484,
                  0.0,
                  210,
                  0
                ],
                [
                  0.3857404021937843,
                  0.0,
                  211,
                  0
                ],
                [
                  0.3875685557586837,
                  0.0,
                  212,
                  0
                ],
                [
                  0.38939670932358317,
                  0.0,
                  213,
                  0
                ],
                [
                  0.3912248628884826,
                  0.0,
                  214,
                  0
                ],
                [
                  0.3930530164533821,
                  0.0,
                  215,
                  0
                ],
                [
                  0.39488117001828155,
                  0.0,
                  216,
                  0
                ],
                [
                  0.396709323583181,
                  0.0,
                  217,
                  0
                ],
                [
                  0.39853747714808047,
                  0.0,
                  218,
                  0
                ],
                [
                  0.40036563071297987,
                  0.0,
                  219,
                  0
                ],
                [
                  0.40219378427787933,
                  0.0,
                  220,
                  0
                ],
                [
                  0.4040219378427788,
                  0.0,
                  221,
                  0
                ],
                [
                  0.40585009140767825,
                  0.0,
                  222,
                  0
                ],
                [
                  0.4076782449725777,
                  0.0,
                  223,
                  0
                ],
                [
                  0.40950639853747717,
                  0.0,
                  224,
                  0
                ],
                [
                  0.4113345521023766,
                  0.0,
                  225,
                  0
                ],
                [
                  0.41316270566727603,
                  0.0,
                  226,
                  0
                ],
                [
                  0.4149908592321755,
                  0.0,
                  227,
                  0
                ],
                [
                  0.41681901279707495,
                  0.0,
                  228,
                  0
                ],
                [
                  0.4186471663619744,
                  0.0,
                  229,
                  0
                ],
                [
                  0.42047531992687387,
                  0.0,
                  230,
                  0
                ],
                [
                  0.42230347349177333,
                  0.0,
                  231,
                  0
                ],
                [
                  0.42413162705667273,
                  0.0,
                  232,
                  0
                ],
                [
                  0.4259597806215722,
                  0.0,
                  233,
                  0
                ],
                [
                  0.42778793418647165,
                  0.0,
                  234,
                  0
                ],
                [
                  0.4296160877513711,
                  0.0,
                  235,
                  0
                ],
                [
                  0.43144424131627057,
                  0.0,
                  236,
                  0
                ],
                [
                  0.43327239488117003,
                  0.0,
                  237,
                  0
                ],
                [
                  0.4351005484460695,
                  0.0,
                  238,
                  0
                ],
                [
                  0.4369287020109689,
                  0.0,
                  239,
                  0
                ],
                [
                  0.43875685557586835,
                  0.0,
                  240,
                  0
                ],
                [
                  0.4405850091407678,
                  0.0,
                  241,
                  0
                ],
                [
                  0.4424131627056673,
                  0.0,
                  242,
                  0
                ],
                [
                  0.44424131627056673,
                  0.0,
                  243,
                  0
                ],
                [
                  0.4460694698354662,
                  0.0,
                  244,
                  0
                ],
                [
                  0.44789762340036565,
                  0.0,
                  245,
                  0
                ],
                [
                  0.44972577696526506,
                  0.0,
                  246,
                  0
                ],
                [
                  0.4515539305301645,
                  0.0,
                  247,
                  0
                ],
                [
                  0.453382084095064,
                  0.0,
                  248,
                  0
                ],
                [
                  0.45521023765996343,
                  0.0,
                  249,
                  0
                ],
                [
                  0.4570383912248629,
                  0.0,
                  250,
                  0
                ],
                [
                  0.45886654478976235,
                  0.0,
                  251,
                  0
                ],
                [
                  0.4606946983546618,
                  0.0,
                  252,
                  0
                ],
                [
                  0.4625228519195612,
                  0.0,
                  253,
                  0
                ],
                [
                  0.4643510054844607,
                  0.0,
                  254,
                  0
                ],
                [
                  0.46617915904936014,
                  0.0,
                  255,
                  0
                ],
                [
                  0.4680073126142596,
                  0.0,
                  256,
                  0
                ],
                [
                  0.46983546617915906,
                  0.0,
                  257,
                  0
                ],
                [
                  0.4716636197440585,
                  0.0,
                  258,
                  0
                ],
                [
                  0.473491773308958,
                  0.0,
                  259,
                  0
                ],
                [
                  0.4753199268738574,
                  0.0,
                  260,
                  0
                ],
                [
                  0.47714808043875684,
                  0.0,
                  261,
                  0
                ],
                [
                  0.4789762340036563,
                  0.0,
                  262,
                  0
                ],
                [
                  0.48080438756855576,
                  0.0,
                  263,
                  0
                ],
                [
                  0.4826325411334552,
                  0.0,
                  264,
                  0
                ],
                [
                  0.4844606946983547,
                  0.0037735849056603774,
                  265,
                  1
                ],
                [
                  0.48628884826325414,
                  0.0037593984962406013,
                  266,
                  1
                ],
                [
                  0.48811700182815354,
                  0.00749063670411985,
                  267,
                  2
                ],
                [
                  0.489945155393053,
                  0.007462686567164179,
                  268,
                  2
                ],
                [
                  0.49177330895795246,
                  0.007434944237918215,
                  269,
                  2
                ],
                [
                  0.4936014625228519,
                  0.007407407407407408,
                  270,
                  2
                ],
                [
                  0.4954296160877514,
                  0.007380073800738007,
                  271,
                  2
                ],
                [
                  0.49725776965265084,
                  0.007352941176470588,
                  272,
                  2
                ],
                [
                  0.4990859232175503,
                  0.007326007326007326,
                  273,
                  2
                ],
                [
                  0.5009140767824497,
                  0.0072992700729927005,
                  274,
                  2
                ],
                [
                  0.5027422303473492,
                  0.007272727272727273,
                  275,
                  2
                ],
                [
                  0.5045703839122486,
                  0.007246376811594203,
                  276,
                  2
                ],
                [
                  0.506398537477148,
                  0.010830324909747292,
                  277,
                  3
                ],
                [
                  0.5082266910420475,
                  0.014388489208633094,
                  278,
                  4
                ],
                [
                  0.5100548446069469,
                  0.014336917562724014,
                  279,
                  4
                ],
                [
                  0.5118829981718465,
                  0.014285714285714285,
                  280,
                  4
                ],
                [
                  0.5137111517367459,
                  0.014234875444839857,
                  281,
                  4
                ],
                [
                  0.5155393053016454,
                  0.014184397163120567,
                  282,
                  4
                ],
                [
                  0.5173674588665448,
                  0.014134275618374558,
                  283,
                  4
                ],
                [
                  0.5191956124314442,
                  0.014084507042253521,
                  284,
                  4
                ],
                [
                  0.5210237659963437,
                  0.014035087719298246,
                  285,
                  4
                ],
                [
                  0.5228519195612431,
                  0.013986013986013986,
                  286,
                  4
                ],
                [
                  0.5246800731261426,
                  0.013937282229965157,
                  287,
                  4
                ],
                [
                  0.526508226691042,
                  0.013888888888888888,
                  288,
                  4
                ],
                [
                  0.5283363802559415,
                  0.01384083044982699,
                  289,
                  4
                ],
                [
                  0.5301645338208409,
                  0.013793103448275862,
                  290,
                  4
                ],
                [
                  0.5319926873857403,
                  0.013745704467353952,
                  291,
                  4
                ],
                [
                  0.5338208409506399,
                  0.0136986301369863,
                  292,
                  4
                ],
                [
                  0.5356489945155393,
                  0.013651877133105802,
                  293,
                  4
                ],
                [
                  0.5374771480804388,
                  0.013605442176870748,
                  294,
                  4
                ],
                [
                  0.5393053016453382,
                  0.01694915254237288,
                  295,
                  5
                ],
                [
                  0.5411334552102377,
                  0.016891891891891893,
                  296,
                  5
                ],
                [
                  0.5429616087751371,
                  0.016835016835016835,
                  297,
                  5
                ],
                [
                  0.5447897623400365,
                  0.016778523489932886,
                  298,
                  5
                ],
                [
                  0.546617915904936,
                  0.016722408026755852,
                  299,
                  5
                ],
                [
                  0.5484460694698354,
                  0.016666666666666666,
                  300,
                  5
                ],
                [
                  0.5502742230347349,
                  0.016611295681063124,
                  301,
                  5
                ],
                [
                  0.5521023765996343,
                  0.016556291390728478,
                  302,
                  5
                ],
                [
                  0.5539305301645339,
                  0.0165016501650165,
                  303,
                  5
                ],
                [
                  0.5557586837294333,
                  0.019736842105263157,
                  304,
                  6
                ],
                [
                  0.5575868372943327,
                  0.019672131147540985,
                  305,
                  6
                ],
                [
                  0.5594149908592322,
                  0.0196078431372549,
                  306,
                  6
                ],
                [
                  0.5612431444241316,
                  0.019543973941368076,
                  307,
                  6
                ],
                [
                  0.5630712979890311,
                  0.01948051948051948,
                  308,
                  6
                ],
                [
                  0.5648994515539305,
                  0.019417475728155338,
                  309,
                  6
                ],
                [
                  0.56672760511883,
                  0.01935483870967742,
                  310,
                  6
                ],
                [
                  0.5685557586837294,
                  0.01929260450160772,
                  311,
                  6
                ],
                [
                  0.5703839122486288,
                  0.019230769230769232,
                  312,
                  6
                ],
                [
                  0.5722120658135283,
                  0.019169329073482427,
                  313,
                  6
                ],
                [
                  0.5740402193784278,
                  0.01910828025477707,
                  314,
                  6
                ],
                [
                  0.5758683729433273,
                  0.01904761904761905,
                  315,
                  6
                ],
                [
                  0.5776965265082267,
                  0.0189873417721519,
                  316,
                  6
                ],
                [
                  0.5795246800731262,
                  0.01892744479495268,
                  317,
                  6
                ],
                [
                  0.5813528336380256,
                  0.018867924528301886,
                  318,
                  6
                ],
                [
                  0.583180987202925,
                  0.018808777429467086,
                  319,
                  6
                ],
                [
                  0.5850091407678245,
                  0.01875,
                  320,
                  6
                ],
                [
                  0.5868372943327239,
                  0.018691588785046728,
                  321,
                  6
                ],
                [
                  0.5886654478976234,
                  0.018633540372670808,
                  322,
                  6
                ],
                [
                  0.5904936014625228,
                  0.018575851393188854,
                  323,
                  6
                ],
                [
                  0.5923217550274223,
                  0.018518518518518517,
                  324,
                  6
                ],
                [
                  0.5941499085923218,
                  0.018461538461538463,
                  325,
                  6
                ],
                [
                  0.5959780621572212,
                  0.02147239263803681,
                  326,
                  7
                ],
                [
                  0.5978062157221207,
                  0.021406727828746176,
                  327,
                  7
                ],
                [
                  0.5996343692870201,
                  0.021341463414634148,
                  328,
                  7
                ],
                [
                  0.6014625228519196,
                  0.02127659574468085,
                  329,
                  7
                ],
                [
                  0.603290676416819,
                  0.021212121212121213,
                  330,
                  7
                ],
                [
                  0.6051188299817185,
                  0.021148036253776436,
                  331,
                  7
                ],
                [
                  0.6069469835466179,
                  0.02108433734939759,
                  332,
                  7
                ],
                [
                  0.6087751371115173,
                  0.021021021021021023,
                  333,
                  7
                ],
                [
                  0.6106032906764168,
                  0.020958083832335328,
                  334,
                  7
                ],
                [
                  0.6124314442413162,
                  0.020895522388059702,
                  335,
                  7
                ],
                [
                  0.6142595978062158,
                  0.020833333333333332,
                  336,
                  7
                ],
                [
                  0.6160877513711152,
                  0.020771513353115726,
                  337,
                  7
                ],
                [
                  0.6179159049360147,
                  0.020710059171597635,
                  338,
                  7
                ],
                [
                  0.6197440585009141,
                  0.02064896755162242,
                  339,
                  7
                ],
                [
                  0.6215722120658135,
                  0.020588235294117647,
                  340,
                  7
                ],
                [
                  0.623400365630713,
                  0.020527859237536656,
                  341,
                  7
                ],
                [
                  0.6252285191956124,
                  0.02046783625730994,
                  342,
                  7
                ],
                [
                  0.6270566727605119,
                  0.02040816326530612,
                  343,
                  7
                ],
                [
                  0.6288848263254113,
                  0.020348837209302327,
                  344,
                  7
                ],
                [
                  0.6307129798903108,
                  0.020289855072463767,
                  345,
                  7
                ],
                [
                  0.6325411334552102,
                  0.02023121387283237,
                  346,
                  7
                ],
                [
                  0.6343692870201096,
                  0.020172910662824207,
                  347,
                  7
                ],
                [
                  0.6361974405850092,
                  0.020114942528735632,
                  348,
                  7
                ],
                [
                  0.6380255941499086,
                  0.02005730659025788,
                  349,
                  7
                ],
                [
                  0.6398537477148081,
                  0.02,
                  350,
                  7
                ],
                [
                  0.6416819012797075,
                  0.019943019943019943,
                  351,
                  7
                ],
                [
                  0.643510054844607,
                  0.019886363636363636,
                  352,
                  7
                ],
                [
                  0.6453382084095064,
                  0.019830028328611898,
                  353,
                  7
                ],
                [
                  0.6471663619744058,
                  0.01977401129943503,
                  354,
                  7
                ],
                [
                  0.6489945155393053,
                  0.01971830985915493,
                  355,
                  7
                ],
                [
                  0.6508226691042047,
                  0.019662921348314606,
                  356,
                  7
                ],
                [
                  0.6526508226691042,
                  0.0196078431372549,
                  357,
                  7
                ],
                [
                  0.6544789762340036,
                  0.019553072625698324,
                  358,
                  7
                ],
                [
                  0.6563071297989032,
                  0.019498607242339833,
                  359,
                  7
                ],
                [
                  0.6581352833638026,
                  0.022222222222222223,
                  360,
                  8
                ],
                [
                  0.659963436928702,
                  0.0221606648199446,
                  361,
                  8
                ],
                [
                  0.6617915904936015,
                  0.022099447513812154,
                  362,
                  8
                ],
                [
                  0.6636197440585009,
                  0.02203856749311295,
                  363,
                  8
                ],
                [
                  0.6654478976234004,
                  0.02197802197802198,
                  364,
                  8
                ],
                [
                  0.6672760511882998,
                  0.021917808219178082,
                  365,
                  8
                ],
                [
                  0.6691042047531993,
                  0.02459016393442623,
                  366,
                  9
                ],
                [
                  0.6709323583180987,
                  0.02452316076294278,
                  367,
                  9
                ],
                [
                  0.6727605118829981,
                  0.024456521739130436,
                  368,
                  9
                ],
                [
                  0.6745886654478976,
                  0.024390243902439025,
                  369,
                  9
                ],
                [
                  0.676416819012797,
                  0.024324324324324326,
                  370,
                  9
                ],
                [
                  0.6782449725776966,
                  0.02425876010781671,
                  371,
                  9
                ],
                [
                  0.680073126142596,
                  0.026881720430107527,
                  372,
                  10
                ],
                [
                  0.6819012797074955,
                  0.02680965147453083,
                  373,
                  10
                ],
                [
                  0.6837294332723949,
                  0.029411764705882353,
                  374,
                  11
                ],
                [
                  0.6855575868372943,
                  0.029333333333333333,
                  375,
                  11
                ],
                [
                  0.6873857404021938,
                  0.02925531914893617,
                  376,
                  11
                ],
                [
                  0.6892138939670932,
                  0.029177718832891247,
                  377,
                  11
                ],
                [
                  0.6910420475319927,
                  0.0291005291005291,
                  378,
                  11
                ],
                [
                  0.6928702010968921,
                  0.029023746701846966,
                  379,
                  11
                ],
                [
                  0.6946983546617916,
                  0.02894736842105263,
                  380,
                  11
                ],
                [
                  0.696526508226691,
                  0.028871391076115485,
                  381,
                  11
                ],
                [
                  0.6983546617915904,
                  0.028795811518324606,
                  382,
                  11
                ],
                [
                  0.70018281535649,
                  0.028720626631853787,
                  383,
                  11
                ],
                [
                  0.7020109689213894,
                  0.028645833333333332,
                  384,
                  11
                ],
                [
                  0.7038391224862889,
                  0.02857142857142857,
                  385,
                  11
                ],
                [
                  0.7056672760511883,
                  0.02849740932642487,
                  386,
                  11
                ],
                [
                  0.7074954296160878,
                  0.028423772609819122,
                  387,
                  11
                ],
                [
                  0.7093235831809872,
                  0.028350515463917526,
                  388,
                  11
                ],
                [
                  0.7111517367458866,
                  0.028277634961439587,
                  389,
                  11
                ],
                [
                  0.7129798903107861,
                  0.028205128205128206,
                  390,
                  11
                ],
                [
                  0.7148080438756855,
                  0.028132992327365727,
                  391,
                  11
                ],
                [
                  0.716636197440585,
                  0.02806122448979592,
                  392,
                  11
                ],
                [
                  0.7184643510054844,
                  0.027989821882951654,
                  393,
                  11
                ],
                [
                  0.720292504570384,
                  0.027918781725888325,
                  394,
                  11
                ],
                [
                  0.7221206581352834,
                  0.027848101265822784,
                  395,
                  11
                ],
                [
                  0.7239488117001828,
                  0.027777777777777776,
                  396,
                  11
                ],
                [
                  0.7257769652650823,
                  0.027707808564231738,
                  397,
                  11
                ],
                [
                  0.7276051188299817,
                  0.02763819095477387,
                  398,
                  11
                ],
                [
                  0.7294332723948812,
                  0.02756892230576441,
                  399,
                  11
                ],
                [
                  0.7312614259597806,
                  0.0275,
                  400,
                  11
                ],
                [
                  0.7330895795246801,
                  0.029925187032418952,
                  401,
                  12
                ],
                [
                  0.7349177330895795,
                  0.029850746268656716,
                  402,
                  12
                ],
                [
                  0.7367458866544789,
                  0.03225806451612903,
                  403,
                  13
                ],
                [
                  0.7385740402193784,
                  0.034653465346534656,
                  404,
                  14
                ],
                [
                  0.7404021937842779,
                  0.0345679012345679,
                  405,
                  14
                ],
                [
                  0.7422303473491774,
                  0.03694581280788178,
                  406,
                  15
                ],
                [
                  0.7440585009140768,
                  0.03931203931203931,
                  407,
                  16
                ],
                [
                  0.7458866544789763,
                  0.0392156862745098,
                  408,
                  16
                ],
                [
                  0.7477148080438757,
                  0.039119804400977995,
                  409,
                  16
                ],
                [
                  0.7495429616087751,
                  0.03902439024390244,
                  410,
                  16
                ],
                [
                  0.7513711151736746,
                  0.038929440389294405,
                  411,
                  16
                ],
                [
                  0.753199268738574,
                  0.038834951456310676,
                  412,
                  16
                ],
                [
                  0.7550274223034735,
                  0.0387409200968523,
                  413,
                  16
                ],
                [
                  0.7568555758683729,
                  0.03864734299516908,
                  414,
                  16
                ],
                [
                  0.7586837294332724,
                  0.04096385542168675,
                  415,
                  17
                ],
                [
                  0.7605118829981719,
                  0.040865384615384616,
                  416,
                  17
                ],
                [
                  0.7623400365630713,
                  0.0407673860911271,
                  417,
                  17
                ],
                [
                  0.7641681901279708,
                  0.04066985645933014,
                  418,
                  17
                ],
                [
                  0.7659963436928702,
                  0.04295942720763723,
                  419,
                  18
                ],
                [
                  0.7678244972577697,
                  0.04523809523809524,
                  420,
                  19
                ],
                [
                  0.7696526508226691,
                  0.04513064133016627,
                  421,
                  19
                ],
                [
                  0.7714808043875686,
                  0.045023696682464455,
                  422,
                  19
                ],
                [
                  0.773308957952468,
                  0.04491725768321513,
                  423,
                  19
                ],
                [
                  0.7751371115173674,
                  0.04716981132075472,
                  424,
                  20
                ],
                [
                  0.7769652650822669,
                  0.047058823529411764,
                  425,
                  20
                ],
                [
                  0.7787934186471663,
                  0.04929577464788732,
                  426,
                  21
                ],
                [
                  0.7806215722120659,
                  0.04918032786885246,
                  427,
                  21
                ],
                [
                  0.7824497257769653,
                  0.04906542056074766,
                  428,
                  21
                ],
                [
                  0.7842778793418648,
                  0.04895104895104895,
                  429,
                  21
                ],
                [
                  0.7861060329067642,
                  0.05116279069767442,
                  430,
                  22
                ],
                [
                  0.7879341864716636,
                  0.05336426914153132,
                  431,
                  23
                ],
                [
                  0.7897623400365631,
                  0.05555555555555555,
                  432,
                  24
                ],
                [
                  0.7915904936014625,
                  0.057736720554272515,
                  433,
                  25
                ],
                [
                  0.793418647166362,
                  0.0576036866359447,
                  434,
                  25
                ],
                [
                  0.7952468007312614,
                  0.05747126436781609,
                  435,
                  25
                ],
                [
                  0.7970749542961609,
                  0.05733944954128441,
                  436,
                  25
                ],
                [
                  0.7989031078610603,
                  0.059496567505720827,
                  437,
                  26
                ],
                [
                  0.8007312614259597,
                  0.06164383561643835,
                  438,
                  27
                ],
                [
                  0.8025594149908593,
                  0.06150341685649203,
                  439,
                  27
                ],
                [
                  0.8043875685557587,
                  0.06136363636363636,
                  440,
                  27
                ],
                [
                  0.8062157221206582,
                  0.061224489795918366,
                  441,
                  27
                ],
                [
                  0.8080438756855576,
                  0.06108597285067873,
                  442,
                  27
                ],
                [
                  0.8098720292504571,
                  0.060948081264108354,
                  443,
                  27
                ],
                [
                  0.8117001828153565,
                  0.060810810810810814,
                  444,
                  27
                ],
                [
                  0.8135283363802559,
                  0.060674157303370786,
                  445,
                  27
                ],
                [
                  0.8153564899451554,
                  0.06053811659192825,
                  446,
                  27
                ],
                [
                  0.8171846435100548,
                  0.06040268456375839,
                  447,
                  27
                ],
                [
                  0.8190127970749543,
                  0.0625,
                  448,
                  28
                ],
                [
                  0.8208409506398537,
                  0.062360801781737196,
                  449,
                  28
                ],
                [
                  0.8226691042047533,
                  0.06222222222222222,
                  450,
                  28
                ],
                [
                  0.8244972577696527,
                  0.06430155210643015,
                  451,
                  29
                ],
                [
                  0.8263254113345521,
                  0.06415929203539823,
                  452,
                  29
                ],
                [
                  0.8281535648994516,
                  0.0640176600441501,
                  453,
                  29
                ],
                [
                  0.829981718464351,
                  0.06387665198237885,
                  454,
                  29
                ],
                [
                  0.8318098720292505,
                  0.06373626373626373,
                  455,
                  29
                ],
                [
                  0.8336380255941499,
                  0.06359649122807018,
                  456,
                  29
                ],
                [
                  0.8354661791590493,
                  0.06345733041575492,
                  457,
                  29
                ],
                [
                  0.8372943327239488,
                  0.06550218340611354,
                  458,
                  30
                ],
                [
                  0.8391224862888482,
                  0.06535947712418301,
                  459,
                  30
                ],
                [
                  0.8409506398537477,
                  0.06521739130434782,
                  460,
                  30
                ],
                [
                  0.8427787934186471,
                  0.0650759219088937,
                  461,
                  30
                ],
                [
                  0.8446069469835467,
                  0.06493506493506493,
                  462,
                  30
                ],
                [
                  0.8464351005484461,
                  0.06479481641468683,
                  463,
                  30
                ],
                [
                  0.8482632541133455,
                  0.0668103448275862,
                  464,
                  31
                ],
                [
                  0.850091407678245,
                  0.06881720430107527,
                  465,
                  32
                ],
                [
                  0.8519195612431444,
                  0.06866952789699571,
                  466,
                  32
                ],
                [
                  0.8537477148080439,
                  0.07066381156316917,
                  467,
                  33
                ],
                [
                  0.8555758683729433,
                  0.07264957264957266,
                  468,
                  34
                ],
                [
                  0.8574040219378428,
                  0.07249466950959488,
                  469,
                  34
                ],
                [
                  0.8592321755027422,
                  0.07446808510638298,
                  470,
                  35
                ],
                [
                  0.8610603290676416,
                  0.07430997876857749,
                  471,
                  35
                ],
                [
                  0.8628884826325411,
                  0.07415254237288135,
                  472,
                  35
                ],
                [
                  0.8647166361974405,
                  0.07399577167019028,
                  473,
                  35
                ],
                [
                  0.8665447897623401,
                  0.07383966244725738,
                  474,
                  35
                ],
                [
                  0.8683729433272395,
                  0.07368421052631578,
                  475,
                  35
                ],
                [
                  0.870201096892139,
                  0.07352941176470588,
                  476,
                  35
                ],
                [
                  0.8720292504570384,
                  0.07337526205450734,
                  477,
                  35
                ],
                [
                  0.8738574040219378,
                  0.07531380753138076,
                  478,
                  36
                ],
                [
                  0.8756855575868373,
                  0.07724425887265135,
                  479,
                  37
                ],
                [
                  0.8775137111517367,
                  0.07916666666666666,
                  480,
                  38
                ],
                [
                  0.8793418647166362,
                  0.08108108108108109,
                  481,
                  39
                ],
                [
                  0.8811700182815356,
                  0.08298755186721991,
                  482,
                  40
                ],
                [
                  0.8829981718464351,
                  0.08488612836438923,
                  483,
                  41
                ],
                [
                  0.8848263254113345,
                  0.08471074380165289,
                  484,
                  41
                ],
                [
                  0.886654478976234,
                  0.08453608247422681,
                  485,
                  41
                ],
                [
                  0.8884826325411335,
                  0.08641975308641975,
                  486,
                  42
                ],
                [
                  0.8903107861060329,
                  0.08829568788501027,
                  487,
                  43
                ],
                [
                  0.8921389396709324,
                  0.08811475409836066,
                  488,
                  43
                ],
                [
                  0.8939670932358318,
                  0.08793456032719836,
                  489,
                  43
                ],
                [
                  0.8957952468007313,
                  0.08775510204081632,
                  490,
                  43
                ],
                [
                  0.8976234003656307,
                  0.08757637474541752,
                  491,
                  43
                ],
                [
                  0.8994515539305301,
                  0.08739837398373984,
                  492,
                  43
                ],
                [
                  0.9012797074954296,
                  0.0872210953346856,
                  493,
                  43
                ],
                [
                  0.903107861060329,
                  0.08906882591093117,
                  494,
                  44
                ],
                [
                  0.9049360146252285,
                  0.08888888888888889,
                  495,
                  44
                ],
                [
                  0.906764168190128,
                  0.08870967741935484,
                  496,
                  44
                ],
                [
                  0.9085923217550275,
                  0.09054325955734406,
                  497,
                  45
                ],
                [
                  0.9104204753199269,
                  0.09236947791164658,
                  498,
                  46
                ],
                [
                  0.9122486288848263,
                  0.09418837675350701,
                  499,
                  47
                ],
                [
                  0.9140767824497258,
                  0.096,
                  500,
                  48
                ],
                [
                  0.9159049360146252,
                  0.09780439121756487,
                  501,
                  49
                ],
                [
                  0.9177330895795247,
                  0.09760956175298804,
                  502,
                  49
                ],
                [
                  0.9195612431444241,
                  0.09940357852882704,
                  503,
                  50
                ],
                [
                  0.9213893967093236,
                  0.10119047619047619,
                  504,
                  51
                ],
                [
                  0.923217550274223,
                  0.100990099009901,
                  505,
                  51
                ],
                [
                  0.9250457038391224,
                  0.10276679841897234,
                  506,
                  52
                ],
                [
                  0.926873857404022,
                  0.10453648915187377,
                  507,
                  53
                ],
                [
                  0.9287020109689214,
                  0.1062992125984252,
                  508,
                  54
                ],
                [
                  0.9305301645338209,
                  0.10609037328094302,
                  509,
                  54
                ],
                [
                  0.9323583180987203,
                  0.10588235294117647,
                  510,
                  54
                ],
                [
                  0.9341864716636198,
                  0.10567514677103718,
                  511,
                  54
                ],
                [
                  0.9360146252285192,
                  0.10546875,
                  512,
                  54
                ],
                [
                  0.9378427787934186,
                  0.10526315789473684,
                  513,
                  54
                ],
                [
                  0.9396709323583181,
                  0.10700389105058365,
                  514,
                  55
                ],
                [
                  0.9414990859232175,
                  0.1087378640776699,
                  515,
                  56
                ],
                [
                  0.943327239488117,
                  0.10852713178294573,
                  516,
                  56
                ],
                [
                  0.9451553930530164,
                  0.1102514506769826,
                  517,
                  57
                ],
                [
                  0.946983546617916,
                  0.11196911196911197,
                  518,
                  58
                ],
                [
                  0.9488117001828154,
                  0.11368015414258188,
                  519,
                  59
                ],
                [
                  0.9506398537477148,
                  0.11346153846153846,
                  520,
                  59
                ],
                [
                  0.9524680073126143,
                  0.11516314779270634,
                  521,
                  60
                ],
                [
                  0.9542961608775137,
                  0.11685823754789272,
                  522,
                  61
                ],
                [
                  0.9561243144424132,
                  0.11854684512428298,
                  523,
                  62
                ],
                [
                  0.9579524680073126,
                  0.12022900763358779,
                  524,
                  63
                ],
                [
                  0.9597806215722121,
                  0.12,
                  525,
                  63
                ],
                [
                  0.9616087751371115,
                  0.11977186311787072,
                  526,
                  63
                ],
                [
                  0.9634369287020109,
                  0.12144212523719165,
                  527,
                  64
                ],
                [
                  0.9652650822669104,
                  0.12121212121212122,
                  528,
                  64
                ],
                [
                  0.9670932358318098,
                  0.12098298676748583,
                  529,
                  64
                ],
                [
                  0.9689213893967094,
                  0.12075471698113208,
                  530,
                  64
                ],
                [
                  0.9707495429616088,
                  0.1224105461393597,
                  531,
                  65
                ],
                [
                  0.9725776965265083,
                  0.12406015037593984,
                  532,
                  66
                ],
                [
                  0.9744058500914077,
                  0.12570356472795496,
                  533,
                  67
                ],
                [
                  0.9762340036563071,
                  0.1254681647940075,
                  534,
                  67
                ],
                [
                  0.9780621572212066,
                  0.1252336448598131,
                  535,
                  67
                ],
                [
                  0.979890310786106,
                  0.12686567164179105,
                  536,
                  68
                ],
                [
                  0.9817184643510055,
                  0.12849162011173185,
                  537,
                  69
                ],
                [
                  0.9835466179159049,
                  0.13011152416356878,
                  538,
                  70
                ],
                [
                  0.9853747714808044,
                  0.12987012987012986,
                  539,
                  70
                ],
                [
                  0.9872029250457038,
                  0.13148148148148148,
                  540,
                  71
                ],
                [
                  0.9890310786106032,
                  0.13123844731977818,
                  541,
                  71
                ],
                [
                  0.9908592321755028,
                  0.13284132841328414,
                  542,
                  72
                ],
                [
                  0.9926873857404022,
                  0.13443830570902393,
                  543,
                  73
                ],
                [
                  0.9945155393053017,
                  0.13602941176470587,
                  544,
                  74
                ],
                [
                  0.9963436928702011,
                  0.13761467889908258,
                  545,
                  75
                ],
                [
                  0.9981718464351006,
                  0.1391941391941392,
                  546,
                  76
                ],
                [
                  1.0,
                  0.14076782449725778,
                  547,
                  77
                ]
              ]
            }
          ]
        },
        {
          "name": "Inception Mercury Decide",
          "text_run": 204,
          "native": [
            {
              "language": "Turkish",
              "wording": "Bilmiyorum",
              "run": 489,
              "full_bank_restricted": false,
              "right": 468,
              "wrong": 71,
              "abstained": 8,
              "invalid": 0,
              "coverage": 0.9853747714808044,
              "risk": 0.13172541743970315
            },
            {
              "language": "Turkish",
              "wording": "I don't know",
              "run": null
            },
            {
              "language": "English",
              "wording": "I don't know",
              "run": 450,
              "full_bank_restricted": false,
              "right": 465,
              "wrong": 71,
              "abstained": 11,
              "invalid": 0,
              "coverage": 0.979890310786106,
              "risk": 0.13246268656716417
            }
          ],
          "curves": [
            {
              "language": "Turkish",
              "run": 204,
              "points": [
                [
                  0.0018281535648994515,
                  0.0,
                  1,
                  0
                ],
                [
                  0.003656307129798903,
                  0.0,
                  2,
                  0
                ],
                [
                  0.005484460694698354,
                  0.0,
                  3,
                  0
                ],
                [
                  0.007312614259597806,
                  0.0,
                  4,
                  0
                ],
                [
                  0.009140767824497258,
                  0.0,
                  5,
                  0
                ],
                [
                  0.010968921389396709,
                  0.0,
                  6,
                  0
                ],
                [
                  0.012797074954296161,
                  0.0,
                  7,
                  0
                ],
                [
                  0.014625228519195612,
                  0.0,
                  8,
                  0
                ],
                [
                  0.016453382084095063,
                  0.0,
                  9,
                  0
                ],
                [
                  0.018281535648994516,
                  0.0,
                  10,
                  0
                ],
                [
                  0.02010968921389397,
                  0.0,
                  11,
                  0
                ],
                [
                  0.021937842778793418,
                  0.0,
                  12,
                  0
                ],
                [
                  0.02376599634369287,
                  0.0,
                  13,
                  0
                ],
                [
                  0.025594149908592323,
                  0.0,
                  14,
                  0
                ],
                [
                  0.027422303473491772,
                  0.0,
                  15,
                  0
                ],
                [
                  0.029250457038391225,
                  0.0,
                  16,
                  0
                ],
                [
                  0.031078610603290677,
                  0.0,
                  17,
                  0
                ],
                [
                  0.03290676416819013,
                  0.0,
                  18,
                  0
                ],
                [
                  0.03473491773308958,
                  0.0,
                  19,
                  0
                ],
                [
                  0.03656307129798903,
                  0.0,
                  20,
                  0
                ],
                [
                  0.038391224862888484,
                  0.0,
                  21,
                  0
                ],
                [
                  0.04021937842778794,
                  0.0,
                  22,
                  0
                ],
                [
                  0.04204753199268738,
                  0.0,
                  23,
                  0
                ],
                [
                  0.043875685557586835,
                  0.0,
                  24,
                  0
                ],
                [
                  0.04570383912248629,
                  0.0,
                  25,
                  0
                ],
                [
                  0.04753199268738574,
                  0.0,
                  26,
                  0
                ],
                [
                  0.04936014625228519,
                  0.0,
                  27,
                  0
                ],
                [
                  0.051188299817184646,
                  0.0,
                  28,
                  0
                ],
                [
                  0.05301645338208409,
                  0.0,
                  29,
                  0
                ],
                [
                  0.054844606946983544,
                  0.0,
                  30,
                  0
                ],
                [
                  0.056672760511883,
                  0.0,
                  31,
                  0
                ],
                [
                  0.05850091407678245,
                  0.0,
                  32,
                  0
                ],
                [
                  0.0603290676416819,
                  0.0,
                  33,
                  0
                ],
                [
                  0.062157221206581355,
                  0.0,
                  34,
                  0
                ],
                [
                  0.06398537477148081,
                  0.0,
                  35,
                  0
                ],
                [
                  0.06581352833638025,
                  0.0,
                  36,
                  0
                ],
                [
                  0.06764168190127971,
                  0.0,
                  37,
                  0
                ],
                [
                  0.06946983546617916,
                  0.0,
                  38,
                  0
                ],
                [
                  0.0712979890310786,
                  0.0,
                  39,
                  0
                ],
                [
                  0.07312614259597806,
                  0.0,
                  40,
                  0
                ],
                [
                  0.07495429616087751,
                  0.0,
                  41,
                  0
                ],
                [
                  0.07678244972577697,
                  0.0,
                  42,
                  0
                ],
                [
                  0.07861060329067641,
                  0.0,
                  43,
                  0
                ],
                [
                  0.08043875685557587,
                  0.0,
                  44,
                  0
                ],
                [
                  0.08226691042047532,
                  0.0,
                  45,
                  0
                ],
                [
                  0.08409506398537477,
                  0.0,
                  46,
                  0
                ],
                [
                  0.08592321755027423,
                  0.0,
                  47,
                  0
                ],
                [
                  0.08775137111517367,
                  0.0,
                  48,
                  0
                ],
                [
                  0.08957952468007313,
                  0.0,
                  49,
                  0
                ],
                [
                  0.09140767824497258,
                  0.0,
                  50,
                  0
                ],
                [
                  0.09323583180987204,
                  0.0,
                  51,
                  0
                ],
                [
                  0.09506398537477148,
                  0.0,
                  52,
                  0
                ],
                [
                  0.09689213893967093,
                  0.0,
                  53,
                  0
                ],
                [
                  0.09872029250457039,
                  0.0,
                  54,
                  0
                ],
                [
                  0.10054844606946983,
                  0.0,
                  55,
                  0
                ],
                [
                  0.10237659963436929,
                  0.0,
                  56,
                  0
                ],
                [
                  0.10420475319926874,
                  0.0,
                  57,
                  0
                ],
                [
                  0.10603290676416818,
                  0.0,
                  58,
                  0
                ],
                [
                  0.10786106032906764,
                  0.0,
                  59,
                  0
                ],
                [
                  0.10968921389396709,
                  0.0,
                  60,
                  0
                ],
                [
                  0.11151736745886655,
                  0.0,
                  61,
                  0
                ],
                [
                  0.113345521023766,
                  0.0,
                  62,
                  0
                ],
                [
                  0.11517367458866545,
                  0.0,
                  63,
                  0
                ],
                [
                  0.1170018281535649,
                  0.0,
                  64,
                  0
                ],
                [
                  0.11882998171846434,
                  0.0,
                  65,
                  0
                ],
                [
                  0.1206581352833638,
                  0.0,
                  66,
                  0
                ],
                [
                  0.12248628884826325,
                  0.0,
                  67,
                  0
                ],
                [
                  0.12431444241316271,
                  0.0,
                  68,
                  0
                ],
                [
                  0.12614259597806216,
                  0.0,
                  69,
                  0
                ],
                [
                  0.12797074954296161,
                  0.0,
                  70,
                  0
                ],
                [
                  0.12979890310786105,
                  0.0,
                  71,
                  0
                ],
                [
                  0.1316270566727605,
                  0.0,
                  72,
                  0
                ],
                [
                  0.13345521023765997,
                  0.0,
                  73,
                  0
                ],
                [
                  0.13528336380255943,
                  0.0,
                  74,
                  0
                ],
                [
                  0.13711151736745886,
                  0.0,
                  75,
                  0
                ],
                [
                  0.13893967093235832,
                  0.0,
                  76,
                  0
                ],
                [
                  0.14076782449725778,
                  0.0,
                  77,
                  0
                ],
                [
                  0.1425959780621572,
                  0.0,
                  78,
                  0
                ],
                [
                  0.14442413162705667,
                  0.0,
                  79,
                  0
                ],
                [
                  0.14625228519195613,
                  0.0,
                  80,
                  0
                ],
                [
                  0.1480804387568556,
                  0.0,
                  81,
                  0
                ],
                [
                  0.14990859232175502,
                  0.0,
                  82,
                  0
                ],
                [
                  0.15173674588665448,
                  0.0,
                  83,
                  0
                ],
                [
                  0.15356489945155394,
                  0.0,
                  84,
                  0
                ],
                [
                  0.15539305301645337,
                  0.0,
                  85,
                  0
                ],
                [
                  0.15722120658135283,
                  0.0,
                  86,
                  0
                ],
                [
                  0.1590493601462523,
                  0.0,
                  87,
                  0
                ],
                [
                  0.16087751371115175,
                  0.0,
                  88,
                  0
                ],
                [
                  0.16270566727605118,
                  0.0,
                  89,
                  0
                ],
                [
                  0.16453382084095064,
                  0.0,
                  90,
                  0
                ],
                [
                  0.1663619744058501,
                  0.0,
                  91,
                  0
                ],
                [
                  0.16819012797074953,
                  0.0,
                  92,
                  0
                ],
                [
                  0.170018281535649,
                  0.0,
                  93,
                  0
                ],
                [
                  0.17184643510054845,
                  0.0,
                  94,
                  0
                ],
                [
                  0.1736745886654479,
                  0.0,
                  95,
                  0
                ],
                [
                  0.17550274223034734,
                  0.0,
                  96,
                  0
                ],
                [
                  0.1773308957952468,
                  0.0,
                  97,
                  0
                ],
                [
                  0.17915904936014626,
                  0.0,
                  98,
                  0
                ],
                [
                  0.1809872029250457,
                  0.0,
                  99,
                  0
                ],
                [
                  0.18281535648994515,
                  0.0,
                  100,
                  0
                ],
                [
                  0.1846435100548446,
                  0.0,
                  101,
                  0
                ],
                [
                  0.18647166361974407,
                  0.0,
                  102,
                  0
                ],
                [
                  0.1882998171846435,
                  0.0,
                  103,
                  0
                ],
                [
                  0.19012797074954296,
                  0.0,
                  104,
                  0
                ],
                [
                  0.19195612431444242,
                  0.0,
                  105,
                  0
                ],
                [
                  0.19378427787934185,
                  0.0,
                  106,
                  0
                ],
                [
                  0.1956124314442413,
                  0.0,
                  107,
                  0
                ],
                [
                  0.19744058500914077,
                  0.0,
                  108,
                  0
                ],
                [
                  0.19926873857404023,
                  0.0,
                  109,
                  0
                ],
                [
                  0.20109689213893966,
                  0.0,
                  110,
                  0
                ],
                [
                  0.20292504570383912,
                  0.0,
                  111,
                  0
                ],
                [
                  0.20475319926873858,
                  0.0,
                  112,
                  0
                ],
                [
                  0.20658135283363802,
                  0.0,
                  113,
                  0
                ],
                [
                  0.20840950639853748,
                  0.0,
                  114,
                  0
                ],
                [
                  0.21023765996343693,
                  0.0,
                  115,
                  0
                ],
                [
                  0.21206581352833637,
                  0.0,
                  116,
                  0
                ],
                [
                  0.21389396709323583,
                  0.0,
                  117,
                  0
                ],
                [
                  0.21572212065813529,
                  0.0,
                  118,
                  0
                ],
                [
                  0.21755027422303475,
                  0.0,
                  119,
                  0
                ],
                [
                  0.21937842778793418,
                  0.0,
                  120,
                  0
                ],
                [
                  0.22120658135283364,
                  0.0,
                  121,
                  0
                ],
                [
                  0.2230347349177331,
                  0.0,
                  122,
                  0
                ],
                [
                  0.22486288848263253,
                  0.0,
                  123,
                  0
                ],
                [
                  0.226691042047532,
                  0.0,
                  124,
                  0
                ],
                [
                  0.22851919561243145,
                  0.0,
                  125,
                  0
                ],
                [
                  0.2303473491773309,
                  0.0,
                  126,
                  0
                ],
                [
                  0.23217550274223034,
                  0.0,
                  127,
                  0
                ],
                [
                  0.2340036563071298,
                  0.0,
                  128,
                  0
                ],
                [
                  0.23583180987202926,
                  0.0,
                  129,
                  0
                ],
                [
                  0.2376599634369287,
                  0.0,
                  130,
                  0
                ],
                [
                  0.23948811700182815,
                  0.0,
                  131,
                  0
                ],
                [
                  0.2413162705667276,
                  0.0,
                  132,
                  0
                ],
                [
                  0.24314442413162707,
                  0.0,
                  133,
                  0
                ],
                [
                  0.2449725776965265,
                  0.0,
                  134,
                  0
                ],
                [
                  0.24680073126142596,
                  0.0,
                  135,
                  0
                ],
                [
                  0.24862888482632542,
                  0.0,
                  136,
                  0
                ],
                [
                  0.25045703839122485,
                  0.0,
                  137,
                  0
                ],
                [
                  0.2522851919561243,
                  0.0,
                  138,
                  0
                ],
                [
                  0.25411334552102377,
                  0.0,
                  139,
                  0
                ],
                [
                  0.25594149908592323,
                  0.0,
                  140,
                  0
                ],
                [
                  0.2577696526508227,
                  0.0070921985815602835,
                  141,
                  1
                ],
                [
                  0.2595978062157221,
                  0.007042253521126761,
                  142,
                  1
                ],
                [
                  0.26142595978062155,
                  0.006993006993006993,
                  143,
                  1
                ],
                [
                  0.263254113345521,
                  0.006944444444444444,
                  144,
                  1
                ],
                [
                  0.26508226691042047,
                  0.006896551724137931,
                  145,
                  1
                ],
                [
                  0.26691042047531993,
                  0.00684931506849315,
                  146,
                  1
                ],
                [
                  0.2687385740402194,
                  0.006802721088435374,
                  147,
                  1
                ],
                [
                  0.27056672760511885,
                  0.006756756756756757,
                  148,
                  1
                ],
                [
                  0.27239488117001825,
                  0.006711409395973154,
                  149,
                  1
                ],
                [
                  0.2742230347349177,
                  0.006666666666666667,
                  150,
                  1
                ],
                [
                  0.2760511882998172,
                  0.006622516556291391,
                  151,
                  1
                ],
                [
                  0.27787934186471663,
                  0.006578947368421052,
                  152,
                  1
                ],
                [
                  0.2797074954296161,
                  0.006535947712418301,
                  153,
                  1
                ],
                [
                  0.28153564899451555,
                  0.006493506493506494,
                  154,
                  1
                ],
                [
                  0.283363802559415,
                  0.0064516129032258064,
                  155,
                  1
                ],
                [
                  0.2851919561243144,
                  0.00641025641025641,
                  156,
                  1
                ],
                [
                  0.2870201096892139,
                  0.006369426751592357,
                  157,
                  1
                ],
                [
                  0.28884826325411334,
                  0.006329113924050633,
                  158,
                  1
                ],
                [
                  0.2906764168190128,
                  0.006289308176100629,
                  159,
                  1
                ],
                [
                  0.29250457038391225,
                  0.00625,
                  160,
                  1
                ],
                [
                  0.2943327239488117,
                  0.006211180124223602,
                  161,
                  1
                ],
                [
                  0.2961608775137112,
                  0.006172839506172839,
                  162,
                  1
                ],
                [
                  0.2979890310786106,
                  0.006134969325153374,
                  163,
                  1
                ],
                [
                  0.29981718464351004,
                  0.006097560975609756,
                  164,
                  1
                ],
                [
                  0.3016453382084095,
                  0.006060606060606061,
                  165,
                  1
                ],
                [
                  0.30347349177330896,
                  0.006024096385542169,
                  166,
                  1
                ],
                [
                  0.3053016453382084,
                  0.005988023952095809,
                  167,
                  1
                ],
                [
                  0.3071297989031079,
                  0.005952380952380952,
                  168,
                  1
                ],
                [
                  0.30895795246800734,
                  0.005917159763313609,
                  169,
                  1
                ],
                [
                  0.31078610603290674,
                  0.0058823529411764705,
                  170,
                  1
                ],
                [
                  0.3126142595978062,
                  0.005847953216374269,
                  171,
                  1
                ],
                [
                  0.31444241316270566,
                  0.005813953488372093,
                  172,
                  1
                ],
                [
                  0.3162705667276051,
                  0.005780346820809248,
                  173,
                  1
                ],
                [
                  0.3180987202925046,
                  0.005747126436781609,
                  174,
                  1
                ],
                [
                  0.31992687385740404,
                  0.005714285714285714,
                  175,
                  1
                ],
                [
                  0.3217550274223035,
                  0.005681818181818182,
                  176,
                  1
                ],
                [
                  0.3235831809872029,
                  0.005649717514124294,
                  177,
                  1
                ],
                [
                  0.32541133455210236,
                  0.0056179775280898875,
                  178,
                  1
                ],
                [
                  0.3272394881170018,
                  0.00558659217877095,
                  179,
                  1
                ],
                [
                  0.3290676416819013,
                  0.005555555555555556,
                  180,
                  1
                ],
                [
                  0.33089579524680074,
                  0.0055248618784530384,
                  181,
                  1
                ],
                [
                  0.3327239488117002,
                  0.005494505494505495,
                  182,
                  1
                ],
                [
                  0.33455210237659966,
                  0.00546448087431694,
                  183,
                  1
                ],
                [
                  0.33638025594149906,
                  0.005434782608695652,
                  184,
                  1
                ],
                [
                  0.3382084095063985,
                  0.005405405405405406,
                  185,
                  1
                ],
                [
                  0.340036563071298,
                  0.005376344086021506,
                  186,
                  1
                ],
                [
                  0.34186471663619744,
                  0.0053475935828877,
                  187,
                  1
                ],
                [
                  0.3436928702010969,
                  0.005319148936170213,
                  188,
                  1
                ],
                [
                  0.34552102376599636,
                  0.005291005291005291,
                  189,
                  1
                ],
                [
                  0.3473491773308958,
                  0.005263157894736842,
                  190,
                  1
                ],
                [
                  0.3491773308957952,
                  0.005235602094240838,
                  191,
                  1
                ],
                [
                  0.3510054844606947,
                  0.005208333333333333,
                  192,
                  1
                ],
                [
                  0.35283363802559414,
                  0.0051813471502590676,
                  193,
                  1
                ],
                [
                  0.3546617915904936,
                  0.005154639175257732,
                  194,
                  1
                ],
                [
                  0.35648994515539306,
                  0.005128205128205128,
                  195,
                  1
                ],
                [
                  0.3583180987202925,
                  0.00510204081632653,
                  196,
                  1
                ],
                [
                  0.360146252285192,
                  0.005076142131979695,
                  197,
                  1
                ],
                [
                  0.3619744058500914,
                  0.005050505050505051,
                  198,
                  1
                ],
                [
                  0.36380255941499084,
                  0.005025125628140704,
                  199,
                  1
                ],
                [
                  0.3656307129798903,
                  0.005,
                  200,
                  1
                ],
                [
                  0.36745886654478976,
                  0.004975124378109453,
                  201,
                  1
                ],
                [
                  0.3692870201096892,
                  0.0049504950495049506,
                  202,
                  1
                ],
                [
                  0.3711151736745887,
                  0.0049261083743842365,
                  203,
                  1
                ],
                [
                  0.37294332723948814,
                  0.004901960784313725,
                  204,
                  1
                ],
                [
                  0.37477148080438755,
                  0.004878048780487805,
                  205,
                  1
                ],
                [
                  0.376599634369287,
                  0.0048543689320388345,
                  206,
                  1
                ],
                [
                  0.37842778793418647,
                  0.004830917874396135,
                  207,
                  1
                ],
                [
                  0.3802559414990859,
                  0.004807692307692308,
                  208,
                  1
                ],
                [
                  0.3820840950639854,
                  0.004784688995215311,
                  209,
                  1
                ],
                [
                  0.38391224862888484,
                  0.004761904761904762,
                  210,
                  1
                ],
                [
                  0.3857404021937843,
                  0.004739336492890996,
                  211,
                  1
                ],
                [
                  0.3875685557586837,
                  0.0047169811320754715,
                  212,
                  1
                ],
                [
                  0.38939670932358317,
                  0.004694835680751174,
                  213,
                  1
                ],
                [
                  0.3912248628884826,
                  0.004672897196261682,
                  214,
                  1
                ],
                [
                  0.3930530164533821,
                  0.004651162790697674,
                  215,
                  1
                ],
                [
                  0.39488117001828155,
                  0.004629629629629629,
                  216,
                  1
                ],
                [
                  0.396709323583181,
                  0.004608294930875576,
                  217,
                  1
                ],
                [
                  0.39853747714808047,
                  0.0045871559633027525,
                  218,
                  1
                ],
                [
                  0.40036563071297987,
                  0.0045662100456621,
                  219,
                  1
                ],
                [
                  0.40219378427787933,
                  0.004545454545454545,
                  220,
                  1
                ],
                [
                  0.4040219378427788,
                  0.004524886877828055,
                  221,
                  1
                ],
                [
                  0.40585009140767825,
                  0.0045045045045045045,
                  222,
                  1
                ],
                [
                  0.4076782449725777,
                  0.004484304932735426,
                  223,
                  1
                ],
                [
                  0.40950639853747717,
                  0.004464285714285714,
                  224,
                  1
                ],
                [
                  0.4113345521023766,
                  0.0044444444444444444,
                  225,
                  1
                ],
                [
                  0.41316270566727603,
                  0.004424778761061947,
                  226,
                  1
                ],
                [
                  0.4149908592321755,
                  0.004405286343612335,
                  227,
                  1
                ],
                [
                  0.41681901279707495,
                  0.0043859649122807015,
                  228,
                  1
                ],
                [
                  0.4186471663619744,
                  0.004366812227074236,
                  229,
                  1
                ],
                [
                  0.42047531992687387,
                  0.004347826086956522,
                  230,
                  1
                ],
                [
                  0.42230347349177333,
                  0.004329004329004329,
                  231,
                  1
                ],
                [
                  0.42413162705667273,
                  0.004310344827586207,
                  232,
                  1
                ],
                [
                  0.4259597806215722,
                  0.004291845493562232,
                  233,
                  1
                ],
                [
                  0.42778793418647165,
                  0.004273504273504274,
                  234,
                  1
                ],
                [
                  0.4296160877513711,
                  0.00425531914893617,
                  235,
                  1
                ],
                [
                  0.43144424131627057,
                  0.00423728813559322,
                  236,
                  1
                ],
                [
                  0.43327239488117003,
                  0.004219409282700422,
                  237,
                  1
                ],
                [
                  0.4351005484460695,
                  0.004201680672268907,
                  238,
                  1
                ],
                [
                  0.4369287020109689,
                  0.0041841004184100415,
                  239,
                  1
                ],
                [
                  0.43875685557586835,
                  0.004166666666666667,
                  240,
                  1
                ],
                [
                  0.4405850091407678,
                  0.004149377593360996,
                  241,
                  1
                ],
                [
                  0.4424131627056673,
                  0.004132231404958678,
                  242,
                  1
                ],
                [
                  0.44424131627056673,
                  0.00411522633744856,
                  243,
                  1
                ],
                [
                  0.4460694698354662,
                  0.004098360655737705,
                  244,
                  1
                ],
                [
                  0.44789762340036565,
                  0.004081632653061225,
                  245,
                  1
                ],
                [
                  0.44972577696526506,
                  0.0040650406504065045,
                  246,
                  1
                ],
                [
                  0.4515539305301645,
                  0.004048582995951417,
                  247,
                  1
                ],
                [
                  0.453382084095064,
                  0.004032258064516129,
                  248,
                  1
                ],
                [
                  0.45521023765996343,
                  0.004016064257028112,
                  249,
                  1
                ],
                [
                  0.4570383912248629,
                  0.004,
                  250,
                  1
                ],
                [
                  0.45886654478976235,
                  0.00398406374501992,
                  251,
                  1
                ],
                [
                  0.4606946983546618,
                  0.007936507936507936,
                  252,
                  2
                ],
                [
                  0.4625228519195612,
                  0.007905138339920948,
                  253,
                  2
                ],
                [
                  0.4643510054844607,
                  0.007874015748031496,
                  254,
                  2
                ],
                [
                  0.46617915904936014,
                  0.00784313725490196,
                  255,
                  2
                ],
                [
                  0.4680073126142596,
                  0.0078125,
                  256,
                  2
                ],
                [
                  0.46983546617915906,
                  0.007782101167315175,
                  257,
                  2
                ],
                [
                  0.4716636197440585,
                  0.007751937984496124,
                  258,
                  2
                ],
                [
                  0.473491773308958,
                  0.007722007722007722,
                  259,
                  2
                ],
                [
                  0.4753199268738574,
                  0.007692307692307693,
                  260,
                  2
                ],
                [
                  0.47714808043875684,
                  0.007662835249042145,
                  261,
                  2
                ],
                [
                  0.4789762340036563,
                  0.007633587786259542,
                  262,
                  2
                ],
                [
                  0.48080438756855576,
                  0.0076045627376425855,
                  263,
                  2
                ],
                [
                  0.4826325411334552,
                  0.007575757575757576,
                  264,
                  2
                ],
                [
                  0.4844606946983547,
                  0.007547169811320755,
                  265,
                  2
                ],
                [
                  0.48628884826325414,
                  0.007518796992481203,
                  266,
                  2
                ],
                [
                  0.48811700182815354,
                  0.00749063670411985,
                  267,
                  2
                ],
                [
                  0.489945155393053,
                  0.007462686567164179,
                  268,
                  2
                ],
                [
                  0.49177330895795246,
                  0.007434944237918215,
                  269,
                  2
                ],
                [
                  0.4936014625228519,
                  0.007407407407407408,
                  270,
                  2
                ],
                [
                  0.4954296160877514,
                  0.007380073800738007,
                  271,
                  2
                ],
                [
                  0.49725776965265084,
                  0.007352941176470588,
                  272,
                  2
                ],
                [
                  0.4990859232175503,
                  0.007326007326007326,
                  273,
                  2
                ],
                [
                  0.5009140767824497,
                  0.0072992700729927005,
                  274,
                  2
                ],
                [
                  0.5027422303473492,
                  0.007272727272727273,
                  275,
                  2
                ],
                [
                  0.5045703839122486,
                  0.007246376811594203,
                  276,
                  2
                ],
                [
                  0.506398537477148,
                  0.007220216606498195,
                  277,
                  2
                ],
                [
                  0.5082266910420475,
                  0.007194244604316547,
                  278,
                  2
                ],
                [
                  0.5100548446069469,
                  0.007168458781362007,
                  279,
                  2
                ],
                [
                  0.5118829981718465,
                  0.007142857142857143,
                  280,
                  2
                ],
                [
                  0.5137111517367459,
                  0.010676156583629894,
                  281,
                  3
                ],
                [
                  0.5155393053016454,
                  0.010638297872340425,
                  282,
                  3
                ],
                [
                  0.5173674588665448,
                  0.01060070671378092,
                  283,
                  3
                ],
                [
                  0.5191956124314442,
                  0.01056338028169014,
                  284,
                  3
                ],
                [
                  0.5210237659963437,
                  0.010526315789473684,
                  285,
                  3
                ],
                [
                  0.5228519195612431,
                  0.01048951048951049,
                  286,
                  3
                ],
                [
                  0.5246800731261426,
                  0.010452961672473868,
                  287,
                  3
                ],
                [
                  0.526508226691042,
                  0.010416666666666666,
                  288,
                  3
                ],
                [
                  0.5283363802559415,
                  0.010380622837370242,
                  289,
                  3
                ],
                [
                  0.5301645338208409,
                  0.010344827586206896,
                  290,
                  3
                ],
                [
                  0.5319926873857403,
                  0.010309278350515464,
                  291,
                  3
                ],
                [
                  0.5338208409506399,
                  0.010273972602739725,
                  292,
                  3
                ],
                [
                  0.5356489945155393,
                  0.010238907849829351,
                  293,
                  3
                ],
                [
                  0.5374771480804388,
                  0.01020408163265306,
                  294,
                  3
                ],
                [
                  0.5393053016453382,
                  0.010169491525423728,
                  295,
                  3
                ],
                [
                  0.5411334552102377,
                  0.010135135135135136,
                  296,
                  3
                ],
                [
                  0.5429616087751371,
                  0.010101010101010102,
                  297,
                  3
                ],
                [
                  0.5447897623400365,
                  0.010067114093959731,
                  298,
                  3
                ],
                [
                  0.546617915904936,
                  0.010033444816053512,
                  299,
                  3
                ],
                [
                  0.5484460694698354,
                  0.01,
                  300,
                  3
                ],
                [
                  0.5502742230347349,
                  0.009966777408637873,
                  301,
                  3
                ],
                [
                  0.5521023765996343,
                  0.009933774834437087,
                  302,
                  3
                ],
                [
                  0.5539305301645339,
                  0.009900990099009901,
                  303,
                  3
                ],
                [
                  0.5557586837294333,
                  0.009868421052631578,
                  304,
                  3
                ],
                [
                  0.5575868372943327,
                  0.009836065573770493,
                  305,
                  3
                ],
                [
                  0.5594149908592322,
                  0.00980392156862745,
                  306,
                  3
                ],
                [
                  0.5612431444241316,
                  0.009771986970684038,
                  307,
                  3
                ],
                [
                  0.5630712979890311,
                  0.00974025974025974,
                  308,
                  3
                ],
                [
                  0.5648994515539305,
                  0.009708737864077669,
                  309,
                  3
                ],
                [
                  0.56672760511883,
                  0.00967741935483871,
                  310,
                  3
                ],
                [
                  0.5685557586837294,
                  0.00964630225080386,
                  311,
                  3
                ],
                [
                  0.5703839122486288,
                  0.009615384615384616,
                  312,
                  3
                ],
                [
                  0.5722120658135283,
                  0.009584664536741214,
                  313,
                  3
                ],
                [
                  0.5740402193784278,
                  0.009554140127388535,
                  314,
                  3
                ],
                [
                  0.5758683729433273,
                  0.009523809523809525,
                  315,
                  3
                ],
                [
                  0.5776965265082267,
                  0.00949367088607595,
                  316,
                  3
                ],
                [
                  0.5795246800731262,
                  0.00946372239747634,
                  317,
                  3
                ],
                [
                  0.5813528336380256,
                  0.009433962264150943,
                  318,
                  3
                ],
                [
                  0.583180987202925,
                  0.009404388714733543,
                  319,
                  3
                ],
                [
                  0.5850091407678245,
                  0.009375,
                  320,
                  3
                ],
                [
                  0.5868372943327239,
                  0.009345794392523364,
                  321,
                  3
                ],
                [
                  0.5886654478976234,
                  0.009316770186335404,
                  322,
                  3
                ],
                [
                  0.5904936014625228,
                  0.009287925696594427,
                  323,
                  3
                ],
                [
                  0.5923217550274223,
                  0.009259259259259259,
                  324,
                  3
                ],
                [
                  0.5941499085923218,
                  0.009230769230769232,
                  325,
                  3
                ],
                [
                  0.5959780621572212,
                  0.009202453987730062,
                  326,
                  3
                ],
                [
                  0.5978062157221207,
                  0.012232415902140673,
                  327,
                  4
                ],
                [
                  0.5996343692870201,
                  0.01524390243902439,
                  328,
                  5
                ],
                [
                  0.6014625228519196,
                  0.015197568389057751,
                  329,
                  5
                ],
                [
                  0.603290676416819,
                  0.015151515151515152,
                  330,
                  5
                ],
                [
                  0.6051188299817185,
                  0.015105740181268883,
                  331,
                  5
                ],
                [
                  0.6069469835466179,
                  0.015060240963855422,
                  332,
                  5
                ],
                [
                  0.6087751371115173,
                  0.015015015015015015,
                  333,
                  5
                ],
                [
                  0.6106032906764168,
                  0.014970059880239521,
                  334,
                  5
                ],
                [
                  0.6124314442413162,
                  0.014925373134328358,
                  335,
                  5
                ],
                [
                  0.6142595978062158,
                  0.01488095238095238,
                  336,
                  5
                ],
                [
                  0.6160877513711152,
                  0.01483679525222552,
                  337,
                  5
                ],
                [
                  0.6179159049360147,
                  0.014792899408284023,
                  338,
                  5
                ],
                [
                  0.6197440585009141,
                  0.014749262536873156,
                  339,
                  5
                ],
                [
                  0.6215722120658135,
                  0.014705882352941176,
                  340,
                  5
                ],
                [
                  0.623400365630713,
                  0.01466275659824047,
                  341,
                  5
                ],
                [
                  0.6252285191956124,
                  0.014619883040935672,
                  342,
                  5
                ],
                [
                  0.6270566727605119,
                  0.014577259475218658,
                  343,
                  5
                ],
                [
                  0.6288848263254113,
                  0.014534883720930232,
                  344,
                  5
                ],
                [
                  0.6307129798903108,
                  0.014492753623188406,
                  345,
                  5
                ],
                [
                  0.6325411334552102,
                  0.014450867052023121,
                  346,
                  5
                ],
                [
                  0.6343692870201096,
                  0.01440922190201729,
                  347,
                  5
                ],
                [
                  0.6361974405850092,
                  0.014367816091954023,
                  348,
                  5
                ],
                [
                  0.6380255941499086,
                  0.017191977077363897,
                  349,
                  6
                ],
                [
                  0.6398537477148081,
                  0.017142857142857144,
                  350,
                  6
                ],
                [
                  0.6416819012797075,
                  0.017094017094017096,
                  351,
                  6
                ],
                [
                  0.643510054844607,
                  0.017045454545454544,
                  352,
                  6
                ],
                [
                  0.6453382084095064,
                  0.0169971671388102,
                  353,
                  6
                ],
                [
                  0.6471663619744058,
                  0.01694915254237288,
                  354,
                  6
                ],
                [
                  0.6489945155393053,
                  0.016901408450704224,
                  355,
                  6
                ],
                [
                  0.6508226691042047,
                  0.016853932584269662,
                  356,
                  6
                ],
                [
                  0.6526508226691042,
                  0.01680672268907563,
                  357,
                  6
                ],
                [
                  0.6544789762340036,
                  0.01675977653631285,
                  358,
                  6
                ],
                [
                  0.6563071297989032,
                  0.016713091922005572,
                  359,
                  6
                ],
                [
                  0.6581352833638026,
                  0.016666666666666666,
                  360,
                  6
                ],
                [
                  0.659963436928702,
                  0.01662049861495845,
                  361,
                  6
                ],
                [
                  0.6617915904936015,
                  0.016574585635359115,
                  362,
                  6
                ],
                [
                  0.6636197440585009,
                  0.01652892561983471,
                  363,
                  6
                ],
                [
                  0.6654478976234004,
                  0.016483516483516484,
                  364,
                  6
                ],
                [
                  0.6672760511882998,
                  0.01643835616438356,
                  365,
                  6
                ],
                [
                  0.6691042047531993,
                  0.01639344262295082,
                  366,
                  6
                ],
                [
                  0.6709323583180987,
                  0.01634877384196185,
                  367,
                  6
                ],
                [
                  0.6727605118829981,
                  0.016304347826086956,
                  368,
                  6
                ],
                [
                  0.6745886654478976,
                  0.016260162601626018,
                  369,
                  6
                ],
                [
                  0.676416819012797,
                  0.016216216216216217,
                  370,
                  6
                ],
                [
                  0.6782449725776966,
                  0.016172506738544475,
                  371,
                  6
                ],
                [
                  0.680073126142596,
                  0.016129032258064516,
                  372,
                  6
                ],
                [
                  0.6819012797074955,
                  0.0160857908847185,
                  373,
                  6
                ],
                [
                  0.6837294332723949,
                  0.016042780748663103,
                  374,
                  6
                ],
                [
                  0.6855575868372943,
                  0.018666666666666668,
                  375,
                  7
                ],
                [
                  0.6873857404021938,
                  0.018617021276595744,
                  376,
                  7
                ],
                [
                  0.6892138939670932,
                  0.01856763925729443,
                  377,
                  7
                ],
                [
                  0.6910420475319927,
                  0.018518518518518517,
                  378,
                  7
                ],
                [
                  0.6928702010968921,
                  0.021108179419525065,
                  379,
                  8
                ],
                [
                  0.6946983546617916,
                  0.021052631578947368,
                  380,
                  8
                ],
                [
                  0.696526508226691,
                  0.02099737532808399,
                  381,
                  8
                ],
                [
                  0.6983546617915904,
                  0.020942408376963352,
                  382,
                  8
                ],
                [
                  0.70018281535649,
                  0.020887728459530026,
                  383,
                  8
                ],
                [
                  0.7020109689213894,
                  0.020833333333333332,
                  384,
                  8
                ],
                [
                  0.7038391224862889,
                  0.02077922077922078,
                  385,
                  8
                ],
                [
                  0.7056672760511883,
                  0.02072538860103627,
                  386,
                  8
                ],
                [
                  0.7074954296160878,
                  0.020671834625323,
                  387,
                  8
                ],
                [
                  0.7093235831809872,
                  0.020618556701030927,
                  388,
                  8
                ],
                [
                  0.7111517367458866,
                  0.02056555269922879,
                  389,
                  8
                ],
                [
                  0.7129798903107861,
                  0.020512820512820513,
                  390,
                  8
                ],
                [
                  0.7148080438756855,
                  0.020460358056265986,
                  391,
                  8
                ],
                [
                  0.716636197440585,
                  0.02040816326530612,
                  392,
                  8
                ],
                [
                  0.7184643510054844,
                  0.020356234096692113,
                  393,
                  8
                ],
                [
                  0.720292504570384,
                  0.02030456852791878,
                  394,
                  8
                ],
                [
                  0.7221206581352834,
                  0.020253164556962026,
                  395,
                  8
                ],
                [
                  0.7239488117001828,
                  0.020202020202020204,
                  396,
                  8
                ],
                [
                  0.7257769652650823,
                  0.020151133501259445,
                  397,
                  8
                ],
                [
                  0.7276051188299817,
                  0.020100502512562814,
                  398,
                  8
                ],
                [
                  0.7294332723948812,
                  0.020050125313283207,
                  399,
                  8
                ],
                [
                  0.7312614259597806,
                  0.02,
                  400,
                  8
                ],
                [
                  0.7330895795246801,
                  0.022443890274314215,
                  401,
                  9
                ],
                [
                  0.7349177330895795,
                  0.022388059701492536,
                  402,
                  9
                ],
                [
                  0.7367458866544789,
                  0.022332506203473945,
                  403,
                  9
                ],
                [
                  0.7385740402193784,
                  0.022277227722772276,
                  404,
                  9
                ],
                [
                  0.7404021937842779,
                  0.022222222222222223,
                  405,
                  9
                ],
                [
                  0.7422303473491774,
                  0.022167487684729065,
                  406,
                  9
                ],
                [
                  0.7440585009140768,
                  0.02457002457002457,
                  407,
                  10
                ],
                [
                  0.7458866544789763,
                  0.02696078431372549,
                  408,
                  11
                ],
                [
                  0.7477148080438757,
                  0.02689486552567237,
                  409,
                  11
                ],
                [
                  0.7495429616087751,
                  0.026829268292682926,
                  410,
                  11
                ],
                [
                  0.7513711151736746,
                  0.0267639902676399,
                  411,
                  11
                ],
                [
                  0.753199268738574,
                  0.02912621359223301,
                  412,
                  12
                ],
                [
                  0.7550274223034735,
                  0.031476997578692496,
                  413,
                  13
                ],
                [
                  0.7568555758683729,
                  0.03140096618357488,
                  414,
                  13
                ],
                [
                  0.7586837294332724,
                  0.03132530120481928,
                  415,
                  13
                ],
                [
                  0.7605118829981719,
                  0.03125,
                  416,
                  13
                ],
                [
                  0.7623400365630713,
                  0.03117505995203837,
                  417,
                  13
                ],
                [
                  0.7641681901279708,
                  0.03110047846889952,
                  418,
                  13
                ],
                [
                  0.7659963436928702,
                  0.031026252983293555,
                  419,
                  13
                ],
                [
                  0.7678244972577697,
                  0.03333333333333333,
                  420,
                  14
                ],
                [
                  0.7696526508226691,
                  0.0332541567695962,
                  421,
                  14
                ],
                [
                  0.7714808043875686,
                  0.03317535545023697,
                  422,
                  14
                ],
                [
                  0.773308957952468,
                  0.03309692671394799,
                  423,
                  14
                ],
                [
                  0.7751371115173674,
                  0.0330188679245283,
                  424,
                  14
                ],
                [
                  0.7769652650822669,
                  0.03529411764705882,
                  425,
                  15
                ],
                [
                  0.7787934186471663,
                  0.035211267605633804,
                  426,
                  15
                ],
                [
                  0.7806215722120659,
                  0.0351288056206089,
                  427,
                  15
                ],
                [
                  0.7824497257769653,
                  0.035046728971962614,
                  428,
                  15
                ],
                [
                  0.7842778793418648,
                  0.03496503496503497,
                  429,
                  15
                ],
                [
                  0.7861060329067642,
                  0.037209302325581395,
                  430,
                  16
                ],
                [
                  0.7879341864716636,
                  0.037122969837587005,
                  431,
                  16
                ],
                [
                  0.7897623400365631,
                  0.037037037037037035,
                  432,
                  16
                ],
                [
                  0.7915904936014625,
                  0.03926096997690531,
                  433,
                  17
                ],
                [
                  0.793418647166362,
                  0.03917050691244239,
                  434,
                  17
                ],
                [
                  0.7952468007312614,
                  0.041379310344827586,
                  435,
                  18
                ],
                [
                  0.7970749542961609,
                  0.04128440366972477,
                  436,
                  18
                ],
                [
                  0.7989031078610603,
                  0.041189931350114416,
                  437,
                  18
                ],
                [
                  0.8007312614259597,
                  0.0410958904109589,
                  438,
                  18
                ],
                [
                  0.8025594149908593,
                  0.04100227790432802,
                  439,
                  18
                ],
                [
                  0.8043875685557587,
                  0.04090909090909091,
                  440,
                  18
                ],
                [
                  0.8062157221206582,
                  0.04308390022675737,
                  441,
                  19
                ],
                [
                  0.8080438756855576,
                  0.04524886877828054,
                  442,
                  20
                ],
                [
                  0.8098720292504571,
                  0.045146726862302484,
                  443,
                  20
                ],
                [
                  0.8117001828153565,
                  0.0472972972972973,
                  444,
                  21
                ],
                [
                  0.8135283363802559,
                  0.04719101123595506,
                  445,
                  21
                ],
                [
                  0.8153564899451554,
                  0.04708520179372197,
                  446,
                  21
                ],
                [
                  0.8171846435100548,
                  0.049217002237136466,
                  447,
                  22
                ],
                [
                  0.8190127970749543,
                  0.05133928571428571,
                  448,
                  23
                ],
                [
                  0.8208409506398537,
                  0.053452115812917596,
                  449,
                  24
                ],
                [
                  0.8226691042047533,
                  0.05333333333333334,
                  450,
                  24
                ],
                [
                  0.8244972577696527,
                  0.05321507760532151,
                  451,
                  24
                ],
                [
                  0.8263254113345521,
                  0.05309734513274336,
                  452,
                  24
                ],
                [
                  0.8281535648994516,
                  0.052980132450331126,
                  453,
                  24
                ],
                [
                  0.829981718464351,
                  0.05286343612334802,
                  454,
                  24
                ],
                [
                  0.8318098720292505,
                  0.05274725274725275,
                  455,
                  24
                ],
                [
                  0.8336380255941499,
                  0.05263157894736842,
                  456,
                  24
                ],
                [
                  0.8354661791590493,
                  0.0525164113785558,
                  457,
                  24
                ],
                [
                  0.8372943327239488,
                  0.05240174672489083,
                  458,
                  24
                ],
                [
                  0.8391224862888482,
                  0.054466230936819175,
                  459,
                  25
                ],
                [
                  0.8409506398537477,
                  0.05434782608695652,
                  460,
                  25
                ],
                [
                  0.8427787934186471,
                  0.05639913232104121,
                  461,
                  26
                ],
                [
                  0.8446069469835467,
                  0.05844155844155844,
                  462,
                  27
                ],
                [
                  0.8464351005484461,
                  0.058315334773218146,
                  463,
                  27
                ],
                [
                  0.8482632541133455,
                  0.05818965517241379,
                  464,
                  27
                ],
                [
                  0.850091407678245,
                  0.05806451612903226,
                  465,
                  27
                ],
                [
                  0.8519195612431444,
                  0.060085836909871244,
                  466,
                  28
                ],
                [
                  0.8537477148080439,
                  0.059957173447537475,
                  467,
                  28
                ],
                [
                  0.8555758683729433,
                  0.05982905982905983,
                  468,
                  28
                ],
                [
                  0.8574040219378428,
                  0.06183368869936034,
                  469,
                  29
                ],
                [
                  0.8592321755027422,
                  0.06170212765957447,
                  470,
                  29
                ],
                [
                  0.8610603290676416,
                  0.06369426751592357,
                  471,
                  30
                ],
                [
                  0.8628884826325411,
                  0.0635593220338983,
                  472,
                  30
                ],
                [
                  0.8647166361974405,
                  0.06553911205073996,
                  473,
                  31
                ],
                [
                  0.8665447897623401,
                  0.06751054852320675,
                  474,
                  32
                ],
                [
                  0.8683729433272395,
                  0.06736842105263158,
                  475,
                  32
                ],
                [
                  0.870201096892139,
                  0.06722689075630252,
                  476,
                  32
                ],
                [
                  0.8720292504570384,
                  0.06708595387840671,
                  477,
                  32
                ],
                [
                  0.8738574040219378,
                  0.06903765690376569,
                  478,
                  33
                ],
                [
                  0.8756855575868373,
                  0.06889352818371608,
                  479,
                  33
                ],
                [
                  0.8775137111517367,
                  0.07083333333333333,
                  480,
                  34
                ],
                [
                  0.8793418647166362,
                  0.07276507276507277,
                  481,
                  35
                ],
                [
                  0.8811700182815356,
                  0.07261410788381743,
                  482,
                  35
                ],
                [
                  0.8829981718464351,
                  0.07453416149068323,
                  483,
                  36
                ],
                [
                  0.8848263254113345,
                  0.07644628099173553,
                  484,
                  37
                ],
                [
                  0.886654478976234,
                  0.07628865979381444,
                  485,
                  37
                ],
                [
                  0.8884826325411335,
                  0.07818930041152264,
                  486,
                  38
                ],
                [
                  0.8903107861060329,
                  0.07802874743326489,
                  487,
                  38
                ],
                [
                  0.8921389396709324,
                  0.07991803278688525,
                  488,
                  39
                ],
                [
                  0.8939670932358318,
                  0.081799591002045,
                  489,
                  40
                ],
                [
                  0.8957952468007313,
                  0.08163265306122448,
                  490,
                  40
                ],
                [
                  0.8976234003656307,
                  0.0835030549898167,
                  491,
                  41
                ],
                [
                  0.8994515539305301,
                  0.08536585365853659,
                  492,
                  42
                ],
                [
                  0.9012797074954296,
                  0.08519269776876268,
                  493,
                  42
                ],
                [
                  0.903107861060329,
                  0.08502024291497975,
                  494,
                  42
                ],
                [
                  0.9049360146252285,
                  0.08686868686868687,
                  495,
                  43
                ],
                [
                  0.906764168190128,
                  0.08669354838709678,
                  496,
                  43
                ],
                [
                  0.9085923217550275,
                  0.08853118712273642,
                  497,
                  44
                ],
                [
                  0.9104204753199269,
                  0.08835341365461848,
                  498,
                  44
                ],
                [
                  0.9122486288848263,
                  0.08817635270541083,
                  499,
                  44
                ],
                [
                  0.9140767824497258,
                  0.09,
                  500,
                  45
                ],
                [
                  0.9159049360146252,
                  0.08982035928143713,
                  501,
                  45
                ],
                [
                  0.9177330895795247,
                  0.08964143426294821,
                  502,
                  45
                ],
                [
                  0.9195612431444241,
                  0.08946322067594434,
                  503,
                  45
                ],
                [
                  0.9213893967093236,
                  0.08928571428571429,
                  504,
                  45
                ],
                [
                  0.923217550274223,
                  0.09108910891089109,
                  505,
                  46
                ],
                [
                  0.9250457038391224,
                  0.09090909090909091,
                  506,
                  46
                ],
                [
                  0.926873857404022,
                  0.09270216962524655,
                  507,
                  47
                ],
                [
                  0.9287020109689214,
                  0.09448818897637795,
                  508,
                  48
                ],
                [
                  0.9305301645338209,
                  0.0962671905697446,
                  509,
                  49
                ],
                [
                  0.9323583180987203,
                  0.09607843137254903,
                  510,
                  49
                ],
                [
                  0.9341864716636198,
                  0.0958904109589041,
                  511,
                  49
                ],
                [
                  0.9360146252285192,
                  0.09765625,
                  512,
                  50
                ],
                [
                  0.9378427787934186,
                  0.09746588693957114,
                  513,
                  50
                ],
                [
                  0.9396709323583181,
                  0.09922178988326848,
                  514,
                  51
                ],
                [
                  0.9414990859232175,
                  0.10097087378640776,
                  515,
                  52
                ],
                [
                  0.943327239488117,
                  0.10077519379844961,
                  516,
                  52
                ],
                [
                  0.9451553930530164,
                  0.10058027079303675,
                  517,
                  52
                ],
                [
                  0.946983546617916,
                  0.10038610038610038,
                  518,
                  52
                ],
                [
                  0.9488117001828154,
                  0.10211946050096339,
                  519,
                  53
                ],
                [
                  0.9506398537477148,
                  0.10192307692307692,
                  520,
                  53
                ],
                [
                  0.9524680073126143,
                  0.1017274472168906,
                  521,
                  53
                ],
                [
                  0.9542961608775137,
                  0.10344827586206896,
                  522,
                  54
                ],
                [
                  0.9561243144424132,
                  0.10325047801147227,
                  523,
                  54
                ],
                [
                  0.9579524680073126,
                  0.1049618320610687,
                  524,
                  55
                ],
                [
                  0.9597806215722121,
                  0.10666666666666667,
                  525,
                  56
                ],
                [
                  0.9616087751371115,
                  0.10646387832699619,
                  526,
                  56
                ],
                [
                  0.9634369287020109,
                  0.10815939278937381,
                  527,
                  57
                ],
                [
                  0.9652650822669104,
                  0.10984848484848485,
                  528,
                  58
                ],
                [
                  0.9670932358318098,
                  0.11153119092627599,
                  529,
                  59
                ],
                [
                  0.9689213893967094,
                  0.11320754716981132,
                  530,
                  60
                ],
                [
                  0.9707495429616088,
                  0.11487758945386065,
                  531,
                  61
                ],
                [
                  0.9725776965265083,
                  0.11654135338345864,
                  532,
                  62
                ],
                [
                  0.9744058500914077,
                  0.11632270168855535,
                  533,
                  62
                ],
                [
                  0.9762340036563071,
                  0.11610486891385768,
                  534,
                  62
                ],
                [
                  0.9780621572212066,
                  0.11775700934579439,
                  535,
                  63
                ],
                [
                  0.979890310786106,
                  0.11753731343283583,
                  536,
                  63
                ],
                [
                  0.9817184643510055,
                  0.11731843575418995,
                  537,
                  63
                ],
                [
                  0.9835466179159049,
                  0.11895910780669144,
                  538,
                  64
                ],
                [
                  0.9853747714808044,
                  0.11873840445269017,
                  539,
                  64
                ],
                [
                  0.9872029250457038,
                  0.12037037037037036,
                  540,
                  65
                ],
                [
                  0.9890310786106032,
                  0.12199630314232902,
                  541,
                  66
                ],
                [
                  0.9908592321755028,
                  0.12361623616236163,
                  542,
                  67
                ],
                [
                  0.9926873857404022,
                  0.12338858195211787,
                  543,
                  67
                ],
                [
                  0.9945155393053017,
                  0.125,
                  544,
                  68
                ],
                [
                  0.9963436928702011,
                  0.12660550458715597,
                  545,
                  69
                ],
                [
                  0.9981718464351006,
                  0.1282051282051282,
                  546,
                  70
                ],
                [
                  1.0,
                  0.12979890310786105,
                  547,
                  71
                ]
              ]
            },
            {
              "language": "English",
              "run": 398,
              "points": [
                [
                  0.0018281535648994515,
                  0.0,
                  1,
                  0
                ],
                [
                  0.003656307129798903,
                  0.0,
                  2,
                  0
                ],
                [
                  0.005484460694698354,
                  0.0,
                  3,
                  0
                ],
                [
                  0.007312614259597806,
                  0.0,
                  4,
                  0
                ],
                [
                  0.009140767824497258,
                  0.0,
                  5,
                  0
                ],
                [
                  0.010968921389396709,
                  0.0,
                  6,
                  0
                ],
                [
                  0.012797074954296161,
                  0.0,
                  7,
                  0
                ],
                [
                  0.014625228519195612,
                  0.0,
                  8,
                  0
                ],
                [
                  0.016453382084095063,
                  0.0,
                  9,
                  0
                ],
                [
                  0.018281535648994516,
                  0.0,
                  10,
                  0
                ],
                [
                  0.02010968921389397,
                  0.0,
                  11,
                  0
                ],
                [
                  0.021937842778793418,
                  0.0,
                  12,
                  0
                ],
                [
                  0.02376599634369287,
                  0.0,
                  13,
                  0
                ],
                [
                  0.025594149908592323,
                  0.0,
                  14,
                  0
                ],
                [
                  0.027422303473491772,
                  0.0,
                  15,
                  0
                ],
                [
                  0.029250457038391225,
                  0.0,
                  16,
                  0
                ],
                [
                  0.031078610603290677,
                  0.0,
                  17,
                  0
                ],
                [
                  0.03290676416819013,
                  0.0,
                  18,
                  0
                ],
                [
                  0.03473491773308958,
                  0.0,
                  19,
                  0
                ],
                [
                  0.03656307129798903,
                  0.0,
                  20,
                  0
                ],
                [
                  0.038391224862888484,
                  0.0,
                  21,
                  0
                ],
                [
                  0.04021937842778794,
                  0.0,
                  22,
                  0
                ],
                [
                  0.04204753199268738,
                  0.0,
                  23,
                  0
                ],
                [
                  0.043875685557586835,
                  0.0,
                  24,
                  0
                ],
                [
                  0.04570383912248629,
                  0.0,
                  25,
                  0
                ],
                [
                  0.04753199268738574,
                  0.0,
                  26,
                  0
                ],
                [
                  0.04936014625228519,
                  0.0,
                  27,
                  0
                ],
                [
                  0.051188299817184646,
                  0.0,
                  28,
                  0
                ],
                [
                  0.05301645338208409,
                  0.0,
                  29,
                  0
                ],
                [
                  0.054844606946983544,
                  0.0,
                  30,
                  0
                ],
                [
                  0.056672760511883,
                  0.0,
                  31,
                  0
                ],
                [
                  0.05850091407678245,
                  0.0,
                  32,
                  0
                ],
                [
                  0.0603290676416819,
                  0.0,
                  33,
                  0
                ],
                [
                  0.062157221206581355,
                  0.0,
                  34,
                  0
                ],
                [
                  0.06398537477148081,
                  0.0,
                  35,
                  0
                ],
                [
                  0.06581352833638025,
                  0.0,
                  36,
                  0
                ],
                [
                  0.06764168190127971,
                  0.0,
                  37,
                  0
                ],
                [
                  0.06946983546617916,
                  0.0,
                  38,
                  0
                ],
                [
                  0.0712979890310786,
                  0.0,
                  39,
                  0
                ],
                [
                  0.07312614259597806,
                  0.0,
                  40,
                  0
                ],
                [
                  0.07495429616087751,
                  0.0,
                  41,
                  0
                ],
                [
                  0.07678244972577697,
                  0.0,
                  42,
                  0
                ],
                [
                  0.07861060329067641,
                  0.0,
                  43,
                  0
                ],
                [
                  0.08043875685557587,
                  0.0,
                  44,
                  0
                ],
                [
                  0.08226691042047532,
                  0.0,
                  45,
                  0
                ],
                [
                  0.08409506398537477,
                  0.0,
                  46,
                  0
                ],
                [
                  0.08592321755027423,
                  0.0,
                  47,
                  0
                ],
                [
                  0.08775137111517367,
                  0.0,
                  48,
                  0
                ],
                [
                  0.08957952468007313,
                  0.0,
                  49,
                  0
                ],
                [
                  0.09140767824497258,
                  0.0,
                  50,
                  0
                ],
                [
                  0.09323583180987204,
                  0.0,
                  51,
                  0
                ],
                [
                  0.09506398537477148,
                  0.0,
                  52,
                  0
                ],
                [
                  0.09689213893967093,
                  0.0,
                  53,
                  0
                ],
                [
                  0.09872029250457039,
                  0.0,
                  54,
                  0
                ],
                [
                  0.10054844606946983,
                  0.0,
                  55,
                  0
                ],
                [
                  0.10237659963436929,
                  0.0,
                  56,
                  0
                ],
                [
                  0.10420475319926874,
                  0.0,
                  57,
                  0
                ],
                [
                  0.10603290676416818,
                  0.0,
                  58,
                  0
                ],
                [
                  0.10786106032906764,
                  0.0,
                  59,
                  0
                ],
                [
                  0.10968921389396709,
                  0.0,
                  60,
                  0
                ],
                [
                  0.11151736745886655,
                  0.0,
                  61,
                  0
                ],
                [
                  0.113345521023766,
                  0.0,
                  62,
                  0
                ],
                [
                  0.11517367458866545,
                  0.0,
                  63,
                  0
                ],
                [
                  0.1170018281535649,
                  0.0,
                  64,
                  0
                ],
                [
                  0.11882998171846434,
                  0.0,
                  65,
                  0
                ],
                [
                  0.1206581352833638,
                  0.0,
                  66,
                  0
                ],
                [
                  0.12248628884826325,
                  0.014925373134328358,
                  67,
                  1
                ],
                [
                  0.12431444241316271,
                  0.014705882352941176,
                  68,
                  1
                ],
                [
                  0.12614259597806216,
                  0.014492753623188406,
                  69,
                  1
                ],
                [
                  0.12797074954296161,
                  0.014285714285714285,
                  70,
                  1
                ],
                [
                  0.12979890310786105,
                  0.014084507042253521,
                  71,
                  1
                ],
                [
                  0.1316270566727605,
                  0.013888888888888888,
                  72,
                  1
                ],
                [
                  0.13345521023765997,
                  0.0136986301369863,
                  73,
                  1
                ],
                [
                  0.13528336380255943,
                  0.013513513513513514,
                  74,
                  1
                ],
                [
                  0.13711151736745886,
                  0.013333333333333334,
                  75,
                  1
                ],
                [
                  0.13893967093235832,
                  0.013157894736842105,
                  76,
                  1
                ],
                [
                  0.14076782449725778,
                  0.012987012987012988,
                  77,
                  1
                ],
                [
                  0.1425959780621572,
                  0.01282051282051282,
                  78,
                  1
                ],
                [
                  0.14442413162705667,
                  0.012658227848101266,
                  79,
                  1
                ],
                [
                  0.14625228519195613,
                  0.0125,
                  80,
                  1
                ],
                [
                  0.1480804387568556,
                  0.012345679012345678,
                  81,
                  1
                ],
                [
                  0.14990859232175502,
                  0.012195121951219513,
                  82,
                  1
                ],
                [
                  0.15173674588665448,
                  0.012048192771084338,
                  83,
                  1
                ],
                [
                  0.15356489945155394,
                  0.011904761904761904,
                  84,
                  1
                ],
                [
                  0.15539305301645337,
                  0.011764705882352941,
                  85,
                  1
                ],
                [
                  0.15722120658135283,
                  0.011627906976744186,
                  86,
                  1
                ],
                [
                  0.1590493601462523,
                  0.011494252873563218,
                  87,
                  1
                ],
                [
                  0.16087751371115175,
                  0.011363636363636364,
                  88,
                  1
                ],
                [
                  0.16270566727605118,
                  0.011235955056179775,
                  89,
                  1
                ],
                [
                  0.16453382084095064,
                  0.011111111111111112,
                  90,
                  1
                ],
                [
                  0.1663619744058501,
                  0.01098901098901099,
                  91,
                  1
                ],
                [
                  0.16819012797074953,
                  0.010869565217391304,
                  92,
                  1
                ],
                [
                  0.170018281535649,
                  0.010752688172043012,
                  93,
                  1
                ],
                [
                  0.17184643510054845,
                  0.010638297872340425,
                  94,
                  1
                ],
                [
                  0.1736745886654479,
                  0.010526315789473684,
                  95,
                  1
                ],
                [
                  0.17550274223034734,
                  0.010416666666666666,
                  96,
                  1
                ],
                [
                  0.1773308957952468,
                  0.010309278350515464,
                  97,
                  1
                ],
                [
                  0.17915904936014626,
                  0.01020408163265306,
                  98,
                  1
                ],
                [
                  0.1809872029250457,
                  0.010101010101010102,
                  99,
                  1
                ],
                [
                  0.18281535648994515,
                  0.01,
                  100,
                  1
                ],
                [
                  0.1846435100548446,
                  0.009900990099009901,
                  101,
                  1
                ],
                [
                  0.18647166361974407,
                  0.00980392156862745,
                  102,
                  1
                ],
                [
                  0.1882998171846435,
                  0.009708737864077669,
                  103,
                  1
                ],
                [
                  0.19012797074954296,
                  0.009615384615384616,
                  104,
                  1
                ],
                [
                  0.19195612431444242,
                  0.009523809523809525,
                  105,
                  1
                ],
                [
                  0.19378427787934185,
                  0.009433962264150943,
                  106,
                  1
                ],
                [
                  0.1956124314442413,
                  0.009345794392523364,
                  107,
                  1
                ],
                [
                  0.19744058500914077,
                  0.009259259259259259,
                  108,
                  1
                ],
                [
                  0.19926873857404023,
                  0.009174311926605505,
                  109,
                  1
                ],
                [
                  0.20109689213893966,
                  0.00909090909090909,
                  110,
                  1
                ],
                [
                  0.20292504570383912,
                  0.009009009009009009,
                  111,
                  1
                ],
                [
                  0.20475319926873858,
                  0.008928571428571428,
                  112,
                  1
                ],
                [
                  0.20658135283363802,
                  0.008849557522123894,
                  113,
                  1
                ],
                [
                  0.20840950639853748,
                  0.008771929824561403,
                  114,
                  1
                ],
                [
                  0.21023765996343693,
                  0.008695652173913044,
                  115,
                  1
                ],
                [
                  0.21206581352833637,
                  0.008620689655172414,
                  116,
                  1
                ],
                [
                  0.21389396709323583,
                  0.008547008547008548,
                  117,
                  1
                ],
                [
                  0.21572212065813529,
                  0.00847457627118644,
                  118,
                  1
                ],
                [
                  0.21755027422303475,
                  0.008403361344537815,
                  119,
                  1
                ],
                [
                  0.21937842778793418,
                  0.008333333333333333,
                  120,
                  1
                ],
                [
                  0.22120658135283364,
                  0.008264462809917356,
                  121,
                  1
                ],
                [
                  0.2230347349177331,
                  0.00819672131147541,
                  122,
                  1
                ],
                [
                  0.22486288848263253,
                  0.008130081300813009,
                  123,
                  1
                ],
                [
                  0.226691042047532,
                  0.008064516129032258,
                  124,
                  1
                ],
                [
                  0.22851919561243145,
                  0.008,
                  125,
                  1
                ],
                [
                  0.2303473491773309,
                  0.007936507936507936,
                  126,
                  1
                ],
                [
                  0.23217550274223034,
                  0.007874015748031496,
                  127,
                  1
                ],
                [
                  0.2340036563071298,
                  0.0078125,
                  128,
                  1
                ],
                [
                  0.23583180987202926,
                  0.007751937984496124,
                  129,
                  1
                ],
                [
                  0.2376599634369287,
                  0.007692307692307693,
                  130,
                  1
                ],
                [
                  0.23948811700182815,
                  0.007633587786259542,
                  131,
                  1
                ],
                [
                  0.2413162705667276,
                  0.007575757575757576,
                  132,
                  1
                ],
                [
                  0.24314442413162707,
                  0.007518796992481203,
                  133,
                  1
                ],
                [
                  0.2449725776965265,
                  0.007462686567164179,
                  134,
                  1
                ],
                [
                  0.24680073126142596,
                  0.007407407407407408,
                  135,
                  1
                ],
                [
                  0.24862888482632542,
                  0.007352941176470588,
                  136,
                  1
                ],
                [
                  0.25045703839122485,
                  0.0072992700729927005,
                  137,
                  1
                ],
                [
                  0.2522851919561243,
                  0.007246376811594203,
                  138,
                  1
                ],
                [
                  0.25411334552102377,
                  0.007194244604316547,
                  139,
                  1
                ],
                [
                  0.25594149908592323,
                  0.007142857142857143,
                  140,
                  1
                ],
                [
                  0.2577696526508227,
                  0.0070921985815602835,
                  141,
                  1
                ],
                [
                  0.2595978062157221,
                  0.007042253521126761,
                  142,
                  1
                ],
                [
                  0.26142595978062155,
                  0.006993006993006993,
                  143,
                  1
                ],
                [
                  0.263254113345521,
                  0.006944444444444444,
                  144,
                  1
                ],
                [
                  0.26508226691042047,
                  0.006896551724137931,
                  145,
                  1
                ],
                [
                  0.26691042047531993,
                  0.00684931506849315,
                  146,
                  1
                ],
                [
                  0.2687385740402194,
                  0.006802721088435374,
                  147,
                  1
                ],
                [
                  0.27056672760511885,
                  0.006756756756756757,
                  148,
                  1
                ],
                [
                  0.27239488117001825,
                  0.006711409395973154,
                  149,
                  1
                ],
                [
                  0.2742230347349177,
                  0.006666666666666667,
                  150,
                  1
                ],
                [
                  0.2760511882998172,
                  0.006622516556291391,
                  151,
                  1
                ],
                [
                  0.27787934186471663,
                  0.006578947368421052,
                  152,
                  1
                ],
                [
                  0.2797074954296161,
                  0.006535947712418301,
                  153,
                  1
                ],
                [
                  0.28153564899451555,
                  0.006493506493506494,
                  154,
                  1
                ],
                [
                  0.283363802559415,
                  0.0064516129032258064,
                  155,
                  1
                ],
                [
                  0.2851919561243144,
                  0.00641025641025641,
                  156,
                  1
                ],
                [
                  0.2870201096892139,
                  0.006369426751592357,
                  157,
                  1
                ],
                [
                  0.28884826325411334,
                  0.006329113924050633,
                  158,
                  1
                ],
                [
                  0.2906764168190128,
                  0.006289308176100629,
                  159,
                  1
                ],
                [
                  0.29250457038391225,
                  0.00625,
                  160,
                  1
                ],
                [
                  0.2943327239488117,
                  0.006211180124223602,
                  161,
                  1
                ],
                [
                  0.2961608775137112,
                  0.006172839506172839,
                  162,
                  1
                ],
                [
                  0.2979890310786106,
                  0.006134969325153374,
                  163,
                  1
                ],
                [
                  0.29981718464351004,
                  0.006097560975609756,
                  164,
                  1
                ],
                [
                  0.3016453382084095,
                  0.006060606060606061,
                  165,
                  1
                ],
                [
                  0.30347349177330896,
                  0.006024096385542169,
                  166,
                  1
                ],
                [
                  0.3053016453382084,
                  0.005988023952095809,
                  167,
                  1
                ],
                [
                  0.3071297989031079,
                  0.005952380952380952,
                  168,
                  1
                ],
                [
                  0.30895795246800734,
                  0.005917159763313609,
                  169,
                  1
                ],
                [
                  0.31078610603290674,
                  0.0058823529411764705,
                  170,
                  1
                ],
                [
                  0.3126142595978062,
                  0.005847953216374269,
                  171,
                  1
                ],
                [
                  0.31444241316270566,
                  0.005813953488372093,
                  172,
                  1
                ],
                [
                  0.3162705667276051,
                  0.005780346820809248,
                  173,
                  1
                ],
                [
                  0.3180987202925046,
                  0.005747126436781609,
                  174,
                  1
                ],
                [
                  0.31992687385740404,
                  0.005714285714285714,
                  175,
                  1
                ],
                [
                  0.3217550274223035,
                  0.005681818181818182,
                  176,
                  1
                ],
                [
                  0.3235831809872029,
                  0.005649717514124294,
                  177,
                  1
                ],
                [
                  0.32541133455210236,
                  0.0056179775280898875,
                  178,
                  1
                ],
                [
                  0.3272394881170018,
                  0.00558659217877095,
                  179,
                  1
                ],
                [
                  0.3290676416819013,
                  0.005555555555555556,
                  180,
                  1
                ],
                [
                  0.33089579524680074,
                  0.0055248618784530384,
                  181,
                  1
                ],
                [
                  0.3327239488117002,
                  0.005494505494505495,
                  182,
                  1
                ],
                [
                  0.33455210237659966,
                  0.00546448087431694,
                  183,
                  1
                ],
                [
                  0.33638025594149906,
                  0.005434782608695652,
                  184,
                  1
                ],
                [
                  0.3382084095063985,
                  0.005405405405405406,
                  185,
                  1
                ],
                [
                  0.340036563071298,
                  0.005376344086021506,
                  186,
                  1
                ],
                [
                  0.34186471663619744,
                  0.0053475935828877,
                  187,
                  1
                ],
                [
                  0.3436928702010969,
                  0.005319148936170213,
                  188,
                  1
                ],
                [
                  0.34552102376599636,
                  0.005291005291005291,
                  189,
                  1
                ],
                [
                  0.3473491773308958,
                  0.005263157894736842,
                  190,
                  1
                ],
                [
                  0.3491773308957952,
                  0.005235602094240838,
                  191,
                  1
                ],
                [
                  0.3510054844606947,
                  0.005208333333333333,
                  192,
                  1
                ],
                [
                  0.35283363802559414,
                  0.0051813471502590676,
                  193,
                  1
                ],
                [
                  0.3546617915904936,
                  0.005154639175257732,
                  194,
                  1
                ],
                [
                  0.35648994515539306,
                  0.005128205128205128,
                  195,
                  1
                ],
                [
                  0.3583180987202925,
                  0.00510204081632653,
                  196,
                  1
                ],
                [
                  0.360146252285192,
                  0.005076142131979695,
                  197,
                  1
                ],
                [
                  0.3619744058500914,
                  0.005050505050505051,
                  198,
                  1
                ],
                [
                  0.36380255941499084,
                  0.005025125628140704,
                  199,
                  1
                ],
                [
                  0.3656307129798903,
                  0.005,
                  200,
                  1
                ],
                [
                  0.36745886654478976,
                  0.004975124378109453,
                  201,
                  1
                ],
                [
                  0.3692870201096892,
                  0.0049504950495049506,
                  202,
                  1
                ],
                [
                  0.3711151736745887,
                  0.0049261083743842365,
                  203,
                  1
                ],
                [
                  0.37294332723948814,
                  0.004901960784313725,
                  204,
                  1
                ],
                [
                  0.37477148080438755,
                  0.004878048780487805,
                  205,
                  1
                ],
                [
                  0.376599634369287,
                  0.0048543689320388345,
                  206,
                  1
                ],
                [
                  0.37842778793418647,
                  0.004830917874396135,
                  207,
                  1
                ],
                [
                  0.3802559414990859,
                  0.004807692307692308,
                  208,
                  1
                ],
                [
                  0.3820840950639854,
                  0.004784688995215311,
                  209,
                  1
                ],
                [
                  0.38391224862888484,
                  0.004761904761904762,
                  210,
                  1
                ],
                [
                  0.3857404021937843,
                  0.004739336492890996,
                  211,
                  1
                ],
                [
                  0.3875685557586837,
                  0.0047169811320754715,
                  212,
                  1
                ],
                [
                  0.38939670932358317,
                  0.004694835680751174,
                  213,
                  1
                ],
                [
                  0.3912248628884826,
                  0.009345794392523364,
                  214,
                  2
                ],
                [
                  0.3930530164533821,
                  0.009302325581395349,
                  215,
                  2
                ],
                [
                  0.39488117001828155,
                  0.009259259259259259,
                  216,
                  2
                ],
                [
                  0.396709323583181,
                  0.009216589861751152,
                  217,
                  2
                ],
                [
                  0.39853747714808047,
                  0.009174311926605505,
                  218,
                  2
                ],
                [
                  0.40036563071297987,
                  0.0091324200913242,
                  219,
                  2
                ],
                [
                  0.40219378427787933,
                  0.00909090909090909,
                  220,
                  2
                ],
                [
                  0.4040219378427788,
                  0.00904977375565611,
                  221,
                  2
                ],
                [
                  0.40585009140767825,
                  0.009009009009009009,
                  222,
                  2
                ],
                [
                  0.4076782449725777,
                  0.008968609865470852,
                  223,
                  2
                ],
                [
                  0.40950639853747717,
                  0.008928571428571428,
                  224,
                  2
                ],
                [
                  0.4113345521023766,
                  0.008888888888888889,
                  225,
                  2
                ],
                [
                  0.41316270566727603,
                  0.008849557522123894,
                  226,
                  2
                ],
                [
                  0.4149908592321755,
                  0.00881057268722467,
                  227,
                  2
                ],
                [
                  0.41681901279707495,
                  0.008771929824561403,
                  228,
                  2
                ],
                [
                  0.4186471663619744,
                  0.008733624454148471,
                  229,
                  2
                ],
                [
                  0.42047531992687387,
                  0.008695652173913044,
                  230,
                  2
                ],
                [
                  0.42230347349177333,
                  0.008658008658008658,
                  231,
                  2
                ],
                [
                  0.42413162705667273,
                  0.008620689655172414,
                  232,
                  2
                ],
                [
                  0.4259597806215722,
                  0.008583690987124463,
                  233,
                  2
                ],
                [
                  0.42778793418647165,
                  0.008547008547008548,
                  234,
                  2
                ],
                [
                  0.4296160877513711,
                  0.00851063829787234,
                  235,
                  2
                ],
                [
                  0.43144424131627057,
                  0.00847457627118644,
                  236,
                  2
                ],
                [
                  0.43327239488117003,
                  0.008438818565400843,
                  237,
                  2
                ],
                [
                  0.4351005484460695,
                  0.008403361344537815,
                  238,
                  2
                ],
                [
                  0.4369287020109689,
                  0.008368200836820083,
                  239,
                  2
                ],
                [
                  0.43875685557586835,
                  0.008333333333333333,
                  240,
                  2
                ],
                [
                  0.4405850091407678,
                  0.008298755186721992,
                  241,
                  2
                ],
                [
                  0.4424131627056673,
                  0.008264462809917356,
                  242,
                  2
                ],
                [
                  0.44424131627056673,
                  0.00823045267489712,
                  243,
                  2
                ],
                [
                  0.4460694698354662,
                  0.00819672131147541,
                  244,
                  2
                ],
                [
                  0.44789762340036565,
                  0.00816326530612245,
                  245,
                  2
                ],
                [
                  0.44972577696526506,
                  0.008130081300813009,
                  246,
                  2
                ],
                [
                  0.4515539305301645,
                  0.008097165991902834,
                  247,
                  2
                ],
                [
                  0.453382084095064,
                  0.008064516129032258,
                  248,
                  2
                ],
                [
                  0.45521023765996343,
                  0.008032128514056224,
                  249,
                  2
                ],
                [
                  0.4570383912248629,
                  0.008,
                  250,
                  2
                ],
                [
                  0.45886654478976235,
                  0.00796812749003984,
                  251,
                  2
                ],
                [
                  0.4606946983546618,
                  0.007936507936507936,
                  252,
                  2
                ],
                [
                  0.4625228519195612,
                  0.007905138339920948,
                  253,
                  2
                ],
                [
                  0.4643510054844607,
                  0.007874015748031496,
                  254,
                  2
                ],
                [
                  0.46617915904936014,
                  0.00784313725490196,
                  255,
                  2
                ],
                [
                  0.4680073126142596,
                  0.0078125,
                  256,
                  2
                ],
                [
                  0.46983546617915906,
                  0.007782101167315175,
                  257,
                  2
                ],
                [
                  0.4716636197440585,
                  0.007751937984496124,
                  258,
                  2
                ],
                [
                  0.473491773308958,
                  0.011583011583011582,
                  259,
                  3
                ],
                [
                  0.4753199268738574,
                  0.011538461538461539,
                  260,
                  3
                ],
                [
                  0.47714808043875684,
                  0.011494252873563218,
                  261,
                  3
                ],
                [
                  0.4789762340036563,
                  0.011450381679389313,
                  262,
                  3
                ],
                [
                  0.48080438756855576,
                  0.011406844106463879,
                  263,
                  3
                ],
                [
                  0.4826325411334552,
                  0.011363636363636364,
                  264,
                  3
                ],
                [
                  0.4844606946983547,
                  0.011320754716981131,
                  265,
                  3
                ],
                [
                  0.48628884826325414,
                  0.011278195488721804,
                  266,
                  3
                ],
                [
                  0.48811700182815354,
                  0.011235955056179775,
                  267,
                  3
                ],
                [
                  0.489945155393053,
                  0.011194029850746268,
                  268,
                  3
                ],
                [
                  0.49177330895795246,
                  0.011152416356877323,
                  269,
                  3
                ],
                [
                  0.4936014625228519,
                  0.011111111111111112,
                  270,
                  3
                ],
                [
                  0.4954296160877514,
                  0.01107011070110701,
                  271,
                  3
                ],
                [
                  0.49725776965265084,
                  0.011029411764705883,
                  272,
                  3
                ],
                [
                  0.4990859232175503,
                  0.01098901098901099,
                  273,
                  3
                ],
                [
                  0.5009140767824497,
                  0.010948905109489052,
                  274,
                  3
                ],
                [
                  0.5027422303473492,
                  0.01090909090909091,
                  275,
                  3
                ],
                [
                  0.5045703839122486,
                  0.010869565217391304,
                  276,
                  3
                ],
                [
                  0.506398537477148,
                  0.010830324909747292,
                  277,
                  3
                ],
                [
                  0.5082266910420475,
                  0.01079136690647482,
                  278,
                  3
                ],
                [
                  0.5100548446069469,
                  0.010752688172043012,
                  279,
                  3
                ],
                [
                  0.5118829981718465,
                  0.010714285714285714,
                  280,
                  3
                ],
                [
                  0.5137111517367459,
                  0.010676156583629894,
                  281,
                  3
                ],
                [
                  0.5155393053016454,
                  0.010638297872340425,
                  282,
                  3
                ],
                [
                  0.5173674588665448,
                  0.01060070671378092,
                  283,
                  3
                ],
                [
                  0.5191956124314442,
                  0.01056338028169014,
                  284,
                  3
                ],
                [
                  0.5210237659963437,
                  0.010526315789473684,
                  285,
                  3
                ],
                [
                  0.5228519195612431,
                  0.01048951048951049,
                  286,
                  3
                ],
                [
                  0.5246800731261426,
                  0.010452961672473868,
                  287,
                  3
                ],
                [
                  0.526508226691042,
                  0.010416666666666666,
                  288,
                  3
                ],
                [
                  0.5283363802559415,
                  0.010380622837370242,
                  289,
                  3
                ],
                [
                  0.5301645338208409,
                  0.010344827586206896,
                  290,
                  3
                ],
                [
                  0.5319926873857403,
                  0.010309278350515464,
                  291,
                  3
                ],
                [
                  0.5338208409506399,
                  0.010273972602739725,
                  292,
                  3
                ],
                [
                  0.5356489945155393,
                  0.010238907849829351,
                  293,
                  3
                ],
                [
                  0.5374771480804388,
                  0.01020408163265306,
                  294,
                  3
                ],
                [
                  0.5393053016453382,
                  0.010169491525423728,
                  295,
                  3
                ],
                [
                  0.5411334552102377,
                  0.010135135135135136,
                  296,
                  3
                ],
                [
                  0.5429616087751371,
                  0.010101010101010102,
                  297,
                  3
                ],
                [
                  0.5447897623400365,
                  0.010067114093959731,
                  298,
                  3
                ],
                [
                  0.546617915904936,
                  0.010033444816053512,
                  299,
                  3
                ],
                [
                  0.5484460694698354,
                  0.01,
                  300,
                  3
                ],
                [
                  0.5502742230347349,
                  0.009966777408637873,
                  301,
                  3
                ],
                [
                  0.5521023765996343,
                  0.009933774834437087,
                  302,
                  3
                ],
                [
                  0.5539305301645339,
                  0.009900990099009901,
                  303,
                  3
                ],
                [
                  0.5557586837294333,
                  0.009868421052631578,
                  304,
                  3
                ],
                [
                  0.5575868372943327,
                  0.009836065573770493,
                  305,
                  3
                ],
                [
                  0.5594149908592322,
                  0.00980392156862745,
                  306,
                  3
                ],
                [
                  0.5612431444241316,
                  0.009771986970684038,
                  307,
                  3
                ],
                [
                  0.5630712979890311,
                  0.012987012987012988,
                  308,
                  4
                ],
                [
                  0.5648994515539305,
                  0.012944983818770227,
                  309,
                  4
                ],
                [
                  0.56672760511883,
                  0.012903225806451613,
                  310,
                  4
                ],
                [
                  0.5685557586837294,
                  0.012861736334405145,
                  311,
                  4
                ],
                [
                  0.5703839122486288,
                  0.01282051282051282,
                  312,
                  4
                ],
                [
                  0.5722120658135283,
                  0.012779552715654952,
                  313,
                  4
                ],
                [
                  0.5740402193784278,
                  0.012738853503184714,
                  314,
                  4
                ],
                [
                  0.5758683729433273,
                  0.012698412698412698,
                  315,
                  4
                ],
                [
                  0.5776965265082267,
                  0.012658227848101266,
                  316,
                  4
                ],
                [
                  0.5795246800731262,
                  0.012618296529968454,
                  317,
                  4
                ],
                [
                  0.5813528336380256,
                  0.012578616352201259,
                  318,
                  4
                ],
                [
                  0.583180987202925,
                  0.012539184952978056,
                  319,
                  4
                ],
                [
                  0.5850091407678245,
                  0.0125,
                  320,
                  4
                ],
                [
                  0.5868372943327239,
                  0.012461059190031152,
                  321,
                  4
                ],
                [
                  0.5886654478976234,
                  0.012422360248447204,
                  322,
                  4
                ],
                [
                  0.5904936014625228,
                  0.015479876160990712,
                  323,
                  5
                ],
                [
                  0.5923217550274223,
                  0.015432098765432098,
                  324,
                  5
                ],
                [
                  0.5941499085923218,
                  0.015384615384615385,
                  325,
                  5
                ],
                [
                  0.5959780621572212,
                  0.015337423312883436,
                  326,
                  5
                ],
                [
                  0.5978062157221207,
                  0.01529051987767584,
                  327,
                  5
                ],
                [
                  0.5996343692870201,
                  0.01524390243902439,
                  328,
                  5
                ],
                [
                  0.6014625228519196,
                  0.015197568389057751,
                  329,
                  5
                ],
                [
                  0.603290676416819,
                  0.015151515151515152,
                  330,
                  5
                ],
                [
                  0.6051188299817185,
                  0.01812688821752266,
                  331,
                  6
                ],
                [
                  0.6069469835466179,
                  0.018072289156626505,
                  332,
                  6
                ],
                [
                  0.6087751371115173,
                  0.018018018018018018,
                  333,
                  6
                ],
                [
                  0.6106032906764168,
                  0.017964071856287425,
                  334,
                  6
                ],
                [
                  0.6124314442413162,
                  0.01791044776119403,
                  335,
                  6
                ],
                [
                  0.6142595978062158,
                  0.017857142857142856,
                  336,
                  6
                ],
                [
                  0.6160877513711152,
                  0.017804154302670624,
                  337,
                  6
                ],
                [
                  0.6179159049360147,
                  0.020710059171597635,
                  338,
                  7
                ],
                [
                  0.6197440585009141,
                  0.02064896755162242,
                  339,
                  7
                ],
                [
                  0.6215722120658135,
                  0.023529411764705882,
                  340,
                  8
                ],
                [
                  0.623400365630713,
                  0.02346041055718475,
                  341,
                  8
                ],
                [
                  0.6252285191956124,
                  0.023391812865497075,
                  342,
                  8
                ],
                [
                  0.6270566727605119,
                  0.023323615160349854,
                  343,
                  8
                ],
                [
                  0.6288848263254113,
                  0.023255813953488372,
                  344,
                  8
                ],
                [
                  0.6307129798903108,
                  0.02318840579710145,
                  345,
                  8
                ],
                [
                  0.6325411334552102,
                  0.023121387283236993,
                  346,
                  8
                ],
                [
                  0.6343692870201096,
                  0.023054755043227664,
                  347,
                  8
                ],
                [
                  0.6361974405850092,
                  0.022988505747126436,
                  348,
                  8
                ],
                [
                  0.6380255941499086,
                  0.022922636103151862,
                  349,
                  8
                ],
                [
                  0.6398537477148081,
                  0.022857142857142857,
                  350,
                  8
                ],
                [
                  0.6416819012797075,
                  0.022792022792022793,
                  351,
                  8
                ],
                [
                  0.643510054844607,
                  0.022727272727272728,
                  352,
                  8
                ],
                [
                  0.6453382084095064,
                  0.0226628895184136,
                  353,
                  8
                ],
                [
                  0.6471663619744058,
                  0.022598870056497175,
                  354,
                  8
                ],
                [
                  0.6489945155393053,
                  0.022535211267605635,
                  355,
                  8
                ],
                [
                  0.6508226691042047,
                  0.02247191011235955,
                  356,
                  8
                ],
                [
                  0.6526508226691042,
                  0.022408963585434174,
                  357,
                  8
                ],
                [
                  0.6544789762340036,
                  0.0223463687150838,
                  358,
                  8
                ],
                [
                  0.6563071297989032,
                  0.022284122562674095,
                  359,
                  8
                ],
                [
                  0.6581352833638026,
                  0.022222222222222223,
                  360,
                  8
                ],
                [
                  0.659963436928702,
                  0.0221606648199446,
                  361,
                  8
                ],
                [
                  0.6617915904936015,
                  0.022099447513812154,
                  362,
                  8
                ],
                [
                  0.6636197440585009,
                  0.02203856749311295,
                  363,
                  8
                ],
                [
                  0.6654478976234004,
                  0.02197802197802198,
                  364,
                  8
                ],
                [
                  0.6672760511882998,
                  0.021917808219178082,
                  365,
                  8
                ],
                [
                  0.6691042047531993,
                  0.02185792349726776,
                  366,
                  8
                ],
                [
                  0.6709323583180987,
                  0.021798365122615803,
                  367,
                  8
                ],
                [
                  0.6727605118829981,
                  0.021739130434782608,
                  368,
                  8
                ],
                [
                  0.6745886654478976,
                  0.02168021680216802,
                  369,
                  8
                ],
                [
                  0.676416819012797,
                  0.021621621621621623,
                  370,
                  8
                ],
                [
                  0.6782449725776966,
                  0.0215633423180593,
                  371,
                  8
                ],
                [
                  0.680073126142596,
                  0.021505376344086023,
                  372,
                  8
                ],
                [
                  0.6819012797074955,
                  0.024128686327077747,
                  373,
                  9
                ],
                [
                  0.6837294332723949,
                  0.02406417112299465,
                  374,
                  9
                ],
                [
                  0.6855575868372943,
                  0.024,
                  375,
                  9
                ],
                [
                  0.6873857404021938,
                  0.023936170212765957,
                  376,
                  9
                ],
                [
                  0.6892138939670932,
                  0.023872679045092837,
                  377,
                  9
                ],
                [
                  0.6910420475319927,
                  0.023809523809523808,
                  378,
                  9
                ],
                [
                  0.6928702010968921,
                  0.023746701846965697,
                  379,
                  9
                ],
                [
                  0.6946983546617916,
                  0.02368421052631579,
                  380,
                  9
                ],
                [
                  0.696526508226691,
                  0.023622047244094488,
                  381,
                  9
                ],
                [
                  0.6983546617915904,
                  0.02617801047120419,
                  382,
                  10
                ],
                [
                  0.70018281535649,
                  0.02610966057441253,
                  383,
                  10
                ],
                [
                  0.7020109689213894,
                  0.026041666666666668,
                  384,
                  10
                ],
                [
                  0.7038391224862889,
                  0.025974025974025976,
                  385,
                  10
                ],
                [
                  0.7056672760511883,
                  0.025906735751295335,
                  386,
                  10
                ],
                [
                  0.7074954296160878,
                  0.025839793281653745,
                  387,
                  10
                ],
                [
                  0.7093235831809872,
                  0.02577319587628866,
                  388,
                  10
                ],
                [
                  0.7111517367458866,
                  0.028277634961439587,
                  389,
                  11
                ],
                [
                  0.7129798903107861,
                  0.028205128205128206,
                  390,
                  11
                ],
                [
                  0.7148080438756855,
                  0.030690537084398978,
                  391,
                  12
                ],
                [
                  0.716636197440585,
                  0.030612244897959183,
                  392,
                  12
                ],
                [
                  0.7184643510054844,
                  0.030534351145038167,
                  393,
                  12
                ],
                [
                  0.720292504570384,
                  0.030456852791878174,
                  394,
                  12
                ],
                [
                  0.7221206581352834,
                  0.03291139240506329,
                  395,
                  13
                ],
                [
                  0.7239488117001828,
                  0.03535353535353535,
                  396,
                  14
                ],
                [
                  0.7257769652650823,
                  0.03526448362720403,
                  397,
                  14
                ],
                [
                  0.7276051188299817,
                  0.03768844221105527,
                  398,
                  15
                ],
                [
                  0.7294332723948812,
                  0.03759398496240601,
                  399,
                  15
                ],
                [
                  0.7312614259597806,
                  0.04,
                  400,
                  16
                ],
                [
                  0.7330895795246801,
                  0.0399002493765586,
                  401,
                  16
                ],
                [
                  0.7349177330895795,
                  0.03980099502487562,
                  402,
                  16
                ],
                [
                  0.7367458866544789,
                  0.04218362282878412,
                  403,
                  17
                ],
                [
                  0.7385740402193784,
                  0.04207920792079208,
                  404,
                  17
                ],
                [
                  0.7404021937842779,
                  0.044444444444444446,
                  405,
                  18
                ],
                [
                  0.7422303473491774,
                  0.046798029556650245,
                  406,
                  19
                ],
                [
                  0.7440585009140768,
                  0.04668304668304668,
                  407,
                  19
                ],
                [
                  0.7458866544789763,
                  0.049019607843137254,
                  408,
                  20
                ],
                [
                  0.7477148080438757,
                  0.0488997555012225,
                  409,
                  20
                ],
                [
                  0.7495429616087751,
                  0.04878048780487805,
                  410,
                  20
                ],
                [
                  0.7513711151736746,
                  0.04866180048661801,
                  411,
                  20
                ],
                [
                  0.753199268738574,
                  0.04854368932038835,
                  412,
                  20
                ],
                [
                  0.7550274223034735,
                  0.048426150121065374,
                  413,
                  20
                ],
                [
                  0.7568555758683729,
                  0.050724637681159424,
                  414,
                  21
                ],
                [
                  0.7586837294332724,
                  0.05060240963855422,
                  415,
                  21
                ],
                [
                  0.7605118829981719,
                  0.05048076923076923,
                  416,
                  21
                ],
                [
                  0.7623400365630713,
                  0.050359712230215826,
                  417,
                  21
                ],
                [
                  0.7641681901279708,
                  0.05263157894736842,
                  418,
                  22
                ],
                [
                  0.7659963436928702,
                  0.05250596658711217,
                  419,
                  22
                ],
                [
                  0.7678244972577697,
                  0.05238095238095238,
                  420,
                  22
                ],
                [
                  0.7696526508226691,
                  0.052256532066508314,
                  421,
                  22
                ],
                [
                  0.7714808043875686,
                  0.052132701421800945,
                  422,
                  22
                ],
                [
                  0.773308957952468,
                  0.05200945626477541,
                  423,
                  22
                ],
                [
                  0.7751371115173674,
                  0.05188679245283019,
                  424,
                  22
                ],
                [
                  0.7769652650822669,
                  0.05176470588235294,
                  425,
                  22
                ],
                [
                  0.7787934186471663,
                  0.051643192488262914,
                  426,
                  22
                ],
                [
                  0.7806215722120659,
                  0.05152224824355972,
                  427,
                  22
                ],
                [
                  0.7824497257769653,
                  0.0514018691588785,
                  428,
                  22
                ],
                [
                  0.7842778793418648,
                  0.05128205128205128,
                  429,
                  22
                ],
                [
                  0.7861060329067642,
                  0.053488372093023255,
                  430,
                  23
                ],
                [
                  0.7879341864716636,
                  0.05336426914153132,
                  431,
                  23
                ],
                [
                  0.7897623400365631,
                  0.05555555555555555,
                  432,
                  24
                ],
                [
                  0.7915904936014625,
                  0.05542725173210162,
                  433,
                  24
                ],
                [
                  0.793418647166362,
                  0.055299539170506916,
                  434,
                  24
                ],
                [
                  0.7952468007312614,
                  0.05517241379310345,
                  435,
                  24
                ],
                [
                  0.7970749542961609,
                  0.05733944954128441,
                  436,
                  25
                ],
                [
                  0.7989031078610603,
                  0.057208237986270026,
                  437,
                  25
                ],
                [
                  0.8007312614259597,
                  0.05707762557077625,
                  438,
                  25
                ],
                [
                  0.8025594149908593,
                  0.05694760820045558,
                  439,
                  25
                ],
                [
                  0.8043875685557587,
                  0.056818181818181816,
                  440,
                  25
                ],
                [
                  0.8062157221206582,
                  0.05895691609977324,
                  441,
                  26
                ],
                [
                  0.8080438756855576,
                  0.058823529411764705,
                  442,
                  26
                ],
                [
                  0.8098720292504571,
                  0.05869074492099323,
                  443,
                  26
                ],
                [
                  0.8117001828153565,
                  0.05855855855855856,
                  444,
                  26
                ],
                [
                  0.8135283363802559,
                  0.058426966292134834,
                  445,
                  26
                ],
                [
                  0.8153564899451554,
                  0.05829596412556054,
                  446,
                  26
                ],
                [
                  0.8171846435100548,
                  0.058165548098434,
                  447,
                  26
                ],
                [
                  0.8190127970749543,
                  0.05803571428571429,
                  448,
                  26
                ],
                [
                  0.8208409506398537,
                  0.05790645879732739,
                  449,
                  26
                ],
                [
                  0.8226691042047533,
                  0.057777777777777775,
                  450,
                  26
                ],
                [
                  0.8244972577696527,
                  0.0598669623059867,
                  451,
                  27
                ],
                [
                  0.8263254113345521,
                  0.061946902654867256,
                  452,
                  28
                ],
                [
                  0.8281535648994516,
                  0.06181015452538632,
                  453,
                  28
                ],
                [
                  0.829981718464351,
                  0.06167400881057269,
                  454,
                  28
                ],
                [
                  0.8318098720292505,
                  0.06373626373626373,
                  455,
                  29
                ],
                [
                  0.8336380255941499,
                  0.06578947368421052,
                  456,
                  30
                ],
                [
                  0.8354661791590493,
                  0.06783369803063458,
                  457,
                  31
                ],
                [
                  0.8372943327239488,
                  0.06768558951965066,
                  458,
                  31
                ],
                [
                  0.8391224862888482,
                  0.06971677559912855,
                  459,
                  32
                ],
                [
                  0.8409506398537477,
                  0.07173913043478261,
                  460,
                  33
                ],
                [
                  0.8427787934186471,
                  0.07158351409978309,
                  461,
                  33
                ],
                [
                  0.8446069469835467,
                  0.07142857142857142,
                  462,
                  33
                ],
                [
                  0.8464351005484461,
                  0.07127429805615551,
                  463,
                  33
                ],
                [
                  0.8482632541133455,
                  0.07327586206896551,
                  464,
                  34
                ],
                [
                  0.850091407678245,
                  0.07311827956989247,
                  465,
                  34
                ],
                [
                  0.8519195612431444,
                  0.07510729613733906,
                  466,
                  35
                ],
                [
                  0.8537477148080439,
                  0.07708779443254818,
                  467,
                  36
                ],
                [
                  0.8555758683729433,
                  0.07692307692307693,
                  468,
                  36
                ],
                [
                  0.8574040219378428,
                  0.0767590618336887,
                  469,
                  36
                ],
                [
                  0.8592321755027422,
                  0.07872340425531915,
                  470,
                  37
                ],
                [
                  0.8610603290676416,
                  0.07855626326963906,
                  471,
                  37
                ],
                [
                  0.8628884826325411,
                  0.08050847457627118,
                  472,
                  38
                ],
                [
                  0.8647166361974405,
                  0.0824524312896406,
                  473,
                  39
                ],
                [
                  0.8665447897623401,
                  0.08227848101265822,
                  474,
                  39
                ],
                [
                  0.8683729433272395,
                  0.08210526315789474,
                  475,
                  39
                ],
                [
                  0.870201096892139,
                  0.0819327731092437,
                  476,
                  39
                ],
                [
                  0.8720292504570384,
                  0.08176100628930817,
                  477,
                  39
                ],
                [
                  0.8738574040219378,
                  0.08368200836820083,
                  478,
                  40
                ],
                [
                  0.8756855575868373,
                  0.08559498956158663,
                  479,
                  41
                ],
                [
                  0.8775137111517367,
                  0.0875,
                  480,
                  42
                ],
                [
                  0.8793418647166362,
                  0.08731808731808732,
                  481,
                  42
                ],
                [
                  0.8811700182815356,
                  0.08713692946058091,
                  482,
                  42
                ],
                [
                  0.8829981718464351,
                  0.08902691511387163,
                  483,
                  43
                ],
                [
                  0.8848263254113345,
                  0.08884297520661157,
                  484,
                  43
                ],
                [
                  0.886654478976234,
                  0.088659793814433,
                  485,
                  43
                ],
                [
                  0.8884826325411335,
                  0.08847736625514403,
                  486,
                  43
                ],
                [
                  0.8903107861060329,
                  0.09034907597535935,
                  487,
                  44
                ],
                [
                  0.8921389396709324,
                  0.09221311475409837,
                  488,
                  45
                ],
                [
                  0.8939670932358318,
                  0.09202453987730061,
                  489,
                  45
                ],
                [
                  0.8957952468007313,
                  0.09387755102040816,
                  490,
                  46
                ],
                [
                  0.8976234003656307,
                  0.09572301425661914,
                  491,
                  47
                ],
                [
                  0.8994515539305301,
                  0.09552845528455285,
                  492,
                  47
                ],
                [
                  0.9012797074954296,
                  0.0973630831643002,
                  493,
                  48
                ],
                [
                  0.903107861060329,
                  0.09919028340080972,
                  494,
                  49
                ],
                [
                  0.9049360146252285,
                  0.09898989898989899,
                  495,
                  49
                ],
                [
                  0.906764168190128,
                  0.10080645161290322,
                  496,
                  50
                ],
                [
                  0.9085923217550275,
                  0.1006036217303823,
                  497,
                  50
                ],
                [
                  0.9104204753199269,
                  0.10240963855421686,
                  498,
                  51
                ],
                [
                  0.9122486288848263,
                  0.10220440881763528,
                  499,
                  51
                ],
                [
                  0.9140767824497258,
                  0.102,
                  500,
                  51
                ],
                [
                  0.9159049360146252,
                  0.10379241516966067,
                  501,
                  52
                ],
                [
                  0.9177330895795247,
                  0.10358565737051793,
                  502,
                  52
                ],
                [
                  0.9195612431444241,
                  0.10337972166998012,
                  503,
                  52
                ],
                [
                  0.9213893967093236,
                  0.10515873015873016,
                  504,
                  53
                ],
                [
                  0.923217550274223,
                  0.10693069306930693,
                  505,
                  54
                ],
                [
                  0.9250457038391224,
                  0.1067193675889328,
                  506,
                  54
                ],
                [
                  0.926873857404022,
                  0.10650887573964497,
                  507,
                  54
                ],
                [
                  0.9287020109689214,
                  0.10826771653543307,
                  508,
                  55
                ],
                [
                  0.9305301645338209,
                  0.10805500982318271,
                  509,
                  55
                ],
                [
                  0.9323583180987203,
                  0.10980392156862745,
                  510,
                  56
                ],
                [
                  0.9341864716636198,
                  0.11154598825831702,
                  511,
                  57
                ],
                [
                  0.9360146252285192,
                  0.111328125,
                  512,
                  57
                ],
                [
                  0.9378427787934186,
                  0.1111111111111111,
                  513,
                  57
                ],
                [
                  0.9396709323583181,
                  0.11284046692607004,
                  514,
                  58
                ],
                [
                  0.9414990859232175,
                  0.11262135922330097,
                  515,
                  58
                ],
                [
                  0.943327239488117,
                  0.1124031007751938,
                  516,
                  58
                ],
                [
                  0.9451553930530164,
                  0.11218568665377177,
                  517,
                  58
                ],
                [
                  0.946983546617916,
                  0.1138996138996139,
                  518,
                  59
                ],
                [
                  0.9488117001828154,
                  0.11368015414258188,
                  519,
                  59
                ],
                [
                  0.9506398537477148,
                  0.11538461538461539,
                  520,
                  60
                ],
                [
                  0.9524680073126143,
                  0.11708253358925144,
                  521,
                  61
                ],
                [
                  0.9542961608775137,
                  0.11877394636015326,
                  522,
                  62
                ],
                [
                  0.9561243144424132,
                  0.12045889101338432,
                  523,
                  63
                ],
                [
                  0.9579524680073126,
                  0.12213740458015267,
                  524,
                  64
                ],
                [
                  0.9597806215722121,
                  0.12380952380952381,
                  525,
                  65
                ],
                [
                  0.9616087751371115,
                  0.12357414448669202,
                  526,
                  65
                ],
                [
                  0.9634369287020109,
                  0.12333965844402277,
                  527,
                  65
                ],
                [
                  0.9652650822669104,
                  0.125,
                  528,
                  66
                ],
                [
                  0.9670932358318098,
                  0.1266540642722117,
                  529,
                  67
                ],
                [
                  0.9689213893967094,
                  0.12830188679245283,
                  530,
                  68
                ],
                [
                  0.9707495429616088,
                  0.128060263653484,
                  531,
                  68
                ],
                [
                  0.9725776965265083,
                  0.12969924812030076,
                  532,
                  69
                ],
                [
                  0.9744058500914077,
                  0.1294559099437148,
                  533,
                  69
                ],
                [
                  0.9762340036563071,
                  0.13108614232209737,
                  534,
                  70
                ],
                [
                  0.9780621572212066,
                  0.13271028037383178,
                  535,
                  71
                ],
                [
                  0.979890310786106,
                  0.13432835820895522,
                  536,
                  72
                ],
                [
                  0.9817184643510055,
                  0.1340782122905028,
                  537,
                  72
                ],
                [
                  0.9835466179159049,
                  0.13382899628252787,
                  538,
                  72
                ],
                [
                  0.9853747714808044,
                  0.13358070500927643,
                  539,
                  72
                ],
                [
                  0.9872029250457038,
                  0.13518518518518519,
                  540,
                  73
                ],
                [
                  0.9890310786106032,
                  0.1367837338262477,
                  541,
                  74
                ],
                [
                  0.9908592321755028,
                  0.13653136531365315,
                  542,
                  74
                ],
                [
                  0.9926873857404022,
                  0.13627992633517497,
                  543,
                  74
                ],
                [
                  0.9945155393053017,
                  0.13786764705882354,
                  544,
                  75
                ],
                [
                  0.9963436928702011,
                  0.13944954128440368,
                  545,
                  76
                ],
                [
                  0.9981718464351006,
                  0.14102564102564102,
                  546,
                  77
                ],
                [
                  1.0,
                  0.1425959780621572,
                  547,
                  78
                ]
              ]
            }
          ]
        },
        {
          "name": "Microsoft-Decision-1",
          "text_run": 205,
          "native": [
            {
              "language": "Turkish",
              "wording": "Bilmiyorum",
              "run": 487,
              "full_bank_restricted": false,
              "right": 457,
              "wrong": 88,
              "abstained": 2,
              "invalid": 0,
              "coverage": 0.9963436928702011,
              "risk": 0.1614678899082569
            },
            {
              "language": "Turkish",
              "wording": "I don't know",
              "run": null
            },
            {
              "language": "English",
              "wording": "I don't know",
              "run": 451,
              "full_bank_restricted": false,
              "right": 467,
              "wrong": 77,
              "abstained": 3,
              "invalid": 0,
              "coverage": 0.9945155393053017,
              "risk": 0.14154411764705882
            }
          ],
          "curves": [
            {
              "language": "Turkish",
              "run": 205,
              "points": [
                [
                  0.0018281535648994515,
                  0.0,
                  1,
                  0
                ],
                [
                  0.003656307129798903,
                  0.0,
                  2,
                  0
                ],
                [
                  0.005484460694698354,
                  0.0,
                  3,
                  0
                ],
                [
                  0.007312614259597806,
                  0.0,
                  4,
                  0
                ],
                [
                  0.009140767824497258,
                  0.0,
                  5,
                  0
                ],
                [
                  0.010968921389396709,
                  0.0,
                  6,
                  0
                ],
                [
                  0.012797074954296161,
                  0.0,
                  7,
                  0
                ],
                [
                  0.014625228519195612,
                  0.0,
                  8,
                  0
                ],
                [
                  0.016453382084095063,
                  0.0,
                  9,
                  0
                ],
                [
                  0.018281535648994516,
                  0.0,
                  10,
                  0
                ],
                [
                  0.02010968921389397,
                  0.0,
                  11,
                  0
                ],
                [
                  0.021937842778793418,
                  0.0,
                  12,
                  0
                ],
                [
                  0.02376599634369287,
                  0.0,
                  13,
                  0
                ],
                [
                  0.025594149908592323,
                  0.0,
                  14,
                  0
                ],
                [
                  0.027422303473491772,
                  0.0,
                  15,
                  0
                ],
                [
                  0.029250457038391225,
                  0.0,
                  16,
                  0
                ],
                [
                  0.031078610603290677,
                  0.0,
                  17,
                  0
                ],
                [
                  0.03290676416819013,
                  0.0,
                  18,
                  0
                ],
                [
                  0.03473491773308958,
                  0.0,
                  19,
                  0
                ],
                [
                  0.03656307129798903,
                  0.0,
                  20,
                  0
                ],
                [
                  0.038391224862888484,
                  0.0,
                  21,
                  0
                ],
                [
                  0.04021937842778794,
                  0.0,
                  22,
                  0
                ],
                [
                  0.04204753199268738,
                  0.0,
                  23,
                  0
                ],
                [
                  0.043875685557586835,
                  0.0,
                  24,
                  0
                ],
                [
                  0.04570383912248629,
                  0.0,
                  25,
                  0
                ],
                [
                  0.04753199268738574,
                  0.0,
                  26,
                  0
                ],
                [
                  0.04936014625228519,
                  0.0,
                  27,
                  0
                ],
                [
                  0.051188299817184646,
                  0.0,
                  28,
                  0
                ],
                [
                  0.05301645338208409,
                  0.0,
                  29,
                  0
                ],
                [
                  0.054844606946983544,
                  0.0,
                  30,
                  0
                ],
                [
                  0.056672760511883,
                  0.0,
                  31,
                  0
                ],
                [
                  0.05850091407678245,
                  0.0,
                  32,
                  0
                ],
                [
                  0.0603290676416819,
                  0.0,
                  33,
                  0
                ],
                [
                  0.062157221206581355,
                  0.0,
                  34,
                  0
                ],
                [
                  0.06398537477148081,
                  0.0,
                  35,
                  0
                ],
                [
                  0.06581352833638025,
                  0.0,
                  36,
                  0
                ],
                [
                  0.06764168190127971,
                  0.0,
                  37,
                  0
                ],
                [
                  0.06946983546617916,
                  0.0,
                  38,
                  0
                ],
                [
                  0.0712979890310786,
                  0.0,
                  39,
                  0
                ],
                [
                  0.07312614259597806,
                  0.0,
                  40,
                  0
                ],
                [
                  0.07495429616087751,
                  0.0,
                  41,
                  0
                ],
                [
                  0.07678244972577697,
                  0.0,
                  42,
                  0
                ],
                [
                  0.07861060329067641,
                  0.0,
                  43,
                  0
                ],
                [
                  0.08043875685557587,
                  0.0,
                  44,
                  0
                ],
                [
                  0.08226691042047532,
                  0.0,
                  45,
                  0
                ],
                [
                  0.08409506398537477,
                  0.0,
                  46,
                  0
                ],
                [
                  0.08592321755027423,
                  0.0,
                  47,
                  0
                ],
                [
                  0.08775137111517367,
                  0.0,
                  48,
                  0
                ],
                [
                  0.08957952468007313,
                  0.0,
                  49,
                  0
                ],
                [
                  0.09140767824497258,
                  0.0,
                  50,
                  0
                ],
                [
                  0.09323583180987204,
                  0.0,
                  51,
                  0
                ],
                [
                  0.09506398537477148,
                  0.0,
                  52,
                  0
                ],
                [
                  0.09689213893967093,
                  0.0,
                  53,
                  0
                ],
                [
                  0.09872029250457039,
                  0.0,
                  54,
                  0
                ],
                [
                  0.10054844606946983,
                  0.0,
                  55,
                  0
                ],
                [
                  0.10237659963436929,
                  0.0,
                  56,
                  0
                ],
                [
                  0.10420475319926874,
                  0.0,
                  57,
                  0
                ],
                [
                  0.10603290676416818,
                  0.0,
                  58,
                  0
                ],
                [
                  0.10786106032906764,
                  0.0,
                  59,
                  0
                ],
                [
                  0.10968921389396709,
                  0.0,
                  60,
                  0
                ],
                [
                  0.11151736745886655,
                  0.0,
                  61,
                  0
                ],
                [
                  0.113345521023766,
                  0.0,
                  62,
                  0
                ],
                [
                  0.11517367458866545,
                  0.0,
                  63,
                  0
                ],
                [
                  0.1170018281535649,
                  0.0,
                  64,
                  0
                ],
                [
                  0.11882998171846434,
                  0.0,
                  65,
                  0
                ],
                [
                  0.1206581352833638,
                  0.0,
                  66,
                  0
                ],
                [
                  0.12248628884826325,
                  0.0,
                  67,
                  0
                ],
                [
                  0.12431444241316271,
                  0.0,
                  68,
                  0
                ],
                [
                  0.12614259597806216,
                  0.0,
                  69,
                  0
                ],
                [
                  0.12797074954296161,
                  0.0,
                  70,
                  0
                ],
                [
                  0.12979890310786105,
                  0.0,
                  71,
                  0
                ],
                [
                  0.1316270566727605,
                  0.0,
                  72,
                  0
                ],
                [
                  0.13345521023765997,
                  0.0,
                  73,
                  0
                ],
                [
                  0.13528336380255943,
                  0.0,
                  74,
                  0
                ],
                [
                  0.13711151736745886,
                  0.0,
                  75,
                  0
                ],
                [
                  0.13893967093235832,
                  0.0,
                  76,
                  0
                ],
                [
                  0.14076782449725778,
                  0.0,
                  77,
                  0
                ],
                [
                  0.1425959780621572,
                  0.0,
                  78,
                  0
                ],
                [
                  0.14442413162705667,
                  0.0,
                  79,
                  0
                ],
                [
                  0.14625228519195613,
                  0.0,
                  80,
                  0
                ],
                [
                  0.1480804387568556,
                  0.0,
                  81,
                  0
                ],
                [
                  0.14990859232175502,
                  0.0,
                  82,
                  0
                ],
                [
                  0.15173674588665448,
                  0.0,
                  83,
                  0
                ],
                [
                  0.15356489945155394,
                  0.0,
                  84,
                  0
                ],
                [
                  0.15539305301645337,
                  0.0,
                  85,
                  0
                ],
                [
                  0.15722120658135283,
                  0.0,
                  86,
                  0
                ],
                [
                  0.1590493601462523,
                  0.0,
                  87,
                  0
                ],
                [
                  0.16087751371115175,
                  0.0,
                  88,
                  0
                ],
                [
                  0.16270566727605118,
                  0.0,
                  89,
                  0
                ],
                [
                  0.16453382084095064,
                  0.0,
                  90,
                  0
                ],
                [
                  0.1663619744058501,
                  0.0,
                  91,
                  0
                ],
                [
                  0.16819012797074953,
                  0.0,
                  92,
                  0
                ],
                [
                  0.170018281535649,
                  0.0,
                  93,
                  0
                ],
                [
                  0.17184643510054845,
                  0.0,
                  94,
                  0
                ],
                [
                  0.1736745886654479,
                  0.0,
                  95,
                  0
                ],
                [
                  0.17550274223034734,
                  0.0,
                  96,
                  0
                ],
                [
                  0.1773308957952468,
                  0.0,
                  97,
                  0
                ],
                [
                  0.17915904936014626,
                  0.0,
                  98,
                  0
                ],
                [
                  0.1809872029250457,
                  0.0,
                  99,
                  0
                ],
                [
                  0.18281535648994515,
                  0.0,
                  100,
                  0
                ],
                [
                  0.1846435100548446,
                  0.0,
                  101,
                  0
                ],
                [
                  0.18647166361974407,
                  0.0,
                  102,
                  0
                ],
                [
                  0.1882998171846435,
                  0.0,
                  103,
                  0
                ],
                [
                  0.19012797074954296,
                  0.0,
                  104,
                  0
                ],
                [
                  0.19195612431444242,
                  0.0,
                  105,
                  0
                ],
                [
                  0.19378427787934185,
                  0.009433962264150943,
                  106,
                  1
                ],
                [
                  0.1956124314442413,
                  0.009345794392523364,
                  107,
                  1
                ],
                [
                  0.19744058500914077,
                  0.009259259259259259,
                  108,
                  1
                ],
                [
                  0.19926873857404023,
                  0.009174311926605505,
                  109,
                  1
                ],
                [
                  0.20109689213893966,
                  0.00909090909090909,
                  110,
                  1
                ],
                [
                  0.20292504570383912,
                  0.009009009009009009,
                  111,
                  1
                ],
                [
                  0.20475319926873858,
                  0.008928571428571428,
                  112,
                  1
                ],
                [
                  0.20658135283363802,
                  0.008849557522123894,
                  113,
                  1
                ],
                [
                  0.20840950639853748,
                  0.008771929824561403,
                  114,
                  1
                ],
                [
                  0.21023765996343693,
                  0.008695652173913044,
                  115,
                  1
                ],
                [
                  0.21206581352833637,
                  0.008620689655172414,
                  116,
                  1
                ],
                [
                  0.21389396709323583,
                  0.008547008547008548,
                  117,
                  1
                ],
                [
                  0.21572212065813529,
                  0.00847457627118644,
                  118,
                  1
                ],
                [
                  0.21755027422303475,
                  0.008403361344537815,
                  119,
                  1
                ],
                [
                  0.21937842778793418,
                  0.008333333333333333,
                  120,
                  1
                ],
                [
                  0.22120658135283364,
                  0.008264462809917356,
                  121,
                  1
                ],
                [
                  0.2230347349177331,
                  0.00819672131147541,
                  122,
                  1
                ],
                [
                  0.22486288848263253,
                  0.008130081300813009,
                  123,
                  1
                ],
                [
                  0.226691042047532,
                  0.008064516129032258,
                  124,
                  1
                ],
                [
                  0.22851919561243145,
                  0.008,
                  125,
                  1
                ],
                [
                  0.2303473491773309,
                  0.007936507936507936,
                  126,
                  1
                ],
                [
                  0.23217550274223034,
                  0.007874015748031496,
                  127,
                  1
                ],
                [
                  0.2340036563071298,
                  0.0078125,
                  128,
                  1
                ],
                [
                  0.23583180987202926,
                  0.007751937984496124,
                  129,
                  1
                ],
                [
                  0.2376599634369287,
                  0.007692307692307693,
                  130,
                  1
                ],
                [
                  0.23948811700182815,
                  0.007633587786259542,
                  131,
                  1
                ],
                [
                  0.2413162705667276,
                  0.007575757575757576,
                  132,
                  1
                ],
                [
                  0.24314442413162707,
                  0.007518796992481203,
                  133,
                  1
                ],
                [
                  0.2449725776965265,
                  0.007462686567164179,
                  134,
                  1
                ],
                [
                  0.24680073126142596,
                  0.007407407407407408,
                  135,
                  1
                ],
                [
                  0.24862888482632542,
                  0.007352941176470588,
                  136,
                  1
                ],
                [
                  0.25045703839122485,
                  0.0072992700729927005,
                  137,
                  1
                ],
                [
                  0.2522851919561243,
                  0.007246376811594203,
                  138,
                  1
                ],
                [
                  0.25411334552102377,
                  0.007194244604316547,
                  139,
                  1
                ],
                [
                  0.25594149908592323,
                  0.007142857142857143,
                  140,
                  1
                ],
                [
                  0.2577696526508227,
                  0.0070921985815602835,
                  141,
                  1
                ],
                [
                  0.2595978062157221,
                  0.007042253521126761,
                  142,
                  1
                ],
                [
                  0.26142595978062155,
                  0.006993006993006993,
                  143,
                  1
                ],
                [
                  0.263254113345521,
                  0.006944444444444444,
                  144,
                  1
                ],
                [
                  0.26508226691042047,
                  0.006896551724137931,
                  145,
                  1
                ],
                [
                  0.26691042047531993,
                  0.00684931506849315,
                  146,
                  1
                ],
                [
                  0.2687385740402194,
                  0.006802721088435374,
                  147,
                  1
                ],
                [
                  0.27056672760511885,
                  0.006756756756756757,
                  148,
                  1
                ],
                [
                  0.27239488117001825,
                  0.006711409395973154,
                  149,
                  1
                ],
                [
                  0.2742230347349177,
                  0.006666666666666667,
                  150,
                  1
                ],
                [
                  0.2760511882998172,
                  0.006622516556291391,
                  151,
                  1
                ],
                [
                  0.27787934186471663,
                  0.013157894736842105,
                  152,
                  2
                ],
                [
                  0.2797074954296161,
                  0.013071895424836602,
                  153,
                  2
                ],
                [
                  0.28153564899451555,
                  0.012987012987012988,
                  154,
                  2
                ],
                [
                  0.283363802559415,
                  0.012903225806451613,
                  155,
                  2
                ],
                [
                  0.2851919561243144,
                  0.01282051282051282,
                  156,
                  2
                ],
                [
                  0.2870201096892139,
                  0.012738853503184714,
                  157,
                  2
                ],
                [
                  0.28884826325411334,
                  0.012658227848101266,
                  158,
                  2
                ],
                [
                  0.2906764168190128,
                  0.012578616352201259,
                  159,
                  2
                ],
                [
                  0.29250457038391225,
                  0.0125,
                  160,
                  2
                ],
                [
                  0.2943327239488117,
                  0.012422360248447204,
                  161,
                  2
                ],
                [
                  0.2961608775137112,
                  0.012345679012345678,
                  162,
                  2
                ],
                [
                  0.2979890310786106,
                  0.012269938650306749,
                  163,
                  2
                ],
                [
                  0.29981718464351004,
                  0.012195121951219513,
                  164,
                  2
                ],
                [
                  0.3016453382084095,
                  0.012121212121212121,
                  165,
                  2
                ],
                [
                  0.30347349177330896,
                  0.012048192771084338,
                  166,
                  2
                ],
                [
                  0.3053016453382084,
                  0.011976047904191617,
                  167,
                  2
                ],
                [
                  0.3071297989031079,
                  0.011904761904761904,
                  168,
                  2
                ],
                [
                  0.30895795246800734,
                  0.011834319526627219,
                  169,
                  2
                ],
                [
                  0.31078610603290674,
                  0.011764705882352941,
                  170,
                  2
                ],
                [
                  0.3126142595978062,
                  0.011695906432748537,
                  171,
                  2
                ],
                [
                  0.31444241316270566,
                  0.011627906976744186,
                  172,
                  2
                ],
                [
                  0.3162705667276051,
                  0.011560693641618497,
                  173,
                  2
                ],
                [
                  0.3180987202925046,
                  0.011494252873563218,
                  174,
                  2
                ],
                [
                  0.31992687385740404,
                  0.011428571428571429,
                  175,
                  2
                ],
                [
                  0.3217550274223035,
                  0.011363636363636364,
                  176,
                  2
                ],
                [
                  0.3235831809872029,
                  0.011299435028248588,
                  177,
                  2
                ],
                [
                  0.32541133455210236,
                  0.011235955056179775,
                  178,
                  2
                ],
                [
                  0.3272394881170018,
                  0.0111731843575419,
                  179,
                  2
                ],
                [
                  0.3290676416819013,
                  0.011111111111111112,
                  180,
                  2
                ],
                [
                  0.33089579524680074,
                  0.011049723756906077,
                  181,
                  2
                ],
                [
                  0.3327239488117002,
                  0.01098901098901099,
                  182,
                  2
                ],
                [
                  0.33455210237659966,
                  0.01092896174863388,
                  183,
                  2
                ],
                [
                  0.33638025594149906,
                  0.010869565217391304,
                  184,
                  2
                ],
                [
                  0.3382084095063985,
                  0.010810810810810811,
                  185,
                  2
                ],
                [
                  0.340036563071298,
                  0.010752688172043012,
                  186,
                  2
                ],
                [
                  0.34186471663619744,
                  0.0106951871657754,
                  187,
                  2
                ],
                [
                  0.3436928702010969,
                  0.010638297872340425,
                  188,
                  2
                ],
                [
                  0.34552102376599636,
                  0.010582010582010581,
                  189,
                  2
                ],
                [
                  0.3473491773308958,
                  0.010526315789473684,
                  190,
                  2
                ],
                [
                  0.3491773308957952,
                  0.010471204188481676,
                  191,
                  2
                ],
                [
                  0.3510054844606947,
                  0.010416666666666666,
                  192,
                  2
                ],
                [
                  0.35283363802559414,
                  0.010362694300518135,
                  193,
                  2
                ],
                [
                  0.3546617915904936,
                  0.010309278350515464,
                  194,
                  2
                ],
                [
                  0.35648994515539306,
                  0.010256410256410256,
                  195,
                  2
                ],
                [
                  0.3583180987202925,
                  0.01020408163265306,
                  196,
                  2
                ],
                [
                  0.360146252285192,
                  0.01015228426395939,
                  197,
                  2
                ],
                [
                  0.3619744058500914,
                  0.010101010101010102,
                  198,
                  2
                ],
                [
                  0.36380255941499084,
                  0.010050251256281407,
                  199,
                  2
                ],
                [
                  0.3656307129798903,
                  0.01,
                  200,
                  2
                ],
                [
                  0.36745886654478976,
                  0.009950248756218905,
                  201,
                  2
                ],
                [
                  0.3692870201096892,
                  0.009900990099009901,
                  202,
                  2
                ],
                [
                  0.3711151736745887,
                  0.009852216748768473,
                  203,
                  2
                ],
                [
                  0.37294332723948814,
                  0.00980392156862745,
                  204,
                  2
                ],
                [
                  0.37477148080438755,
                  0.00975609756097561,
                  205,
                  2
                ],
                [
                  0.376599634369287,
                  0.009708737864077669,
                  206,
                  2
                ],
                [
                  0.37842778793418647,
                  0.00966183574879227,
                  207,
                  2
                ],
                [
                  0.3802559414990859,
                  0.009615384615384616,
                  208,
                  2
                ],
                [
                  0.3820840950639854,
                  0.014354066985645933,
                  209,
                  3
                ],
                [
                  0.38391224862888484,
                  0.014285714285714285,
                  210,
                  3
                ],
                [
                  0.3857404021937843,
                  0.014218009478672985,
                  211,
                  3
                ],
                [
                  0.3875685557586837,
                  0.014150943396226415,
                  212,
                  3
                ],
                [
                  0.38939670932358317,
                  0.014084507042253521,
                  213,
                  3
                ],
                [
                  0.3912248628884826,
                  0.014018691588785047,
                  214,
                  3
                ],
                [
                  0.3930530164533821,
                  0.013953488372093023,
                  215,
                  3
                ],
                [
                  0.39488117001828155,
                  0.018518518518518517,
                  216,
                  4
                ],
                [
                  0.396709323583181,
                  0.018433179723502304,
                  217,
                  4
                ],
                [
                  0.39853747714808047,
                  0.01834862385321101,
                  218,
                  4
                ],
                [
                  0.40036563071297987,
                  0.0182648401826484,
                  219,
                  4
                ],
                [
                  0.40219378427787933,
                  0.01818181818181818,
                  220,
                  4
                ],
                [
                  0.4040219378427788,
                  0.01809954751131222,
                  221,
                  4
                ],
                [
                  0.40585009140767825,
                  0.018018018018018018,
                  222,
                  4
                ],
                [
                  0.4076782449725777,
                  0.017937219730941704,
                  223,
                  4
                ],
                [
                  0.40950639853747717,
                  0.017857142857142856,
                  224,
                  4
                ],
                [
                  0.4113345521023766,
                  0.017777777777777778,
                  225,
                  4
                ],
                [
                  0.41316270566727603,
                  0.017699115044247787,
                  226,
                  4
                ],
                [
                  0.4149908592321755,
                  0.01762114537444934,
                  227,
                  4
                ],
                [
                  0.41681901279707495,
                  0.017543859649122806,
                  228,
                  4
                ],
                [
                  0.4186471663619744,
                  0.017467248908296942,
                  229,
                  4
                ],
                [
                  0.42047531992687387,
                  0.017391304347826087,
                  230,
                  4
                ],
                [
                  0.42230347349177333,
                  0.017316017316017316,
                  231,
                  4
                ],
                [
                  0.42413162705667273,
                  0.017241379310344827,
                  232,
                  4
                ],
                [
                  0.4259597806215722,
                  0.017167381974248927,
                  233,
                  4
                ],
                [
                  0.42778793418647165,
                  0.017094017094017096,
                  234,
                  4
                ],
                [
                  0.4296160877513711,
                  0.01702127659574468,
                  235,
                  4
                ],
                [
                  0.43144424131627057,
                  0.01694915254237288,
                  236,
                  4
                ],
                [
                  0.43327239488117003,
                  0.016877637130801686,
                  237,
                  4
                ],
                [
                  0.4351005484460695,
                  0.01680672268907563,
                  238,
                  4
                ],
                [
                  0.4369287020109689,
                  0.016736401673640166,
                  239,
                  4
                ],
                [
                  0.43875685557586835,
                  0.016666666666666666,
                  240,
                  4
                ],
                [
                  0.4405850091407678,
                  0.016597510373443983,
                  241,
                  4
                ],
                [
                  0.4424131627056673,
                  0.01652892561983471,
                  242,
                  4
                ],
                [
                  0.44424131627056673,
                  0.01646090534979424,
                  243,
                  4
                ],
                [
                  0.4460694698354662,
                  0.01639344262295082,
                  244,
                  4
                ],
                [
                  0.44789762340036565,
                  0.0163265306122449,
                  245,
                  4
                ],
                [
                  0.44972577696526506,
                  0.016260162601626018,
                  246,
                  4
                ],
                [
                  0.4515539305301645,
                  0.016194331983805668,
                  247,
                  4
                ],
                [
                  0.453382084095064,
                  0.016129032258064516,
                  248,
                  4
                ],
                [
                  0.45521023765996343,
                  0.01606425702811245,
                  249,
                  4
                ],
                [
                  0.4570383912248629,
                  0.016,
                  250,
                  4
                ],
                [
                  0.45886654478976235,
                  0.01593625498007968,
                  251,
                  4
                ],
                [
                  0.4606946983546618,
                  0.01984126984126984,
                  252,
                  5
                ],
                [
                  0.4625228519195612,
                  0.019762845849802372,
                  253,
                  5
                ],
                [
                  0.4643510054844607,
                  0.01968503937007874,
                  254,
                  5
                ],
                [
                  0.46617915904936014,
                  0.0196078431372549,
                  255,
                  5
                ],
                [
                  0.4680073126142596,
                  0.0234375,
                  256,
                  6
                ],
                [
                  0.46983546617915906,
                  0.023346303501945526,
                  257,
                  6
                ],
                [
                  0.4716636197440585,
                  0.023255813953488372,
                  258,
                  6
                ],
                [
                  0.473491773308958,
                  0.023166023166023165,
                  259,
                  6
                ],
                [
                  0.4753199268738574,
                  0.023076923076923078,
                  260,
                  6
                ],
                [
                  0.47714808043875684,
                  0.022988505747126436,
                  261,
                  6
                ],
                [
                  0.4789762340036563,
                  0.022900763358778626,
                  262,
                  6
                ],
                [
                  0.48080438756855576,
                  0.022813688212927757,
                  263,
                  6
                ],
                [
                  0.4826325411334552,
                  0.022727272727272728,
                  264,
                  6
                ],
                [
                  0.4844606946983547,
                  0.022641509433962263,
                  265,
                  6
                ],
                [
                  0.48628884826325414,
                  0.022556390977443608,
                  266,
                  6
                ],
                [
                  0.48811700182815354,
                  0.02247191011235955,
                  267,
                  6
                ],
                [
                  0.489945155393053,
                  0.022388059701492536,
                  268,
                  6
                ],
                [
                  0.49177330895795246,
                  0.022304832713754646,
                  269,
                  6
                ],
                [
                  0.4936014625228519,
                  0.022222222222222223,
                  270,
                  6
                ],
                [
                  0.4954296160877514,
                  0.02214022140221402,
                  271,
                  6
                ],
                [
                  0.49725776965265084,
                  0.022058823529411766,
                  272,
                  6
                ],
                [
                  0.4990859232175503,
                  0.02197802197802198,
                  273,
                  6
                ],
                [
                  0.5009140767824497,
                  0.021897810218978103,
                  274,
                  6
                ],
                [
                  0.5027422303473492,
                  0.02181818181818182,
                  275,
                  6
                ],
                [
                  0.5045703839122486,
                  0.021739130434782608,
                  276,
                  6
                ],
                [
                  0.506398537477148,
                  0.021660649819494584,
                  277,
                  6
                ],
                [
                  0.5082266910420475,
                  0.02158273381294964,
                  278,
                  6
                ],
                [
                  0.5100548446069469,
                  0.021505376344086023,
                  279,
                  6
                ],
                [
                  0.5118829981718465,
                  0.025,
                  280,
                  7
                ],
                [
                  0.5137111517367459,
                  0.02491103202846975,
                  281,
                  7
                ],
                [
                  0.5155393053016454,
                  0.024822695035460994,
                  282,
                  7
                ],
                [
                  0.5173674588665448,
                  0.024734982332155476,
                  283,
                  7
                ],
                [
                  0.5191956124314442,
                  0.02464788732394366,
                  284,
                  7
                ],
                [
                  0.5210237659963437,
                  0.02456140350877193,
                  285,
                  7
                ],
                [
                  0.5228519195612431,
                  0.024475524475524476,
                  286,
                  7
                ],
                [
                  0.5246800731261426,
                  0.024390243902439025,
                  287,
                  7
                ],
                [
                  0.526508226691042,
                  0.024305555555555556,
                  288,
                  7
                ],
                [
                  0.5283363802559415,
                  0.02422145328719723,
                  289,
                  7
                ],
                [
                  0.5301645338208409,
                  0.02413793103448276,
                  290,
                  7
                ],
                [
                  0.5319926873857403,
                  0.024054982817869417,
                  291,
                  7
                ],
                [
                  0.5338208409506399,
                  0.023972602739726026,
                  292,
                  7
                ],
                [
                  0.5356489945155393,
                  0.023890784982935155,
                  293,
                  7
                ],
                [
                  0.5374771480804388,
                  0.023809523809523808,
                  294,
                  7
                ],
                [
                  0.5393053016453382,
                  0.023728813559322035,
                  295,
                  7
                ],
                [
                  0.5411334552102377,
                  0.02702702702702703,
                  296,
                  8
                ],
                [
                  0.5429616087751371,
                  0.026936026936026935,
                  297,
                  8
                ],
                [
                  0.5447897623400365,
                  0.026845637583892617,
                  298,
                  8
                ],
                [
                  0.546617915904936,
                  0.030100334448160536,
                  299,
                  9
                ],
                [
                  0.5484460694698354,
                  0.03,
                  300,
                  9
                ],
                [
                  0.5502742230347349,
                  0.029900332225913623,
                  301,
                  9
                ],
                [
                  0.5521023765996343,
                  0.029801324503311258,
                  302,
                  9
                ],
                [
                  0.5539305301645339,
                  0.0297029702970297,
                  303,
                  9
                ],
                [
                  0.5557586837294333,
                  0.029605263157894735,
                  304,
                  9
                ],
                [
                  0.5575868372943327,
                  0.029508196721311476,
                  305,
                  9
                ],
                [
                  0.5594149908592322,
                  0.029411764705882353,
                  306,
                  9
                ],
                [
                  0.5612431444241316,
                  0.029315960912052116,
                  307,
                  9
                ],
                [
                  0.5630712979890311,
                  0.032467532467532464,
                  308,
                  10
                ],
                [
                  0.5648994515539305,
                  0.032362459546925564,
                  309,
                  10
                ],
                [
                  0.56672760511883,
                  0.03225806451612903,
                  310,
                  10
                ],
                [
                  0.5685557586837294,
                  0.03215434083601286,
                  311,
                  10
                ],
                [
                  0.5703839122486288,
                  0.03205128205128205,
                  312,
                  10
                ],
                [
                  0.5722120658135283,
                  0.03514376996805112,
                  313,
                  11
                ],
                [
                  0.5740402193784278,
                  0.03503184713375796,
                  314,
                  11
                ],
                [
                  0.5758683729433273,
                  0.03492063492063492,
                  315,
                  11
                ],
                [
                  0.5776965265082267,
                  0.03481012658227848,
                  316,
                  11
                ],
                [
                  0.5795246800731262,
                  0.03785488958990536,
                  317,
                  12
                ],
                [
                  0.5813528336380256,
                  0.03773584905660377,
                  318,
                  12
                ],
                [
                  0.583180987202925,
                  0.04075235109717868,
                  319,
                  13
                ],
                [
                  0.5850091407678245,
                  0.04375,
                  320,
                  14
                ],
                [
                  0.5868372943327239,
                  0.04672897196261682,
                  321,
                  15
                ],
                [
                  0.5886654478976234,
                  0.046583850931677016,
                  322,
                  15
                ],
                [
                  0.5904936014625228,
                  0.04643962848297214,
                  323,
                  15
                ],
                [
                  0.5923217550274223,
                  0.046296296296296294,
                  324,
                  15
                ],
                [
                  0.5941499085923218,
                  0.046153846153846156,
                  325,
                  15
                ],
                [
                  0.5959780621572212,
                  0.046012269938650305,
                  326,
                  15
                ],
                [
                  0.5978062157221207,
                  0.045871559633027525,
                  327,
                  15
                ],
                [
                  0.5996343692870201,
                  0.04878048780487805,
                  328,
                  16
                ],
                [
                  0.6014625228519196,
                  0.0486322188449848,
                  329,
                  16
                ],
                [
                  0.603290676416819,
                  0.048484848484848485,
                  330,
                  16
                ],
                [
                  0.6051188299817185,
                  0.04833836858006042,
                  331,
                  16
                ],
                [
                  0.6069469835466179,
                  0.04819277108433735,
                  332,
                  16
                ],
                [
                  0.6087751371115173,
                  0.04804804804804805,
                  333,
                  16
                ],
                [
                  0.6106032906764168,
                  0.04790419161676647,
                  334,
                  16
                ],
                [
                  0.6124314442413162,
                  0.050746268656716415,
                  335,
                  17
                ],
                [
                  0.6142595978062158,
                  0.050595238095238096,
                  336,
                  17
                ],
                [
                  0.6160877513711152,
                  0.050445103857566766,
                  337,
                  17
                ],
                [
                  0.6179159049360147,
                  0.05325443786982249,
                  338,
                  18
                ],
                [
                  0.6197440585009141,
                  0.05604719764011799,
                  339,
                  19
                ],
                [
                  0.6215722120658135,
                  0.05588235294117647,
                  340,
                  19
                ],
                [
                  0.623400365630713,
                  0.05571847507331378,
                  341,
                  19
                ],
                [
                  0.6252285191956124,
                  0.05555555555555555,
                  342,
                  19
                ],
                [
                  0.6270566727605119,
                  0.05539358600583091,
                  343,
                  19
                ],
                [
                  0.6288848263254113,
                  0.05813953488372093,
                  344,
                  20
                ],
                [
                  0.6307129798903108,
                  0.057971014492753624,
                  345,
                  20
                ],
                [
                  0.6325411334552102,
                  0.057803468208092484,
                  346,
                  20
                ],
                [
                  0.6343692870201096,
                  0.05763688760806916,
                  347,
                  20
                ],
                [
                  0.6361974405850092,
                  0.0603448275862069,
                  348,
                  21
                ],
                [
                  0.6380255941499086,
                  0.06017191977077364,
                  349,
                  21
                ],
                [
                  0.6398537477148081,
                  0.06,
                  350,
                  21
                ],
                [
                  0.6416819012797075,
                  0.05982905982905983,
                  351,
                  21
                ],
                [
                  0.643510054844607,
                  0.0625,
                  352,
                  22
                ],
                [
                  0.6453382084095064,
                  0.06515580736543909,
                  353,
                  23
                ],
                [
                  0.6471663619744058,
                  0.06497175141242938,
                  354,
                  23
                ],
                [
                  0.6489945155393053,
                  0.0647887323943662,
                  355,
                  23
                ],
                [
                  0.6508226691042047,
                  0.06741573033707865,
                  356,
                  24
                ],
                [
                  0.6526508226691042,
                  0.0700280112044818,
                  357,
                  25
                ],
                [
                  0.6544789762340036,
                  0.07262569832402235,
                  358,
                  26
                ],
                [
                  0.6563071297989032,
                  0.07242339832869081,
                  359,
                  26
                ],
                [
                  0.6581352833638026,
                  0.07222222222222222,
                  360,
                  26
                ],
                [
                  0.659963436928702,
                  0.07479224376731301,
                  361,
                  27
                ],
                [
                  0.6617915904936015,
                  0.07734806629834254,
                  362,
                  28
                ],
                [
                  0.6636197440585009,
                  0.07988980716253444,
                  363,
                  29
                ],
                [
                  0.6654478976234004,
                  0.07967032967032966,
                  364,
                  29
                ],
                [
                  0.6672760511882998,
                  0.07945205479452055,
                  365,
                  29
                ],
                [
                  0.6691042047531993,
                  0.08196721311475409,
                  366,
                  30
                ],
                [
                  0.6709323583180987,
                  0.08174386920980926,
                  367,
                  30
                ],
                [
                  0.6727605118829981,
                  0.08152173913043478,
                  368,
                  30
                ],
                [
                  0.6745886654478976,
                  0.08401084010840108,
                  369,
                  31
                ],
                [
                  0.676416819012797,
                  0.08648648648648649,
                  370,
                  32
                ],
                [
                  0.6782449725776966,
                  0.0862533692722372,
                  371,
                  32
                ],
                [
                  0.680073126142596,
                  0.08602150537634409,
                  372,
                  32
                ],
                [
                  0.6819012797074955,
                  0.08579088471849866,
                  373,
                  32
                ],
                [
                  0.6837294332723949,
                  0.08823529411764706,
                  374,
                  33
                ],
                [
                  0.6855575868372943,
                  0.09066666666666667,
                  375,
                  34
                ],
                [
                  0.6873857404021938,
                  0.09042553191489362,
                  376,
                  34
                ],
                [
                  0.6892138939670932,
                  0.09283819628647215,
                  377,
                  35
                ],
                [
                  0.6910420475319927,
                  0.09523809523809523,
                  378,
                  36
                ],
                [
                  0.6928702010968921,
                  0.09762532981530343,
                  379,
                  37
                ],
                [
                  0.6946983546617916,
                  0.09736842105263158,
                  380,
                  37
                ],
                [
                  0.696526508226691,
                  0.09711286089238845,
                  381,
                  37
                ],
                [
                  0.6983546617915904,
                  0.09947643979057591,
                  382,
                  38
                ],
                [
                  0.70018281535649,
                  0.09921671018276762,
                  383,
                  38
                ],
                [
                  0.7020109689213894,
                  0.09895833333333333,
                  384,
                  38
                ],
                [
                  0.7038391224862889,
                  0.0987012987012987,
                  385,
                  38
                ],
                [
                  0.7056672760511883,
                  0.10103626943005181,
                  386,
                  39
                ],
                [
                  0.7074954296160878,
                  0.10077519379844961,
                  387,
                  39
                ],
                [
                  0.7093235831809872,
                  0.10051546391752578,
                  388,
                  39
                ],
                [
                  0.7111517367458866,
                  0.10282776349614396,
                  389,
                  40
                ],
                [
                  0.7129798903107861,
                  0.10256410256410256,
                  390,
                  40
                ],
                [
                  0.7148080438756855,
                  0.10230179028132992,
                  391,
                  40
                ],
                [
                  0.716636197440585,
                  0.10204081632653061,
                  392,
                  40
                ],
                [
                  0.7184643510054844,
                  0.10178117048346055,
                  393,
                  40
                ],
                [
                  0.720292504570384,
                  0.10152284263959391,
                  394,
                  40
                ],
                [
                  0.7221206581352834,
                  0.10126582278481013,
                  395,
                  40
                ],
                [
                  0.7239488117001828,
                  0.10353535353535354,
                  396,
                  41
                ],
                [
                  0.7257769652650823,
                  0.10579345088161209,
                  397,
                  42
                ],
                [
                  0.7276051188299817,
                  0.10804020100502512,
                  398,
                  43
                ],
                [
                  0.7294332723948812,
                  0.11027568922305764,
                  399,
                  44
                ],
                [
                  0.7312614259597806,
                  0.11,
                  400,
                  44
                ],
                [
                  0.7330895795246801,
                  0.11221945137157108,
                  401,
                  45
                ],
                [
                  0.7349177330895795,
                  0.11194029850746269,
                  402,
                  45
                ],
                [
                  0.7367458866544789,
                  0.1141439205955335,
                  403,
                  46
                ],
                [
                  0.7385740402193784,
                  0.11633663366336634,
                  404,
                  47
                ],
                [
                  0.7404021937842779,
                  0.11851851851851852,
                  405,
                  48
                ],
                [
                  0.7422303473491774,
                  0.11822660098522167,
                  406,
                  48
                ],
                [
                  0.7440585009140768,
                  0.11793611793611794,
                  407,
                  48
                ],
                [
                  0.7458866544789763,
                  0.12009803921568628,
                  408,
                  49
                ],
                [
                  0.7477148080438757,
                  0.1198044009779951,
                  409,
                  49
                ],
                [
                  0.7495429616087751,
                  0.11951219512195121,
                  410,
                  49
                ],
                [
                  0.7513711151736746,
                  0.1192214111922141,
                  411,
                  49
                ],
                [
                  0.753199268738574,
                  0.11893203883495146,
                  412,
                  49
                ],
                [
                  0.7550274223034735,
                  0.12106537530266344,
                  413,
                  50
                ],
                [
                  0.7568555758683729,
                  0.12318840579710146,
                  414,
                  51
                ],
                [
                  0.7586837294332724,
                  0.12289156626506025,
                  415,
                  51
                ],
                [
                  0.7605118829981719,
                  0.12259615384615384,
                  416,
                  51
                ],
                [
                  0.7623400365630713,
                  0.12470023980815348,
                  417,
                  52
                ],
                [
                  0.7641681901279708,
                  0.12440191387559808,
                  418,
                  52
                ],
                [
                  0.7659963436928702,
                  0.12649164677804295,
                  419,
                  53
                ],
                [
                  0.7678244972577697,
                  0.12857142857142856,
                  420,
                  54
                ],
                [
                  0.7696526508226691,
                  0.12826603325415678,
                  421,
                  54
                ],
                [
                  0.7714808043875686,
                  0.13033175355450238,
                  422,
                  55
                ],
                [
                  0.773308957952468,
                  0.13002364066193853,
                  423,
                  55
                ],
                [
                  0.7751371115173674,
                  0.12971698113207547,
                  424,
                  55
                ],
                [
                  0.7769652650822669,
                  0.12941176470588237,
                  425,
                  55
                ],
                [
                  0.7787934186471663,
                  0.13145539906103287,
                  426,
                  56
                ],
                [
                  0.7806215722120659,
                  0.13348946135831383,
                  427,
                  57
                ],
                [
                  0.7824497257769653,
                  0.13317757009345793,
                  428,
                  57
                ],
                [
                  0.7842778793418648,
                  0.1351981351981352,
                  429,
                  58
                ],
                [
                  0.7861060329067642,
                  0.13488372093023257,
                  430,
                  58
                ],
                [
                  0.7879341864716636,
                  0.1345707656612529,
                  431,
                  58
                ],
                [
                  0.7897623400365631,
                  0.13657407407407407,
                  432,
                  59
                ],
                [
                  0.7915904936014625,
                  0.13625866050808313,
                  433,
                  59
                ],
                [
                  0.793418647166362,
                  0.1359447004608295,
                  434,
                  59
                ],
                [
                  0.7952468007312614,
                  0.13793103448275862,
                  435,
                  60
                ],
                [
                  0.7970749542961609,
                  0.13761467889908258,
                  436,
                  60
                ],
                [
                  0.7989031078610603,
                  0.13958810068649885,
                  437,
                  61
                ],
                [
                  0.8007312614259597,
                  0.1415525114155251,
                  438,
                  62
                ],
                [
                  0.8025594149908593,
                  0.14123006833712984,
                  439,
                  62
                ],
                [
                  0.8043875685557587,
                  0.1409090909090909,
                  440,
                  62
                ],
                [
                  0.8062157221206582,
                  0.14285714285714285,
                  441,
                  63
                ],
                [
                  0.8080438756855576,
                  0.1425339366515837,
                  442,
                  63
                ],
                [
                  0.8098720292504571,
                  0.14446952595936793,
                  443,
                  64
                ],
                [
                  0.8117001828153565,
                  0.1463963963963964,
                  444,
                  65
                ],
                [
                  0.8135283363802559,
                  0.14831460674157304,
                  445,
                  66
                ],
                [
                  0.8153564899451554,
                  0.15022421524663676,
                  446,
                  67
                ],
                [
                  0.8171846435100548,
                  0.15212527964205816,
                  447,
                  68
                ],
                [
                  0.8190127970749543,
                  0.15178571428571427,
                  448,
                  68
                ],
                [
                  0.8208409506398537,
                  0.1514476614699332,
                  449,
                  68
                ],
                [
                  0.8226691042047533,
                  0.15333333333333332,
                  450,
                  69
                ],
                [
                  0.8244972577696527,
                  0.15299334811529933,
                  451,
                  69
                ],
                [
                  0.8263254113345521,
                  0.15486725663716813,
                  452,
                  70
                ],
                [
                  0.8281535648994516,
                  0.15673289183222958,
                  453,
                  71
                ],
                [
                  0.829981718464351,
                  0.15638766519823788,
                  454,
                  71
                ],
                [
                  0.8318098720292505,
                  0.15824175824175823,
                  455,
                  72
                ],
                [
                  0.8336380255941499,
                  0.15789473684210525,
                  456,
                  72
                ],
                [
                  0.8354661791590493,
                  0.15973741794310722,
                  457,
                  73
                ],
                [
                  0.8372943327239488,
                  0.15938864628820962,
                  458,
                  73
                ],
                [
                  0.8391224862888482,
                  0.15904139433551198,
                  459,
                  73
                ],
                [
                  0.8409506398537477,
                  0.15869565217391304,
                  460,
                  73
                ],
                [
                  0.8427787934186471,
                  0.16052060737527116,
                  461,
                  74
                ],
                [
                  0.8446069469835467,
                  0.16233766233766234,
                  462,
                  75
                ],
                [
                  0.8464351005484461,
                  0.16198704103671707,
                  463,
                  75
                ],
                [
                  0.8482632541133455,
                  0.16163793103448276,
                  464,
                  75
                ],
                [
                  0.850091407678245,
                  0.16344086021505377,
                  465,
                  76
                ],
                [
                  0.8519195612431444,
                  0.1630901287553648,
                  466,
                  76
                ],
                [
                  0.8537477148080439,
                  0.16488222698072805,
                  467,
                  77
                ],
                [
                  0.8555758683729433,
                  0.16452991452991453,
                  468,
                  77
                ],
                [
                  0.8574040219378428,
                  0.16417910447761194,
                  469,
                  77
                ],
                [
                  0.8592321755027422,
                  0.16382978723404254,
                  470,
                  77
                ],
                [
                  0.8610603290676416,
                  0.16560509554140126,
                  471,
                  78
                ],
                [
                  0.8628884826325411,
                  0.1652542372881356,
                  472,
                  78
                ],
                [
                  0.8647166361974405,
                  0.16701902748414377,
                  473,
                  79
                ],
                [
                  0.8665447897623401,
                  0.16877637130801687,
                  474,
                  80
                ],
                [
                  0.8683729433272395,
                  0.1705263157894737,
                  475,
                  81
                ],
                [
                  0.870201096892139,
                  0.1722689075630252,
                  476,
                  82
                ],
                [
                  0.8720292504570384,
                  0.1740041928721174,
                  477,
                  83
                ],
                [
                  0.8738574040219378,
                  0.17573221757322174,
                  478,
                  84
                ],
                [
                  0.8756855575868373,
                  0.17745302713987474,
                  479,
                  85
                ],
                [
                  0.8775137111517367,
                  0.17708333333333334,
                  480,
                  85
                ],
                [
                  0.8793418647166362,
                  0.1787941787941788,
                  481,
                  86
                ],
                [
                  0.8811700182815356,
                  0.18049792531120332,
                  482,
                  87
                ],
                [
                  0.8829981718464351,
                  0.18012422360248448,
                  483,
                  87
                ],
                [
                  0.8848263254113345,
                  0.18181818181818182,
                  484,
                  88
                ],
                [
                  0.886654478976234,
                  0.18144329896907216,
                  485,
                  88
                ],
                [
                  0.8884826325411335,
                  0.18106995884773663,
                  486,
                  88
                ],
                [
                  0.8903107861060329,
                  0.18275154004106775,
                  487,
                  89
                ],
                [
                  0.8921389396709324,
                  0.18237704918032788,
                  488,
                  89
                ],
                [
                  0.8939670932358318,
                  0.18404907975460122,
                  489,
                  90
                ],
                [
                  0.8957952468007313,
                  0.1836734693877551,
                  490,
                  90
                ],
                [
                  0.8976234003656307,
                  0.18329938900203666,
                  491,
                  90
                ],
                [
                  0.8994515539305301,
                  0.18495934959349594,
                  492,
                  91
                ],
                [
                  0.9012797074954296,
                  0.1845841784989858,
                  493,
                  91
                ],
                [
                  0.903107861060329,
                  0.18421052631578946,
                  494,
                  91
                ],
                [
                  0.9049360146252285,
                  0.18585858585858586,
                  495,
                  92
                ],
                [
                  0.906764168190128,
                  0.1875,
                  496,
                  93
                ],
                [
                  0.9085923217550275,
                  0.1891348088531187,
                  497,
                  94
                ],
                [
                  0.9104204753199269,
                  0.19076305220883535,
                  498,
                  95
                ],
                [
                  0.9122486288848263,
                  0.19238476953907815,
                  499,
                  96
                ],
                [
                  0.9140767824497258,
                  0.194,
                  500,
                  97
                ],
                [
                  0.9159049360146252,
                  0.1936127744510978,
                  501,
                  97
                ],
                [
                  0.9177330895795247,
                  0.19322709163346613,
                  502,
                  97
                ],
                [
                  0.9195612431444241,
                  0.19284294234592445,
                  503,
                  97
                ],
                [
                  0.9213893967093236,
                  0.19444444444444445,
                  504,
                  98
                ],
                [
                  0.923217550274223,
                  0.19603960396039605,
                  505,
                  99
                ],
                [
                  0.9250457038391224,
                  0.1956521739130435,
                  506,
                  99
                ],
                [
                  0.926873857404022,
                  0.19723865877712032,
                  507,
                  100
                ],
                [
                  0.9287020109689214,
                  0.1968503937007874,
                  508,
                  100
                ],
                [
                  0.9305301645338209,
                  0.19842829076620824,
                  509,
                  101
                ],
                [
                  0.9323583180987203,
                  0.2,
                  510,
                  102
                ],
                [
                  0.9341864716636198,
                  0.20156555772994128,
                  511,
                  103
                ],
                [
                  0.9360146252285192,
                  0.201171875,
                  512,
                  103
                ],
                [
                  0.9378427787934186,
                  0.20272904483430798,
                  513,
                  104
                ],
                [
                  0.9396709323583181,
                  0.20428015564202334,
                  514,
                  105
                ],
                [
                  0.9414990859232175,
                  0.2058252427184466,
                  515,
                  106
                ],
                [
                  0.943327239488117,
                  0.2054263565891473,
                  516,
                  106
                ],
                [
                  0.9451553930530164,
                  0.20696324951644102,
                  517,
                  107
                ],
                [
                  0.946983546617916,
                  0.2084942084942085,
                  518,
                  108
                ],
                [
                  0.9488117001828154,
                  0.21001926782273603,
                  519,
                  109
                ],
                [
                  0.9506398537477148,
                  0.20961538461538462,
                  520,
                  109
                ],
                [
                  0.9524680073126143,
                  0.21113243761996162,
                  521,
                  110
                ],
                [
                  0.9542961608775137,
                  0.210727969348659,
                  522,
                  110
                ],
                [
                  0.9561243144424132,
                  0.21032504780114722,
                  523,
                  110
                ],
                [
                  0.9579524680073126,
                  0.2099236641221374,
                  524,
                  110
                ],
                [
                  0.9597806215722121,
                  0.21142857142857144,
                  525,
                  111
                ],
                [
                  0.9616087751371115,
                  0.21292775665399238,
                  526,
                  112
                ],
                [
                  0.9634369287020109,
                  0.2144212523719165,
                  527,
                  113
                ],
                [
                  0.9652650822669104,
                  0.2159090909090909,
                  528,
                  114
                ],
                [
                  0.9670932358318098,
                  0.21739130434782608,
                  529,
                  115
                ],
                [
                  0.9689213893967094,
                  0.2188679245283019,
                  530,
                  116
                ],
                [
                  0.9707495429616088,
                  0.22033898305084745,
                  531,
                  117
                ],
                [
                  0.9725776965265083,
                  0.2199248120300752,
                  532,
                  117
                ],
                [
                  0.9744058500914077,
                  0.22138836772983114,
                  533,
                  118
                ],
                [
                  0.9762340036563071,
                  0.22284644194756553,
                  534,
                  119
                ],
                [
                  0.9780621572212066,
                  0.22242990654205608,
                  535,
                  119
                ],
                [
                  0.979890310786106,
                  0.22201492537313433,
                  536,
                  119
                ],
                [
                  0.9817184643510055,
                  0.22346368715083798,
                  537,
                  120
                ],
                [
                  0.9835466179159049,
                  0.22490706319702602,
                  538,
                  121
                ],
                [
                  0.9853747714808044,
                  0.22634508348794063,
                  539,
                  122
                ],
                [
                  0.9872029250457038,
                  0.22777777777777777,
                  540,
                  123
                ],
                [
                  0.9890310786106032,
                  0.22920517560073936,
                  541,
                  124
                ],
                [
                  0.9908592321755028,
                  0.23062730627306274,
                  542,
                  125
                ],
                [
                  0.9926873857404022,
                  0.2302025782688766,
                  543,
                  125
                ],
                [
                  0.9945155393053017,
                  0.23161764705882354,
                  544,
                  126
                ],
                [
                  0.9963436928702011,
                  0.23302752293577983,
                  545,
                  127
                ],
                [
                  0.9981718464351006,
                  0.23443223443223443,
                  546,
                  128
                ],
                [
                  1.0,
                  0.23583180987202926,
                  547,
                  129
                ]
              ]
            },
            {
              "language": "English",
              "run": 412,
              "points": [
                [
                  0.0018281535648994515,
                  0.0,
                  1,
                  0
                ],
                [
                  0.003656307129798903,
                  0.0,
                  2,
                  0
                ],
                [
                  0.005484460694698354,
                  0.0,
                  3,
                  0
                ],
                [
                  0.007312614259597806,
                  0.0,
                  4,
                  0
                ],
                [
                  0.009140767824497258,
                  0.0,
                  5,
                  0
                ],
                [
                  0.010968921389396709,
                  0.0,
                  6,
                  0
                ],
                [
                  0.012797074954296161,
                  0.0,
                  7,
                  0
                ],
                [
                  0.014625228519195612,
                  0.0,
                  8,
                  0
                ],
                [
                  0.016453382084095063,
                  0.0,
                  9,
                  0
                ],
                [
                  0.018281535648994516,
                  0.0,
                  10,
                  0
                ],
                [
                  0.02010968921389397,
                  0.0,
                  11,
                  0
                ],
                [
                  0.021937842778793418,
                  0.0,
                  12,
                  0
                ],
                [
                  0.02376599634369287,
                  0.0,
                  13,
                  0
                ],
                [
                  0.025594149908592323,
                  0.0,
                  14,
                  0
                ],
                [
                  0.027422303473491772,
                  0.0,
                  15,
                  0
                ],
                [
                  0.029250457038391225,
                  0.0,
                  16,
                  0
                ],
                [
                  0.031078610603290677,
                  0.0,
                  17,
                  0
                ],
                [
                  0.03290676416819013,
                  0.0,
                  18,
                  0
                ],
                [
                  0.03473491773308958,
                  0.0,
                  19,
                  0
                ],
                [
                  0.03656307129798903,
                  0.0,
                  20,
                  0
                ],
                [
                  0.038391224862888484,
                  0.0,
                  21,
                  0
                ],
                [
                  0.04021937842778794,
                  0.0,
                  22,
                  0
                ],
                [
                  0.04204753199268738,
                  0.0,
                  23,
                  0
                ],
                [
                  0.043875685557586835,
                  0.0,
                  24,
                  0
                ],
                [
                  0.04570383912248629,
                  0.0,
                  25,
                  0
                ],
                [
                  0.04753199268738574,
                  0.0,
                  26,
                  0
                ],
                [
                  0.04936014625228519,
                  0.0,
                  27,
                  0
                ],
                [
                  0.051188299817184646,
                  0.0,
                  28,
                  0
                ],
                [
                  0.05301645338208409,
                  0.0,
                  29,
                  0
                ],
                [
                  0.054844606946983544,
                  0.0,
                  30,
                  0
                ],
                [
                  0.056672760511883,
                  0.0,
                  31,
                  0
                ],
                [
                  0.05850091407678245,
                  0.0,
                  32,
                  0
                ],
                [
                  0.0603290676416819,
                  0.0,
                  33,
                  0
                ],
                [
                  0.062157221206581355,
                  0.0,
                  34,
                  0
                ],
                [
                  0.06398537477148081,
                  0.0,
                  35,
                  0
                ],
                [
                  0.06581352833638025,
                  0.0,
                  36,
                  0
                ],
                [
                  0.06764168190127971,
                  0.0,
                  37,
                  0
                ],
                [
                  0.06946983546617916,
                  0.0,
                  38,
                  0
                ],
                [
                  0.0712979890310786,
                  0.0,
                  39,
                  0
                ],
                [
                  0.07312614259597806,
                  0.0,
                  40,
                  0
                ],
                [
                  0.07495429616087751,
                  0.0,
                  41,
                  0
                ],
                [
                  0.07678244972577697,
                  0.0,
                  42,
                  0
                ],
                [
                  0.07861060329067641,
                  0.0,
                  43,
                  0
                ],
                [
                  0.08043875685557587,
                  0.0,
                  44,
                  0
                ],
                [
                  0.08226691042047532,
                  0.0,
                  45,
                  0
                ],
                [
                  0.08409506398537477,
                  0.0,
                  46,
                  0
                ],
                [
                  0.08592321755027423,
                  0.0,
                  47,
                  0
                ],
                [
                  0.08775137111517367,
                  0.0,
                  48,
                  0
                ],
                [
                  0.08957952468007313,
                  0.0,
                  49,
                  0
                ],
                [
                  0.09140767824497258,
                  0.0,
                  50,
                  0
                ],
                [
                  0.09323583180987204,
                  0.0,
                  51,
                  0
                ],
                [
                  0.09506398537477148,
                  0.0,
                  52,
                  0
                ],
                [
                  0.09689213893967093,
                  0.0,
                  53,
                  0
                ],
                [
                  0.09872029250457039,
                  0.0,
                  54,
                  0
                ],
                [
                  0.10054844606946983,
                  0.0,
                  55,
                  0
                ],
                [
                  0.10237659963436929,
                  0.0,
                  56,
                  0
                ],
                [
                  0.10420475319926874,
                  0.0,
                  57,
                  0
                ],
                [
                  0.10603290676416818,
                  0.0,
                  58,
                  0
                ],
                [
                  0.10786106032906764,
                  0.0,
                  59,
                  0
                ],
                [
                  0.10968921389396709,
                  0.0,
                  60,
                  0
                ],
                [
                  0.11151736745886655,
                  0.0,
                  61,
                  0
                ],
                [
                  0.113345521023766,
                  0.0,
                  62,
                  0
                ],
                [
                  0.11517367458866545,
                  0.0,
                  63,
                  0
                ],
                [
                  0.1170018281535649,
                  0.0,
                  64,
                  0
                ],
                [
                  0.11882998171846434,
                  0.0,
                  65,
                  0
                ],
                [
                  0.1206581352833638,
                  0.0,
                  66,
                  0
                ],
                [
                  0.12248628884826325,
                  0.0,
                  67,
                  0
                ],
                [
                  0.12431444241316271,
                  0.0,
                  68,
                  0
                ],
                [
                  0.12614259597806216,
                  0.0,
                  69,
                  0
                ],
                [
                  0.12797074954296161,
                  0.0,
                  70,
                  0
                ],
                [
                  0.12979890310786105,
                  0.0,
                  71,
                  0
                ],
                [
                  0.1316270566727605,
                  0.0,
                  72,
                  0
                ],
                [
                  0.13345521023765997,
                  0.0,
                  73,
                  0
                ],
                [
                  0.13528336380255943,
                  0.0,
                  74,
                  0
                ],
                [
                  0.13711151736745886,
                  0.0,
                  75,
                  0
                ],
                [
                  0.13893967093235832,
                  0.0,
                  76,
                  0
                ],
                [
                  0.14076782449725778,
                  0.0,
                  77,
                  0
                ],
                [
                  0.1425959780621572,
                  0.0,
                  78,
                  0
                ],
                [
                  0.14442413162705667,
                  0.0,
                  79,
                  0
                ],
                [
                  0.14625228519195613,
                  0.0,
                  80,
                  0
                ],
                [
                  0.1480804387568556,
                  0.0,
                  81,
                  0
                ],
                [
                  0.14990859232175502,
                  0.0,
                  82,
                  0
                ],
                [
                  0.15173674588665448,
                  0.0,
                  83,
                  0
                ],
                [
                  0.15356489945155394,
                  0.0,
                  84,
                  0
                ],
                [
                  0.15539305301645337,
                  0.0,
                  85,
                  0
                ],
                [
                  0.15722120658135283,
                  0.0,
                  86,
                  0
                ],
                [
                  0.1590493601462523,
                  0.0,
                  87,
                  0
                ],
                [
                  0.16087751371115175,
                  0.0,
                  88,
                  0
                ],
                [
                  0.16270566727605118,
                  0.0,
                  89,
                  0
                ],
                [
                  0.16453382084095064,
                  0.0,
                  90,
                  0
                ],
                [
                  0.1663619744058501,
                  0.0,
                  91,
                  0
                ],
                [
                  0.16819012797074953,
                  0.0,
                  92,
                  0
                ],
                [
                  0.170018281535649,
                  0.0,
                  93,
                  0
                ],
                [
                  0.17184643510054845,
                  0.010638297872340425,
                  94,
                  1
                ],
                [
                  0.1736745886654479,
                  0.010526315789473684,
                  95,
                  1
                ],
                [
                  0.17550274223034734,
                  0.010416666666666666,
                  96,
                  1
                ],
                [
                  0.1773308957952468,
                  0.010309278350515464,
                  97,
                  1
                ],
                [
                  0.17915904936014626,
                  0.01020408163265306,
                  98,
                  1
                ],
                [
                  0.1809872029250457,
                  0.010101010101010102,
                  99,
                  1
                ],
                [
                  0.18281535648994515,
                  0.01,
                  100,
                  1
                ],
                [
                  0.1846435100548446,
                  0.009900990099009901,
                  101,
                  1
                ],
                [
                  0.18647166361974407,
                  0.00980392156862745,
                  102,
                  1
                ],
                [
                  0.1882998171846435,
                  0.009708737864077669,
                  103,
                  1
                ],
                [
                  0.19012797074954296,
                  0.009615384615384616,
                  104,
                  1
                ],
                [
                  0.19195612431444242,
                  0.009523809523809525,
                  105,
                  1
                ],
                [
                  0.19378427787934185,
                  0.009433962264150943,
                  106,
                  1
                ],
                [
                  0.1956124314442413,
                  0.018691588785046728,
                  107,
                  2
                ],
                [
                  0.19744058500914077,
                  0.018518518518518517,
                  108,
                  2
                ],
                [
                  0.19926873857404023,
                  0.01834862385321101,
                  109,
                  2
                ],
                [
                  0.20109689213893966,
                  0.01818181818181818,
                  110,
                  2
                ],
                [
                  0.20292504570383912,
                  0.018018018018018018,
                  111,
                  2
                ],
                [
                  0.20475319926873858,
                  0.017857142857142856,
                  112,
                  2
                ],
                [
                  0.20658135283363802,
                  0.017699115044247787,
                  113,
                  2
                ],
                [
                  0.20840950639853748,
                  0.017543859649122806,
                  114,
                  2
                ],
                [
                  0.21023765996343693,
                  0.017391304347826087,
                  115,
                  2
                ],
                [
                  0.21206581352833637,
                  0.017241379310344827,
                  116,
                  2
                ],
                [
                  0.21389396709323583,
                  0.017094017094017096,
                  117,
                  2
                ],
                [
                  0.21572212065813529,
                  0.01694915254237288,
                  118,
                  2
                ],
                [
                  0.21755027422303475,
                  0.01680672268907563,
                  119,
                  2
                ],
                [
                  0.21937842778793418,
                  0.016666666666666666,
                  120,
                  2
                ],
                [
                  0.22120658135283364,
                  0.01652892561983471,
                  121,
                  2
                ],
                [
                  0.2230347349177331,
                  0.01639344262295082,
                  122,
                  2
                ],
                [
                  0.22486288848263253,
                  0.016260162601626018,
                  123,
                  2
                ],
                [
                  0.226691042047532,
                  0.016129032258064516,
                  124,
                  2
                ],
                [
                  0.22851919561243145,
                  0.016,
                  125,
                  2
                ],
                [
                  0.2303473491773309,
                  0.015873015873015872,
                  126,
                  2
                ],
                [
                  0.23217550274223034,
                  0.015748031496062992,
                  127,
                  2
                ],
                [
                  0.2340036563071298,
                  0.015625,
                  128,
                  2
                ],
                [
                  0.23583180987202926,
                  0.015503875968992248,
                  129,
                  2
                ],
                [
                  0.2376599634369287,
                  0.015384615384615385,
                  130,
                  2
                ],
                [
                  0.23948811700182815,
                  0.015267175572519083,
                  131,
                  2
                ],
                [
                  0.2413162705667276,
                  0.015151515151515152,
                  132,
                  2
                ],
                [
                  0.24314442413162707,
                  0.015037593984962405,
                  133,
                  2
                ],
                [
                  0.2449725776965265,
                  0.014925373134328358,
                  134,
                  2
                ],
                [
                  0.24680073126142596,
                  0.014814814814814815,
                  135,
                  2
                ],
                [
                  0.24862888482632542,
                  0.014705882352941176,
                  136,
                  2
                ],
                [
                  0.25045703839122485,
                  0.014598540145985401,
                  137,
                  2
                ],
                [
                  0.2522851919561243,
                  0.014492753623188406,
                  138,
                  2
                ],
                [
                  0.25411334552102377,
                  0.014388489208633094,
                  139,
                  2
                ],
                [
                  0.25594149908592323,
                  0.014285714285714285,
                  140,
                  2
                ],
                [
                  0.2577696526508227,
                  0.014184397163120567,
                  141,
                  2
                ],
                [
                  0.2595978062157221,
                  0.014084507042253521,
                  142,
                  2
                ],
                [
                  0.26142595978062155,
                  0.013986013986013986,
                  143,
                  2
                ],
                [
                  0.263254113345521,
                  0.013888888888888888,
                  144,
                  2
                ],
                [
                  0.26508226691042047,
                  0.013793103448275862,
                  145,
                  2
                ],
                [
                  0.26691042047531993,
                  0.0136986301369863,
                  146,
                  2
                ],
                [
                  0.2687385740402194,
                  0.013605442176870748,
                  147,
                  2
                ],
                [
                  0.27056672760511885,
                  0.013513513513513514,
                  148,
                  2
                ],
                [
                  0.27239488117001825,
                  0.013422818791946308,
                  149,
                  2
                ],
                [
                  0.2742230347349177,
                  0.013333333333333334,
                  150,
                  2
                ],
                [
                  0.2760511882998172,
                  0.013245033112582781,
                  151,
                  2
                ],
                [
                  0.27787934186471663,
                  0.013157894736842105,
                  152,
                  2
                ],
                [
                  0.2797074954296161,
                  0.013071895424836602,
                  153,
                  2
                ],
                [
                  0.28153564899451555,
                  0.012987012987012988,
                  154,
                  2
                ],
                [
                  0.283363802559415,
                  0.012903225806451613,
                  155,
                  2
                ],
                [
                  0.2851919561243144,
                  0.01282051282051282,
                  156,
                  2
                ],
                [
                  0.2870201096892139,
                  0.012738853503184714,
                  157,
                  2
                ],
                [
                  0.28884826325411334,
                  0.012658227848101266,
                  158,
                  2
                ],
                [
                  0.2906764168190128,
                  0.012578616352201259,
                  159,
                  2
                ],
                [
                  0.29250457038391225,
                  0.0125,
                  160,
                  2
                ],
                [
                  0.2943327239488117,
                  0.012422360248447204,
                  161,
                  2
                ],
                [
                  0.2961608775137112,
                  0.012345679012345678,
                  162,
                  2
                ],
                [
                  0.2979890310786106,
                  0.012269938650306749,
                  163,
                  2
                ],
                [
                  0.29981718464351004,
                  0.012195121951219513,
                  164,
                  2
                ],
                [
                  0.3016453382084095,
                  0.012121212121212121,
                  165,
                  2
                ],
                [
                  0.30347349177330896,
                  0.012048192771084338,
                  166,
                  2
                ],
                [
                  0.3053016453382084,
                  0.011976047904191617,
                  167,
                  2
                ],
                [
                  0.3071297989031079,
                  0.011904761904761904,
                  168,
                  2
                ],
                [
                  0.30895795246800734,
                  0.011834319526627219,
                  169,
                  2
                ],
                [
                  0.31078610603290674,
                  0.011764705882352941,
                  170,
                  2
                ],
                [
                  0.3126142595978062,
                  0.011695906432748537,
                  171,
                  2
                ],
                [
                  0.31444241316270566,
                  0.011627906976744186,
                  172,
                  2
                ],
                [
                  0.3162705667276051,
                  0.011560693641618497,
                  173,
                  2
                ],
                [
                  0.3180987202925046,
                  0.011494252873563218,
                  174,
                  2
                ],
                [
                  0.31992687385740404,
                  0.011428571428571429,
                  175,
                  2
                ],
                [
                  0.3217550274223035,
                  0.011363636363636364,
                  176,
                  2
                ],
                [
                  0.3235831809872029,
                  0.011299435028248588,
                  177,
                  2
                ],
                [
                  0.32541133455210236,
                  0.011235955056179775,
                  178,
                  2
                ],
                [
                  0.3272394881170018,
                  0.0111731843575419,
                  179,
                  2
                ],
                [
                  0.3290676416819013,
                  0.011111111111111112,
                  180,
                  2
                ],
                [
                  0.33089579524680074,
                  0.011049723756906077,
                  181,
                  2
                ],
                [
                  0.3327239488117002,
                  0.016483516483516484,
                  182,
                  3
                ],
                [
                  0.33455210237659966,
                  0.01639344262295082,
                  183,
                  3
                ],
                [
                  0.33638025594149906,
                  0.016304347826086956,
                  184,
                  3
                ],
                [
                  0.3382084095063985,
                  0.016216216216216217,
                  185,
                  3
                ],
                [
                  0.340036563071298,
                  0.016129032258064516,
                  186,
                  3
                ],
                [
                  0.34186471663619744,
                  0.016042780748663103,
                  187,
                  3
                ],
                [
                  0.3436928702010969,
                  0.015957446808510637,
                  188,
                  3
                ],
                [
                  0.34552102376599636,
                  0.015873015873015872,
                  189,
                  3
                ],
                [
                  0.3473491773308958,
                  0.015789473684210527,
                  190,
                  3
                ],
                [
                  0.3491773308957952,
                  0.015706806282722512,
                  191,
                  3
                ],
                [
                  0.3510054844606947,
                  0.015625,
                  192,
                  3
                ],
                [
                  0.35283363802559414,
                  0.015544041450777202,
                  193,
                  3
                ],
                [
                  0.3546617915904936,
                  0.015463917525773196,
                  194,
                  3
                ],
                [
                  0.35648994515539306,
                  0.015384615384615385,
                  195,
                  3
                ],
                [
                  0.3583180987202925,
                  0.015306122448979591,
                  196,
                  3
                ],
                [
                  0.360146252285192,
                  0.015228426395939087,
                  197,
                  3
                ],
                [
                  0.3619744058500914,
                  0.015151515151515152,
                  198,
                  3
                ],
                [
                  0.36380255941499084,
                  0.01507537688442211,
                  199,
                  3
                ],
                [
                  0.3656307129798903,
                  0.015,
                  200,
                  3
                ],
                [
                  0.36745886654478976,
                  0.014925373134328358,
                  201,
                  3
                ],
                [
                  0.3692870201096892,
                  0.01485148514851485,
                  202,
                  3
                ],
                [
                  0.3711151736745887,
                  0.014778325123152709,
                  203,
                  3
                ],
                [
                  0.37294332723948814,
                  0.014705882352941176,
                  204,
                  3
                ],
                [
                  0.37477148080438755,
                  0.014634146341463415,
                  205,
                  3
                ],
                [
                  0.376599634369287,
                  0.014563106796116505,
                  206,
                  3
                ],
                [
                  0.37842778793418647,
                  0.014492753623188406,
                  207,
                  3
                ],
                [
                  0.3802559414990859,
                  0.019230769230769232,
                  208,
                  4
                ],
                [
                  0.3820840950639854,
                  0.019138755980861243,
                  209,
                  4
                ],
                [
                  0.38391224862888484,
                  0.01904761904761905,
                  210,
                  4
                ],
                [
                  0.3857404021937843,
                  0.018957345971563982,
                  211,
                  4
                ],
                [
                  0.3875685557586837,
                  0.018867924528301886,
                  212,
                  4
                ],
                [
                  0.38939670932358317,
                  0.018779342723004695,
                  213,
                  4
                ],
                [
                  0.3912248628884826,
                  0.018691588785046728,
                  214,
                  4
                ],
                [
                  0.3930530164533821,
                  0.018604651162790697,
                  215,
                  4
                ],
                [
                  0.39488117001828155,
                  0.018518518518518517,
                  216,
                  4
                ],
                [
                  0.396709323583181,
                  0.018433179723502304,
                  217,
                  4
                ],
                [
                  0.39853747714808047,
                  0.01834862385321101,
                  218,
                  4
                ],
                [
                  0.40036563071297987,
                  0.0182648401826484,
                  219,
                  4
                ],
                [
                  0.40219378427787933,
                  0.01818181818181818,
                  220,
                  4
                ],
                [
                  0.4040219378427788,
                  0.01809954751131222,
                  221,
                  4
                ],
                [
                  0.40585009140767825,
                  0.018018018018018018,
                  222,
                  4
                ],
                [
                  0.4076782449725777,
                  0.017937219730941704,
                  223,
                  4
                ],
                [
                  0.40950639853747717,
                  0.017857142857142856,
                  224,
                  4
                ],
                [
                  0.4113345521023766,
                  0.017777777777777778,
                  225,
                  4
                ],
                [
                  0.41316270566727603,
                  0.017699115044247787,
                  226,
                  4
                ],
                [
                  0.4149908592321755,
                  0.01762114537444934,
                  227,
                  4
                ],
                [
                  0.41681901279707495,
                  0.017543859649122806,
                  228,
                  4
                ],
                [
                  0.4186471663619744,
                  0.017467248908296942,
                  229,
                  4
                ],
                [
                  0.42047531992687387,
                  0.017391304347826087,
                  230,
                  4
                ],
                [
                  0.42230347349177333,
                  0.017316017316017316,
                  231,
                  4
                ],
                [
                  0.42413162705667273,
                  0.017241379310344827,
                  232,
                  4
                ],
                [
                  0.4259597806215722,
                  0.017167381974248927,
                  233,
                  4
                ],
                [
                  0.42778793418647165,
                  0.017094017094017096,
                  234,
                  4
                ],
                [
                  0.4296160877513711,
                  0.01702127659574468,
                  235,
                  4
                ],
                [
                  0.43144424131627057,
                  0.01694915254237288,
                  236,
                  4
                ],
                [
                  0.43327239488117003,
                  0.016877637130801686,
                  237,
                  4
                ],
                [
                  0.4351005484460695,
                  0.01680672268907563,
                  238,
                  4
                ],
                [
                  0.4369287020109689,
                  0.016736401673640166,
                  239,
                  4
                ],
                [
                  0.43875685557586835,
                  0.016666666666666666,
                  240,
                  4
                ],
                [
                  0.4405850091407678,
                  0.016597510373443983,
                  241,
                  4
                ],
                [
                  0.4424131627056673,
                  0.01652892561983471,
                  242,
                  4
                ],
                [
                  0.44424131627056673,
                  0.01646090534979424,
                  243,
                  4
                ],
                [
                  0.4460694698354662,
                  0.01639344262295082,
                  244,
                  4
                ],
                [
                  0.44789762340036565,
                  0.0163265306122449,
                  245,
                  4
                ],
                [
                  0.44972577696526506,
                  0.016260162601626018,
                  246,
                  4
                ],
                [
                  0.4515539305301645,
                  0.016194331983805668,
                  247,
                  4
                ],
                [
                  0.453382084095064,
                  0.016129032258064516,
                  248,
                  4
                ],
                [
                  0.45521023765996343,
                  0.01606425702811245,
                  249,
                  4
                ],
                [
                  0.4570383912248629,
                  0.016,
                  250,
                  4
                ],
                [
                  0.45886654478976235,
                  0.01593625498007968,
                  251,
                  4
                ],
                [
                  0.4606946983546618,
                  0.015873015873015872,
                  252,
                  4
                ],
                [
                  0.4625228519195612,
                  0.015810276679841896,
                  253,
                  4
                ],
                [
                  0.4643510054844607,
                  0.015748031496062992,
                  254,
                  4
                ],
                [
                  0.46617915904936014,
                  0.01568627450980392,
                  255,
                  4
                ],
                [
                  0.4680073126142596,
                  0.015625,
                  256,
                  4
                ],
                [
                  0.46983546617915906,
                  0.01556420233463035,
                  257,
                  4
                ],
                [
                  0.4716636197440585,
                  0.015503875968992248,
                  258,
                  4
                ],
                [
                  0.473491773308958,
                  0.019305019305019305,
                  259,
                  5
                ],
                [
                  0.4753199268738574,
                  0.019230769230769232,
                  260,
                  5
                ],
                [
                  0.47714808043875684,
                  0.022988505747126436,
                  261,
                  6
                ],
                [
                  0.4789762340036563,
                  0.022900763358778626,
                  262,
                  6
                ],
                [
                  0.48080438756855576,
                  0.022813688212927757,
                  263,
                  6
                ],
                [
                  0.4826325411334552,
                  0.022727272727272728,
                  264,
                  6
                ],
                [
                  0.4844606946983547,
                  0.022641509433962263,
                  265,
                  6
                ],
                [
                  0.48628884826325414,
                  0.022556390977443608,
                  266,
                  6
                ],
                [
                  0.48811700182815354,
                  0.02247191011235955,
                  267,
                  6
                ],
                [
                  0.489945155393053,
                  0.022388059701492536,
                  268,
                  6
                ],
                [
                  0.49177330895795246,
                  0.022304832713754646,
                  269,
                  6
                ],
                [
                  0.4936014625228519,
                  0.022222222222222223,
                  270,
                  6
                ],
                [
                  0.4954296160877514,
                  0.02214022140221402,
                  271,
                  6
                ],
                [
                  0.49725776965265084,
                  0.022058823529411766,
                  272,
                  6
                ],
                [
                  0.4990859232175503,
                  0.02197802197802198,
                  273,
                  6
                ],
                [
                  0.5009140767824497,
                  0.021897810218978103,
                  274,
                  6
                ],
                [
                  0.5027422303473492,
                  0.02181818181818182,
                  275,
                  6
                ],
                [
                  0.5045703839122486,
                  0.021739130434782608,
                  276,
                  6
                ],
                [
                  0.506398537477148,
                  0.021660649819494584,
                  277,
                  6
                ],
                [
                  0.5082266910420475,
                  0.02158273381294964,
                  278,
                  6
                ],
                [
                  0.5100548446069469,
                  0.021505376344086023,
                  279,
                  6
                ],
                [
                  0.5118829981718465,
                  0.02142857142857143,
                  280,
                  6
                ],
                [
                  0.5137111517367459,
                  0.021352313167259787,
                  281,
                  6
                ],
                [
                  0.5155393053016454,
                  0.02127659574468085,
                  282,
                  6
                ],
                [
                  0.5173674588665448,
                  0.02120141342756184,
                  283,
                  6
                ],
                [
                  0.5191956124314442,
                  0.02112676056338028,
                  284,
                  6
                ],
                [
                  0.5210237659963437,
                  0.021052631578947368,
                  285,
                  6
                ],
                [
                  0.5228519195612431,
                  0.02097902097902098,
                  286,
                  6
                ],
                [
                  0.5246800731261426,
                  0.020905923344947737,
                  287,
                  6
                ],
                [
                  0.526508226691042,
                  0.020833333333333332,
                  288,
                  6
                ],
                [
                  0.5283363802559415,
                  0.020761245674740483,
                  289,
                  6
                ],
                [
                  0.5301645338208409,
                  0.020689655172413793,
                  290,
                  6
                ],
                [
                  0.5319926873857403,
                  0.020618556701030927,
                  291,
                  6
                ],
                [
                  0.5338208409506399,
                  0.02054794520547945,
                  292,
                  6
                ],
                [
                  0.5356489945155393,
                  0.020477815699658702,
                  293,
                  6
                ],
                [
                  0.5374771480804388,
                  0.02040816326530612,
                  294,
                  6
                ],
                [
                  0.5393053016453382,
                  0.020338983050847456,
                  295,
                  6
                ],
                [
                  0.5411334552102377,
                  0.02364864864864865,
                  296,
                  7
                ],
                [
                  0.5429616087751371,
                  0.02356902356902357,
                  297,
                  7
                ],
                [
                  0.5447897623400365,
                  0.02348993288590604,
                  298,
                  7
                ],
                [
                  0.546617915904936,
                  0.023411371237458192,
                  299,
                  7
                ],
                [
                  0.5484460694698354,
                  0.023333333333333334,
                  300,
                  7
                ],
                [
                  0.5502742230347349,
                  0.023255813953488372,
                  301,
                  7
                ],
                [
                  0.5521023765996343,
                  0.026490066225165563,
                  302,
                  8
                ],
                [
                  0.5539305301645339,
                  0.026402640264026403,
                  303,
                  8
                ],
                [
                  0.5557586837294333,
                  0.02631578947368421,
                  304,
                  8
                ],
                [
                  0.5575868372943327,
                  0.02622950819672131,
                  305,
                  8
                ],
                [
                  0.5594149908592322,
                  0.026143790849673203,
                  306,
                  8
                ],
                [
                  0.5612431444241316,
                  0.026058631921824105,
                  307,
                  8
                ],
                [
                  0.5630712979890311,
                  0.025974025974025976,
                  308,
                  8
                ],
                [
                  0.5648994515539305,
                  0.025889967637540454,
                  309,
                  8
                ],
                [
                  0.56672760511883,
                  0.025806451612903226,
                  310,
                  8
                ],
                [
                  0.5685557586837294,
                  0.02572347266881029,
                  311,
                  8
                ],
                [
                  0.5703839122486288,
                  0.02564102564102564,
                  312,
                  8
                ],
                [
                  0.5722120658135283,
                  0.025559105431309903,
                  313,
                  8
                ],
                [
                  0.5740402193784278,
                  0.025477707006369428,
                  314,
                  8
                ],
                [
                  0.5758683729433273,
                  0.025396825396825397,
                  315,
                  8
                ],
                [
                  0.5776965265082267,
                  0.02531645569620253,
                  316,
                  8
                ],
                [
                  0.5795246800731262,
                  0.025236593059936908,
                  317,
                  8
                ],
                [
                  0.5813528336380256,
                  0.025157232704402517,
                  318,
                  8
                ],
                [
                  0.583180987202925,
                  0.025078369905956112,
                  319,
                  8
                ],
                [
                  0.5850091407678245,
                  0.025,
                  320,
                  8
                ],
                [
                  0.5868372943327239,
                  0.024922118380062305,
                  321,
                  8
                ],
                [
                  0.5886654478976234,
                  0.024844720496894408,
                  322,
                  8
                ],
                [
                  0.5904936014625228,
                  0.02476780185758514,
                  323,
                  8
                ],
                [
                  0.5923217550274223,
                  0.024691358024691357,
                  324,
                  8
                ],
                [
                  0.5941499085923218,
                  0.024615384615384615,
                  325,
                  8
                ],
                [
                  0.5959780621572212,
                  0.024539877300613498,
                  326,
                  8
                ],
                [
                  0.5978062157221207,
                  0.024464831804281346,
                  327,
                  8
                ],
                [
                  0.5996343692870201,
                  0.024390243902439025,
                  328,
                  8
                ],
                [
                  0.6014625228519196,
                  0.02735562310030395,
                  329,
                  9
                ],
                [
                  0.603290676416819,
                  0.02727272727272727,
                  330,
                  9
                ],
                [
                  0.6051188299817185,
                  0.027190332326283987,
                  331,
                  9
                ],
                [
                  0.6069469835466179,
                  0.02710843373493976,
                  332,
                  9
                ],
                [
                  0.6087751371115173,
                  0.03003003003003003,
                  333,
                  10
                ],
                [
                  0.6106032906764168,
                  0.03293413173652695,
                  334,
                  11
                ],
                [
                  0.6124314442413162,
                  0.03582089552238806,
                  335,
                  12
                ],
                [
                  0.6142595978062158,
                  0.03571428571428571,
                  336,
                  12
                ],
                [
                  0.6160877513711152,
                  0.03560830860534125,
                  337,
                  12
                ],
                [
                  0.6179159049360147,
                  0.038461538461538464,
                  338,
                  13
                ],
                [
                  0.6197440585009141,
                  0.038348082595870206,
                  339,
                  13
                ],
                [
                  0.6215722120658135,
                  0.03823529411764706,
                  340,
                  13
                ],
                [
                  0.623400365630713,
                  0.03812316715542522,
                  341,
                  13
                ],
                [
                  0.6252285191956124,
                  0.04093567251461988,
                  342,
                  14
                ],
                [
                  0.6270566727605119,
                  0.04081632653061224,
                  343,
                  14
                ],
                [
                  0.6288848263254113,
                  0.040697674418604654,
                  344,
                  14
                ],
                [
                  0.6307129798903108,
                  0.04057971014492753,
                  345,
                  14
                ],
                [
                  0.6325411334552102,
                  0.04046242774566474,
                  346,
                  14
                ],
                [
                  0.6343692870201096,
                  0.040345821325648415,
                  347,
                  14
                ],
                [
                  0.6361974405850092,
                  0.040229885057471264,
                  348,
                  14
                ],
                [
                  0.6380255941499086,
                  0.04011461318051576,
                  349,
                  14
                ],
                [
                  0.6398537477148081,
                  0.04,
                  350,
                  14
                ],
                [
                  0.6416819012797075,
                  0.039886039886039885,
                  351,
                  14
                ],
                [
                  0.643510054844607,
                  0.03977272727272727,
                  352,
                  14
                ],
                [
                  0.6453382084095064,
                  0.039660056657223795,
                  353,
                  14
                ],
                [
                  0.6471663619744058,
                  0.0423728813559322,
                  354,
                  15
                ],
                [
                  0.6489945155393053,
                  0.04225352112676056,
                  355,
                  15
                ],
                [
                  0.6508226691042047,
                  0.042134831460674156,
                  356,
                  15
                ],
                [
                  0.6526508226691042,
                  0.04201680672268908,
                  357,
                  15
                ],
                [
                  0.6544789762340036,
                  0.0446927374301676,
                  358,
                  16
                ],
                [
                  0.6563071297989032,
                  0.04456824512534819,
                  359,
                  16
                ],
                [
                  0.6581352833638026,
                  0.044444444444444446,
                  360,
                  16
                ],
                [
                  0.659963436928702,
                  0.0443213296398892,
                  361,
                  16
                ],
                [
                  0.6617915904936015,
                  0.04419889502762431,
                  362,
                  16
                ],
                [
                  0.6636197440585009,
                  0.0440771349862259,
                  363,
                  16
                ],
                [
                  0.6654478976234004,
                  0.04395604395604396,
                  364,
                  16
                ],
                [
                  0.6672760511882998,
                  0.043835616438356165,
                  365,
                  16
                ],
                [
                  0.6691042047531993,
                  0.04371584699453552,
                  366,
                  16
                ],
                [
                  0.6709323583180987,
                  0.043596730245231606,
                  367,
                  16
                ],
                [
                  0.6727605118829981,
                  0.043478260869565216,
                  368,
                  16
                ],
                [
                  0.6745886654478976,
                  0.04607046070460705,
                  369,
                  17
                ],
                [
                  0.676416819012797,
                  0.04594594594594595,
                  370,
                  17
                ],
                [
                  0.6782449725776966,
                  0.04582210242587601,
                  371,
                  17
                ],
                [
                  0.680073126142596,
                  0.0456989247311828,
                  372,
                  17
                ],
                [
                  0.6819012797074955,
                  0.045576407506702415,
                  373,
                  17
                ],
                [
                  0.6837294332723949,
                  0.045454545454545456,
                  374,
                  17
                ],
                [
                  0.6855575868372943,
                  0.04533333333333334,
                  375,
                  17
                ],
                [
                  0.6873857404021938,
                  0.04521276595744681,
                  376,
                  17
                ],
                [
                  0.6892138939670932,
                  0.04774535809018567,
                  377,
                  18
                ],
                [
                  0.6910420475319927,
                  0.047619047619047616,
                  378,
                  18
                ],
                [
                  0.6928702010968921,
                  0.047493403693931395,
                  379,
                  18
                ],
                [
                  0.6946983546617916,
                  0.04736842105263158,
                  380,
                  18
                ],
                [
                  0.696526508226691,
                  0.049868766404199474,
                  381,
                  19
                ],
                [
                  0.6983546617915904,
                  0.049738219895287955,
                  382,
                  19
                ],
                [
                  0.70018281535649,
                  0.04960835509138381,
                  383,
                  19
                ],
                [
                  0.7020109689213894,
                  0.049479166666666664,
                  384,
                  19
                ],
                [
                  0.7038391224862889,
                  0.04935064935064935,
                  385,
                  19
                ],
                [
                  0.7056672760511883,
                  0.04922279792746114,
                  386,
                  19
                ],
                [
                  0.7074954296160878,
                  0.04909560723514212,
                  387,
                  19
                ],
                [
                  0.7093235831809872,
                  0.04896907216494845,
                  388,
                  19
                ],
                [
                  0.7111517367458866,
                  0.04884318766066838,
                  389,
                  19
                ],
                [
                  0.7129798903107861,
                  0.05128205128205128,
                  390,
                  20
                ],
                [
                  0.7148080438756855,
                  0.05115089514066496,
                  391,
                  20
                ],
                [
                  0.716636197440585,
                  0.05102040816326531,
                  392,
                  20
                ],
                [
                  0.7184643510054844,
                  0.05089058524173028,
                  393,
                  20
                ],
                [
                  0.720292504570384,
                  0.050761421319796954,
                  394,
                  20
                ],
                [
                  0.7221206581352834,
                  0.053164556962025315,
                  395,
                  21
                ],
                [
                  0.7239488117001828,
                  0.05555555555555555,
                  396,
                  22
                ],
                [
                  0.7257769652650823,
                  0.055415617128463476,
                  397,
                  22
                ],
                [
                  0.7276051188299817,
                  0.05527638190954774,
                  398,
                  22
                ],
                [
                  0.7294332723948812,
                  0.05513784461152882,
                  399,
                  22
                ],
                [
                  0.7312614259597806,
                  0.055,
                  400,
                  22
                ],
                [
                  0.7330895795246801,
                  0.05486284289276808,
                  401,
                  22
                ],
                [
                  0.7349177330895795,
                  0.05472636815920398,
                  402,
                  22
                ],
                [
                  0.7367458866544789,
                  0.05459057071960298,
                  403,
                  22
                ],
                [
                  0.7385740402193784,
                  0.05693069306930693,
                  404,
                  23
                ],
                [
                  0.7404021937842779,
                  0.05925925925925926,
                  405,
                  24
                ],
                [
                  0.7422303473491774,
                  0.059113300492610835,
                  406,
                  24
                ],
                [
                  0.7440585009140768,
                  0.05896805896805897,
                  407,
                  24
                ],
                [
                  0.7458866544789763,
                  0.058823529411764705,
                  408,
                  24
                ],
                [
                  0.7477148080438757,
                  0.05867970660146699,
                  409,
                  24
                ],
                [
                  0.7495429616087751,
                  0.05853658536585366,
                  410,
                  24
                ],
                [
                  0.7513711151736746,
                  0.058394160583941604,
                  411,
                  24
                ],
                [
                  0.753199268738574,
                  0.05825242718446602,
                  412,
                  24
                ],
                [
                  0.7550274223034735,
                  0.05811138014527845,
                  413,
                  24
                ],
                [
                  0.7568555758683729,
                  0.057971014492753624,
                  414,
                  24
                ],
                [
                  0.7586837294332724,
                  0.05783132530120482,
                  415,
                  24
                ],
                [
                  0.7605118829981719,
                  0.057692307692307696,
                  416,
                  24
                ],
                [
                  0.7623400365630713,
                  0.05755395683453238,
                  417,
                  24
                ],
                [
                  0.7641681901279708,
                  0.05741626794258373,
                  418,
                  24
                ],
                [
                  0.7659963436928702,
                  0.059665871121718374,
                  419,
                  25
                ],
                [
                  0.7678244972577697,
                  0.05952380952380952,
                  420,
                  25
                ],
                [
                  0.7696526508226691,
                  0.05938242280285035,
                  421,
                  25
                ],
                [
                  0.7714808043875686,
                  0.05924170616113744,
                  422,
                  25
                ],
                [
                  0.773308957952468,
                  0.0591016548463357,
                  423,
                  25
                ],
                [
                  0.7751371115173674,
                  0.0589622641509434,
                  424,
                  25
                ],
                [
                  0.7769652650822669,
                  0.058823529411764705,
                  425,
                  25
                ],
                [
                  0.7787934186471663,
                  0.05868544600938967,
                  426,
                  25
                ],
                [
                  0.7806215722120659,
                  0.06088992974238876,
                  427,
                  26
                ],
                [
                  0.7824497257769653,
                  0.06074766355140187,
                  428,
                  26
                ],
                [
                  0.7842778793418648,
                  0.06060606060606061,
                  429,
                  26
                ],
                [
                  0.7861060329067642,
                  0.06279069767441861,
                  430,
                  27
                ],
                [
                  0.7879341864716636,
                  0.06264501160092807,
                  431,
                  27
                ],
                [
                  0.7897623400365631,
                  0.0625,
                  432,
                  27
                ],
                [
                  0.7915904936014625,
                  0.06235565819861432,
                  433,
                  27
                ],
                [
                  0.793418647166362,
                  0.06451612903225806,
                  434,
                  28
                ],
                [
                  0.7952468007312614,
                  0.06436781609195402,
                  435,
                  28
                ],
                [
                  0.7970749542961609,
                  0.06651376146788991,
                  436,
                  29
                ],
                [
                  0.7989031078610603,
                  0.06636155606407322,
                  437,
                  29
                ],
                [
                  0.8007312614259597,
                  0.06621004566210045,
                  438,
                  29
                ],
                [
                  0.8025594149908593,
                  0.06605922551252848,
                  439,
                  29
                ],
                [
                  0.8043875685557587,
                  0.06818181818181818,
                  440,
                  30
                ],
                [
                  0.8062157221206582,
                  0.07029478458049887,
                  441,
                  31
                ],
                [
                  0.8080438756855576,
                  0.07013574660633484,
                  442,
                  31
                ],
                [
                  0.8098720292504571,
                  0.06997742663656885,
                  443,
                  31
                ],
                [
                  0.8117001828153565,
                  0.06981981981981981,
                  444,
                  31
                ],
                [
                  0.8135283363802559,
                  0.0696629213483146,
                  445,
                  31
                ],
                [
                  0.8153564899451554,
                  0.07174887892376682,
                  446,
                  32
                ],
                [
                  0.8171846435100548,
                  0.0738255033557047,
                  447,
                  33
                ],
                [
                  0.8190127970749543,
                  0.07366071428571429,
                  448,
                  33
                ],
                [
                  0.8208409506398537,
                  0.07349665924276169,
                  449,
                  33
                ],
                [
                  0.8226691042047533,
                  0.07333333333333333,
                  450,
                  33
                ],
                [
                  0.8244972577696527,
                  0.07538802660753881,
                  451,
                  34
                ],
                [
                  0.8263254113345521,
                  0.0752212389380531,
                  452,
                  34
                ],
                [
                  0.8281535648994516,
                  0.07505518763796909,
                  453,
                  34
                ],
                [
                  0.829981718464351,
                  0.07488986784140969,
                  454,
                  34
                ],
                [
                  0.8318098720292505,
                  0.07472527472527472,
                  455,
                  34
                ],
                [
                  0.8336380255941499,
                  0.07675438596491228,
                  456,
                  35
                ],
                [
                  0.8354661791590493,
                  0.0787746170678337,
                  457,
                  36
                ],
                [
                  0.8372943327239488,
                  0.07860262008733625,
                  458,
                  36
                ],
                [
                  0.8391224862888482,
                  0.0784313725490196,
                  459,
                  36
                ],
                [
                  0.8409506398537477,
                  0.08043478260869565,
                  460,
                  37
                ],
                [
                  0.8427787934186471,
                  0.08026030368763558,
                  461,
                  37
                ],
                [
                  0.8446069469835467,
                  0.08225108225108226,
                  462,
                  38
                ],
                [
                  0.8464351005484461,
                  0.08207343412526998,
                  463,
                  38
                ],
                [
                  0.8482632541133455,
                  0.08405172413793104,
                  464,
                  39
                ],
                [
                  0.850091407678245,
                  0.08602150537634409,
                  465,
                  40
                ],
                [
                  0.8519195612431444,
                  0.08583690987124463,
                  466,
                  40
                ],
                [
                  0.8537477148080439,
                  0.08779443254817987,
                  467,
                  41
                ],
                [
                  0.8555758683729433,
                  0.0876068376068376,
                  468,
                  41
                ],
                [
                  0.8574040219378428,
                  0.08955223880597014,
                  469,
                  42
                ],
                [
                  0.8592321755027422,
                  0.08936170212765958,
                  470,
                  42
                ],
                [
                  0.8610603290676416,
                  0.09129511677282377,
                  471,
                  43
                ],
                [
                  0.8628884826325411,
                  0.09322033898305085,
                  472,
                  44
                ],
                [
                  0.8647166361974405,
                  0.09513742071881606,
                  473,
                  45
                ],
                [
                  0.8665447897623401,
                  0.0949367088607595,
                  474,
                  45
                ],
                [
                  0.8683729433272395,
                  0.0968421052631579,
                  475,
                  46
                ],
                [
                  0.870201096892139,
                  0.09663865546218488,
                  476,
                  46
                ],
                [
                  0.8720292504570384,
                  0.09853249475890985,
                  477,
                  47
                ],
                [
                  0.8738574040219378,
                  0.09832635983263599,
                  478,
                  47
                ],
                [
                  0.8756855575868373,
                  0.09812108559498957,
                  479,
                  47
                ],
                [
                  0.8775137111517367,
                  0.09791666666666667,
                  480,
                  47
                ],
                [
                  0.8793418647166362,
                  0.0997920997920998,
                  481,
                  48
                ],
                [
                  0.8811700182815356,
                  0.0995850622406639,
                  482,
                  48
                ],
                [
                  0.8829981718464351,
                  0.09937888198757763,
                  483,
                  48
                ],
                [
                  0.8848263254113345,
                  0.09917355371900827,
                  484,
                  48
                ],
                [
                  0.886654478976234,
                  0.09896907216494845,
                  485,
                  48
                ],
                [
                  0.8884826325411335,
                  0.09876543209876543,
                  486,
                  48
                ],
                [
                  0.8903107861060329,
                  0.09856262833675565,
                  487,
                  48
                ],
                [
                  0.8921389396709324,
                  0.09836065573770492,
                  488,
                  48
                ],
                [
                  0.8939670932358318,
                  0.10020449897750511,
                  489,
                  49
                ],
                [
                  0.8957952468007313,
                  0.1,
                  490,
                  49
                ],
                [
                  0.8976234003656307,
                  0.10183299389002037,
                  491,
                  50
                ],
                [
                  0.8994515539305301,
                  0.10365853658536585,
                  492,
                  51
                ],
                [
                  0.9012797074954296,
                  0.10344827586206896,
                  493,
                  51
                ],
                [
                  0.903107861060329,
                  0.10323886639676114,
                  494,
                  51
                ],
                [
                  0.9049360146252285,
                  0.10505050505050505,
                  495,
                  52
                ],
                [
                  0.906764168190128,
                  0.10483870967741936,
                  496,
                  52
                ],
                [
                  0.9085923217550275,
                  0.10663983903420524,
                  497,
                  53
                ],
                [
                  0.9104204753199269,
                  0.10843373493975904,
                  498,
                  54
                ],
                [
                  0.9122486288848263,
                  0.10821643286573146,
                  499,
                  54
                ],
                [
                  0.9140767824497258,
                  0.108,
                  500,
                  54
                ],
                [
                  0.9159049360146252,
                  0.10978043912175649,
                  501,
                  55
                ],
                [
                  0.9177330895795247,
                  0.10956175298804781,
                  502,
                  55
                ],
                [
                  0.9195612431444241,
                  0.10934393638170974,
                  503,
                  55
                ],
                [
                  0.9213893967093236,
                  0.10912698412698413,
                  504,
                  55
                ],
                [
                  0.923217550274223,
                  0.10891089108910891,
                  505,
                  55
                ],
                [
                  0.9250457038391224,
                  0.11067193675889328,
                  506,
                  56
                ],
                [
                  0.926873857404022,
                  0.11045364891518737,
                  507,
                  56
                ],
                [
                  0.9287020109689214,
                  0.11220472440944881,
                  508,
                  57
                ],
                [
                  0.9305301645338209,
                  0.11394891944990176,
                  509,
                  58
                ],
                [
                  0.9323583180987203,
                  0.11568627450980393,
                  510,
                  59
                ],
                [
                  0.9341864716636198,
                  0.11545988258317025,
                  511,
                  59
                ],
                [
                  0.9360146252285192,
                  0.115234375,
                  512,
                  59
                ],
                [
                  0.9378427787934186,
                  0.11500974658869395,
                  513,
                  59
                ],
                [
                  0.9396709323583181,
                  0.11478599221789883,
                  514,
                  59
                ],
                [
                  0.9414990859232175,
                  0.1145631067961165,
                  515,
                  59
                ],
                [
                  0.943327239488117,
                  0.11434108527131782,
                  516,
                  59
                ],
                [
                  0.9451553930530164,
                  0.11605415860735009,
                  517,
                  60
                ],
                [
                  0.946983546617916,
                  0.11776061776061776,
                  518,
                  61
                ],
                [
                  0.9488117001828154,
                  0.11946050096339114,
                  519,
                  62
                ],
                [
                  0.9506398537477148,
                  0.12115384615384615,
                  520,
                  63
                ],
                [
                  0.9524680073126143,
                  0.12284069097888675,
                  521,
                  64
                ],
                [
                  0.9542961608775137,
                  0.12452107279693486,
                  522,
                  65
                ],
                [
                  0.9561243144424132,
                  0.12619502868068833,
                  523,
                  66
                ],
                [
                  0.9579524680073126,
                  0.12786259541984732,
                  524,
                  67
                ],
                [
                  0.9597806215722121,
                  0.12761904761904763,
                  525,
                  67
                ],
                [
                  0.9616087751371115,
                  0.12737642585551331,
                  526,
                  67
                ],
                [
                  0.9634369287020109,
                  0.12903225806451613,
                  527,
                  68
                ],
                [
                  0.9652650822669104,
                  0.12878787878787878,
                  528,
                  68
                ],
                [
                  0.9670932358318098,
                  0.1285444234404537,
                  529,
                  68
                ],
                [
                  0.9689213893967094,
                  0.13018867924528302,
                  530,
                  69
                ],
                [
                  0.9707495429616088,
                  0.1318267419962335,
                  531,
                  70
                ],
                [
                  0.9725776965265083,
                  0.13157894736842105,
                  532,
                  70
                ],
                [
                  0.9744058500914077,
                  0.13133208255159476,
                  533,
                  70
                ],
                [
                  0.9762340036563071,
                  0.13295880149812733,
                  534,
                  71
                ],
                [
                  0.9780621572212066,
                  0.13457943925233645,
                  535,
                  72
                ],
                [
                  0.979890310786106,
                  0.13432835820895522,
                  536,
                  72
                ],
                [
                  0.9817184643510055,
                  0.1340782122905028,
                  537,
                  72
                ],
                [
                  0.9835466179159049,
                  0.13382899628252787,
                  538,
                  72
                ],
                [
                  0.9853747714808044,
                  0.13358070500927643,
                  539,
                  72
                ],
                [
                  0.9872029250457038,
                  0.13333333333333333,
                  540,
                  72
                ],
                [
                  0.9890310786106032,
                  0.13493530499075784,
                  541,
                  73
                ],
                [
                  0.9908592321755028,
                  0.13653136531365315,
                  542,
                  74
                ],
                [
                  0.9926873857404022,
                  0.13812154696132597,
                  543,
                  75
                ],
                [
                  0.9945155393053017,
                  0.13970588235294118,
                  544,
                  76
                ],
                [
                  0.9963436928702011,
                  0.14128440366972478,
                  545,
                  77
                ],
                [
                  0.9981718464351006,
                  0.14102564102564102,
                  546,
                  77
                ],
                [
                  1.0,
                  0.1425959780621572,
                  547,
                  78
                ]
              ]
            }
          ]
        },
        {
          "name": "Elephant-1",
          "text_run": 373,
          "native": [
            {
              "language": "Turkish",
              "wording": "Bilmiyorum",
              "run": 382,
              "full_bank_restricted": true,
              "right": 450,
              "wrong": 82,
              "abstained": 15,
              "invalid": 0,
              "coverage": 0.9725776965265083,
              "risk": 0.15413533834586465
            },
            {
              "language": "Turkish",
              "wording": "I don't know",
              "run": 515,
              "full_bank_restricted": true,
              "right": 441,
              "wrong": 82,
              "abstained": 24,
              "invalid": 0,
              "coverage": 0.9561243144424132,
              "risk": 0.15678776290630975
            },
            {
              "language": "English",
              "wording": "I don't know",
              "run": 501,
              "full_bank_restricted": false,
              "right": 457,
              "wrong": 67,
              "abstained": 23,
              "invalid": 0,
              "coverage": 0.9579524680073126,
              "risk": 0.12786259541984732
            }
          ],
          "curves": [
            {
              "language": "Turkish",
              "run": 373,
              "points": [
                [
                  0.0018281535648994515,
                  0.0,
                  1,
                  0
                ],
                [
                  0.003656307129798903,
                  0.0,
                  2,
                  0
                ],
                [
                  0.005484460694698354,
                  0.0,
                  3,
                  0
                ],
                [
                  0.007312614259597806,
                  0.0,
                  4,
                  0
                ],
                [
                  0.010968921389396709,
                  0.0,
                  6,
                  0
                ],
                [
                  0.012797074954296161,
                  0.0,
                  7,
                  0
                ],
                [
                  0.014625228519195612,
                  0.0,
                  8,
                  0
                ],
                [
                  0.016453382084095063,
                  0.0,
                  9,
                  0
                ],
                [
                  0.018281535648994516,
                  0.0,
                  10,
                  0
                ],
                [
                  0.02010968921389397,
                  0.0,
                  11,
                  0
                ],
                [
                  0.021937842778793418,
                  0.0,
                  12,
                  0
                ],
                [
                  0.02376599634369287,
                  0.0,
                  13,
                  0
                ],
                [
                  0.025594149908592323,
                  0.0,
                  14,
                  0
                ],
                [
                  0.027422303473491772,
                  0.0,
                  15,
                  0
                ],
                [
                  0.029250457038391225,
                  0.0,
                  16,
                  0
                ],
                [
                  0.031078610603290677,
                  0.0,
                  17,
                  0
                ],
                [
                  0.03290676416819013,
                  0.0,
                  18,
                  0
                ],
                [
                  0.03473491773308958,
                  0.0,
                  19,
                  0
                ],
                [
                  0.03656307129798903,
                  0.0,
                  20,
                  0
                ],
                [
                  0.038391224862888484,
                  0.0,
                  21,
                  0
                ],
                [
                  0.04021937842778794,
                  0.0,
                  22,
                  0
                ],
                [
                  0.04204753199268738,
                  0.0,
                  23,
                  0
                ],
                [
                  0.043875685557586835,
                  0.0,
                  24,
                  0
                ],
                [
                  0.04570383912248629,
                  0.0,
                  25,
                  0
                ],
                [
                  0.04753199268738574,
                  0.0,
                  26,
                  0
                ],
                [
                  0.04936014625228519,
                  0.0,
                  27,
                  0
                ],
                [
                  0.051188299817184646,
                  0.0,
                  28,
                  0
                ],
                [
                  0.05301645338208409,
                  0.0,
                  29,
                  0
                ],
                [
                  0.054844606946983544,
                  0.0,
                  30,
                  0
                ],
                [
                  0.056672760511883,
                  0.0,
                  31,
                  0
                ],
                [
                  0.05850091407678245,
                  0.0,
                  32,
                  0
                ],
                [
                  0.0603290676416819,
                  0.0,
                  33,
                  0
                ],
                [
                  0.062157221206581355,
                  0.0,
                  34,
                  0
                ],
                [
                  0.06398537477148081,
                  0.0,
                  35,
                  0
                ],
                [
                  0.06581352833638025,
                  0.0,
                  36,
                  0
                ],
                [
                  0.06764168190127971,
                  0.0,
                  37,
                  0
                ],
                [
                  0.06946983546617916,
                  0.0,
                  38,
                  0
                ],
                [
                  0.0712979890310786,
                  0.0,
                  39,
                  0
                ],
                [
                  0.07312614259597806,
                  0.0,
                  40,
                  0
                ],
                [
                  0.07495429616087751,
                  0.0,
                  41,
                  0
                ],
                [
                  0.07678244972577697,
                  0.0,
                  42,
                  0
                ],
                [
                  0.07861060329067641,
                  0.0,
                  43,
                  0
                ],
                [
                  0.08043875685557587,
                  0.0,
                  44,
                  0
                ],
                [
                  0.08226691042047532,
                  0.0,
                  45,
                  0
                ],
                [
                  0.08409506398537477,
                  0.0,
                  46,
                  0
                ],
                [
                  0.08592321755027423,
                  0.0,
                  47,
                  0
                ],
                [
                  0.08775137111517367,
                  0.0,
                  48,
                  0
                ],
                [
                  0.08957952468007313,
                  0.0,
                  49,
                  0
                ],
                [
                  0.09140767824497258,
                  0.0,
                  50,
                  0
                ],
                [
                  0.09323583180987204,
                  0.0,
                  51,
                  0
                ],
                [
                  0.09506398537477148,
                  0.0,
                  52,
                  0
                ],
                [
                  0.09689213893967093,
                  0.0,
                  53,
                  0
                ],
                [
                  0.09872029250457039,
                  0.0,
                  54,
                  0
                ],
                [
                  0.10054844606946983,
                  0.0,
                  55,
                  0
                ],
                [
                  0.10237659963436929,
                  0.0,
                  56,
                  0
                ],
                [
                  0.10420475319926874,
                  0.0,
                  57,
                  0
                ],
                [
                  0.10603290676416818,
                  0.0,
                  58,
                  0
                ],
                [
                  0.10786106032906764,
                  0.0,
                  59,
                  0
                ],
                [
                  0.10968921389396709,
                  0.0,
                  60,
                  0
                ],
                [
                  0.11151736745886655,
                  0.0,
                  61,
                  0
                ],
                [
                  0.113345521023766,
                  0.0,
                  62,
                  0
                ],
                [
                  0.11517367458866545,
                  0.0,
                  63,
                  0
                ],
                [
                  0.1170018281535649,
                  0.0,
                  64,
                  0
                ],
                [
                  0.11882998171846434,
                  0.0,
                  65,
                  0
                ],
                [
                  0.1206581352833638,
                  0.0,
                  66,
                  0
                ],
                [
                  0.12248628884826325,
                  0.0,
                  67,
                  0
                ],
                [
                  0.12431444241316271,
                  0.0,
                  68,
                  0
                ],
                [
                  0.12614259597806216,
                  0.0,
                  69,
                  0
                ],
                [
                  0.12797074954296161,
                  0.0,
                  70,
                  0
                ],
                [
                  0.12979890310786105,
                  0.0,
                  71,
                  0
                ],
                [
                  0.1316270566727605,
                  0.0,
                  72,
                  0
                ],
                [
                  0.13345521023765997,
                  0.0,
                  73,
                  0
                ],
                [
                  0.13528336380255943,
                  0.0,
                  74,
                  0
                ],
                [
                  0.13711151736745886,
                  0.0,
                  75,
                  0
                ],
                [
                  0.13893967093235832,
                  0.0,
                  76,
                  0
                ],
                [
                  0.14076782449725778,
                  0.0,
                  77,
                  0
                ],
                [
                  0.1425959780621572,
                  0.0,
                  78,
                  0
                ],
                [
                  0.14442413162705667,
                  0.0,
                  79,
                  0
                ],
                [
                  0.14625228519195613,
                  0.0,
                  80,
                  0
                ],
                [
                  0.1480804387568556,
                  0.0,
                  81,
                  0
                ],
                [
                  0.14990859232175502,
                  0.0,
                  82,
                  0
                ],
                [
                  0.15173674588665448,
                  0.0,
                  83,
                  0
                ],
                [
                  0.15356489945155394,
                  0.0,
                  84,
                  0
                ],
                [
                  0.15539305301645337,
                  0.0,
                  85,
                  0
                ],
                [
                  0.15722120658135283,
                  0.0,
                  86,
                  0
                ],
                [
                  0.1590493601462523,
                  0.0,
                  87,
                  0
                ],
                [
                  0.16087751371115175,
                  0.0,
                  88,
                  0
                ],
                [
                  0.16270566727605118,
                  0.0,
                  89,
                  0
                ],
                [
                  0.16453382084095064,
                  0.0,
                  90,
                  0
                ],
                [
                  0.1663619744058501,
                  0.0,
                  91,
                  0
                ],
                [
                  0.16819012797074953,
                  0.0,
                  92,
                  0
                ],
                [
                  0.170018281535649,
                  0.0,
                  93,
                  0
                ],
                [
                  0.17184643510054845,
                  0.0,
                  94,
                  0
                ],
                [
                  0.1736745886654479,
                  0.0,
                  95,
                  0
                ],
                [
                  0.17550274223034734,
                  0.0,
                  96,
                  0
                ],
                [
                  0.1773308957952468,
                  0.0,
                  97,
                  0
                ],
                [
                  0.17915904936014626,
                  0.0,
                  98,
                  0
                ],
                [
                  0.1809872029250457,
                  0.0,
                  99,
                  0
                ],
                [
                  0.18281535648994515,
                  0.0,
                  100,
                  0
                ],
                [
                  0.1846435100548446,
                  0.0,
                  101,
                  0
                ],
                [
                  0.18647166361974407,
                  0.0,
                  102,
                  0
                ],
                [
                  0.1882998171846435,
                  0.0,
                  103,
                  0
                ],
                [
                  0.19012797074954296,
                  0.0,
                  104,
                  0
                ],
                [
                  0.19195612431444242,
                  0.0,
                  105,
                  0
                ],
                [
                  0.19378427787934185,
                  0.0,
                  106,
                  0
                ],
                [
                  0.1956124314442413,
                  0.0,
                  107,
                  0
                ],
                [
                  0.19744058500914077,
                  0.0,
                  108,
                  0
                ],
                [
                  0.19926873857404023,
                  0.0,
                  109,
                  0
                ],
                [
                  0.20109689213893966,
                  0.0,
                  110,
                  0
                ],
                [
                  0.20292504570383912,
                  0.0,
                  111,
                  0
                ],
                [
                  0.20475319926873858,
                  0.0,
                  112,
                  0
                ],
                [
                  0.20658135283363802,
                  0.0,
                  113,
                  0
                ],
                [
                  0.20840950639853748,
                  0.0,
                  114,
                  0
                ],
                [
                  0.21023765996343693,
                  0.0,
                  115,
                  0
                ],
                [
                  0.21206581352833637,
                  0.0,
                  116,
                  0
                ],
                [
                  0.21389396709323583,
                  0.0,
                  117,
                  0
                ],
                [
                  0.21572212065813529,
                  0.0,
                  118,
                  0
                ],
                [
                  0.21937842778793418,
                  0.0,
                  120,
                  0
                ],
                [
                  0.22120658135283364,
                  0.0,
                  121,
                  0
                ],
                [
                  0.2230347349177331,
                  0.0,
                  122,
                  0
                ],
                [
                  0.22486288848263253,
                  0.0,
                  123,
                  0
                ],
                [
                  0.226691042047532,
                  0.0,
                  124,
                  0
                ],
                [
                  0.22851919561243145,
                  0.0,
                  125,
                  0
                ],
                [
                  0.2303473491773309,
                  0.0,
                  126,
                  0
                ],
                [
                  0.23217550274223034,
                  0.0,
                  127,
                  0
                ],
                [
                  0.2340036563071298,
                  0.0,
                  128,
                  0
                ],
                [
                  0.23583180987202926,
                  0.0,
                  129,
                  0
                ],
                [
                  0.2376599634369287,
                  0.0,
                  130,
                  0
                ],
                [
                  0.23948811700182815,
                  0.0,
                  131,
                  0
                ],
                [
                  0.2413162705667276,
                  0.0,
                  132,
                  0
                ],
                [
                  0.24314442413162707,
                  0.0,
                  133,
                  0
                ],
                [
                  0.2449725776965265,
                  0.0,
                  134,
                  0
                ],
                [
                  0.24680073126142596,
                  0.0,
                  135,
                  0
                ],
                [
                  0.24862888482632542,
                  0.0,
                  136,
                  0
                ],
                [
                  0.25045703839122485,
                  0.0,
                  137,
                  0
                ],
                [
                  0.2522851919561243,
                  0.0,
                  138,
                  0
                ],
                [
                  0.25411334552102377,
                  0.0,
                  139,
                  0
                ],
                [
                  0.25594149908592323,
                  0.0,
                  140,
                  0
                ],
                [
                  0.2577696526508227,
                  0.0,
                  141,
                  0
                ],
                [
                  0.2595978062157221,
                  0.0,
                  142,
                  0
                ],
                [
                  0.26142595978062155,
                  0.0,
                  143,
                  0
                ],
                [
                  0.263254113345521,
                  0.0,
                  144,
                  0
                ],
                [
                  0.26508226691042047,
                  0.0,
                  145,
                  0
                ],
                [
                  0.26691042047531993,
                  0.0,
                  146,
                  0
                ],
                [
                  0.2687385740402194,
                  0.0,
                  147,
                  0
                ],
                [
                  0.27056672760511885,
                  0.0,
                  148,
                  0
                ],
                [
                  0.27239488117001825,
                  0.0,
                  149,
                  0
                ],
                [
                  0.2742230347349177,
                  0.0,
                  150,
                  0
                ],
                [
                  0.2760511882998172,
                  0.0,
                  151,
                  0
                ],
                [
                  0.27787934186471663,
                  0.0,
                  152,
                  0
                ],
                [
                  0.2797074954296161,
                  0.0,
                  153,
                  0
                ],
                [
                  0.28153564899451555,
                  0.0,
                  154,
                  0
                ],
                [
                  0.283363802559415,
                  0.0,
                  155,
                  0
                ],
                [
                  0.2851919561243144,
                  0.0,
                  156,
                  0
                ],
                [
                  0.2870201096892139,
                  0.0,
                  157,
                  0
                ],
                [
                  0.28884826325411334,
                  0.0,
                  158,
                  0
                ],
                [
                  0.2906764168190128,
                  0.0,
                  159,
                  0
                ],
                [
                  0.29250457038391225,
                  0.0,
                  160,
                  0
                ],
                [
                  0.2943327239488117,
                  0.0,
                  161,
                  0
                ],
                [
                  0.2961608775137112,
                  0.0,
                  162,
                  0
                ],
                [
                  0.2979890310786106,
                  0.0,
                  163,
                  0
                ],
                [
                  0.29981718464351004,
                  0.0,
                  164,
                  0
                ],
                [
                  0.3016453382084095,
                  0.0,
                  165,
                  0
                ],
                [
                  0.30347349177330896,
                  0.0,
                  166,
                  0
                ],
                [
                  0.3053016453382084,
                  0.0,
                  167,
                  0
                ],
                [
                  0.3071297989031079,
                  0.0,
                  168,
                  0
                ],
                [
                  0.30895795246800734,
                  0.0,
                  169,
                  0
                ],
                [
                  0.31078610603290674,
                  0.0,
                  170,
                  0
                ],
                [
                  0.3126142595978062,
                  0.0,
                  171,
                  0
                ],
                [
                  0.31444241316270566,
                  0.0,
                  172,
                  0
                ],
                [
                  0.3162705667276051,
                  0.0,
                  173,
                  0
                ],
                [
                  0.3180987202925046,
                  0.0,
                  174,
                  0
                ],
                [
                  0.31992687385740404,
                  0.0,
                  175,
                  0
                ],
                [
                  0.3217550274223035,
                  0.0,
                  176,
                  0
                ],
                [
                  0.3235831809872029,
                  0.0,
                  177,
                  0
                ],
                [
                  0.32541133455210236,
                  0.0,
                  178,
                  0
                ],
                [
                  0.3272394881170018,
                  0.0,
                  179,
                  0
                ],
                [
                  0.3290676416819013,
                  0.0,
                  180,
                  0
                ],
                [
                  0.33089579524680074,
                  0.0,
                  181,
                  0
                ],
                [
                  0.3327239488117002,
                  0.0,
                  182,
                  0
                ],
                [
                  0.33455210237659966,
                  0.0,
                  183,
                  0
                ],
                [
                  0.33638025594149906,
                  0.0,
                  184,
                  0
                ],
                [
                  0.3382084095063985,
                  0.0,
                  185,
                  0
                ],
                [
                  0.340036563071298,
                  0.0,
                  186,
                  0
                ],
                [
                  0.34186471663619744,
                  0.0,
                  187,
                  0
                ],
                [
                  0.3436928702010969,
                  0.0,
                  188,
                  0
                ],
                [
                  0.34552102376599636,
                  0.0,
                  189,
                  0
                ],
                [
                  0.3473491773308958,
                  0.0,
                  190,
                  0
                ],
                [
                  0.3491773308957952,
                  0.0,
                  191,
                  0
                ],
                [
                  0.3510054844606947,
                  0.0,
                  192,
                  0
                ],
                [
                  0.35283363802559414,
                  0.0,
                  193,
                  0
                ],
                [
                  0.3546617915904936,
                  0.0,
                  194,
                  0
                ],
                [
                  0.35648994515539306,
                  0.0,
                  195,
                  0
                ],
                [
                  0.3583180987202925,
                  0.0,
                  196,
                  0
                ],
                [
                  0.360146252285192,
                  0.0,
                  197,
                  0
                ],
                [
                  0.3619744058500914,
                  0.005050505050505051,
                  198,
                  1
                ],
                [
                  0.36380255941499084,
                  0.005025125628140704,
                  199,
                  1
                ],
                [
                  0.3656307129798903,
                  0.005,
                  200,
                  1
                ],
                [
                  0.36745886654478976,
                  0.004975124378109453,
                  201,
                  1
                ],
                [
                  0.3692870201096892,
                  0.0049504950495049506,
                  202,
                  1
                ],
                [
                  0.3711151736745887,
                  0.0049261083743842365,
                  203,
                  1
                ],
                [
                  0.37294332723948814,
                  0.004901960784313725,
                  204,
                  1
                ],
                [
                  0.37477148080438755,
                  0.004878048780487805,
                  205,
                  1
                ],
                [
                  0.376599634369287,
                  0.0048543689320388345,
                  206,
                  1
                ],
                [
                  0.37842778793418647,
                  0.004830917874396135,
                  207,
                  1
                ],
                [
                  0.3802559414990859,
                  0.004807692307692308,
                  208,
                  1
                ],
                [
                  0.3820840950639854,
                  0.004784688995215311,
                  209,
                  1
                ],
                [
                  0.38391224862888484,
                  0.004761904761904762,
                  210,
                  1
                ],
                [
                  0.3857404021937843,
                  0.004739336492890996,
                  211,
                  1
                ],
                [
                  0.3875685557586837,
                  0.0047169811320754715,
                  212,
                  1
                ],
                [
                  0.38939670932358317,
                  0.004694835680751174,
                  213,
                  1
                ],
                [
                  0.3912248628884826,
                  0.004672897196261682,
                  214,
                  1
                ],
                [
                  0.3930530164533821,
                  0.004651162790697674,
                  215,
                  1
                ],
                [
                  0.39488117001828155,
                  0.004629629629629629,
                  216,
                  1
                ],
                [
                  0.396709323583181,
                  0.004608294930875576,
                  217,
                  1
                ],
                [
                  0.39853747714808047,
                  0.0045871559633027525,
                  218,
                  1
                ],
                [
                  0.40036563071297987,
                  0.0045662100456621,
                  219,
                  1
                ],
                [
                  0.40219378427787933,
                  0.004545454545454545,
                  220,
                  1
                ],
                [
                  0.4040219378427788,
                  0.004524886877828055,
                  221,
                  1
                ],
                [
                  0.40585009140767825,
                  0.0045045045045045045,
                  222,
                  1
                ],
                [
                  0.4076782449725777,
                  0.004484304932735426,
                  223,
                  1
                ],
                [
                  0.40950639853747717,
                  0.004464285714285714,
                  224,
                  1
                ],
                [
                  0.4113345521023766,
                  0.0044444444444444444,
                  225,
                  1
                ],
                [
                  0.41316270566727603,
                  0.004424778761061947,
                  226,
                  1
                ],
                [
                  0.4149908592321755,
                  0.004405286343612335,
                  227,
                  1
                ],
                [
                  0.41681901279707495,
                  0.0043859649122807015,
                  228,
                  1
                ],
                [
                  0.4186471663619744,
                  0.004366812227074236,
                  229,
                  1
                ],
                [
                  0.42047531992687387,
                  0.004347826086956522,
                  230,
                  1
                ],
                [
                  0.42230347349177333,
                  0.004329004329004329,
                  231,
                  1
                ],
                [
                  0.42413162705667273,
                  0.004310344827586207,
                  232,
                  1
                ],
                [
                  0.4259597806215722,
                  0.004291845493562232,
                  233,
                  1
                ],
                [
                  0.42778793418647165,
                  0.004273504273504274,
                  234,
                  1
                ],
                [
                  0.4296160877513711,
                  0.00425531914893617,
                  235,
                  1
                ],
                [
                  0.43144424131627057,
                  0.00423728813559322,
                  236,
                  1
                ],
                [
                  0.43327239488117003,
                  0.004219409282700422,
                  237,
                  1
                ],
                [
                  0.4351005484460695,
                  0.004201680672268907,
                  238,
                  1
                ],
                [
                  0.4369287020109689,
                  0.0041841004184100415,
                  239,
                  1
                ],
                [
                  0.43875685557586835,
                  0.004166666666666667,
                  240,
                  1
                ],
                [
                  0.4405850091407678,
                  0.004149377593360996,
                  241,
                  1
                ],
                [
                  0.4424131627056673,
                  0.004132231404958678,
                  242,
                  1
                ],
                [
                  0.44424131627056673,
                  0.00411522633744856,
                  243,
                  1
                ],
                [
                  0.4460694698354662,
                  0.004098360655737705,
                  244,
                  1
                ],
                [
                  0.44789762340036565,
                  0.004081632653061225,
                  245,
                  1
                ],
                [
                  0.44972577696526506,
                  0.0040650406504065045,
                  246,
                  1
                ],
                [
                  0.4515539305301645,
                  0.004048582995951417,
                  247,
                  1
                ],
                [
                  0.453382084095064,
                  0.004032258064516129,
                  248,
                  1
                ],
                [
                  0.45521023765996343,
                  0.004016064257028112,
                  249,
                  1
                ],
                [
                  0.4570383912248629,
                  0.004,
                  250,
                  1
                ],
                [
                  0.45886654478976235,
                  0.00398406374501992,
                  251,
                  1
                ],
                [
                  0.4606946983546618,
                  0.003968253968253968,
                  252,
                  1
                ],
                [
                  0.4625228519195612,
                  0.003952569169960474,
                  253,
                  1
                ],
                [
                  0.4643510054844607,
                  0.003937007874015748,
                  254,
                  1
                ],
                [
                  0.46617915904936014,
                  0.00392156862745098,
                  255,
                  1
                ],
                [
                  0.4680073126142596,
                  0.00390625,
                  256,
                  1
                ],
                [
                  0.46983546617915906,
                  0.0038910505836575876,
                  257,
                  1
                ],
                [
                  0.4716636197440585,
                  0.003875968992248062,
                  258,
                  1
                ],
                [
                  0.473491773308958,
                  0.003861003861003861,
                  259,
                  1
                ],
                [
                  0.4753199268738574,
                  0.0038461538461538464,
                  260,
                  1
                ],
                [
                  0.47714808043875684,
                  0.0038314176245210726,
                  261,
                  1
                ],
                [
                  0.4789762340036563,
                  0.003816793893129771,
                  262,
                  1
                ],
                [
                  0.48080438756855576,
                  0.0038022813688212928,
                  263,
                  1
                ],
                [
                  0.4826325411334552,
                  0.003787878787878788,
                  264,
                  1
                ],
                [
                  0.4844606946983547,
                  0.0037735849056603774,
                  265,
                  1
                ],
                [
                  0.48628884826325414,
                  0.0037593984962406013,
                  266,
                  1
                ],
                [
                  0.48811700182815354,
                  0.003745318352059925,
                  267,
                  1
                ],
                [
                  0.489945155393053,
                  0.0037313432835820895,
                  268,
                  1
                ],
                [
                  0.49177330895795246,
                  0.007434944237918215,
                  269,
                  2
                ],
                [
                  0.4936014625228519,
                  0.007407407407407408,
                  270,
                  2
                ],
                [
                  0.4954296160877514,
                  0.007380073800738007,
                  271,
                  2
                ],
                [
                  0.49725776965265084,
                  0.007352941176470588,
                  272,
                  2
                ],
                [
                  0.4990859232175503,
                  0.007326007326007326,
                  273,
                  2
                ],
                [
                  0.5009140767824497,
                  0.010948905109489052,
                  274,
                  3
                ],
                [
                  0.5027422303473492,
                  0.01090909090909091,
                  275,
                  3
                ],
                [
                  0.5045703839122486,
                  0.010869565217391304,
                  276,
                  3
                ],
                [
                  0.506398537477148,
                  0.010830324909747292,
                  277,
                  3
                ],
                [
                  0.5082266910420475,
                  0.01079136690647482,
                  278,
                  3
                ],
                [
                  0.5100548446069469,
                  0.010752688172043012,
                  279,
                  3
                ],
                [
                  0.5118829981718465,
                  0.010714285714285714,
                  280,
                  3
                ],
                [
                  0.5137111517367459,
                  0.010676156583629894,
                  281,
                  3
                ],
                [
                  0.5155393053016454,
                  0.010638297872340425,
                  282,
                  3
                ],
                [
                  0.5173674588665448,
                  0.01060070671378092,
                  283,
                  3
                ],
                [
                  0.5191956124314442,
                  0.01056338028169014,
                  284,
                  3
                ],
                [
                  0.5210237659963437,
                  0.010526315789473684,
                  285,
                  3
                ],
                [
                  0.5228519195612431,
                  0.01048951048951049,
                  286,
                  3
                ],
                [
                  0.5246800731261426,
                  0.010452961672473868,
                  287,
                  3
                ],
                [
                  0.526508226691042,
                  0.010416666666666666,
                  288,
                  3
                ],
                [
                  0.5283363802559415,
                  0.01384083044982699,
                  289,
                  4
                ],
                [
                  0.5301645338208409,
                  0.013793103448275862,
                  290,
                  4
                ],
                [
                  0.5319926873857403,
                  0.013745704467353952,
                  291,
                  4
                ],
                [
                  0.5338208409506399,
                  0.0136986301369863,
                  292,
                  4
                ],
                [
                  0.5356489945155393,
                  0.013651877133105802,
                  293,
                  4
                ],
                [
                  0.5374771480804388,
                  0.013605442176870748,
                  294,
                  4
                ],
                [
                  0.5393053016453382,
                  0.013559322033898305,
                  295,
                  4
                ],
                [
                  0.5411334552102377,
                  0.013513513513513514,
                  296,
                  4
                ],
                [
                  0.5429616087751371,
                  0.013468013468013467,
                  297,
                  4
                ],
                [
                  0.5447897623400365,
                  0.013422818791946308,
                  298,
                  4
                ],
                [
                  0.546617915904936,
                  0.013377926421404682,
                  299,
                  4
                ],
                [
                  0.5484460694698354,
                  0.013333333333333334,
                  300,
                  4
                ],
                [
                  0.5502742230347349,
                  0.013289036544850499,
                  301,
                  4
                ],
                [
                  0.5521023765996343,
                  0.013245033112582781,
                  302,
                  4
                ],
                [
                  0.5539305301645339,
                  0.013201320132013201,
                  303,
                  4
                ],
                [
                  0.5557586837294333,
                  0.013157894736842105,
                  304,
                  4
                ],
                [
                  0.5575868372943327,
                  0.013114754098360656,
                  305,
                  4
                ],
                [
                  0.5594149908592322,
                  0.013071895424836602,
                  306,
                  4
                ],
                [
                  0.5612431444241316,
                  0.016286644951140065,
                  307,
                  5
                ],
                [
                  0.5630712979890311,
                  0.016233766233766232,
                  308,
                  5
                ],
                [
                  0.5648994515539305,
                  0.016181229773462782,
                  309,
                  5
                ],
                [
                  0.56672760511883,
                  0.016129032258064516,
                  310,
                  5
                ],
                [
                  0.5685557586837294,
                  0.01929260450160772,
                  311,
                  6
                ],
                [
                  0.5703839122486288,
                  0.019230769230769232,
                  312,
                  6
                ],
                [
                  0.5722120658135283,
                  0.019169329073482427,
                  313,
                  6
                ],
                [
                  0.5740402193784278,
                  0.01910828025477707,
                  314,
                  6
                ],
                [
                  0.5758683729433273,
                  0.01904761904761905,
                  315,
                  6
                ],
                [
                  0.5776965265082267,
                  0.0189873417721519,
                  316,
                  6
                ],
                [
                  0.5795246800731262,
                  0.01892744479495268,
                  317,
                  6
                ],
                [
                  0.5813528336380256,
                  0.018867924528301886,
                  318,
                  6
                ],
                [
                  0.583180987202925,
                  0.018808777429467086,
                  319,
                  6
                ],
                [
                  0.5850091407678245,
                  0.01875,
                  320,
                  6
                ],
                [
                  0.5868372943327239,
                  0.018691588785046728,
                  321,
                  6
                ],
                [
                  0.5886654478976234,
                  0.018633540372670808,
                  322,
                  6
                ],
                [
                  0.5904936014625228,
                  0.021671826625386997,
                  323,
                  7
                ],
                [
                  0.5923217550274223,
                  0.021604938271604937,
                  324,
                  7
                ],
                [
                  0.5941499085923218,
                  0.021538461538461538,
                  325,
                  7
                ],
                [
                  0.5959780621572212,
                  0.02147239263803681,
                  326,
                  7
                ],
                [
                  0.5978062157221207,
                  0.024464831804281346,
                  327,
                  8
                ],
                [
                  0.5996343692870201,
                  0.024390243902439025,
                  328,
                  8
                ],
                [
                  0.6014625228519196,
                  0.0243161094224924,
                  329,
                  8
                ],
                [
                  0.603290676416819,
                  0.024242424242424242,
                  330,
                  8
                ],
                [
                  0.6051188299817185,
                  0.027190332326283987,
                  331,
                  9
                ],
                [
                  0.6069469835466179,
                  0.02710843373493976,
                  332,
                  9
                ],
                [
                  0.6087751371115173,
                  0.02702702702702703,
                  333,
                  9
                ],
                [
                  0.6106032906764168,
                  0.02694610778443114,
                  334,
                  9
                ],
                [
                  0.6124314442413162,
                  0.029850746268656716,
                  335,
                  10
                ],
                [
                  0.6142595978062158,
                  0.02976190476190476,
                  336,
                  10
                ],
                [
                  0.6160877513711152,
                  0.02967359050445104,
                  337,
                  10
                ],
                [
                  0.6179159049360147,
                  0.029585798816568046,
                  338,
                  10
                ],
                [
                  0.6197440585009141,
                  0.029498525073746312,
                  339,
                  10
                ],
                [
                  0.6215722120658135,
                  0.029411764705882353,
                  340,
                  10
                ],
                [
                  0.623400365630713,
                  0.02932551319648094,
                  341,
                  10
                ],
                [
                  0.6252285191956124,
                  0.029239766081871343,
                  342,
                  10
                ],
                [
                  0.6270566727605119,
                  0.03206997084548105,
                  343,
                  11
                ],
                [
                  0.6288848263254113,
                  0.03488372093023256,
                  344,
                  12
                ],
                [
                  0.6307129798903108,
                  0.034782608695652174,
                  345,
                  12
                ],
                [
                  0.6325411334552102,
                  0.03468208092485549,
                  346,
                  12
                ],
                [
                  0.6343692870201096,
                  0.0345821325648415,
                  347,
                  12
                ],
                [
                  0.6361974405850092,
                  0.034482758620689655,
                  348,
                  12
                ],
                [
                  0.6380255941499086,
                  0.034383954154727794,
                  349,
                  12
                ],
                [
                  0.6398537477148081,
                  0.03428571428571429,
                  350,
                  12
                ],
                [
                  0.6416819012797075,
                  0.037037037037037035,
                  351,
                  13
                ],
                [
                  0.643510054844607,
                  0.036931818181818184,
                  352,
                  13
                ],
                [
                  0.6453382084095064,
                  0.036827195467422094,
                  353,
                  13
                ],
                [
                  0.6471663619744058,
                  0.03672316384180791,
                  354,
                  13
                ],
                [
                  0.6489945155393053,
                  0.036619718309859155,
                  355,
                  13
                ],
                [
                  0.6508226691042047,
                  0.03651685393258427,
                  356,
                  13
                ],
                [
                  0.6526508226691042,
                  0.036414565826330535,
                  357,
                  13
                ],
                [
                  0.6544789762340036,
                  0.036312849162011177,
                  358,
                  13
                ],
                [
                  0.6563071297989032,
                  0.036211699164345405,
                  359,
                  13
                ],
                [
                  0.6581352833638026,
                  0.03611111111111111,
                  360,
                  13
                ],
                [
                  0.659963436928702,
                  0.036011080332409975,
                  361,
                  13
                ],
                [
                  0.6617915904936015,
                  0.03591160220994475,
                  362,
                  13
                ],
                [
                  0.6636197440585009,
                  0.03581267217630854,
                  363,
                  13
                ],
                [
                  0.6654478976234004,
                  0.03571428571428571,
                  364,
                  13
                ],
                [
                  0.6672760511882998,
                  0.03561643835616438,
                  365,
                  13
                ],
                [
                  0.6691042047531993,
                  0.03551912568306011,
                  366,
                  13
                ],
                [
                  0.6709323583180987,
                  0.035422343324250684,
                  367,
                  13
                ],
                [
                  0.6727605118829981,
                  0.035326086956521736,
                  368,
                  13
                ],
                [
                  0.6745886654478976,
                  0.03523035230352303,
                  369,
                  13
                ],
                [
                  0.676416819012797,
                  0.03513513513513514,
                  370,
                  13
                ],
                [
                  0.6782449725776966,
                  0.03504043126684636,
                  371,
                  13
                ],
                [
                  0.680073126142596,
                  0.03763440860215054,
                  372,
                  14
                ],
                [
                  0.6819012797074955,
                  0.040214477211796246,
                  373,
                  15
                ],
                [
                  0.6837294332723949,
                  0.040106951871657755,
                  374,
                  15
                ],
                [
                  0.6855575868372943,
                  0.04,
                  375,
                  15
                ],
                [
                  0.6873857404021938,
                  0.0425531914893617,
                  376,
                  16
                ],
                [
                  0.6892138939670932,
                  0.042440318302387266,
                  377,
                  16
                ],
                [
                  0.6910420475319927,
                  0.042328042328042326,
                  378,
                  16
                ],
                [
                  0.6928702010968921,
                  0.04221635883905013,
                  379,
                  16
                ],
                [
                  0.6946983546617916,
                  0.042105263157894736,
                  380,
                  16
                ],
                [
                  0.696526508226691,
                  0.04199475065616798,
                  381,
                  16
                ],
                [
                  0.6983546617915904,
                  0.041884816753926704,
                  382,
                  16
                ],
                [
                  0.70018281535649,
                  0.04177545691906005,
                  383,
                  16
                ],
                [
                  0.7020109689213894,
                  0.041666666666666664,
                  384,
                  16
                ],
                [
                  0.7038391224862889,
                  0.04155844155844156,
                  385,
                  16
                ],
                [
                  0.7056672760511883,
                  0.04145077720207254,
                  386,
                  16
                ],
                [
                  0.7074954296160878,
                  0.041343669250646,
                  387,
                  16
                ],
                [
                  0.7093235831809872,
                  0.041237113402061855,
                  388,
                  16
                ],
                [
                  0.7111517367458866,
                  0.04113110539845758,
                  389,
                  16
                ],
                [
                  0.7129798903107861,
                  0.041025641025641026,
                  390,
                  16
                ],
                [
                  0.7148080438756855,
                  0.04092071611253197,
                  391,
                  16
                ],
                [
                  0.716636197440585,
                  0.04081632653061224,
                  392,
                  16
                ],
                [
                  0.7184643510054844,
                  0.04071246819338423,
                  393,
                  16
                ],
                [
                  0.720292504570384,
                  0.04314720812182741,
                  394,
                  17
                ],
                [
                  0.7221206581352834,
                  0.043037974683544304,
                  395,
                  17
                ],
                [
                  0.7239488117001828,
                  0.04292929292929293,
                  396,
                  17
                ],
                [
                  0.7257769652650823,
                  0.042821158690176324,
                  397,
                  17
                ],
                [
                  0.7276051188299817,
                  0.04522613065326633,
                  398,
                  18
                ],
                [
                  0.7294332723948812,
                  0.047619047619047616,
                  399,
                  19
                ],
                [
                  0.7312614259597806,
                  0.0475,
                  400,
                  19
                ],
                [
                  0.7330895795246801,
                  0.04987531172069826,
                  401,
                  20
                ],
                [
                  0.7349177330895795,
                  0.04975124378109453,
                  402,
                  20
                ],
                [
                  0.7367458866544789,
                  0.04962779156327544,
                  403,
                  20
                ],
                [
                  0.7385740402193784,
                  0.05198019801980198,
                  404,
                  21
                ],
                [
                  0.7404021937842779,
                  0.05185185185185185,
                  405,
                  21
                ],
                [
                  0.7422303473491774,
                  0.05172413793103448,
                  406,
                  21
                ],
                [
                  0.7440585009140768,
                  0.051597051597051594,
                  407,
                  21
                ],
                [
                  0.7458866544789763,
                  0.051470588235294115,
                  408,
                  21
                ],
                [
                  0.7477148080438757,
                  0.05134474327628362,
                  409,
                  21
                ],
                [
                  0.7495429616087751,
                  0.05121951219512195,
                  410,
                  21
                ],
                [
                  0.7513711151736746,
                  0.051094890510948905,
                  411,
                  21
                ],
                [
                  0.753199268738574,
                  0.05339805825242718,
                  412,
                  22
                ],
                [
                  0.7550274223034735,
                  0.053268765133171914,
                  413,
                  22
                ],
                [
                  0.7568555758683729,
                  0.05314009661835749,
                  414,
                  22
                ],
                [
                  0.7586837294332724,
                  0.05301204819277108,
                  415,
                  22
                ],
                [
                  0.7605118829981719,
                  0.055288461538461536,
                  416,
                  23
                ],
                [
                  0.7623400365630713,
                  0.05755395683453238,
                  417,
                  24
                ],
                [
                  0.7641681901279708,
                  0.05980861244019139,
                  418,
                  25
                ],
                [
                  0.7659963436928702,
                  0.059665871121718374,
                  419,
                  25
                ],
                [
                  0.7678244972577697,
                  0.05952380952380952,
                  420,
                  25
                ],
                [
                  0.7696526508226691,
                  0.06175771971496437,
                  421,
                  26
                ],
                [
                  0.7714808043875686,
                  0.061611374407582936,
                  422,
                  26
                ],
                [
                  0.773308957952468,
                  0.061465721040189124,
                  423,
                  26
                ],
                [
                  0.7751371115173674,
                  0.06132075471698113,
                  424,
                  26
                ],
                [
                  0.7769652650822669,
                  0.06352941176470588,
                  425,
                  27
                ],
                [
                  0.7787934186471663,
                  0.06572769953051644,
                  426,
                  28
                ],
                [
                  0.7806215722120659,
                  0.06791569086651054,
                  427,
                  29
                ],
                [
                  0.7824497257769653,
                  0.06775700934579439,
                  428,
                  29
                ],
                [
                  0.7842778793418648,
                  0.06993006993006994,
                  429,
                  30
                ],
                [
                  0.7861060329067642,
                  0.06976744186046512,
                  430,
                  30
                ],
                [
                  0.7879341864716636,
                  0.06960556844547564,
                  431,
                  30
                ],
                [
                  0.7897623400365631,
                  0.06944444444444445,
                  432,
                  30
                ],
                [
                  0.7915904936014625,
                  0.06928406466512702,
                  433,
                  30
                ],
                [
                  0.793418647166362,
                  0.06912442396313365,
                  434,
                  30
                ],
                [
                  0.7952468007312614,
                  0.06896551724137931,
                  435,
                  30
                ],
                [
                  0.7970749542961609,
                  0.07110091743119266,
                  436,
                  31
                ],
                [
                  0.7989031078610603,
                  0.07322654462242563,
                  437,
                  32
                ],
                [
                  0.8007312614259597,
                  0.07534246575342465,
                  438,
                  33
                ],
                [
                  0.8025594149908593,
                  0.07517084282460136,
                  439,
                  33
                ],
                [
                  0.8043875685557587,
                  0.07727272727272727,
                  440,
                  34
                ],
                [
                  0.8062157221206582,
                  0.07709750566893424,
                  441,
                  34
                ],
                [
                  0.8080438756855576,
                  0.07918552036199095,
                  442,
                  35
                ],
                [
                  0.8098720292504571,
                  0.07900677200902935,
                  443,
                  35
                ],
                [
                  0.8117001828153565,
                  0.08108108108108109,
                  444,
                  36
                ],
                [
                  0.8135283363802559,
                  0.08089887640449438,
                  445,
                  36
                ],
                [
                  0.8153564899451554,
                  0.08071748878923767,
                  446,
                  36
                ],
                [
                  0.8171846435100548,
                  0.08053691275167785,
                  447,
                  36
                ],
                [
                  0.8190127970749543,
                  0.08258928571428571,
                  448,
                  37
                ],
                [
                  0.8208409506398537,
                  0.08463251670378619,
                  449,
                  38
                ],
                [
                  0.8226691042047533,
                  0.08444444444444445,
                  450,
                  38
                ],
                [
                  0.8244972577696527,
                  0.08647450110864745,
                  451,
                  39
                ],
                [
                  0.8263254113345521,
                  0.08628318584070796,
                  452,
                  39
                ],
                [
                  0.8281535648994516,
                  0.08609271523178808,
                  453,
                  39
                ],
                [
                  0.829981718464351,
                  0.08590308370044053,
                  454,
                  39
                ],
                [
                  0.8318098720292505,
                  0.08791208791208792,
                  455,
                  40
                ],
                [
                  0.8336380255941499,
                  0.08771929824561403,
                  456,
                  40
                ],
                [
                  0.8354661791590493,
                  0.08971553610503283,
                  457,
                  41
                ],
                [
                  0.8372943327239488,
                  0.09170305676855896,
                  458,
                  42
                ],
                [
                  0.8391224862888482,
                  0.09368191721132897,
                  459,
                  43
                ],
                [
                  0.8409506398537477,
                  0.09347826086956522,
                  460,
                  43
                ],
                [
                  0.8427787934186471,
                  0.09327548806941431,
                  461,
                  43
                ],
                [
                  0.8446069469835467,
                  0.09523809523809523,
                  462,
                  44
                ],
                [
                  0.8464351005484461,
                  0.09503239740820735,
                  463,
                  44
                ],
                [
                  0.8482632541133455,
                  0.09698275862068965,
                  464,
                  45
                ],
                [
                  0.850091407678245,
                  0.0989247311827957,
                  465,
                  46
                ],
                [
                  0.8519195612431444,
                  0.10085836909871244,
                  466,
                  47
                ],
                [
                  0.8537477148080439,
                  0.10278372591006424,
                  467,
                  48
                ],
                [
                  0.8555758683729433,
                  0.10256410256410256,
                  468,
                  48
                ],
                [
                  0.8574040219378428,
                  0.1023454157782516,
                  469,
                  48
                ],
                [
                  0.8592321755027422,
                  0.10212765957446808,
                  470,
                  48
                ],
                [
                  0.8610603290676416,
                  0.1040339702760085,
                  471,
                  49
                ],
                [
                  0.8628884826325411,
                  0.1059322033898305,
                  472,
                  50
                ],
                [
                  0.8647166361974405,
                  0.10782241014799154,
                  473,
                  51
                ],
                [
                  0.8665447897623401,
                  0.10970464135021098,
                  474,
                  52
                ],
                [
                  0.8683729433272395,
                  0.11157894736842106,
                  475,
                  53
                ],
                [
                  0.870201096892139,
                  0.11134453781512606,
                  476,
                  53
                ],
                [
                  0.8720292504570384,
                  0.1111111111111111,
                  477,
                  53
                ],
                [
                  0.8738574040219378,
                  0.11297071129707113,
                  478,
                  54
                ],
                [
                  0.8756855575868373,
                  0.11482254697286012,
                  479,
                  55
                ],
                [
                  0.8775137111517367,
                  0.11458333333333333,
                  480,
                  55
                ],
                [
                  0.8793418647166362,
                  0.11642411642411643,
                  481,
                  56
                ],
                [
                  0.8811700182815356,
                  0.11618257261410789,
                  482,
                  56
                ],
                [
                  0.8829981718464351,
                  0.11594202898550725,
                  483,
                  56
                ],
                [
                  0.8848263254113345,
                  0.11776859504132231,
                  484,
                  57
                ],
                [
                  0.886654478976234,
                  0.11958762886597939,
                  485,
                  58
                ],
                [
                  0.8884826325411335,
                  0.12139917695473251,
                  486,
                  59
                ],
                [
                  0.8903107861060329,
                  0.12114989733059549,
                  487,
                  59
                ],
                [
                  0.8921389396709324,
                  0.12295081967213115,
                  488,
                  60
                ],
                [
                  0.8939670932358318,
                  0.12269938650306748,
                  489,
                  60
                ],
                [
                  0.8957952468007313,
                  0.12244897959183673,
                  490,
                  60
                ],
                [
                  0.8976234003656307,
                  0.12219959266802444,
                  491,
                  60
                ],
                [
                  0.8994515539305301,
                  0.12195121951219512,
                  492,
                  60
                ],
                [
                  0.9012797074954296,
                  0.12373225152129817,
                  493,
                  61
                ],
                [
                  0.903107861060329,
                  0.12348178137651822,
                  494,
                  61
                ],
                [
                  0.9049360146252285,
                  0.12525252525252525,
                  495,
                  62
                ],
                [
                  0.906764168190128,
                  0.12701612903225806,
                  496,
                  63
                ],
                [
                  0.9085923217550275,
                  0.12877263581488935,
                  497,
                  64
                ],
                [
                  0.9104204753199269,
                  0.13052208835341367,
                  498,
                  65
                ],
                [
                  0.9122486288848263,
                  0.13226452905811623,
                  499,
                  66
                ],
                [
                  0.9140767824497258,
                  0.134,
                  500,
                  67
                ],
                [
                  0.9159049360146252,
                  0.13373253493013973,
                  501,
                  67
                ],
                [
                  0.9177330895795247,
                  0.13545816733067728,
                  502,
                  68
                ],
                [
                  0.9195612431444241,
                  0.13518886679920478,
                  503,
                  68
                ],
                [
                  0.9213893967093236,
                  0.1349206349206349,
                  504,
                  68
                ],
                [
                  0.923217550274223,
                  0.13465346534653466,
                  505,
                  68
                ],
                [
                  0.9250457038391224,
                  0.13438735177865613,
                  506,
                  68
                ],
                [
                  0.926873857404022,
                  0.13609467455621302,
                  507,
                  69
                ],
                [
                  0.9287020109689214,
                  0.13582677165354332,
                  508,
                  69
                ],
                [
                  0.9305301645338209,
                  0.13555992141453832,
                  509,
                  69
                ],
                [
                  0.9323583180987203,
                  0.13529411764705881,
                  510,
                  69
                ],
                [
                  0.9341864716636198,
                  0.136986301369863,
                  511,
                  70
                ],
                [
                  0.9360146252285192,
                  0.13671875,
                  512,
                  70
                ],
                [
                  0.9378427787934186,
                  0.1364522417153996,
                  513,
                  70
                ],
                [
                  0.9396709323583181,
                  0.13618677042801555,
                  514,
                  70
                ],
                [
                  0.9414990859232175,
                  0.1378640776699029,
                  515,
                  71
                ],
                [
                  0.943327239488117,
                  0.13953488372093023,
                  516,
                  72
                ],
                [
                  0.9451553930530164,
                  0.13926499032882012,
                  517,
                  72
                ],
                [
                  0.946983546617916,
                  0.138996138996139,
                  518,
                  72
                ],
                [
                  0.9488117001828154,
                  0.13872832369942195,
                  519,
                  72
                ],
                [
                  0.9506398537477148,
                  0.13846153846153847,
                  520,
                  72
                ],
                [
                  0.9524680073126143,
                  0.1401151631477927,
                  521,
                  73
                ],
                [
                  0.9542961608775137,
                  0.13984674329501914,
                  522,
                  73
                ],
                [
                  0.9561243144424132,
                  0.13957934990439771,
                  523,
                  73
                ],
                [
                  0.9579524680073126,
                  0.13931297709923665,
                  524,
                  73
                ],
                [
                  0.9597806215722121,
                  0.14095238095238094,
                  525,
                  74
                ],
                [
                  0.9616087751371115,
                  0.14258555133079848,
                  526,
                  75
                ],
                [
                  0.9634369287020109,
                  0.14231499051233396,
                  527,
                  75
                ],
                [
                  0.9652650822669104,
                  0.14204545454545456,
                  528,
                  75
                ],
                [
                  0.9670932358318098,
                  0.14366729678638943,
                  529,
                  76
                ],
                [
                  0.9689213893967094,
                  0.14528301886792452,
                  530,
                  77
                ],
                [
                  0.9707495429616088,
                  0.14689265536723164,
                  531,
                  78
                ],
                [
                  0.9725776965265083,
                  0.14849624060150377,
                  532,
                  79
                ],
                [
                  0.9744058500914077,
                  0.150093808630394,
                  533,
                  80
                ],
                [
                  0.9762340036563071,
                  0.15168539325842698,
                  534,
                  81
                ],
                [
                  0.9780621572212066,
                  0.15327102803738318,
                  535,
                  82
                ],
                [
                  0.979890310786106,
                  0.15485074626865672,
                  536,
                  83
                ],
                [
                  0.9817184643510055,
                  0.15456238361266295,
                  537,
                  83
                ],
                [
                  0.9835466179159049,
                  0.15613382899628253,
                  538,
                  84
                ],
                [
                  0.9853747714808044,
                  0.15584415584415584,
                  539,
                  84
                ],
                [
                  0.9872029250457038,
                  0.1574074074074074,
                  540,
                  85
                ],
                [
                  0.9890310786106032,
                  0.15711645101663585,
                  541,
                  85
                ],
                [
                  0.9908592321755028,
                  0.15867158671586715,
                  542,
                  86
                ],
                [
                  0.9926873857404022,
                  0.16022099447513813,
                  543,
                  87
                ],
                [
                  0.9945155393053017,
                  0.16176470588235295,
                  544,
                  88
                ],
                [
                  0.9963436928702011,
                  0.163302752293578,
                  545,
                  89
                ],
                [
                  0.9981718464351006,
                  0.163003663003663,
                  546,
                  89
                ],
                [
                  1.0,
                  0.16453382084095064,
                  547,
                  90
                ]
              ]
            },
            {
              "language": "English",
              "run": 499,
              "points": [
                [
                  0.0018281535648994515,
                  0.0,
                  1,
                  0
                ],
                [
                  0.003656307129798903,
                  0.0,
                  2,
                  0
                ],
                [
                  0.005484460694698354,
                  0.0,
                  3,
                  0
                ],
                [
                  0.007312614259597806,
                  0.0,
                  4,
                  0
                ],
                [
                  0.009140767824497258,
                  0.0,
                  5,
                  0
                ],
                [
                  0.010968921389396709,
                  0.0,
                  6,
                  0
                ],
                [
                  0.012797074954296161,
                  0.0,
                  7,
                  0
                ],
                [
                  0.014625228519195612,
                  0.0,
                  8,
                  0
                ],
                [
                  0.016453382084095063,
                  0.0,
                  9,
                  0
                ],
                [
                  0.018281535648994516,
                  0.0,
                  10,
                  0
                ],
                [
                  0.02010968921389397,
                  0.0,
                  11,
                  0
                ],
                [
                  0.021937842778793418,
                  0.0,
                  12,
                  0
                ],
                [
                  0.02376599634369287,
                  0.0,
                  13,
                  0
                ],
                [
                  0.025594149908592323,
                  0.0,
                  14,
                  0
                ],
                [
                  0.027422303473491772,
                  0.0,
                  15,
                  0
                ],
                [
                  0.029250457038391225,
                  0.0,
                  16,
                  0
                ],
                [
                  0.031078610603290677,
                  0.0,
                  17,
                  0
                ],
                [
                  0.03290676416819013,
                  0.0,
                  18,
                  0
                ],
                [
                  0.03473491773308958,
                  0.0,
                  19,
                  0
                ],
                [
                  0.03656307129798903,
                  0.0,
                  20,
                  0
                ],
                [
                  0.038391224862888484,
                  0.0,
                  21,
                  0
                ],
                [
                  0.04021937842778794,
                  0.0,
                  22,
                  0
                ],
                [
                  0.04204753199268738,
                  0.0,
                  23,
                  0
                ],
                [
                  0.043875685557586835,
                  0.0,
                  24,
                  0
                ],
                [
                  0.04570383912248629,
                  0.0,
                  25,
                  0
                ],
                [
                  0.04753199268738574,
                  0.0,
                  26,
                  0
                ],
                [
                  0.04936014625228519,
                  0.0,
                  27,
                  0
                ],
                [
                  0.051188299817184646,
                  0.0,
                  28,
                  0
                ],
                [
                  0.05301645338208409,
                  0.0,
                  29,
                  0
                ],
                [
                  0.054844606946983544,
                  0.0,
                  30,
                  0
                ],
                [
                  0.056672760511883,
                  0.0,
                  31,
                  0
                ],
                [
                  0.05850091407678245,
                  0.0,
                  32,
                  0
                ],
                [
                  0.0603290676416819,
                  0.0,
                  33,
                  0
                ],
                [
                  0.062157221206581355,
                  0.0,
                  34,
                  0
                ],
                [
                  0.06398537477148081,
                  0.0,
                  35,
                  0
                ],
                [
                  0.06581352833638025,
                  0.0,
                  36,
                  0
                ],
                [
                  0.06764168190127971,
                  0.0,
                  37,
                  0
                ],
                [
                  0.06946983546617916,
                  0.0,
                  38,
                  0
                ],
                [
                  0.0712979890310786,
                  0.0,
                  39,
                  0
                ],
                [
                  0.07312614259597806,
                  0.0,
                  40,
                  0
                ],
                [
                  0.07495429616087751,
                  0.0,
                  41,
                  0
                ],
                [
                  0.07678244972577697,
                  0.0,
                  42,
                  0
                ],
                [
                  0.07861060329067641,
                  0.0,
                  43,
                  0
                ],
                [
                  0.08043875685557587,
                  0.0,
                  44,
                  0
                ],
                [
                  0.08226691042047532,
                  0.0,
                  45,
                  0
                ],
                [
                  0.08409506398537477,
                  0.0,
                  46,
                  0
                ],
                [
                  0.08592321755027423,
                  0.0,
                  47,
                  0
                ],
                [
                  0.08775137111517367,
                  0.0,
                  48,
                  0
                ],
                [
                  0.08957952468007313,
                  0.0,
                  49,
                  0
                ],
                [
                  0.09140767824497258,
                  0.0,
                  50,
                  0
                ],
                [
                  0.09323583180987204,
                  0.0,
                  51,
                  0
                ],
                [
                  0.09506398537477148,
                  0.0,
                  52,
                  0
                ],
                [
                  0.09689213893967093,
                  0.0,
                  53,
                  0
                ],
                [
                  0.09872029250457039,
                  0.0,
                  54,
                  0
                ],
                [
                  0.10054844606946983,
                  0.0,
                  55,
                  0
                ],
                [
                  0.10237659963436929,
                  0.0,
                  56,
                  0
                ],
                [
                  0.10420475319926874,
                  0.0,
                  57,
                  0
                ],
                [
                  0.10603290676416818,
                  0.0,
                  58,
                  0
                ],
                [
                  0.10786106032906764,
                  0.0,
                  59,
                  0
                ],
                [
                  0.10968921389396709,
                  0.0,
                  60,
                  0
                ],
                [
                  0.11151736745886655,
                  0.0,
                  61,
                  0
                ],
                [
                  0.113345521023766,
                  0.0,
                  62,
                  0
                ],
                [
                  0.11517367458866545,
                  0.0,
                  63,
                  0
                ],
                [
                  0.1170018281535649,
                  0.0,
                  64,
                  0
                ],
                [
                  0.11882998171846434,
                  0.0,
                  65,
                  0
                ],
                [
                  0.1206581352833638,
                  0.0,
                  66,
                  0
                ],
                [
                  0.12248628884826325,
                  0.0,
                  67,
                  0
                ],
                [
                  0.12431444241316271,
                  0.0,
                  68,
                  0
                ],
                [
                  0.12614259597806216,
                  0.0,
                  69,
                  0
                ],
                [
                  0.12797074954296161,
                  0.0,
                  70,
                  0
                ],
                [
                  0.12979890310786105,
                  0.0,
                  71,
                  0
                ],
                [
                  0.1316270566727605,
                  0.0,
                  72,
                  0
                ],
                [
                  0.13345521023765997,
                  0.0,
                  73,
                  0
                ],
                [
                  0.13528336380255943,
                  0.0,
                  74,
                  0
                ],
                [
                  0.13711151736745886,
                  0.0,
                  75,
                  0
                ],
                [
                  0.13893967093235832,
                  0.0,
                  76,
                  0
                ],
                [
                  0.14076782449725778,
                  0.0,
                  77,
                  0
                ],
                [
                  0.1425959780621572,
                  0.0,
                  78,
                  0
                ],
                [
                  0.14442413162705667,
                  0.0,
                  79,
                  0
                ],
                [
                  0.14625228519195613,
                  0.0,
                  80,
                  0
                ],
                [
                  0.1480804387568556,
                  0.0,
                  81,
                  0
                ],
                [
                  0.14990859232175502,
                  0.0,
                  82,
                  0
                ],
                [
                  0.15173674588665448,
                  0.0,
                  83,
                  0
                ],
                [
                  0.15356489945155394,
                  0.0,
                  84,
                  0
                ],
                [
                  0.15539305301645337,
                  0.0,
                  85,
                  0
                ],
                [
                  0.15722120658135283,
                  0.0,
                  86,
                  0
                ],
                [
                  0.1590493601462523,
                  0.0,
                  87,
                  0
                ],
                [
                  0.16087751371115175,
                  0.0,
                  88,
                  0
                ],
                [
                  0.16270566727605118,
                  0.0,
                  89,
                  0
                ],
                [
                  0.16453382084095064,
                  0.0,
                  90,
                  0
                ],
                [
                  0.1663619744058501,
                  0.0,
                  91,
                  0
                ],
                [
                  0.16819012797074953,
                  0.0,
                  92,
                  0
                ],
                [
                  0.170018281535649,
                  0.0,
                  93,
                  0
                ],
                [
                  0.17184643510054845,
                  0.0,
                  94,
                  0
                ],
                [
                  0.1736745886654479,
                  0.0,
                  95,
                  0
                ],
                [
                  0.17550274223034734,
                  0.0,
                  96,
                  0
                ],
                [
                  0.1773308957952468,
                  0.0,
                  97,
                  0
                ],
                [
                  0.17915904936014626,
                  0.0,
                  98,
                  0
                ],
                [
                  0.1809872029250457,
                  0.0,
                  99,
                  0
                ],
                [
                  0.18281535648994515,
                  0.0,
                  100,
                  0
                ],
                [
                  0.1846435100548446,
                  0.0,
                  101,
                  0
                ],
                [
                  0.18647166361974407,
                  0.0,
                  102,
                  0
                ],
                [
                  0.1882998171846435,
                  0.0,
                  103,
                  0
                ],
                [
                  0.19012797074954296,
                  0.0,
                  104,
                  0
                ],
                [
                  0.19195612431444242,
                  0.0,
                  105,
                  0
                ],
                [
                  0.19378427787934185,
                  0.0,
                  106,
                  0
                ],
                [
                  0.1956124314442413,
                  0.0,
                  107,
                  0
                ],
                [
                  0.19744058500914077,
                  0.0,
                  108,
                  0
                ],
                [
                  0.19926873857404023,
                  0.0,
                  109,
                  0
                ],
                [
                  0.20109689213893966,
                  0.0,
                  110,
                  0
                ],
                [
                  0.20292504570383912,
                  0.0,
                  111,
                  0
                ],
                [
                  0.20475319926873858,
                  0.0,
                  112,
                  0
                ],
                [
                  0.20658135283363802,
                  0.0,
                  113,
                  0
                ],
                [
                  0.20840950639853748,
                  0.0,
                  114,
                  0
                ],
                [
                  0.21023765996343693,
                  0.0,
                  115,
                  0
                ],
                [
                  0.21206581352833637,
                  0.0,
                  116,
                  0
                ],
                [
                  0.21389396709323583,
                  0.0,
                  117,
                  0
                ],
                [
                  0.21572212065813529,
                  0.0,
                  118,
                  0
                ],
                [
                  0.21755027422303475,
                  0.0,
                  119,
                  0
                ],
                [
                  0.21937842778793418,
                  0.0,
                  120,
                  0
                ],
                [
                  0.22120658135283364,
                  0.0,
                  121,
                  0
                ],
                [
                  0.2230347349177331,
                  0.0,
                  122,
                  0
                ],
                [
                  0.22486288848263253,
                  0.0,
                  123,
                  0
                ],
                [
                  0.226691042047532,
                  0.0,
                  124,
                  0
                ],
                [
                  0.22851919561243145,
                  0.0,
                  125,
                  0
                ],
                [
                  0.2303473491773309,
                  0.0,
                  126,
                  0
                ],
                [
                  0.23217550274223034,
                  0.0,
                  127,
                  0
                ],
                [
                  0.2340036563071298,
                  0.0,
                  128,
                  0
                ],
                [
                  0.23583180987202926,
                  0.0,
                  129,
                  0
                ],
                [
                  0.2376599634369287,
                  0.0,
                  130,
                  0
                ],
                [
                  0.23948811700182815,
                  0.0,
                  131,
                  0
                ],
                [
                  0.2413162705667276,
                  0.0,
                  132,
                  0
                ],
                [
                  0.24314442413162707,
                  0.0,
                  133,
                  0
                ],
                [
                  0.2449725776965265,
                  0.0,
                  134,
                  0
                ],
                [
                  0.24680073126142596,
                  0.0,
                  135,
                  0
                ],
                [
                  0.24862888482632542,
                  0.0,
                  136,
                  0
                ],
                [
                  0.25045703839122485,
                  0.0,
                  137,
                  0
                ],
                [
                  0.2522851919561243,
                  0.0,
                  138,
                  0
                ],
                [
                  0.25411334552102377,
                  0.0,
                  139,
                  0
                ],
                [
                  0.25594149908592323,
                  0.0,
                  140,
                  0
                ],
                [
                  0.2577696526508227,
                  0.0,
                  141,
                  0
                ],
                [
                  0.2595978062157221,
                  0.0,
                  142,
                  0
                ],
                [
                  0.26142595978062155,
                  0.0,
                  143,
                  0
                ],
                [
                  0.263254113345521,
                  0.0,
                  144,
                  0
                ],
                [
                  0.26508226691042047,
                  0.0,
                  145,
                  0
                ],
                [
                  0.26691042047531993,
                  0.0,
                  146,
                  0
                ],
                [
                  0.2687385740402194,
                  0.0,
                  147,
                  0
                ],
                [
                  0.27056672760511885,
                  0.0,
                  148,
                  0
                ],
                [
                  0.27239488117001825,
                  0.0,
                  149,
                  0
                ],
                [
                  0.2742230347349177,
                  0.0,
                  150,
                  0
                ],
                [
                  0.2760511882998172,
                  0.0,
                  151,
                  0
                ],
                [
                  0.27787934186471663,
                  0.0,
                  152,
                  0
                ],
                [
                  0.2797074954296161,
                  0.0,
                  153,
                  0
                ],
                [
                  0.28153564899451555,
                  0.0,
                  154,
                  0
                ],
                [
                  0.283363802559415,
                  0.0,
                  155,
                  0
                ],
                [
                  0.2851919561243144,
                  0.0,
                  156,
                  0
                ],
                [
                  0.2870201096892139,
                  0.0,
                  157,
                  0
                ],
                [
                  0.28884826325411334,
                  0.0,
                  158,
                  0
                ],
                [
                  0.2906764168190128,
                  0.0,
                  159,
                  0
                ],
                [
                  0.29250457038391225,
                  0.0,
                  160,
                  0
                ],
                [
                  0.2943327239488117,
                  0.0,
                  161,
                  0
                ],
                [
                  0.2961608775137112,
                  0.0,
                  162,
                  0
                ],
                [
                  0.2979890310786106,
                  0.0,
                  163,
                  0
                ],
                [
                  0.29981718464351004,
                  0.0,
                  164,
                  0
                ],
                [
                  0.3016453382084095,
                  0.0,
                  165,
                  0
                ],
                [
                  0.30347349177330896,
                  0.0,
                  166,
                  0
                ],
                [
                  0.3053016453382084,
                  0.0,
                  167,
                  0
                ],
                [
                  0.3071297989031079,
                  0.0,
                  168,
                  0
                ],
                [
                  0.30895795246800734,
                  0.0,
                  169,
                  0
                ],
                [
                  0.31078610603290674,
                  0.0,
                  170,
                  0
                ],
                [
                  0.3126142595978062,
                  0.0,
                  171,
                  0
                ],
                [
                  0.31444241316270566,
                  0.0,
                  172,
                  0
                ],
                [
                  0.3162705667276051,
                  0.0,
                  173,
                  0
                ],
                [
                  0.3180987202925046,
                  0.0,
                  174,
                  0
                ],
                [
                  0.31992687385740404,
                  0.0,
                  175,
                  0
                ],
                [
                  0.3217550274223035,
                  0.0,
                  176,
                  0
                ],
                [
                  0.3235831809872029,
                  0.0,
                  177,
                  0
                ],
                [
                  0.32541133455210236,
                  0.0,
                  178,
                  0
                ],
                [
                  0.3272394881170018,
                  0.0,
                  179,
                  0
                ],
                [
                  0.3290676416819013,
                  0.0,
                  180,
                  0
                ],
                [
                  0.33089579524680074,
                  0.0,
                  181,
                  0
                ],
                [
                  0.3327239488117002,
                  0.0,
                  182,
                  0
                ],
                [
                  0.33455210237659966,
                  0.0,
                  183,
                  0
                ],
                [
                  0.33638025594149906,
                  0.0,
                  184,
                  0
                ],
                [
                  0.3382084095063985,
                  0.0,
                  185,
                  0
                ],
                [
                  0.340036563071298,
                  0.0,
                  186,
                  0
                ],
                [
                  0.34186471663619744,
                  0.0,
                  187,
                  0
                ],
                [
                  0.3436928702010969,
                  0.0,
                  188,
                  0
                ],
                [
                  0.34552102376599636,
                  0.0,
                  189,
                  0
                ],
                [
                  0.3473491773308958,
                  0.0,
                  190,
                  0
                ],
                [
                  0.3491773308957952,
                  0.0,
                  191,
                  0
                ],
                [
                  0.3510054844606947,
                  0.0,
                  192,
                  0
                ],
                [
                  0.35283363802559414,
                  0.0,
                  193,
                  0
                ],
                [
                  0.3546617915904936,
                  0.0,
                  194,
                  0
                ],
                [
                  0.35648994515539306,
                  0.0,
                  195,
                  0
                ],
                [
                  0.3583180987202925,
                  0.0,
                  196,
                  0
                ],
                [
                  0.360146252285192,
                  0.0,
                  197,
                  0
                ],
                [
                  0.3619744058500914,
                  0.0,
                  198,
                  0
                ],
                [
                  0.36380255941499084,
                  0.0,
                  199,
                  0
                ],
                [
                  0.3656307129798903,
                  0.0,
                  200,
                  0
                ],
                [
                  0.36745886654478976,
                  0.0,
                  201,
                  0
                ],
                [
                  0.3692870201096892,
                  0.0,
                  202,
                  0
                ],
                [
                  0.3711151736745887,
                  0.0,
                  203,
                  0
                ],
                [
                  0.37294332723948814,
                  0.0,
                  204,
                  0
                ],
                [
                  0.37477148080438755,
                  0.0,
                  205,
                  0
                ],
                [
                  0.376599634369287,
                  0.0,
                  206,
                  0
                ],
                [
                  0.37842778793418647,
                  0.0,
                  207,
                  0
                ],
                [
                  0.3802559414990859,
                  0.0,
                  208,
                  0
                ],
                [
                  0.3820840950639854,
                  0.0,
                  209,
                  0
                ],
                [
                  0.38391224862888484,
                  0.0,
                  210,
                  0
                ],
                [
                  0.3857404021937843,
                  0.0,
                  211,
                  0
                ],
                [
                  0.3875685557586837,
                  0.0,
                  212,
                  0
                ],
                [
                  0.38939670932358317,
                  0.0,
                  213,
                  0
                ],
                [
                  0.3912248628884826,
                  0.0,
                  214,
                  0
                ],
                [
                  0.3930530164533821,
                  0.0,
                  215,
                  0
                ],
                [
                  0.39488117001828155,
                  0.0,
                  216,
                  0
                ],
                [
                  0.396709323583181,
                  0.0,
                  217,
                  0
                ],
                [
                  0.39853747714808047,
                  0.0,
                  218,
                  0
                ],
                [
                  0.40036563071297987,
                  0.0,
                  219,
                  0
                ],
                [
                  0.40219378427787933,
                  0.0,
                  220,
                  0
                ],
                [
                  0.4040219378427788,
                  0.0,
                  221,
                  0
                ],
                [
                  0.40585009140767825,
                  0.0,
                  222,
                  0
                ],
                [
                  0.4076782449725777,
                  0.0,
                  223,
                  0
                ],
                [
                  0.40950639853747717,
                  0.0,
                  224,
                  0
                ],
                [
                  0.4113345521023766,
                  0.0,
                  225,
                  0
                ],
                [
                  0.41316270566727603,
                  0.0,
                  226,
                  0
                ],
                [
                  0.4149908592321755,
                  0.0,
                  227,
                  0
                ],
                [
                  0.41681901279707495,
                  0.0,
                  228,
                  0
                ],
                [
                  0.4186471663619744,
                  0.0,
                  229,
                  0
                ],
                [
                  0.42047531992687387,
                  0.004347826086956522,
                  230,
                  1
                ],
                [
                  0.42230347349177333,
                  0.004329004329004329,
                  231,
                  1
                ],
                [
                  0.42413162705667273,
                  0.004310344827586207,
                  232,
                  1
                ],
                [
                  0.4259597806215722,
                  0.004291845493562232,
                  233,
                  1
                ],
                [
                  0.42778793418647165,
                  0.004273504273504274,
                  234,
                  1
                ],
                [
                  0.4296160877513711,
                  0.00425531914893617,
                  235,
                  1
                ],
                [
                  0.43144424131627057,
                  0.00423728813559322,
                  236,
                  1
                ],
                [
                  0.43327239488117003,
                  0.004219409282700422,
                  237,
                  1
                ],
                [
                  0.4351005484460695,
                  0.004201680672268907,
                  238,
                  1
                ],
                [
                  0.4369287020109689,
                  0.0041841004184100415,
                  239,
                  1
                ],
                [
                  0.43875685557586835,
                  0.004166666666666667,
                  240,
                  1
                ],
                [
                  0.4405850091407678,
                  0.004149377593360996,
                  241,
                  1
                ],
                [
                  0.4424131627056673,
                  0.004132231404958678,
                  242,
                  1
                ],
                [
                  0.44424131627056673,
                  0.00411522633744856,
                  243,
                  1
                ],
                [
                  0.4460694698354662,
                  0.004098360655737705,
                  244,
                  1
                ],
                [
                  0.44789762340036565,
                  0.004081632653061225,
                  245,
                  1
                ],
                [
                  0.44972577696526506,
                  0.0040650406504065045,
                  246,
                  1
                ],
                [
                  0.4515539305301645,
                  0.004048582995951417,
                  247,
                  1
                ],
                [
                  0.453382084095064,
                  0.004032258064516129,
                  248,
                  1
                ],
                [
                  0.45521023765996343,
                  0.004016064257028112,
                  249,
                  1
                ],
                [
                  0.4570383912248629,
                  0.004,
                  250,
                  1
                ],
                [
                  0.45886654478976235,
                  0.00398406374501992,
                  251,
                  1
                ],
                [
                  0.4606946983546618,
                  0.003968253968253968,
                  252,
                  1
                ],
                [
                  0.4625228519195612,
                  0.003952569169960474,
                  253,
                  1
                ],
                [
                  0.4643510054844607,
                  0.003937007874015748,
                  254,
                  1
                ],
                [
                  0.46617915904936014,
                  0.00392156862745098,
                  255,
                  1
                ],
                [
                  0.4680073126142596,
                  0.00390625,
                  256,
                  1
                ],
                [
                  0.46983546617915906,
                  0.0038910505836575876,
                  257,
                  1
                ],
                [
                  0.4716636197440585,
                  0.003875968992248062,
                  258,
                  1
                ],
                [
                  0.473491773308958,
                  0.003861003861003861,
                  259,
                  1
                ],
                [
                  0.4753199268738574,
                  0.0038461538461538464,
                  260,
                  1
                ],
                [
                  0.47714808043875684,
                  0.0038314176245210726,
                  261,
                  1
                ],
                [
                  0.4789762340036563,
                  0.003816793893129771,
                  262,
                  1
                ],
                [
                  0.48080438756855576,
                  0.0038022813688212928,
                  263,
                  1
                ],
                [
                  0.4826325411334552,
                  0.003787878787878788,
                  264,
                  1
                ],
                [
                  0.4844606946983547,
                  0.0037735849056603774,
                  265,
                  1
                ],
                [
                  0.48628884826325414,
                  0.0037593984962406013,
                  266,
                  1
                ],
                [
                  0.48811700182815354,
                  0.003745318352059925,
                  267,
                  1
                ],
                [
                  0.489945155393053,
                  0.0037313432835820895,
                  268,
                  1
                ],
                [
                  0.49177330895795246,
                  0.0037174721189591076,
                  269,
                  1
                ],
                [
                  0.4936014625228519,
                  0.003703703703703704,
                  270,
                  1
                ],
                [
                  0.4954296160877514,
                  0.0036900369003690036,
                  271,
                  1
                ],
                [
                  0.49725776965265084,
                  0.003676470588235294,
                  272,
                  1
                ],
                [
                  0.4990859232175503,
                  0.003663003663003663,
                  273,
                  1
                ],
                [
                  0.5009140767824497,
                  0.0036496350364963502,
                  274,
                  1
                ],
                [
                  0.5027422303473492,
                  0.0036363636363636364,
                  275,
                  1
                ],
                [
                  0.5045703839122486,
                  0.0036231884057971015,
                  276,
                  1
                ],
                [
                  0.506398537477148,
                  0.0036101083032490976,
                  277,
                  1
                ],
                [
                  0.5082266910420475,
                  0.0035971223021582736,
                  278,
                  1
                ],
                [
                  0.5100548446069469,
                  0.0035842293906810036,
                  279,
                  1
                ],
                [
                  0.5118829981718465,
                  0.0035714285714285713,
                  280,
                  1
                ],
                [
                  0.5137111517367459,
                  0.0071174377224199285,
                  281,
                  2
                ],
                [
                  0.5155393053016454,
                  0.0070921985815602835,
                  282,
                  2
                ],
                [
                  0.5173674588665448,
                  0.007067137809187279,
                  283,
                  2
                ],
                [
                  0.5191956124314442,
                  0.007042253521126761,
                  284,
                  2
                ],
                [
                  0.5210237659963437,
                  0.007017543859649123,
                  285,
                  2
                ],
                [
                  0.5228519195612431,
                  0.006993006993006993,
                  286,
                  2
                ],
                [
                  0.5246800731261426,
                  0.006968641114982578,
                  287,
                  2
                ],
                [
                  0.526508226691042,
                  0.006944444444444444,
                  288,
                  2
                ],
                [
                  0.5283363802559415,
                  0.006920415224913495,
                  289,
                  2
                ],
                [
                  0.5301645338208409,
                  0.006896551724137931,
                  290,
                  2
                ],
                [
                  0.5319926873857403,
                  0.006872852233676976,
                  291,
                  2
                ],
                [
                  0.5338208409506399,
                  0.00684931506849315,
                  292,
                  2
                ],
                [
                  0.5356489945155393,
                  0.006825938566552901,
                  293,
                  2
                ],
                [
                  0.5374771480804388,
                  0.006802721088435374,
                  294,
                  2
                ],
                [
                  0.5393053016453382,
                  0.006779661016949152,
                  295,
                  2
                ],
                [
                  0.5411334552102377,
                  0.006756756756756757,
                  296,
                  2
                ],
                [
                  0.5429616087751371,
                  0.006734006734006734,
                  297,
                  2
                ],
                [
                  0.5447897623400365,
                  0.006711409395973154,
                  298,
                  2
                ],
                [
                  0.546617915904936,
                  0.006688963210702341,
                  299,
                  2
                ],
                [
                  0.5484460694698354,
                  0.006666666666666667,
                  300,
                  2
                ],
                [
                  0.5502742230347349,
                  0.006644518272425249,
                  301,
                  2
                ],
                [
                  0.5521023765996343,
                  0.006622516556291391,
                  302,
                  2
                ],
                [
                  0.5539305301645339,
                  0.006600660066006601,
                  303,
                  2
                ],
                [
                  0.5557586837294333,
                  0.006578947368421052,
                  304,
                  2
                ],
                [
                  0.5575868372943327,
                  0.006557377049180328,
                  305,
                  2
                ],
                [
                  0.5594149908592322,
                  0.006535947712418301,
                  306,
                  2
                ],
                [
                  0.5612431444241316,
                  0.006514657980456026,
                  307,
                  2
                ],
                [
                  0.5630712979890311,
                  0.006493506493506494,
                  308,
                  2
                ],
                [
                  0.5648994515539305,
                  0.006472491909385114,
                  309,
                  2
                ],
                [
                  0.56672760511883,
                  0.0064516129032258064,
                  310,
                  2
                ],
                [
                  0.5685557586837294,
                  0.006430868167202572,
                  311,
                  2
                ],
                [
                  0.5703839122486288,
                  0.009615384615384616,
                  312,
                  3
                ],
                [
                  0.5722120658135283,
                  0.009584664536741214,
                  313,
                  3
                ],
                [
                  0.5740402193784278,
                  0.009554140127388535,
                  314,
                  3
                ],
                [
                  0.5758683729433273,
                  0.009523809523809525,
                  315,
                  3
                ],
                [
                  0.5776965265082267,
                  0.00949367088607595,
                  316,
                  3
                ],
                [
                  0.5795246800731262,
                  0.00946372239747634,
                  317,
                  3
                ],
                [
                  0.5813528336380256,
                  0.009433962264150943,
                  318,
                  3
                ],
                [
                  0.583180987202925,
                  0.009404388714733543,
                  319,
                  3
                ],
                [
                  0.5850091407678245,
                  0.009375,
                  320,
                  3
                ],
                [
                  0.5868372943327239,
                  0.009345794392523364,
                  321,
                  3
                ],
                [
                  0.5886654478976234,
                  0.009316770186335404,
                  322,
                  3
                ],
                [
                  0.5904936014625228,
                  0.009287925696594427,
                  323,
                  3
                ],
                [
                  0.5923217550274223,
                  0.009259259259259259,
                  324,
                  3
                ],
                [
                  0.5941499085923218,
                  0.009230769230769232,
                  325,
                  3
                ],
                [
                  0.5959780621572212,
                  0.012269938650306749,
                  326,
                  4
                ],
                [
                  0.5978062157221207,
                  0.012232415902140673,
                  327,
                  4
                ],
                [
                  0.5996343692870201,
                  0.012195121951219513,
                  328,
                  4
                ],
                [
                  0.6014625228519196,
                  0.0121580547112462,
                  329,
                  4
                ],
                [
                  0.603290676416819,
                  0.012121212121212121,
                  330,
                  4
                ],
                [
                  0.6051188299817185,
                  0.012084592145015106,
                  331,
                  4
                ],
                [
                  0.6069469835466179,
                  0.012048192771084338,
                  332,
                  4
                ],
                [
                  0.6087751371115173,
                  0.012012012012012012,
                  333,
                  4
                ],
                [
                  0.6106032906764168,
                  0.011976047904191617,
                  334,
                  4
                ],
                [
                  0.6124314442413162,
                  0.011940298507462687,
                  335,
                  4
                ],
                [
                  0.6142595978062158,
                  0.011904761904761904,
                  336,
                  4
                ],
                [
                  0.6160877513711152,
                  0.011869436201780416,
                  337,
                  4
                ],
                [
                  0.6179159049360147,
                  0.011834319526627219,
                  338,
                  4
                ],
                [
                  0.6197440585009141,
                  0.014749262536873156,
                  339,
                  5
                ],
                [
                  0.6215722120658135,
                  0.01764705882352941,
                  340,
                  6
                ],
                [
                  0.623400365630713,
                  0.017595307917888565,
                  341,
                  6
                ],
                [
                  0.6252285191956124,
                  0.017543859649122806,
                  342,
                  6
                ],
                [
                  0.6270566727605119,
                  0.01749271137026239,
                  343,
                  6
                ],
                [
                  0.6288848263254113,
                  0.020348837209302327,
                  344,
                  7
                ],
                [
                  0.6307129798903108,
                  0.02318840579710145,
                  345,
                  8
                ],
                [
                  0.6325411334552102,
                  0.023121387283236993,
                  346,
                  8
                ],
                [
                  0.6343692870201096,
                  0.023054755043227664,
                  347,
                  8
                ],
                [
                  0.6361974405850092,
                  0.022988505747126436,
                  348,
                  8
                ],
                [
                  0.6380255941499086,
                  0.022922636103151862,
                  349,
                  8
                ],
                [
                  0.6398537477148081,
                  0.025714285714285714,
                  350,
                  9
                ],
                [
                  0.6416819012797075,
                  0.02564102564102564,
                  351,
                  9
                ],
                [
                  0.643510054844607,
                  0.028409090909090908,
                  352,
                  10
                ],
                [
                  0.6453382084095064,
                  0.028328611898016998,
                  353,
                  10
                ],
                [
                  0.6471663619744058,
                  0.02824858757062147,
                  354,
                  10
                ],
                [
                  0.6489945155393053,
                  0.028169014084507043,
                  355,
                  10
                ],
                [
                  0.6508226691042047,
                  0.028089887640449437,
                  356,
                  10
                ],
                [
                  0.6526508226691042,
                  0.028011204481792718,
                  357,
                  10
                ],
                [
                  0.6544789762340036,
                  0.030726256983240222,
                  358,
                  11
                ],
                [
                  0.6563071297989032,
                  0.03064066852367688,
                  359,
                  11
                ],
                [
                  0.6581352833638026,
                  0.030555555555555555,
                  360,
                  11
                ],
                [
                  0.659963436928702,
                  0.030470914127423823,
                  361,
                  11
                ],
                [
                  0.6617915904936015,
                  0.03038674033149171,
                  362,
                  11
                ],
                [
                  0.6636197440585009,
                  0.03305785123966942,
                  363,
                  12
                ],
                [
                  0.6654478976234004,
                  0.03296703296703297,
                  364,
                  12
                ],
                [
                  0.6672760511882998,
                  0.03287671232876712,
                  365,
                  12
                ],
                [
                  0.6691042047531993,
                  0.03278688524590164,
                  366,
                  12
                ],
                [
                  0.6709323583180987,
                  0.035422343324250684,
                  367,
                  13
                ],
                [
                  0.6727605118829981,
                  0.035326086956521736,
                  368,
                  13
                ],
                [
                  0.6745886654478976,
                  0.03523035230352303,
                  369,
                  13
                ],
                [
                  0.676416819012797,
                  0.03513513513513514,
                  370,
                  13
                ],
                [
                  0.6782449725776966,
                  0.03504043126684636,
                  371,
                  13
                ],
                [
                  0.680073126142596,
                  0.03494623655913978,
                  372,
                  13
                ],
                [
                  0.6819012797074955,
                  0.03485254691689008,
                  373,
                  13
                ],
                [
                  0.6837294332723949,
                  0.034759358288770054,
                  374,
                  13
                ],
                [
                  0.6855575868372943,
                  0.034666666666666665,
                  375,
                  13
                ],
                [
                  0.6873857404021938,
                  0.03723404255319149,
                  376,
                  14
                ],
                [
                  0.6892138939670932,
                  0.03713527851458886,
                  377,
                  14
                ],
                [
                  0.6910420475319927,
                  0.03968253968253968,
                  378,
                  15
                ],
                [
                  0.6928702010968921,
                  0.0395778364116095,
                  379,
                  15
                ],
                [
                  0.6946983546617916,
                  0.039473684210526314,
                  380,
                  15
                ],
                [
                  0.696526508226691,
                  0.03937007874015748,
                  381,
                  15
                ],
                [
                  0.6983546617915904,
                  0.03926701570680628,
                  382,
                  15
                ],
                [
                  0.70018281535649,
                  0.04177545691906005,
                  383,
                  16
                ],
                [
                  0.7020109689213894,
                  0.041666666666666664,
                  384,
                  16
                ],
                [
                  0.7038391224862889,
                  0.04415584415584416,
                  385,
                  17
                ],
                [
                  0.7056672760511883,
                  0.046632124352331605,
                  386,
                  18
                ],
                [
                  0.7074954296160878,
                  0.046511627906976744,
                  387,
                  18
                ],
                [
                  0.7093235831809872,
                  0.04639175257731959,
                  388,
                  18
                ],
                [
                  0.7111517367458866,
                  0.04627249357326478,
                  389,
                  18
                ],
                [
                  0.7129798903107861,
                  0.046153846153846156,
                  390,
                  18
                ],
                [
                  0.7148080438756855,
                  0.04603580562659847,
                  391,
                  18
                ],
                [
                  0.716636197440585,
                  0.04591836734693878,
                  392,
                  18
                ],
                [
                  0.7184643510054844,
                  0.04834605597964377,
                  393,
                  19
                ],
                [
                  0.720292504570384,
                  0.050761421319796954,
                  394,
                  20
                ],
                [
                  0.7221206581352834,
                  0.05063291139240506,
                  395,
                  20
                ],
                [
                  0.7239488117001828,
                  0.050505050505050504,
                  396,
                  20
                ],
                [
                  0.7257769652650823,
                  0.05037783375314862,
                  397,
                  20
                ],
                [
                  0.7276051188299817,
                  0.05025125628140704,
                  398,
                  20
                ],
                [
                  0.7294332723948812,
                  0.05012531328320802,
                  399,
                  20
                ],
                [
                  0.7312614259597806,
                  0.05,
                  400,
                  20
                ],
                [
                  0.7330895795246801,
                  0.05236907730673317,
                  401,
                  21
                ],
                [
                  0.7349177330895795,
                  0.05223880597014925,
                  402,
                  21
                ],
                [
                  0.7367458866544789,
                  0.05459057071960298,
                  403,
                  22
                ],
                [
                  0.7385740402193784,
                  0.05693069306930693,
                  404,
                  23
                ],
                [
                  0.7404021937842779,
                  0.056790123456790124,
                  405,
                  23
                ],
                [
                  0.7422303473491774,
                  0.059113300492610835,
                  406,
                  24
                ],
                [
                  0.7440585009140768,
                  0.05896805896805897,
                  407,
                  24
                ],
                [
                  0.7458866544789763,
                  0.058823529411764705,
                  408,
                  24
                ],
                [
                  0.7477148080438757,
                  0.061124694376528114,
                  409,
                  25
                ],
                [
                  0.7495429616087751,
                  0.06097560975609756,
                  410,
                  25
                ],
                [
                  0.7513711151736746,
                  0.06326034063260341,
                  411,
                  26
                ],
                [
                  0.753199268738574,
                  0.06310679611650485,
                  412,
                  26
                ],
                [
                  0.7550274223034735,
                  0.06295399515738499,
                  413,
                  26
                ],
                [
                  0.7568555758683729,
                  0.06280193236714976,
                  414,
                  26
                ],
                [
                  0.7586837294332724,
                  0.06265060240963856,
                  415,
                  26
                ],
                [
                  0.7605118829981719,
                  0.0625,
                  416,
                  26
                ],
                [
                  0.7623400365630713,
                  0.06235011990407674,
                  417,
                  26
                ],
                [
                  0.7641681901279708,
                  0.06220095693779904,
                  418,
                  26
                ],
                [
                  0.7659963436928702,
                  0.06205250596658711,
                  419,
                  26
                ],
                [
                  0.7678244972577697,
                  0.06190476190476191,
                  420,
                  26
                ],
                [
                  0.7696526508226691,
                  0.06175771971496437,
                  421,
                  26
                ],
                [
                  0.7714808043875686,
                  0.061611374407582936,
                  422,
                  26
                ],
                [
                  0.773308957952468,
                  0.061465721040189124,
                  423,
                  26
                ],
                [
                  0.7751371115173674,
                  0.06132075471698113,
                  424,
                  26
                ],
                [
                  0.7769652650822669,
                  0.06352941176470588,
                  425,
                  27
                ],
                [
                  0.7787934186471663,
                  0.06338028169014084,
                  426,
                  27
                ],
                [
                  0.7806215722120659,
                  0.06323185011709602,
                  427,
                  27
                ],
                [
                  0.7824497257769653,
                  0.06542056074766354,
                  428,
                  28
                ],
                [
                  0.7842778793418648,
                  0.06526806526806526,
                  429,
                  28
                ],
                [
                  0.7861060329067642,
                  0.06511627906976744,
                  430,
                  28
                ],
                [
                  0.7879341864716636,
                  0.06728538283062645,
                  431,
                  29
                ],
                [
                  0.7897623400365631,
                  0.06712962962962964,
                  432,
                  29
                ],
                [
                  0.7915904936014625,
                  0.06697459584295612,
                  433,
                  29
                ],
                [
                  0.793418647166362,
                  0.06912442396313365,
                  434,
                  30
                ],
                [
                  0.7952468007312614,
                  0.06896551724137931,
                  435,
                  30
                ],
                [
                  0.7970749542961609,
                  0.06880733944954129,
                  436,
                  30
                ],
                [
                  0.7989031078610603,
                  0.07093821510297482,
                  437,
                  31
                ],
                [
                  0.8007312614259597,
                  0.07077625570776255,
                  438,
                  31
                ],
                [
                  0.8025594149908593,
                  0.07061503416856492,
                  439,
                  31
                ],
                [
                  0.8043875685557587,
                  0.07272727272727272,
                  440,
                  32
                ],
                [
                  0.8062157221206582,
                  0.07482993197278912,
                  441,
                  33
                ],
                [
                  0.8080438756855576,
                  0.0746606334841629,
                  442,
                  33
                ],
                [
                  0.8098720292504571,
                  0.0744920993227991,
                  443,
                  33
                ],
                [
                  0.8117001828153565,
                  0.07432432432432433,
                  444,
                  33
                ],
                [
                  0.8135283363802559,
                  0.07415730337078652,
                  445,
                  33
                ],
                [
                  0.8153564899451554,
                  0.07399103139013453,
                  446,
                  33
                ],
                [
                  0.8171846435100548,
                  0.0738255033557047,
                  447,
                  33
                ],
                [
                  0.8190127970749543,
                  0.07589285714285714,
                  448,
                  34
                ],
                [
                  0.8208409506398537,
                  0.0757238307349666,
                  449,
                  34
                ],
                [
                  0.8226691042047533,
                  0.07777777777777778,
                  450,
                  35
                ],
                [
                  0.8244972577696527,
                  0.07760532150776053,
                  451,
                  35
                ],
                [
                  0.8263254113345521,
                  0.07964601769911504,
                  452,
                  36
                ],
                [
                  0.8281535648994516,
                  0.07947019867549669,
                  453,
                  36
                ],
                [
                  0.829981718464351,
                  0.07929515418502203,
                  454,
                  36
                ],
                [
                  0.8318098720292505,
                  0.07912087912087912,
                  455,
                  36
                ],
                [
                  0.8336380255941499,
                  0.07894736842105263,
                  456,
                  36
                ],
                [
                  0.8354661791590493,
                  0.0787746170678337,
                  457,
                  36
                ],
                [
                  0.8372943327239488,
                  0.08078602620087336,
                  458,
                  37
                ],
                [
                  0.8391224862888482,
                  0.08061002178649238,
                  459,
                  37
                ],
                [
                  0.8409506398537477,
                  0.08043478260869565,
                  460,
                  37
                ],
                [
                  0.8427787934186471,
                  0.08026030368763558,
                  461,
                  37
                ],
                [
                  0.8446069469835467,
                  0.08225108225108226,
                  462,
                  38
                ],
                [
                  0.8464351005484461,
                  0.08207343412526998,
                  463,
                  38
                ],
                [
                  0.8482632541133455,
                  0.08405172413793104,
                  464,
                  39
                ],
                [
                  0.850091407678245,
                  0.08602150537634409,
                  465,
                  40
                ],
                [
                  0.8519195612431444,
                  0.08798283261802575,
                  466,
                  41
                ],
                [
                  0.8537477148080439,
                  0.08779443254817987,
                  467,
                  41
                ],
                [
                  0.8555758683729433,
                  0.08974358974358974,
                  468,
                  42
                ],
                [
                  0.8574040219378428,
                  0.09168443496801706,
                  469,
                  43
                ],
                [
                  0.8592321755027422,
                  0.09148936170212765,
                  470,
                  43
                ],
                [
                  0.8610603290676416,
                  0.09129511677282377,
                  471,
                  43
                ],
                [
                  0.8628884826325411,
                  0.09322033898305085,
                  472,
                  44
                ],
                [
                  0.8647166361974405,
                  0.09302325581395349,
                  473,
                  44
                ],
                [
                  0.8665447897623401,
                  0.0949367088607595,
                  474,
                  45
                ],
                [
                  0.8683729433272395,
                  0.09473684210526316,
                  475,
                  45
                ],
                [
                  0.870201096892139,
                  0.09663865546218488,
                  476,
                  46
                ],
                [
                  0.8720292504570384,
                  0.09643605870020965,
                  477,
                  46
                ],
                [
                  0.8738574040219378,
                  0.09832635983263599,
                  478,
                  47
                ],
                [
                  0.8756855575868373,
                  0.10020876826722339,
                  479,
                  48
                ],
                [
                  0.8775137111517367,
                  0.10208333333333333,
                  480,
                  49
                ],
                [
                  0.8793418647166362,
                  0.10187110187110188,
                  481,
                  49
                ],
                [
                  0.8811700182815356,
                  0.1016597510373444,
                  482,
                  49
                ],
                [
                  0.8829981718464351,
                  0.10144927536231885,
                  483,
                  49
                ],
                [
                  0.8848263254113345,
                  0.1012396694214876,
                  484,
                  49
                ],
                [
                  0.886654478976234,
                  0.10103092783505155,
                  485,
                  49
                ],
                [
                  0.8884826325411335,
                  0.10082304526748971,
                  486,
                  49
                ],
                [
                  0.8903107861060329,
                  0.1026694045174538,
                  487,
                  50
                ],
                [
                  0.8921389396709324,
                  0.10450819672131148,
                  488,
                  51
                ],
                [
                  0.8939670932358318,
                  0.10429447852760736,
                  489,
                  51
                ],
                [
                  0.8957952468007313,
                  0.10612244897959183,
                  490,
                  52
                ],
                [
                  0.8976234003656307,
                  0.10590631364562118,
                  491,
                  52
                ],
                [
                  0.8994515539305301,
                  0.10772357723577236,
                  492,
                  53
                ],
                [
                  0.9012797074954296,
                  0.1075050709939148,
                  493,
                  53
                ],
                [
                  0.903107861060329,
                  0.10931174089068826,
                  494,
                  54
                ],
                [
                  0.9049360146252285,
                  0.10909090909090909,
                  495,
                  54
                ],
                [
                  0.906764168190128,
                  0.11088709677419355,
                  496,
                  55
                ],
                [
                  0.9085923217550275,
                  0.11267605633802817,
                  497,
                  56
                ],
                [
                  0.9104204753199269,
                  0.1144578313253012,
                  498,
                  57
                ],
                [
                  0.9122486288848263,
                  0.11422845691382766,
                  499,
                  57
                ],
                [
                  0.9140767824497258,
                  0.114,
                  500,
                  57
                ],
                [
                  0.9159049360146252,
                  0.11377245508982035,
                  501,
                  57
                ],
                [
                  0.9177330895795247,
                  0.11354581673306773,
                  502,
                  57
                ],
                [
                  0.9195612431444241,
                  0.11332007952286283,
                  503,
                  57
                ],
                [
                  0.9213893967093236,
                  0.1130952380952381,
                  504,
                  57
                ],
                [
                  0.923217550274223,
                  0.11287128712871287,
                  505,
                  57
                ],
                [
                  0.9250457038391224,
                  0.11264822134387352,
                  506,
                  57
                ],
                [
                  0.926873857404022,
                  0.11439842209072978,
                  507,
                  58
                ],
                [
                  0.9287020109689214,
                  0.1141732283464567,
                  508,
                  58
                ],
                [
                  0.9305301645338209,
                  0.11591355599214145,
                  509,
                  59
                ],
                [
                  0.9323583180987203,
                  0.11568627450980393,
                  510,
                  59
                ],
                [
                  0.9341864716636198,
                  0.11545988258317025,
                  511,
                  59
                ],
                [
                  0.9360146252285192,
                  0.115234375,
                  512,
                  59
                ],
                [
                  0.9378427787934186,
                  0.11500974658869395,
                  513,
                  59
                ],
                [
                  0.9396709323583181,
                  0.11673151750972763,
                  514,
                  60
                ],
                [
                  0.9414990859232175,
                  0.11844660194174757,
                  515,
                  61
                ],
                [
                  0.943327239488117,
                  0.1182170542635659,
                  516,
                  61
                ],
                [
                  0.9451553930530164,
                  0.11992263056092843,
                  517,
                  62
                ],
                [
                  0.946983546617916,
                  0.11969111969111969,
                  518,
                  62
                ],
                [
                  0.9488117001828154,
                  0.12138728323699421,
                  519,
                  63
                ],
                [
                  0.9506398537477148,
                  0.12115384615384615,
                  520,
                  63
                ],
                [
                  0.9524680073126143,
                  0.12284069097888675,
                  521,
                  64
                ],
                [
                  0.9542961608775137,
                  0.12260536398467432,
                  522,
                  64
                ],
                [
                  0.9561243144424132,
                  0.124282982791587,
                  523,
                  65
                ],
                [
                  0.9579524680073126,
                  0.12404580152671756,
                  524,
                  65
                ],
                [
                  0.9597806215722121,
                  0.12571428571428572,
                  525,
                  66
                ],
                [
                  0.9616087751371115,
                  0.12737642585551331,
                  526,
                  67
                ],
                [
                  0.9634369287020109,
                  0.12903225806451613,
                  527,
                  68
                ],
                [
                  0.9652650822669104,
                  0.13068181818181818,
                  528,
                  69
                ],
                [
                  0.9670932358318098,
                  0.13043478260869565,
                  529,
                  69
                ],
                [
                  0.9689213893967094,
                  0.1320754716981132,
                  530,
                  70
                ],
                [
                  0.9707495429616088,
                  0.1337099811676083,
                  531,
                  71
                ],
                [
                  0.9725776965265083,
                  0.13533834586466165,
                  532,
                  72
                ],
                [
                  0.9744058500914077,
                  0.1350844277673546,
                  533,
                  72
                ],
                [
                  0.9762340036563071,
                  0.1348314606741573,
                  534,
                  72
                ],
                [
                  0.9780621572212066,
                  0.13457943925233645,
                  535,
                  72
                ],
                [
                  0.979890310786106,
                  0.13432835820895522,
                  536,
                  72
                ],
                [
                  0.9817184643510055,
                  0.13594040968342644,
                  537,
                  73
                ],
                [
                  0.9835466179159049,
                  0.137546468401487,
                  538,
                  74
                ],
                [
                  0.9853747714808044,
                  0.137291280148423,
                  539,
                  74
                ],
                [
                  0.9872029250457038,
                  0.1388888888888889,
                  540,
                  75
                ],
                [
                  0.9890310786106032,
                  0.14048059149722736,
                  541,
                  76
                ],
                [
                  0.9908592321755028,
                  0.14206642066420663,
                  542,
                  77
                ],
                [
                  0.9926873857404022,
                  0.141804788213628,
                  543,
                  77
                ],
                [
                  0.9945155393053017,
                  0.14338235294117646,
                  544,
                  78
                ],
                [
                  0.9963436928702011,
                  0.14495412844036698,
                  545,
                  79
                ],
                [
                  0.9981718464351006,
                  0.1446886446886447,
                  546,
                  79
                ],
                [
                  1.0,
                  0.14625228519195613,
                  547,
                  80
                ]
              ]
            }
          ]
        },
        {
          "name": "Celeris-1 Decision",
          "text_run": 150,
          "native": [
            {
              "language": "Turkish",
              "wording": "Bilmiyorum",
              "run": null
            },
            {
              "language": "Turkish",
              "wording": "I don't know",
              "run": null
            },
            {
              "language": "English",
              "wording": "I don't know",
              "run": 548,
              "full_bank_restricted": false,
              "right": 418,
              "wrong": 37,
              "abstained": 92,
              "invalid": 0,
              "coverage": 0.8318098720292505,
              "risk": 0.08131868131868132
            }
          ],
          "curves": [
            {
              "language": "Turkish",
              "run": 150,
              "points": [
                [
                  0.005484460694698354,
                  0.0,
                  3,
                  0
                ],
                [
                  0.0603290676416819,
                  0.0,
                  33,
                  0
                ],
                [
                  0.16087751371115175,
                  0.0,
                  88,
                  0
                ],
                [
                  0.27239488117001825,
                  0.0,
                  149,
                  0
                ],
                [
                  0.37842778793418647,
                  0.0,
                  207,
                  0
                ],
                [
                  0.43875685557586835,
                  0.0,
                  240,
                  0
                ],
                [
                  0.4954296160877514,
                  0.0,
                  271,
                  0
                ],
                [
                  0.5283363802559415,
                  0.0,
                  289,
                  0
                ],
                [
                  0.5539305301645339,
                  0.0,
                  303,
                  0
                ],
                [
                  0.5776965265082267,
                  0.0,
                  316,
                  0
                ],
                [
                  0.5886654478976234,
                  0.003105590062111801,
                  322,
                  1
                ],
                [
                  0.6069469835466179,
                  0.0030120481927710845,
                  332,
                  1
                ],
                [
                  0.6179159049360147,
                  0.005917159763313609,
                  338,
                  2
                ],
                [
                  0.6252285191956124,
                  0.011695906432748537,
                  342,
                  4
                ],
                [
                  0.643510054844607,
                  0.014204545454545454,
                  352,
                  5
                ],
                [
                  0.659963436928702,
                  0.01662049861495845,
                  361,
                  6
                ],
                [
                  0.6654478976234004,
                  0.019230769230769232,
                  364,
                  7
                ],
                [
                  0.680073126142596,
                  0.021505376344086023,
                  372,
                  8
                ],
                [
                  0.6892138939670932,
                  0.023872679045092837,
                  377,
                  9
                ],
                [
                  0.696526508226691,
                  0.023622047244094488,
                  381,
                  9
                ],
                [
                  0.7148080438756855,
                  0.030690537084398978,
                  391,
                  12
                ],
                [
                  0.7221206581352834,
                  0.03291139240506329,
                  395,
                  13
                ],
                [
                  0.7349177330895795,
                  0.03482587064676617,
                  402,
                  14
                ],
                [
                  0.7422303473491774,
                  0.03694581280788178,
                  406,
                  15
                ],
                [
                  0.7513711151736746,
                  0.0413625304136253,
                  411,
                  17
                ],
                [
                  0.7550274223034735,
                  0.04116222760290557,
                  413,
                  17
                ],
                [
                  0.7586837294332724,
                  0.043373493975903614,
                  415,
                  18
                ],
                [
                  0.7714808043875686,
                  0.045023696682464455,
                  422,
                  19
                ],
                [
                  0.7806215722120659,
                  0.04918032786885246,
                  427,
                  21
                ],
                [
                  0.7879341864716636,
                  0.05104408352668213,
                  431,
                  22
                ],
                [
                  0.7970749542961609,
                  0.05045871559633028,
                  436,
                  22
                ],
                [
                  0.7989031078610603,
                  0.05034324942791762,
                  437,
                  22
                ],
                [
                  0.8153564899451554,
                  0.053811659192825115,
                  446,
                  24
                ],
                [
                  0.8190127970749543,
                  0.05580357142857143,
                  448,
                  25
                ],
                [
                  0.8244972577696527,
                  0.0598669623059867,
                  451,
                  27
                ],
                [
                  0.8409506398537477,
                  0.06521739130434782,
                  460,
                  30
                ],
                [
                  0.8446069469835467,
                  0.0670995670995671,
                  462,
                  31
                ],
                [
                  0.850091407678245,
                  0.07096774193548387,
                  465,
                  33
                ],
                [
                  0.8519195612431444,
                  0.07081545064377683,
                  466,
                  33
                ],
                [
                  0.8574040219378428,
                  0.07036247334754797,
                  469,
                  33
                ],
                [
                  0.8628884826325411,
                  0.06991525423728813,
                  472,
                  33
                ],
                [
                  0.8738574040219378,
                  0.07531380753138076,
                  478,
                  36
                ],
                [
                  0.8775137111517367,
                  0.075,
                  480,
                  36
                ],
                [
                  0.886654478976234,
                  0.07835051546391752,
                  485,
                  38
                ],
                [
                  0.8939670932358318,
                  0.07770961145194274,
                  489,
                  38
                ],
                [
                  0.8976234003656307,
                  0.07739307535641547,
                  491,
                  38
                ],
                [
                  0.9049360146252285,
                  0.08080808080808081,
                  495,
                  40
                ],
                [
                  0.9085923217550275,
                  0.08249496981891348,
                  497,
                  41
                ],
                [
                  0.9195612431444241,
                  0.0874751491053678,
                  503,
                  44
                ],
                [
                  0.9323583180987203,
                  0.09411764705882353,
                  510,
                  48
                ],
                [
                  0.9360146252285192,
                  0.09765625,
                  512,
                  50
                ],
                [
                  0.9451553930530164,
                  0.10058027079303675,
                  517,
                  52
                ],
                [
                  0.946983546617916,
                  0.10231660231660232,
                  518,
                  53
                ],
                [
                  0.9542961608775137,
                  0.10153256704980843,
                  522,
                  53
                ],
                [
                  0.9561243144424132,
                  0.10133843212237094,
                  523,
                  53
                ],
                [
                  0.9616087751371115,
                  0.10456273764258556,
                  526,
                  55
                ],
                [
                  0.9634369287020109,
                  0.1062618595825427,
                  527,
                  56
                ],
                [
                  0.9652650822669104,
                  0.10795454545454546,
                  528,
                  57
                ],
                [
                  0.9689213893967094,
                  0.11132075471698114,
                  530,
                  59
                ],
                [
                  0.9725776965265083,
                  0.11278195488721804,
                  532,
                  60
                ],
                [
                  0.9762340036563071,
                  0.11235955056179775,
                  534,
                  60
                ],
                [
                  0.9780621572212066,
                  0.11214953271028037,
                  535,
                  60
                ],
                [
                  0.9835466179159049,
                  0.11524163568773234,
                  538,
                  62
                ],
                [
                  0.9853747714808044,
                  0.1150278293135436,
                  539,
                  62
                ],
                [
                  0.9872029250457038,
                  0.11666666666666667,
                  540,
                  63
                ],
                [
                  0.9890310786106032,
                  0.11829944547134935,
                  541,
                  64
                ],
                [
                  0.9926873857404022,
                  0.12154696132596685,
                  543,
                  66
                ],
                [
                  0.9945155393053017,
                  0.12316176470588236,
                  544,
                  67
                ],
                [
                  0.9981718464351006,
                  0.12454212454212454,
                  546,
                  68
                ],
                [
                  1.0,
                  0.12614259597806216,
                  547,
                  69
                ]
              ]
            },
            {
              "language": "English",
              "run": 546,
              "points": [
                [
                  0.029250457038391225,
                  0.0,
                  16,
                  0
                ],
                [
                  0.2303473491773309,
                  0.007936507936507936,
                  126,
                  1
                ],
                [
                  0.37477148080438755,
                  0.004878048780487805,
                  205,
                  1
                ],
                [
                  0.43875685557586835,
                  0.004166666666666667,
                  240,
                  1
                ],
                [
                  0.5009140767824497,
                  0.0072992700729927005,
                  274,
                  2
                ],
                [
                  0.5319926873857403,
                  0.013745704467353952,
                  291,
                  4
                ],
                [
                  0.5539305301645339,
                  0.013201320132013201,
                  303,
                  4
                ],
                [
                  0.5813528336380256,
                  0.015723270440251572,
                  318,
                  5
                ],
                [
                  0.6051188299817185,
                  0.01812688821752266,
                  331,
                  6
                ],
                [
                  0.6160877513711152,
                  0.020771513353115726,
                  337,
                  7
                ],
                [
                  0.6215722120658135,
                  0.023529411764705882,
                  340,
                  8
                ],
                [
                  0.6398537477148081,
                  0.025714285714285714,
                  350,
                  9
                ],
                [
                  0.6471663619744058,
                  0.025423728813559324,
                  354,
                  9
                ],
                [
                  0.6654478976234004,
                  0.03021978021978022,
                  364,
                  11
                ],
                [
                  0.680073126142596,
                  0.03494623655913978,
                  372,
                  13
                ],
                [
                  0.6910420475319927,
                  0.03968253968253968,
                  378,
                  15
                ],
                [
                  0.70018281535649,
                  0.04177545691906005,
                  383,
                  16
                ],
                [
                  0.7074954296160878,
                  0.041343669250646,
                  387,
                  16
                ],
                [
                  0.7221206581352834,
                  0.05063291139240506,
                  395,
                  20
                ],
                [
                  0.7257769652650823,
                  0.05289672544080604,
                  397,
                  21
                ],
                [
                  0.7276051188299817,
                  0.052763819095477386,
                  398,
                  21
                ],
                [
                  0.7312614259597806,
                  0.0525,
                  400,
                  21
                ],
                [
                  0.7367458866544789,
                  0.05707196029776675,
                  403,
                  23
                ],
                [
                  0.7458866544789763,
                  0.061274509803921566,
                  408,
                  25
                ],
                [
                  0.7605118829981719,
                  0.0673076923076923,
                  416,
                  28
                ],
                [
                  0.7714808043875686,
                  0.07109004739336493,
                  422,
                  30
                ],
                [
                  0.7861060329067642,
                  0.07674418604651163,
                  430,
                  33
                ],
                [
                  0.7970749542961609,
                  0.07568807339449542,
                  436,
                  33
                ],
                [
                  0.8043875685557587,
                  0.07954545454545454,
                  440,
                  35
                ],
                [
                  0.8117001828153565,
                  0.08108108108108109,
                  444,
                  36
                ],
                [
                  0.8171846435100548,
                  0.08053691275167785,
                  447,
                  36
                ],
                [
                  0.8281535648994516,
                  0.08388520971302428,
                  453,
                  38
                ],
                [
                  0.8354661791590493,
                  0.087527352297593,
                  457,
                  40
                ],
                [
                  0.8409506398537477,
                  0.0891304347826087,
                  460,
                  41
                ],
                [
                  0.8446069469835467,
                  0.09090909090909091,
                  462,
                  42
                ],
                [
                  0.8519195612431444,
                  0.09012875536480687,
                  466,
                  42
                ],
                [
                  0.8555758683729433,
                  0.09188034188034189,
                  468,
                  43
                ],
                [
                  0.8610603290676416,
                  0.09341825902335456,
                  471,
                  44
                ],
                [
                  0.8683729433272395,
                  0.09473684210526316,
                  475,
                  45
                ],
                [
                  0.8884826325411335,
                  0.10493827160493827,
                  486,
                  51
                ],
                [
                  0.8921389396709324,
                  0.10450819672131148,
                  488,
                  51
                ],
                [
                  0.9012797074954296,
                  0.1075050709939148,
                  493,
                  53
                ],
                [
                  0.906764168190128,
                  0.11290322580645161,
                  496,
                  56
                ],
                [
                  0.9140767824497258,
                  0.112,
                  500,
                  56
                ],
                [
                  0.9250457038391224,
                  0.1225296442687747,
                  506,
                  62
                ],
                [
                  0.9323583180987203,
                  0.12745098039215685,
                  510,
                  65
                ],
                [
                  0.9360146252285192,
                  0.130859375,
                  512,
                  67
                ],
                [
                  0.9414990859232175,
                  0.13009708737864079,
                  515,
                  67
                ],
                [
                  0.9451553930530164,
                  0.13152804642166344,
                  517,
                  68
                ],
                [
                  0.946983546617916,
                  0.13127413127413126,
                  518,
                  68
                ],
                [
                  0.9542961608775137,
                  0.13218390804597702,
                  522,
                  69
                ],
                [
                  0.9561243144424132,
                  0.1338432122370937,
                  523,
                  70
                ],
                [
                  0.9579524680073126,
                  0.13549618320610687,
                  524,
                  71
                ],
                [
                  0.9670932358318098,
                  0.13799621928166353,
                  529,
                  73
                ],
                [
                  0.9744058500914077,
                  0.1425891181988743,
                  533,
                  76
                ],
                [
                  0.9762340036563071,
                  0.1441947565543071,
                  534,
                  77
                ],
                [
                  0.979890310786106,
                  0.1455223880597015,
                  536,
                  78
                ],
                [
                  0.9835466179159049,
                  0.14684014869888476,
                  538,
                  79
                ],
                [
                  0.9872029250457038,
                  0.15,
                  540,
                  81
                ],
                [
                  0.9890310786106032,
                  0.15157116451016636,
                  541,
                  82
                ],
                [
                  0.9908592321755028,
                  0.15313653136531366,
                  542,
                  83
                ],
                [
                  0.9945155393053017,
                  0.15625,
                  544,
                  85
                ],
                [
                  0.9963436928702011,
                  0.1577981651376147,
                  545,
                  86
                ],
                [
                  0.9981718464351006,
                  0.15934065934065933,
                  546,
                  87
                ],
                [
                  1.0,
                  0.1590493601462523,
                  547,
                  87
                ]
              ]
            }
          ]
        },
        {
          "name": "GPT-6 Luna",
          "text_run": 143,
          "native": [
            {
              "language": "Turkish",
              "wording": "Bilmiyorum",
              "run": 453,
              "full_bank_restricted": false,
              "right": 463,
              "wrong": 78,
              "abstained": 6,
              "invalid": 0,
              "coverage": 0.9890310786106032,
              "risk": 0.14417744916820702
            },
            {
              "language": "Turkish",
              "wording": "I don't know",
              "run": null
            },
            {
              "language": "English",
              "wording": "I don't know",
              "run": 444,
              "full_bank_restricted": false,
              "right": 441,
              "wrong": 96,
              "abstained": 10,
              "invalid": 0,
              "coverage": 0.9817184643510055,
              "risk": 0.1787709497206704
            }
          ],
          "curves": [
            {
              "language": "Turkish",
              "run": 143,
              "points": [
                [
                  0.3656307129798903,
                  0.01,
                  200,
                  2
                ],
                [
                  0.44972577696526506,
                  0.008130081300813009,
                  246,
                  2
                ],
                [
                  0.5009140767824497,
                  0.010948905109489052,
                  274,
                  3
                ],
                [
                  0.546617915904936,
                  0.013377926421404682,
                  299,
                  4
                ],
                [
                  0.5703839122486288,
                  0.016025641025641024,
                  312,
                  5
                ],
                [
                  0.5904936014625228,
                  0.021671826625386997,
                  323,
                  7
                ],
                [
                  0.603290676416819,
                  0.021212121212121213,
                  330,
                  7
                ],
                [
                  0.6215722120658135,
                  0.029411764705882353,
                  340,
                  10
                ],
                [
                  0.6361974405850092,
                  0.028735632183908046,
                  348,
                  10
                ],
                [
                  0.6526508226691042,
                  0.03081232492997199,
                  357,
                  11
                ],
                [
                  0.6672760511882998,
                  0.03287671232876712,
                  365,
                  12
                ],
                [
                  0.6745886654478976,
                  0.032520325203252036,
                  369,
                  12
                ],
                [
                  0.6819012797074955,
                  0.032171581769437,
                  373,
                  12
                ],
                [
                  0.6910420475319927,
                  0.037037037037037035,
                  378,
                  14
                ],
                [
                  0.6946983546617916,
                  0.03684210526315789,
                  380,
                  14
                ],
                [
                  0.7020109689213894,
                  0.036458333333333336,
                  384,
                  14
                ],
                [
                  0.7074954296160878,
                  0.03617571059431524,
                  387,
                  14
                ],
                [
                  0.7129798903107861,
                  0.038461538461538464,
                  390,
                  15
                ],
                [
                  0.7294332723948812,
                  0.040100250626566414,
                  399,
                  16
                ],
                [
                  0.7367458866544789,
                  0.04466501240694789,
                  403,
                  18
                ],
                [
                  0.7404021937842779,
                  0.044444444444444446,
                  405,
                  18
                ],
                [
                  0.7495429616087751,
                  0.05121951219512195,
                  410,
                  21
                ],
                [
                  0.7586837294332724,
                  0.05060240963855422,
                  415,
                  21
                ],
                [
                  0.7678244972577697,
                  0.05238095238095238,
                  420,
                  22
                ],
                [
                  0.7696526508226691,
                  0.05463182897862233,
                  421,
                  23
                ],
                [
                  0.7769652650822669,
                  0.05411764705882353,
                  425,
                  23
                ],
                [
                  0.7806215722120659,
                  0.053864168618266976,
                  427,
                  23
                ],
                [
                  0.7842778793418648,
                  0.055944055944055944,
                  429,
                  24
                ],
                [
                  0.7879341864716636,
                  0.058004640371229696,
                  431,
                  25
                ],
                [
                  0.7915904936014625,
                  0.057736720554272515,
                  433,
                  25
                ],
                [
                  0.7989031078610603,
                  0.06407322654462243,
                  437,
                  28
                ],
                [
                  0.8025594149908593,
                  0.06605922551252848,
                  439,
                  29
                ],
                [
                  0.8153564899451554,
                  0.06726457399103139,
                  446,
                  30
                ],
                [
                  0.8208409506398537,
                  0.06904231625835189,
                  449,
                  31
                ],
                [
                  0.8281535648994516,
                  0.0684326710816777,
                  453,
                  31
                ],
                [
                  0.8354661791590493,
                  0.0700218818380744,
                  457,
                  32
                ],
                [
                  0.8372943327239488,
                  0.06986899563318777,
                  458,
                  32
                ],
                [
                  0.8409506398537477,
                  0.07173913043478261,
                  460,
                  33
                ],
                [
                  0.8464351005484461,
                  0.0734341252699784,
                  463,
                  34
                ],
                [
                  0.8592321755027422,
                  0.08085106382978724,
                  470,
                  38
                ],
                [
                  0.8628884826325411,
                  0.08050847457627118,
                  472,
                  38
                ],
                [
                  0.8647166361974405,
                  0.080338266384778,
                  473,
                  38
                ],
                [
                  0.8665447897623401,
                  0.08016877637130802,
                  474,
                  38
                ],
                [
                  0.8738574040219378,
                  0.08368200836820083,
                  478,
                  40
                ],
                [
                  0.8775137111517367,
                  0.08333333333333333,
                  480,
                  40
                ],
                [
                  0.8848263254113345,
                  0.08677685950413223,
                  484,
                  42
                ],
                [
                  0.8921389396709324,
                  0.09221311475409837,
                  488,
                  45
                ],
                [
                  0.903107861060329,
                  0.09109311740890688,
                  494,
                  45
                ],
                [
                  0.906764168190128,
                  0.0907258064516129,
                  496,
                  45
                ],
                [
                  0.9159049360146252,
                  0.09181636726546906,
                  501,
                  46
                ],
                [
                  0.9250457038391224,
                  0.09288537549407115,
                  506,
                  47
                ],
                [
                  0.9305301645338209,
                  0.0962671905697446,
                  509,
                  49
                ],
                [
                  0.9360146252285192,
                  0.099609375,
                  512,
                  51
                ],
                [
                  0.9488117001828154,
                  0.10404624277456648,
                  519,
                  54
                ],
                [
                  0.9506398537477148,
                  0.10576923076923077,
                  520,
                  55
                ],
                [
                  0.9542961608775137,
                  0.10919540229885058,
                  522,
                  57
                ],
                [
                  0.9597806215722121,
                  0.11238095238095239,
                  525,
                  59
                ],
                [
                  0.9670932358318098,
                  0.11720226843100189,
                  529,
                  62
                ],
                [
                  0.9762340036563071,
                  0.1198501872659176,
                  534,
                  64
                ],
                [
                  0.9835466179159049,
                  0.12453531598513011,
                  538,
                  67
                ],
                [
                  0.9890310786106032,
                  0.1256931608133087,
                  541,
                  68
                ],
                [
                  0.9945155393053017,
                  0.13051470588235295,
                  544,
                  71
                ],
                [
                  0.9963436928702011,
                  0.13027522935779817,
                  545,
                  71
                ],
                [
                  0.9981718464351006,
                  0.13186813186813187,
                  546,
                  72
                ]
              ]
            },
            {
              "language": "English",
              "run": 397,
              "points": [
                [
                  0.3327239488117002,
                  0.005494505494505495,
                  182,
                  1
                ],
                [
                  0.41316270566727603,
                  0.008849557522123894,
                  226,
                  2
                ],
                [
                  0.453382084095064,
                  0.008064516129032258,
                  248,
                  2
                ],
                [
                  0.4936014625228519,
                  0.011111111111111112,
                  270,
                  3
                ],
                [
                  0.5210237659963437,
                  0.021052631578947368,
                  285,
                  6
                ],
                [
                  0.546617915904936,
                  0.023411371237458192,
                  299,
                  7
                ],
                [
                  0.5612431444241316,
                  0.026058631921824105,
                  307,
                  8
                ],
                [
                  0.5795246800731262,
                  0.03470031545741325,
                  317,
                  11
                ],
                [
                  0.5904936014625228,
                  0.03715170278637771,
                  323,
                  12
                ],
                [
                  0.6124314442413162,
                  0.041791044776119404,
                  335,
                  14
                ],
                [
                  0.6307129798903108,
                  0.04057971014492753,
                  345,
                  14
                ],
                [
                  0.6398537477148081,
                  0.04285714285714286,
                  350,
                  15
                ],
                [
                  0.6544789762340036,
                  0.0446927374301676,
                  358,
                  16
                ],
                [
                  0.6563071297989032,
                  0.04456824512534819,
                  359,
                  16
                ],
                [
                  0.6672760511882998,
                  0.04657534246575343,
                  365,
                  17
                ],
                [
                  0.6727605118829981,
                  0.04891304347826087,
                  368,
                  18
                ],
                [
                  0.680073126142596,
                  0.04838709677419355,
                  372,
                  18
                ],
                [
                  0.6928702010968921,
                  0.05013192612137203,
                  379,
                  19
                ],
                [
                  0.70018281535649,
                  0.05483028720626632,
                  383,
                  21
                ],
                [
                  0.7056672760511883,
                  0.05958549222797927,
                  386,
                  23
                ],
                [
                  0.7093235831809872,
                  0.061855670103092786,
                  388,
                  24
                ],
                [
                  0.7184643510054844,
                  0.061068702290076333,
                  393,
                  24
                ],
                [
                  0.7276051188299817,
                  0.06030150753768844,
                  398,
                  24
                ],
                [
                  0.7312614259597806,
                  0.06,
                  400,
                  24
                ],
                [
                  0.7349177330895795,
                  0.05970149253731343,
                  402,
                  24
                ],
                [
                  0.7404021937842779,
                  0.05925925925925926,
                  405,
                  24
                ],
                [
                  0.7458866544789763,
                  0.06372549019607843,
                  408,
                  26
                ],
                [
                  0.7495429616087751,
                  0.06341463414634146,
                  410,
                  26
                ],
                [
                  0.7623400365630713,
                  0.0671462829736211,
                  417,
                  28
                ],
                [
                  0.7659963436928702,
                  0.06921241050119331,
                  419,
                  29
                ],
                [
                  0.7751371115173674,
                  0.07311320754716981,
                  424,
                  31
                ],
                [
                  0.7787934186471663,
                  0.07511737089201878,
                  426,
                  32
                ],
                [
                  0.7824497257769653,
                  0.07476635514018691,
                  428,
                  32
                ],
                [
                  0.7915904936014625,
                  0.08083140877598152,
                  433,
                  35
                ],
                [
                  0.8025594149908593,
                  0.08428246013667426,
                  439,
                  37
                ],
                [
                  0.8135283363802559,
                  0.0853932584269663,
                  445,
                  38
                ],
                [
                  0.8190127970749543,
                  0.08482142857142858,
                  448,
                  38
                ],
                [
                  0.8226691042047533,
                  0.08666666666666667,
                  450,
                  39
                ],
                [
                  0.8318098720292505,
                  0.09010989010989011,
                  455,
                  41
                ],
                [
                  0.8427787934186471,
                  0.09327548806941431,
                  461,
                  43
                ],
                [
                  0.8464351005484461,
                  0.09287257019438445,
                  463,
                  43
                ],
                [
                  0.8519195612431444,
                  0.0944206008583691,
                  466,
                  44
                ],
                [
                  0.8574040219378428,
                  0.09808102345415778,
                  469,
                  46
                ],
                [
                  0.8628884826325411,
                  0.1038135593220339,
                  472,
                  49
                ],
                [
                  0.8683729433272395,
                  0.10526315789473684,
                  475,
                  50
                ],
                [
                  0.8738574040219378,
                  0.10669456066945607,
                  478,
                  51
                ],
                [
                  0.8775137111517367,
                  0.10625,
                  480,
                  51
                ],
                [
                  0.8811700182815356,
                  0.1078838174273859,
                  482,
                  52
                ],
                [
                  0.8884826325411335,
                  0.10905349794238683,
                  486,
                  53
                ],
                [
                  0.9012797074954296,
                  0.11561866125760649,
                  493,
                  57
                ],
                [
                  0.9140767824497258,
                  0.12,
                  500,
                  60
                ],
                [
                  0.923217550274223,
                  0.12673267326732673,
                  505,
                  64
                ],
                [
                  0.9287020109689214,
                  0.12992125984251968,
                  508,
                  66
                ],
                [
                  0.9341864716636198,
                  0.1350293542074364,
                  511,
                  69
                ],
                [
                  0.9378427787934186,
                  0.13450292397660818,
                  513,
                  69
                ],
                [
                  0.9506398537477148,
                  0.1423076923076923,
                  520,
                  74
                ],
                [
                  0.9561243144424132,
                  0.14340344168260039,
                  523,
                  75
                ],
                [
                  0.9670932358318098,
                  0.14366729678638943,
                  529,
                  76
                ],
                [
                  0.9707495429616088,
                  0.14500941619585686,
                  531,
                  77
                ],
                [
                  0.9725776965265083,
                  0.14661654135338345,
                  532,
                  78
                ],
                [
                  0.9744058500914077,
                  0.14634146341463414,
                  533,
                  78
                ],
                [
                  0.979890310786106,
                  0.14738805970149255,
                  536,
                  79
                ],
                [
                  0.9835466179159049,
                  0.14869888475836432,
                  538,
                  80
                ],
                [
                  0.9853747714808044,
                  0.150278293135436,
                  539,
                  81
                ],
                [
                  0.9908592321755028,
                  0.15313653136531366,
                  542,
                  83
                ],
                [
                  0.9926873857404022,
                  0.15469613259668508,
                  543,
                  84
                ],
                [
                  0.9945155393053017,
                  0.15441176470588236,
                  544,
                  84
                ],
                [
                  0.9963436928702011,
                  0.1559633027522936,
                  545,
                  85
                ]
              ]
            }
          ]
        },
        {
          "name": "Perplexity Decider Flash v1.2",
          "text_run": 223,
          "native": [
            {
              "language": "Turkish",
              "wording": "Bilmiyorum",
              "run": 537,
              "full_bank_restricted": false,
              "right": 408,
              "wrong": 133,
              "abstained": 6,
              "invalid": 0,
              "coverage": 0.9890310786106032,
              "risk": 0.24584103512014788
            },
            {
              "language": "Turkish",
              "wording": "I don't know",
              "run": null
            },
            {
              "language": "English",
              "wording": "I don't know",
              "run": 535,
              "full_bank_restricted": false,
              "right": 442,
              "wrong": 100,
              "abstained": 5,
              "invalid": 0,
              "coverage": 0.9908592321755028,
              "risk": 0.18450184501845018
            }
          ],
          "curves": [
            {
              "language": "Turkish",
              "run": 223,
              "points": [
                [
                  0.0018281535648994515,
                  0.0,
                  1,
                  0
                ],
                [
                  0.003656307129798903,
                  0.0,
                  2,
                  0
                ],
                [
                  0.005484460694698354,
                  0.0,
                  3,
                  0
                ],
                [
                  0.007312614259597806,
                  0.0,
                  4,
                  0
                ],
                [
                  0.009140767824497258,
                  0.0,
                  5,
                  0
                ],
                [
                  0.010968921389396709,
                  0.0,
                  6,
                  0
                ],
                [
                  0.012797074954296161,
                  0.0,
                  7,
                  0
                ],
                [
                  0.014625228519195612,
                  0.0,
                  8,
                  0
                ],
                [
                  0.016453382084095063,
                  0.0,
                  9,
                  0
                ],
                [
                  0.018281535648994516,
                  0.0,
                  10,
                  0
                ],
                [
                  0.02010968921389397,
                  0.0,
                  11,
                  0
                ],
                [
                  0.021937842778793418,
                  0.0,
                  12,
                  0
                ],
                [
                  0.02376599634369287,
                  0.0,
                  13,
                  0
                ],
                [
                  0.025594149908592323,
                  0.0,
                  14,
                  0
                ],
                [
                  0.027422303473491772,
                  0.0,
                  15,
                  0
                ],
                [
                  0.029250457038391225,
                  0.0,
                  16,
                  0
                ],
                [
                  0.031078610603290677,
                  0.0,
                  17,
                  0
                ],
                [
                  0.03290676416819013,
                  0.0,
                  18,
                  0
                ],
                [
                  0.03473491773308958,
                  0.0,
                  19,
                  0
                ],
                [
                  0.03656307129798903,
                  0.0,
                  20,
                  0
                ],
                [
                  0.038391224862888484,
                  0.0,
                  21,
                  0
                ],
                [
                  0.04021937842778794,
                  0.0,
                  22,
                  0
                ],
                [
                  0.04204753199268738,
                  0.0,
                  23,
                  0
                ],
                [
                  0.043875685557586835,
                  0.0,
                  24,
                  0
                ],
                [
                  0.04570383912248629,
                  0.0,
                  25,
                  0
                ],
                [
                  0.04753199268738574,
                  0.0,
                  26,
                  0
                ],
                [
                  0.04936014625228519,
                  0.0,
                  27,
                  0
                ],
                [
                  0.051188299817184646,
                  0.0,
                  28,
                  0
                ],
                [
                  0.05301645338208409,
                  0.0,
                  29,
                  0
                ],
                [
                  0.054844606946983544,
                  0.0,
                  30,
                  0
                ],
                [
                  0.05850091407678245,
                  0.0,
                  32,
                  0
                ],
                [
                  0.0603290676416819,
                  0.0,
                  33,
                  0
                ],
                [
                  0.062157221206581355,
                  0.0,
                  34,
                  0
                ],
                [
                  0.06398537477148081,
                  0.0,
                  35,
                  0
                ],
                [
                  0.06581352833638025,
                  0.0,
                  36,
                  0
                ],
                [
                  0.06764168190127971,
                  0.0,
                  37,
                  0
                ],
                [
                  0.06946983546617916,
                  0.0,
                  38,
                  0
                ],
                [
                  0.0712979890310786,
                  0.0,
                  39,
                  0
                ],
                [
                  0.07312614259597806,
                  0.0,
                  40,
                  0
                ],
                [
                  0.07495429616087751,
                  0.0,
                  41,
                  0
                ],
                [
                  0.07678244972577697,
                  0.0,
                  42,
                  0
                ],
                [
                  0.07861060329067641,
                  0.0,
                  43,
                  0
                ],
                [
                  0.08043875685557587,
                  0.0,
                  44,
                  0
                ],
                [
                  0.08226691042047532,
                  0.0,
                  45,
                  0
                ],
                [
                  0.08409506398537477,
                  0.0,
                  46,
                  0
                ],
                [
                  0.08592321755027423,
                  0.0,
                  47,
                  0
                ],
                [
                  0.08775137111517367,
                  0.0,
                  48,
                  0
                ],
                [
                  0.08957952468007313,
                  0.0,
                  49,
                  0
                ],
                [
                  0.09140767824497258,
                  0.0,
                  50,
                  0
                ],
                [
                  0.09323583180987204,
                  0.0,
                  51,
                  0
                ],
                [
                  0.09506398537477148,
                  0.0,
                  52,
                  0
                ],
                [
                  0.09689213893967093,
                  0.0,
                  53,
                  0
                ],
                [
                  0.09872029250457039,
                  0.0,
                  54,
                  0
                ],
                [
                  0.10054844606946983,
                  0.0,
                  55,
                  0
                ],
                [
                  0.10237659963436929,
                  0.0,
                  56,
                  0
                ],
                [
                  0.10420475319926874,
                  0.0,
                  57,
                  0
                ],
                [
                  0.10603290676416818,
                  0.0,
                  58,
                  0
                ],
                [
                  0.10786106032906764,
                  0.0,
                  59,
                  0
                ],
                [
                  0.10968921389396709,
                  0.0,
                  60,
                  0
                ],
                [
                  0.11151736745886655,
                  0.0,
                  61,
                  0
                ],
                [
                  0.113345521023766,
                  0.0,
                  62,
                  0
                ],
                [
                  0.11517367458866545,
                  0.0,
                  63,
                  0
                ],
                [
                  0.1170018281535649,
                  0.0,
                  64,
                  0
                ],
                [
                  0.11882998171846434,
                  0.0,
                  65,
                  0
                ],
                [
                  0.1206581352833638,
                  0.0,
                  66,
                  0
                ],
                [
                  0.12248628884826325,
                  0.0,
                  67,
                  0
                ],
                [
                  0.12431444241316271,
                  0.0,
                  68,
                  0
                ],
                [
                  0.12614259597806216,
                  0.0,
                  69,
                  0
                ],
                [
                  0.12797074954296161,
                  0.0,
                  70,
                  0
                ],
                [
                  0.12979890310786105,
                  0.0,
                  71,
                  0
                ],
                [
                  0.1316270566727605,
                  0.0,
                  72,
                  0
                ],
                [
                  0.13345521023765997,
                  0.0,
                  73,
                  0
                ],
                [
                  0.13528336380255943,
                  0.0,
                  74,
                  0
                ],
                [
                  0.13711151736745886,
                  0.0,
                  75,
                  0
                ],
                [
                  0.13893967093235832,
                  0.0,
                  76,
                  0
                ],
                [
                  0.14076782449725778,
                  0.0,
                  77,
                  0
                ],
                [
                  0.1425959780621572,
                  0.0,
                  78,
                  0
                ],
                [
                  0.14442413162705667,
                  0.0,
                  79,
                  0
                ],
                [
                  0.14625228519195613,
                  0.0,
                  80,
                  0
                ],
                [
                  0.1480804387568556,
                  0.0,
                  81,
                  0
                ],
                [
                  0.14990859232175502,
                  0.0,
                  82,
                  0
                ],
                [
                  0.15173674588665448,
                  0.0,
                  83,
                  0
                ],
                [
                  0.15356489945155394,
                  0.0,
                  84,
                  0
                ],
                [
                  0.15539305301645337,
                  0.0,
                  85,
                  0
                ],
                [
                  0.15722120658135283,
                  0.0,
                  86,
                  0
                ],
                [
                  0.1590493601462523,
                  0.0,
                  87,
                  0
                ],
                [
                  0.16087751371115175,
                  0.0,
                  88,
                  0
                ],
                [
                  0.16270566727605118,
                  0.0,
                  89,
                  0
                ],
                [
                  0.16453382084095064,
                  0.0,
                  90,
                  0
                ],
                [
                  0.1663619744058501,
                  0.0,
                  91,
                  0
                ],
                [
                  0.16819012797074953,
                  0.0,
                  92,
                  0
                ],
                [
                  0.170018281535649,
                  0.0,
                  93,
                  0
                ],
                [
                  0.17184643510054845,
                  0.0,
                  94,
                  0
                ],
                [
                  0.1736745886654479,
                  0.0,
                  95,
                  0
                ],
                [
                  0.17550274223034734,
                  0.0,
                  96,
                  0
                ],
                [
                  0.1773308957952468,
                  0.0,
                  97,
                  0
                ],
                [
                  0.17915904936014626,
                  0.0,
                  98,
                  0
                ],
                [
                  0.1809872029250457,
                  0.0,
                  99,
                  0
                ],
                [
                  0.18281535648994515,
                  0.0,
                  100,
                  0
                ],
                [
                  0.1846435100548446,
                  0.0,
                  101,
                  0
                ],
                [
                  0.18647166361974407,
                  0.0,
                  102,
                  0
                ],
                [
                  0.1882998171846435,
                  0.0,
                  103,
                  0
                ],
                [
                  0.19012797074954296,
                  0.0,
                  104,
                  0
                ],
                [
                  0.19195612431444242,
                  0.0,
                  105,
                  0
                ],
                [
                  0.19378427787934185,
                  0.0,
                  106,
                  0
                ],
                [
                  0.1956124314442413,
                  0.0,
                  107,
                  0
                ],
                [
                  0.19744058500914077,
                  0.0,
                  108,
                  0
                ],
                [
                  0.19926873857404023,
                  0.0,
                  109,
                  0
                ],
                [
                  0.20109689213893966,
                  0.0,
                  110,
                  0
                ],
                [
                  0.20292504570383912,
                  0.0,
                  111,
                  0
                ],
                [
                  0.20475319926873858,
                  0.0,
                  112,
                  0
                ],
                [
                  0.20658135283363802,
                  0.0,
                  113,
                  0
                ],
                [
                  0.20840950639853748,
                  0.0,
                  114,
                  0
                ],
                [
                  0.21023765996343693,
                  0.0,
                  115,
                  0
                ],
                [
                  0.21206581352833637,
                  0.0,
                  116,
                  0
                ],
                [
                  0.21389396709323583,
                  0.0,
                  117,
                  0
                ],
                [
                  0.21572212065813529,
                  0.0,
                  118,
                  0
                ],
                [
                  0.21755027422303475,
                  0.0,
                  119,
                  0
                ],
                [
                  0.21937842778793418,
                  0.0,
                  120,
                  0
                ],
                [
                  0.22120658135283364,
                  0.0,
                  121,
                  0
                ],
                [
                  0.2230347349177331,
                  0.0,
                  122,
                  0
                ],
                [
                  0.22486288848263253,
                  0.0,
                  123,
                  0
                ],
                [
                  0.226691042047532,
                  0.0,
                  124,
                  0
                ],
                [
                  0.22851919561243145,
                  0.0,
                  125,
                  0
                ],
                [
                  0.2303473491773309,
                  0.0,
                  126,
                  0
                ],
                [
                  0.23217550274223034,
                  0.0,
                  127,
                  0
                ],
                [
                  0.2340036563071298,
                  0.0,
                  128,
                  0
                ],
                [
                  0.23583180987202926,
                  0.0,
                  129,
                  0
                ],
                [
                  0.2376599634369287,
                  0.0,
                  130,
                  0
                ],
                [
                  0.23948811700182815,
                  0.0,
                  131,
                  0
                ],
                [
                  0.2413162705667276,
                  0.0,
                  132,
                  0
                ],
                [
                  0.24314442413162707,
                  0.0,
                  133,
                  0
                ],
                [
                  0.2449725776965265,
                  0.0,
                  134,
                  0
                ],
                [
                  0.24680073126142596,
                  0.0,
                  135,
                  0
                ],
                [
                  0.24862888482632542,
                  0.0,
                  136,
                  0
                ],
                [
                  0.25045703839122485,
                  0.0,
                  137,
                  0
                ],
                [
                  0.2522851919561243,
                  0.0,
                  138,
                  0
                ],
                [
                  0.25411334552102377,
                  0.0,
                  139,
                  0
                ],
                [
                  0.25594149908592323,
                  0.0,
                  140,
                  0
                ],
                [
                  0.2577696526508227,
                  0.0,
                  141,
                  0
                ],
                [
                  0.2595978062157221,
                  0.0,
                  142,
                  0
                ],
                [
                  0.26142595978062155,
                  0.0,
                  143,
                  0
                ],
                [
                  0.263254113345521,
                  0.0,
                  144,
                  0
                ],
                [
                  0.26508226691042047,
                  0.006896551724137931,
                  145,
                  1
                ],
                [
                  0.26691042047531993,
                  0.00684931506849315,
                  146,
                  1
                ],
                [
                  0.2687385740402194,
                  0.006802721088435374,
                  147,
                  1
                ],
                [
                  0.27056672760511885,
                  0.006756756756756757,
                  148,
                  1
                ],
                [
                  0.27239488117001825,
                  0.006711409395973154,
                  149,
                  1
                ],
                [
                  0.2742230347349177,
                  0.006666666666666667,
                  150,
                  1
                ],
                [
                  0.2760511882998172,
                  0.006622516556291391,
                  151,
                  1
                ],
                [
                  0.27787934186471663,
                  0.006578947368421052,
                  152,
                  1
                ],
                [
                  0.2797074954296161,
                  0.006535947712418301,
                  153,
                  1
                ],
                [
                  0.28153564899451555,
                  0.006493506493506494,
                  154,
                  1
                ],
                [
                  0.283363802559415,
                  0.0064516129032258064,
                  155,
                  1
                ],
                [
                  0.2851919561243144,
                  0.00641025641025641,
                  156,
                  1
                ],
                [
                  0.2870201096892139,
                  0.006369426751592357,
                  157,
                  1
                ],
                [
                  0.28884826325411334,
                  0.006329113924050633,
                  158,
                  1
                ],
                [
                  0.2906764168190128,
                  0.006289308176100629,
                  159,
                  1
                ],
                [
                  0.29250457038391225,
                  0.00625,
                  160,
                  1
                ],
                [
                  0.2943327239488117,
                  0.012422360248447204,
                  161,
                  2
                ],
                [
                  0.2961608775137112,
                  0.012345679012345678,
                  162,
                  2
                ],
                [
                  0.2979890310786106,
                  0.012269938650306749,
                  163,
                  2
                ],
                [
                  0.29981718464351004,
                  0.012195121951219513,
                  164,
                  2
                ],
                [
                  0.3016453382084095,
                  0.012121212121212121,
                  165,
                  2
                ],
                [
                  0.30347349177330896,
                  0.012048192771084338,
                  166,
                  2
                ],
                [
                  0.3053016453382084,
                  0.011976047904191617,
                  167,
                  2
                ],
                [
                  0.3071297989031079,
                  0.011904761904761904,
                  168,
                  2
                ],
                [
                  0.30895795246800734,
                  0.011834319526627219,
                  169,
                  2
                ],
                [
                  0.31078610603290674,
                  0.011764705882352941,
                  170,
                  2
                ],
                [
                  0.3126142595978062,
                  0.011695906432748537,
                  171,
                  2
                ],
                [
                  0.31444241316270566,
                  0.011627906976744186,
                  172,
                  2
                ],
                [
                  0.3162705667276051,
                  0.011560693641618497,
                  173,
                  2
                ],
                [
                  0.3180987202925046,
                  0.017241379310344827,
                  174,
                  3
                ],
                [
                  0.31992687385740404,
                  0.022857142857142857,
                  175,
                  4
                ],
                [
                  0.3217550274223035,
                  0.022727272727272728,
                  176,
                  4
                ],
                [
                  0.3235831809872029,
                  0.022598870056497175,
                  177,
                  4
                ],
                [
                  0.32541133455210236,
                  0.02247191011235955,
                  178,
                  4
                ],
                [
                  0.3272394881170018,
                  0.0223463687150838,
                  179,
                  4
                ],
                [
                  0.3290676416819013,
                  0.022222222222222223,
                  180,
                  4
                ],
                [
                  0.33089579524680074,
                  0.022099447513812154,
                  181,
                  4
                ],
                [
                  0.3327239488117002,
                  0.027472527472527472,
                  182,
                  5
                ],
                [
                  0.33455210237659966,
                  0.0273224043715847,
                  183,
                  5
                ],
                [
                  0.33638025594149906,
                  0.02717391304347826,
                  184,
                  5
                ],
                [
                  0.3382084095063985,
                  0.02702702702702703,
                  185,
                  5
                ],
                [
                  0.340036563071298,
                  0.026881720430107527,
                  186,
                  5
                ],
                [
                  0.34186471663619744,
                  0.026737967914438502,
                  187,
                  5
                ],
                [
                  0.3436928702010969,
                  0.026595744680851064,
                  188,
                  5
                ],
                [
                  0.34552102376599636,
                  0.026455026455026454,
                  189,
                  5
                ],
                [
                  0.3473491773308958,
                  0.02631578947368421,
                  190,
                  5
                ],
                [
                  0.3491773308957952,
                  0.02617801047120419,
                  191,
                  5
                ],
                [
                  0.3510054844606947,
                  0.026041666666666668,
                  192,
                  5
                ],
                [
                  0.35283363802559414,
                  0.025906735751295335,
                  193,
                  5
                ],
                [
                  0.3546617915904936,
                  0.02577319587628866,
                  194,
                  5
                ],
                [
                  0.35648994515539306,
                  0.02564102564102564,
                  195,
                  5
                ],
                [
                  0.3583180987202925,
                  0.025510204081632654,
                  196,
                  5
                ],
                [
                  0.360146252285192,
                  0.025380710659898477,
                  197,
                  5
                ],
                [
                  0.3619744058500914,
                  0.025252525252525252,
                  198,
                  5
                ],
                [
                  0.36380255941499084,
                  0.02512562814070352,
                  199,
                  5
                ],
                [
                  0.3656307129798903,
                  0.025,
                  200,
                  5
                ],
                [
                  0.36745886654478976,
                  0.024875621890547265,
                  201,
                  5
                ],
                [
                  0.3692870201096892,
                  0.024752475247524754,
                  202,
                  5
                ],
                [
                  0.3711151736745887,
                  0.024630541871921183,
                  203,
                  5
                ],
                [
                  0.37294332723948814,
                  0.024509803921568627,
                  204,
                  5
                ],
                [
                  0.37477148080438755,
                  0.024390243902439025,
                  205,
                  5
                ],
                [
                  0.376599634369287,
                  0.024271844660194174,
                  206,
                  5
                ],
                [
                  0.37842778793418647,
                  0.024154589371980676,
                  207,
                  5
                ],
                [
                  0.3802559414990859,
                  0.028846153846153848,
                  208,
                  6
                ],
                [
                  0.3820840950639854,
                  0.028708133971291867,
                  209,
                  6
                ],
                [
                  0.38391224862888484,
                  0.02857142857142857,
                  210,
                  6
                ],
                [
                  0.3857404021937843,
                  0.02843601895734597,
                  211,
                  6
                ],
                [
                  0.3875685557586837,
                  0.02830188679245283,
                  212,
                  6
                ],
                [
                  0.38939670932358317,
                  0.028169014084507043,
                  213,
                  6
                ],
                [
                  0.3912248628884826,
                  0.028037383177570093,
                  214,
                  6
                ],
                [
                  0.3930530164533821,
                  0.027906976744186046,
                  215,
                  6
                ],
                [
                  0.39488117001828155,
                  0.027777777777777776,
                  216,
                  6
                ],
                [
                  0.396709323583181,
                  0.027649769585253458,
                  217,
                  6
                ],
                [
                  0.39853747714808047,
                  0.027522935779816515,
                  218,
                  6
                ],
                [
                  0.40036563071297987,
                  0.0273972602739726,
                  219,
                  6
                ],
                [
                  0.40219378427787933,
                  0.02727272727272727,
                  220,
                  6
                ],
                [
                  0.4040219378427788,
                  0.027149321266968326,
                  221,
                  6
                ],
                [
                  0.40585009140767825,
                  0.02702702702702703,
                  222,
                  6
                ],
                [
                  0.4076782449725777,
                  0.026905829596412557,
                  223,
                  6
                ],
                [
                  0.40950639853747717,
                  0.026785714285714284,
                  224,
                  6
                ],
                [
                  0.4113345521023766,
                  0.03111111111111111,
                  225,
                  7
                ],
                [
                  0.41316270566727603,
                  0.030973451327433628,
                  226,
                  7
                ],
                [
                  0.4149908592321755,
                  0.030837004405286344,
                  227,
                  7
                ],
                [
                  0.41681901279707495,
                  0.03070175438596491,
                  228,
                  7
                ],
                [
                  0.4186471663619744,
                  0.03056768558951965,
                  229,
                  7
                ],
                [
                  0.42047531992687387,
                  0.030434782608695653,
                  230,
                  7
                ],
                [
                  0.42230347349177333,
                  0.030303030303030304,
                  231,
                  7
                ],
                [
                  0.42413162705667273,
                  0.03017241379310345,
                  232,
                  7
                ],
                [
                  0.4259597806215722,
                  0.030042918454935622,
                  233,
                  7
                ],
                [
                  0.42778793418647165,
                  0.029914529914529916,
                  234,
                  7
                ],
                [
                  0.4296160877513711,
                  0.03404255319148936,
                  235,
                  8
                ],
                [
                  0.43144424131627057,
                  0.03389830508474576,
                  236,
                  8
                ],
                [
                  0.43327239488117003,
                  0.03375527426160337,
                  237,
                  8
                ],
                [
                  0.4351005484460695,
                  0.03361344537815126,
                  238,
                  8
                ],
                [
                  0.4369287020109689,
                  0.03347280334728033,
                  239,
                  8
                ],
                [
                  0.43875685557586835,
                  0.03333333333333333,
                  240,
                  8
                ],
                [
                  0.4405850091407678,
                  0.03319502074688797,
                  241,
                  8
                ],
                [
                  0.4424131627056673,
                  0.03305785123966942,
                  242,
                  8
                ],
                [
                  0.44424131627056673,
                  0.03292181069958848,
                  243,
                  8
                ],
                [
                  0.4460694698354662,
                  0.03278688524590164,
                  244,
                  8
                ],
                [
                  0.44789762340036565,
                  0.036734693877551024,
                  245,
                  9
                ],
                [
                  0.44972577696526506,
                  0.04065040650406504,
                  246,
                  10
                ],
                [
                  0.4515539305301645,
                  0.04048582995951417,
                  247,
                  10
                ],
                [
                  0.453382084095064,
                  0.04435483870967742,
                  248,
                  11
                ],
                [
                  0.45521023765996343,
                  0.04417670682730924,
                  249,
                  11
                ],
                [
                  0.4570383912248629,
                  0.044,
                  250,
                  11
                ],
                [
                  0.45886654478976235,
                  0.043824701195219126,
                  251,
                  11
                ],
                [
                  0.4606946983546618,
                  0.047619047619047616,
                  252,
                  12
                ],
                [
                  0.4625228519195612,
                  0.04743083003952569,
                  253,
                  12
                ],
                [
                  0.4643510054844607,
                  0.047244094488188976,
                  254,
                  12
                ],
                [
                  0.46617915904936014,
                  0.047058823529411764,
                  255,
                  12
                ],
                [
                  0.4680073126142596,
                  0.046875,
                  256,
                  12
                ],
                [
                  0.46983546617915906,
                  0.05058365758754864,
                  257,
                  13
                ],
                [
                  0.4716636197440585,
                  0.050387596899224806,
                  258,
                  13
                ],
                [
                  0.473491773308958,
                  0.05019305019305019,
                  259,
                  13
                ],
                [
                  0.4753199268738574,
                  0.05,
                  260,
                  13
                ],
                [
                  0.47714808043875684,
                  0.04980842911877394,
                  261,
                  13
                ],
                [
                  0.4789762340036563,
                  0.05343511450381679,
                  262,
                  14
                ],
                [
                  0.48080438756855576,
                  0.053231939163498096,
                  263,
                  14
                ],
                [
                  0.4826325411334552,
                  0.05303030303030303,
                  264,
                  14
                ],
                [
                  0.4844606946983547,
                  0.052830188679245285,
                  265,
                  14
                ],
                [
                  0.48628884826325414,
                  0.05263157894736842,
                  266,
                  14
                ],
                [
                  0.48811700182815354,
                  0.052434456928838954,
                  267,
                  14
                ],
                [
                  0.489945155393053,
                  0.05223880597014925,
                  268,
                  14
                ],
                [
                  0.49177330895795246,
                  0.05204460966542751,
                  269,
                  14
                ],
                [
                  0.4936014625228519,
                  0.05555555555555555,
                  270,
                  15
                ],
                [
                  0.4954296160877514,
                  0.055350553505535055,
                  271,
                  15
                ],
                [
                  0.49725776965265084,
                  0.058823529411764705,
                  272,
                  16
                ],
                [
                  0.4990859232175503,
                  0.05860805860805861,
                  273,
                  16
                ],
                [
                  0.5009140767824497,
                  0.058394160583941604,
                  274,
                  16
                ],
                [
                  0.5027422303473492,
                  0.06181818181818182,
                  275,
                  17
                ],
                [
                  0.5045703839122486,
                  0.06159420289855073,
                  276,
                  17
                ],
                [
                  0.506398537477148,
                  0.061371841155234655,
                  277,
                  17
                ],
                [
                  0.5082266910420475,
                  0.06115107913669065,
                  278,
                  17
                ],
                [
                  0.5100548446069469,
                  0.06093189964157706,
                  279,
                  17
                ],
                [
                  0.5118829981718465,
                  0.060714285714285714,
                  280,
                  17
                ],
                [
                  0.5137111517367459,
                  0.060498220640569395,
                  281,
                  17
                ],
                [
                  0.5155393053016454,
                  0.06028368794326241,
                  282,
                  17
                ],
                [
                  0.5173674588665448,
                  0.06007067137809187,
                  283,
                  17
                ],
                [
                  0.5191956124314442,
                  0.05985915492957746,
                  284,
                  17
                ],
                [
                  0.5210237659963437,
                  0.05964912280701754,
                  285,
                  17
                ],
                [
                  0.5228519195612431,
                  0.05944055944055944,
                  286,
                  17
                ],
                [
                  0.5246800731261426,
                  0.059233449477351915,
                  287,
                  17
                ],
                [
                  0.526508226691042,
                  0.059027777777777776,
                  288,
                  17
                ],
                [
                  0.5283363802559415,
                  0.058823529411764705,
                  289,
                  17
                ],
                [
                  0.5301645338208409,
                  0.05862068965517241,
                  290,
                  17
                ],
                [
                  0.5319926873857403,
                  0.061855670103092786,
                  291,
                  18
                ],
                [
                  0.5338208409506399,
                  0.06164383561643835,
                  292,
                  18
                ],
                [
                  0.5356489945155393,
                  0.06143344709897611,
                  293,
                  18
                ],
                [
                  0.5374771480804388,
                  0.061224489795918366,
                  294,
                  18
                ],
                [
                  0.5393053016453382,
                  0.061016949152542375,
                  295,
                  18
                ],
                [
                  0.5411334552102377,
                  0.060810810810810814,
                  296,
                  18
                ],
                [
                  0.5429616087751371,
                  0.06060606060606061,
                  297,
                  18
                ],
                [
                  0.5447897623400365,
                  0.06040268456375839,
                  298,
                  18
                ],
                [
                  0.546617915904936,
                  0.06354515050167224,
                  299,
                  19
                ],
                [
                  0.5484460694698354,
                  0.06333333333333334,
                  300,
                  19
                ],
                [
                  0.5502742230347349,
                  0.06312292358803986,
                  301,
                  19
                ],
                [
                  0.5521023765996343,
                  0.06291390728476821,
                  302,
                  19
                ],
                [
                  0.5539305301645339,
                  0.0627062706270627,
                  303,
                  19
                ],
                [
                  0.5557586837294333,
                  0.0625,
                  304,
                  19
                ],
                [
                  0.5575868372943327,
                  0.06229508196721312,
                  305,
                  19
                ],
                [
                  0.5594149908592322,
                  0.06209150326797386,
                  306,
                  19
                ],
                [
                  0.5612431444241316,
                  0.06188925081433225,
                  307,
                  19
                ],
                [
                  0.5630712979890311,
                  0.06493506493506493,
                  308,
                  20
                ],
                [
                  0.5648994515539305,
                  0.06796116504854369,
                  309,
                  21
                ],
                [
                  0.56672760511883,
                  0.07096774193548387,
                  310,
                  22
                ],
                [
                  0.5685557586837294,
                  0.0707395498392283,
                  311,
                  22
                ],
                [
                  0.5703839122486288,
                  0.07051282051282051,
                  312,
                  22
                ],
                [
                  0.5722120658135283,
                  0.07028753993610223,
                  313,
                  22
                ],
                [
                  0.5740402193784278,
                  0.0732484076433121,
                  314,
                  23
                ],
                [
                  0.5758683729433273,
                  0.07301587301587302,
                  315,
                  23
                ],
                [
                  0.5776965265082267,
                  0.07278481012658228,
                  316,
                  23
                ],
                [
                  0.5795246800731262,
                  0.07570977917981073,
                  317,
                  24
                ],
                [
                  0.5813528336380256,
                  0.07547169811320754,
                  318,
                  24
                ],
                [
                  0.583180987202925,
                  0.07836990595611286,
                  319,
                  25
                ],
                [
                  0.5850091407678245,
                  0.078125,
                  320,
                  25
                ],
                [
                  0.5868372943327239,
                  0.0778816199376947,
                  321,
                  25
                ],
                [
                  0.5886654478976234,
                  0.07763975155279502,
                  322,
                  25
                ],
                [
                  0.5904936014625228,
                  0.07739938080495357,
                  323,
                  25
                ],
                [
                  0.5923217550274223,
                  0.08024691358024691,
                  324,
                  26
                ],
                [
                  0.5941499085923218,
                  0.08,
                  325,
                  26
                ],
                [
                  0.5959780621572212,
                  0.07975460122699386,
                  326,
                  26
                ],
                [
                  0.5978062157221207,
                  0.07951070336391437,
                  327,
                  26
                ],
                [
                  0.5996343692870201,
                  0.07926829268292683,
                  328,
                  26
                ],
                [
                  0.6014625228519196,
                  0.0790273556231003,
                  329,
                  26
                ],
                [
                  0.603290676416819,
                  0.08181818181818182,
                  330,
                  27
                ],
                [
                  0.6051188299817185,
                  0.08157099697885196,
                  331,
                  27
                ],
                [
                  0.6069469835466179,
                  0.08132530120481928,
                  332,
                  27
                ],
                [
                  0.6087751371115173,
                  0.08108108108108109,
                  333,
                  27
                ],
                [
                  0.6106032906764168,
                  0.08083832335329341,
                  334,
                  27
                ],
                [
                  0.6124314442413162,
                  0.08059701492537313,
                  335,
                  27
                ],
                [
                  0.6142595978062158,
                  0.08035714285714286,
                  336,
                  27
                ],
                [
                  0.6160877513711152,
                  0.08011869436201781,
                  337,
                  27
                ],
                [
                  0.6179159049360147,
                  0.07988165680473373,
                  338,
                  27
                ],
                [
                  0.6197440585009141,
                  0.08259587020648967,
                  339,
                  28
                ],
                [
                  0.6215722120658135,
                  0.08235294117647059,
                  340,
                  28
                ],
                [
                  0.623400365630713,
                  0.08211143695014662,
                  341,
                  28
                ],
                [
                  0.6252285191956124,
                  0.0847953216374269,
                  342,
                  29
                ],
                [
                  0.6270566727605119,
                  0.08454810495626822,
                  343,
                  29
                ],
                [
                  0.6288848263254113,
                  0.08430232558139535,
                  344,
                  29
                ],
                [
                  0.6307129798903108,
                  0.08405797101449275,
                  345,
                  29
                ],
                [
                  0.6325411334552102,
                  0.0838150289017341,
                  346,
                  29
                ],
                [
                  0.6343692870201096,
                  0.08357348703170028,
                  347,
                  29
                ],
                [
                  0.6361974405850092,
                  0.08333333333333333,
                  348,
                  29
                ],
                [
                  0.6380255941499086,
                  0.0830945558739255,
                  349,
                  29
                ],
                [
                  0.6398537477148081,
                  0.08571428571428572,
                  350,
                  30
                ],
                [
                  0.6416819012797075,
                  0.08547008547008547,
                  351,
                  30
                ],
                [
                  0.643510054844607,
                  0.08522727272727272,
                  352,
                  30
                ],
                [
                  0.6453382084095064,
                  0.08781869688385269,
                  353,
                  31
                ],
                [
                  0.6471663619744058,
                  0.08757062146892655,
                  354,
                  31
                ],
                [
                  0.6489945155393053,
                  0.08732394366197183,
                  355,
                  31
                ],
                [
                  0.6508226691042047,
                  0.0898876404494382,
                  356,
                  32
                ],
                [
                  0.6526508226691042,
                  0.09243697478991597,
                  357,
                  33
                ],
                [
                  0.6544789762340036,
                  0.09217877094972067,
                  358,
                  33
                ],
                [
                  0.6563071297989032,
                  0.09192200557103064,
                  359,
                  33
                ],
                [
                  0.6581352833638026,
                  0.09166666666666666,
                  360,
                  33
                ],
                [
                  0.659963436928702,
                  0.09141274238227147,
                  361,
                  33
                ],
                [
                  0.6617915904936015,
                  0.09116022099447514,
                  362,
                  33
                ],
                [
                  0.6636197440585009,
                  0.09090909090909091,
                  363,
                  33
                ],
                [
                  0.6654478976234004,
                  0.09065934065934066,
                  364,
                  33
                ],
                [
                  0.6672760511882998,
                  0.09315068493150686,
                  365,
                  34
                ],
                [
                  0.6691042047531993,
                  0.09289617486338798,
                  366,
                  34
                ],
                [
                  0.6709323583180987,
                  0.09264305177111716,
                  367,
                  34
                ],
                [
                  0.6727605118829981,
                  0.09239130434782608,
                  368,
                  34
                ],
                [
                  0.6745886654478976,
                  0.0921409214092141,
                  369,
                  34
                ],
                [
                  0.676416819012797,
                  0.0918918918918919,
                  370,
                  34
                ],
                [
                  0.6782449725776966,
                  0.09164420485175202,
                  371,
                  34
                ],
                [
                  0.680073126142596,
                  0.0913978494623656,
                  372,
                  34
                ],
                [
                  0.6819012797074955,
                  0.09115281501340483,
                  373,
                  34
                ],
                [
                  0.6837294332723949,
                  0.09358288770053476,
                  374,
                  35
                ],
                [
                  0.6855575868372943,
                  0.096,
                  375,
                  36
                ],
                [
                  0.6873857404021938,
                  0.09574468085106383,
                  376,
                  36
                ],
                [
                  0.6892138939670932,
                  0.09549071618037135,
                  377,
                  36
                ],
                [
                  0.6910420475319927,
                  0.09523809523809523,
                  378,
                  36
                ],
                [
                  0.6928702010968921,
                  0.09762532981530343,
                  379,
                  37
                ],
                [
                  0.6946983546617916,
                  0.09736842105263158,
                  380,
                  37
                ],
                [
                  0.696526508226691,
                  0.09973753280839895,
                  381,
                  38
                ],
                [
                  0.6983546617915904,
                  0.09947643979057591,
                  382,
                  38
                ],
                [
                  0.70018281535649,
                  0.10182767624020887,
                  383,
                  39
                ],
                [
                  0.7020109689213894,
                  0.1015625,
                  384,
                  39
                ],
                [
                  0.7038391224862889,
                  0.1012987012987013,
                  385,
                  39
                ],
                [
                  0.7056672760511883,
                  0.10362694300518134,
                  386,
                  40
                ],
                [
                  0.7074954296160878,
                  0.10594315245478036,
                  387,
                  41
                ],
                [
                  0.7093235831809872,
                  0.10824742268041238,
                  388,
                  42
                ],
                [
                  0.7111517367458866,
                  0.10796915167095116,
                  389,
                  42
                ],
                [
                  0.7129798903107861,
                  0.11025641025641025,
                  390,
                  43
                ],
                [
                  0.7148080438756855,
                  0.10997442455242967,
                  391,
                  43
                ],
                [
                  0.716636197440585,
                  0.1096938775510204,
                  392,
                  43
                ],
                [
                  0.7184643510054844,
                  0.11195928753180662,
                  393,
                  44
                ],
                [
                  0.720292504570384,
                  0.1116751269035533,
                  394,
                  44
                ],
                [
                  0.7221206581352834,
                  0.11139240506329114,
                  395,
                  44
                ],
                [
                  0.7239488117001828,
                  0.1111111111111111,
                  396,
                  44
                ],
                [
                  0.7257769652650823,
                  0.11083123425692695,
                  397,
                  44
                ],
                [
                  0.7276051188299817,
                  0.11306532663316583,
                  398,
                  45
                ],
                [
                  0.7294332723948812,
                  0.11278195488721804,
                  399,
                  45
                ],
                [
                  0.7312614259597806,
                  0.1125,
                  400,
                  45
                ],
                [
                  0.7330895795246801,
                  0.11221945137157108,
                  401,
                  45
                ],
                [
                  0.7349177330895795,
                  0.11442786069651742,
                  402,
                  46
                ],
                [
                  0.7367458866544789,
                  0.11662531017369727,
                  403,
                  47
                ],
                [
                  0.7385740402193784,
                  0.1188118811881188,
                  404,
                  48
                ],
                [
                  0.7404021937842779,
                  0.11851851851851852,
                  405,
                  48
                ],
                [
                  0.7422303473491774,
                  0.1206896551724138,
                  406,
                  49
                ],
                [
                  0.7440585009140768,
                  0.12039312039312039,
                  407,
                  49
                ],
                [
                  0.7458866544789763,
                  0.12009803921568628,
                  408,
                  49
                ],
                [
                  0.7477148080438757,
                  0.1198044009779951,
                  409,
                  49
                ],
                [
                  0.7495429616087751,
                  0.11951219512195121,
                  410,
                  49
                ],
                [
                  0.7513711151736746,
                  0.12165450121654502,
                  411,
                  50
                ],
                [
                  0.753199268738574,
                  0.12378640776699029,
                  412,
                  51
                ],
                [
                  0.7550274223034735,
                  0.12590799031476999,
                  413,
                  52
                ],
                [
                  0.7568555758683729,
                  0.1280193236714976,
                  414,
                  53
                ],
                [
                  0.7586837294332724,
                  0.12771084337349398,
                  415,
                  53
                ],
                [
                  0.7605118829981719,
                  0.12980769230769232,
                  416,
                  54
                ],
                [
                  0.7623400365630713,
                  0.13189448441247004,
                  417,
                  55
                ],
                [
                  0.7641681901279708,
                  0.1339712918660287,
                  418,
                  56
                ],
                [
                  0.7659963436928702,
                  0.1360381861575179,
                  419,
                  57
                ],
                [
                  0.7678244972577697,
                  0.1357142857142857,
                  420,
                  57
                ],
                [
                  0.7696526508226691,
                  0.1377672209026128,
                  421,
                  58
                ],
                [
                  0.7714808043875686,
                  0.13981042654028436,
                  422,
                  59
                ],
                [
                  0.773308957952468,
                  0.13947990543735225,
                  423,
                  59
                ],
                [
                  0.7751371115173674,
                  0.1391509433962264,
                  424,
                  59
                ],
                [
                  0.7769652650822669,
                  0.1388235294117647,
                  425,
                  59
                ],
                [
                  0.7787934186471663,
                  0.13849765258215962,
                  426,
                  59
                ],
                [
                  0.7806215722120659,
                  0.13817330210772832,
                  427,
                  59
                ],
                [
                  0.7824497257769653,
                  0.1378504672897196,
                  428,
                  59
                ],
                [
                  0.7842778793418648,
                  0.13986013986013987,
                  429,
                  60
                ],
                [
                  0.7861060329067642,
                  0.13953488372093023,
                  430,
                  60
                ],
                [
                  0.7879341864716636,
                  0.14153132250580047,
                  431,
                  61
                ],
                [
                  0.7897623400365631,
                  0.14351851851851852,
                  432,
                  62
                ],
                [
                  0.7915904936014625,
                  0.14318706697459585,
                  433,
                  62
                ],
                [
                  0.793418647166362,
                  0.14285714285714285,
                  434,
                  62
                ],
                [
                  0.7952468007312614,
                  0.1425287356321839,
                  435,
                  62
                ],
                [
                  0.7970749542961609,
                  0.14220183486238533,
                  436,
                  62
                ],
                [
                  0.7989031078610603,
                  0.14416475972540047,
                  437,
                  63
                ],
                [
                  0.8007312614259597,
                  0.14383561643835616,
                  438,
                  63
                ],
                [
                  0.8025594149908593,
                  0.14578587699316628,
                  439,
                  64
                ],
                [
                  0.8043875685557587,
                  0.14545454545454545,
                  440,
                  64
                ],
                [
                  0.8062157221206582,
                  0.14512471655328799,
                  441,
                  64
                ],
                [
                  0.8080438756855576,
                  0.14479638009049775,
                  442,
                  64
                ],
                [
                  0.8098720292504571,
                  0.14672686230248308,
                  443,
                  65
                ],
                [
                  0.8117001828153565,
                  0.14864864864864866,
                  444,
                  66
                ],
                [
                  0.8135283363802559,
                  0.15056179775280898,
                  445,
                  67
                ],
                [
                  0.8153564899451554,
                  0.15246636771300448,
                  446,
                  68
                ],
                [
                  0.8171846435100548,
                  0.15212527964205816,
                  447,
                  68
                ],
                [
                  0.8190127970749543,
                  0.15178571428571427,
                  448,
                  68
                ],
                [
                  0.8208409506398537,
                  0.15367483296213807,
                  449,
                  69
                ],
                [
                  0.8226691042047533,
                  0.15333333333333332,
                  450,
                  69
                ],
                [
                  0.8244972577696527,
                  0.15299334811529933,
                  451,
                  69
                ],
                [
                  0.8263254113345521,
                  0.15486725663716813,
                  452,
                  70
                ],
                [
                  0.8281535648994516,
                  0.1545253863134658,
                  453,
                  70
                ],
                [
                  0.829981718464351,
                  0.15418502202643172,
                  454,
                  70
                ],
                [
                  0.8318098720292505,
                  0.15384615384615385,
                  455,
                  70
                ],
                [
                  0.8336380255941499,
                  0.15350877192982457,
                  456,
                  70
                ],
                [
                  0.8354661791590493,
                  0.15536105032822758,
                  457,
                  71
                ],
                [
                  0.8372943327239488,
                  0.1572052401746725,
                  458,
                  72
                ],
                [
                  0.8391224862888482,
                  0.1568627450980392,
                  459,
                  72
                ],
                [
                  0.8409506398537477,
                  0.1565217391304348,
                  460,
                  72
                ],
                [
                  0.8427787934186471,
                  0.15835140997830802,
                  461,
                  73
                ],
                [
                  0.8446069469835467,
                  0.15800865800865802,
                  462,
                  73
                ],
                [
                  0.8464351005484461,
                  0.15982721382289417,
                  463,
                  74
                ],
                [
                  0.8482632541133455,
                  0.16163793103448276,
                  464,
                  75
                ],
                [
                  0.850091407678245,
                  0.16344086021505377,
                  465,
                  76
                ],
                [
                  0.8519195612431444,
                  0.1630901287553648,
                  466,
                  76
                ],
                [
                  0.8537477148080439,
                  0.16274089935760172,
                  467,
                  76
                ],
                [
                  0.8555758683729433,
                  0.16452991452991453,
                  468,
                  77
                ],
                [
                  0.8574040219378428,
                  0.16631130063965885,
                  469,
                  78
                ],
                [
                  0.8592321755027422,
                  0.16595744680851063,
                  470,
                  78
                ],
                [
                  0.8610603290676416,
                  0.16560509554140126,
                  471,
                  78
                ],
                [
                  0.8628884826325411,
                  0.1652542372881356,
                  472,
                  78
                ],
                [
                  0.8647166361974405,
                  0.16701902748414377,
                  473,
                  79
                ],
                [
                  0.8665447897623401,
                  0.16877637130801687,
                  474,
                  80
                ],
                [
                  0.8683729433272395,
                  0.16842105263157894,
                  475,
                  80
                ],
                [
                  0.870201096892139,
                  0.17016806722689076,
                  476,
                  81
                ],
                [
                  0.8720292504570384,
                  0.1719077568134172,
                  477,
                  82
                ],
                [
                  0.8738574040219378,
                  0.17154811715481172,
                  478,
                  82
                ],
                [
                  0.8756855575868373,
                  0.1732776617954071,
                  479,
                  83
                ],
                [
                  0.8775137111517367,
                  0.175,
                  480,
                  84
                ],
                [
                  0.8793418647166362,
                  0.17463617463617465,
                  481,
                  84
                ],
                [
                  0.8811700182815356,
                  0.17427385892116182,
                  482,
                  84
                ],
                [
                  0.8829981718464351,
                  0.17391304347826086,
                  483,
                  84
                ],
                [
                  0.8848263254113345,
                  0.1756198347107438,
                  484,
                  85
                ],
                [
                  0.886654478976234,
                  0.17525773195876287,
                  485,
                  85
                ],
                [
                  0.8884826325411335,
                  0.17695473251028807,
                  486,
                  86
                ],
                [
                  0.8903107861060329,
                  0.17864476386036962,
                  487,
                  87
                ],
                [
                  0.8921389396709324,
                  0.18032786885245902,
                  488,
                  88
                ],
                [
                  0.8939670932358318,
                  0.17995910020449898,
                  489,
                  88
                ],
                [
                  0.8957952468007313,
                  0.17959183673469387,
                  490,
                  88
                ],
                [
                  0.8976234003656307,
                  0.18126272912423624,
                  491,
                  89
                ],
                [
                  0.8994515539305301,
                  0.18089430894308944,
                  492,
                  89
                ],
                [
                  0.9012797074954296,
                  0.18255578093306288,
                  493,
                  90
                ],
                [
                  0.903107861060329,
                  0.18421052631578946,
                  494,
                  91
                ],
                [
                  0.9049360146252285,
                  0.18383838383838383,
                  495,
                  91
                ],
                [
                  0.906764168190128,
                  0.18346774193548387,
                  496,
                  91
                ],
                [
                  0.9085923217550275,
                  0.1851106639839034,
                  497,
                  92
                ],
                [
                  0.9104204753199269,
                  0.18674698795180722,
                  498,
                  93
                ],
                [
                  0.9122486288848263,
                  0.18637274549098196,
                  499,
                  93
                ],
                [
                  0.9140767824497258,
                  0.188,
                  500,
                  94
                ],
                [
                  0.9159049360146252,
                  0.18962075848303392,
                  501,
                  95
                ],
                [
                  0.9177330895795247,
                  0.1892430278884462,
                  502,
                  95
                ],
                [
                  0.9195612431444241,
                  0.18886679920477137,
                  503,
                  95
                ],
                [
                  0.9213893967093236,
                  0.19047619047619047,
                  504,
                  96
                ],
                [
                  0.923217550274223,
                  0.19207920792079208,
                  505,
                  97
                ],
                [
                  0.9250457038391224,
                  0.19367588932806323,
                  506,
                  98
                ],
                [
                  0.926873857404022,
                  0.1952662721893491,
                  507,
                  99
                ],
                [
                  0.9287020109689214,
                  0.1968503937007874,
                  508,
                  100
                ],
                [
                  0.9305301645338209,
                  0.19842829076620824,
                  509,
                  101
                ],
                [
                  0.9323583180987203,
                  0.2,
                  510,
                  102
                ],
                [
                  0.9341864716636198,
                  0.19960861056751467,
                  511,
                  102
                ],
                [
                  0.9360146252285192,
                  0.201171875,
                  512,
                  103
                ],
                [
                  0.9378427787934186,
                  0.20272904483430798,
                  513,
                  104
                ],
                [
                  0.9396709323583181,
                  0.20233463035019456,
                  514,
                  104
                ],
                [
                  0.9414990859232175,
                  0.20388349514563106,
                  515,
                  105
                ],
                [
                  0.943327239488117,
                  0.2054263565891473,
                  516,
                  106
                ],
                [
                  0.9451553930530164,
                  0.20696324951644102,
                  517,
                  107
                ],
                [
                  0.946983546617916,
                  0.20656370656370657,
                  518,
                  107
                ],
                [
                  0.9488117001828154,
                  0.20616570327552985,
                  519,
                  107
                ],
                [
                  0.9506398537477148,
                  0.2076923076923077,
                  520,
                  108
                ],
                [
                  0.9524680073126143,
                  0.2072936660268714,
                  521,
                  108
                ],
                [
                  0.9542961608775137,
                  0.20881226053639848,
                  522,
                  109
                ],
                [
                  0.9561243144424132,
                  0.21032504780114722,
                  523,
                  110
                ],
                [
                  0.9579524680073126,
                  0.21183206106870228,
                  524,
                  111
                ],
                [
                  0.9597806215722121,
                  0.21142857142857144,
                  525,
                  111
                ],
                [
                  0.9616087751371115,
                  0.21292775665399238,
                  526,
                  112
                ],
                [
                  0.9634369287020109,
                  0.2144212523719165,
                  527,
                  113
                ],
                [
                  0.9652650822669104,
                  0.2159090909090909,
                  528,
                  114
                ],
                [
                  0.9670932358318098,
                  0.21739130434782608,
                  529,
                  115
                ],
                [
                  0.9689213893967094,
                  0.2188679245283019,
                  530,
                  116
                ],
                [
                  0.9707495429616088,
                  0.22033898305084745,
                  531,
                  117
                ],
                [
                  0.9725776965265083,
                  0.22180451127819548,
                  532,
                  118
                ],
                [
                  0.9744058500914077,
                  0.22326454033771106,
                  533,
                  119
                ],
                [
                  0.9762340036563071,
                  0.22284644194756553,
                  534,
                  119
                ],
                [
                  0.9780621572212066,
                  0.22242990654205608,
                  535,
                  119
                ],
                [
                  0.979890310786106,
                  0.22201492537313433,
                  536,
                  119
                ],
                [
                  0.9817184643510055,
                  0.22346368715083798,
                  537,
                  120
                ],
                [
                  0.9835466179159049,
                  0.22304832713754646,
                  538,
                  120
                ],
                [
                  0.9853747714808044,
                  0.22448979591836735,
                  539,
                  121
                ],
                [
                  0.9872029250457038,
                  0.22592592592592592,
                  540,
                  122
                ],
                [
                  0.9890310786106032,
                  0.2255083179297597,
                  541,
                  122
                ],
                [
                  0.9908592321755028,
                  0.22693726937269373,
                  542,
                  123
                ],
                [
                  0.9926873857404022,
                  0.2283609576427256,
                  543,
                  124
                ],
                [
                  0.9945155393053017,
                  0.22977941176470587,
                  544,
                  125
                ],
                [
                  0.9963436928702011,
                  0.23119266055045873,
                  545,
                  126
                ],
                [
                  0.9981718464351006,
                  0.2326007326007326,
                  546,
                  127
                ],
                [
                  1.0,
                  0.2340036563071298,
                  547,
                  128
                ]
              ]
            },
            {
              "language": "English",
              "run": 533,
              "points": [
                [
                  0.0018281535648994515,
                  0.0,
                  1,
                  0
                ],
                [
                  0.003656307129798903,
                  0.0,
                  2,
                  0
                ],
                [
                  0.005484460694698354,
                  0.0,
                  3,
                  0
                ],
                [
                  0.007312614259597806,
                  0.0,
                  4,
                  0
                ],
                [
                  0.009140767824497258,
                  0.0,
                  5,
                  0
                ],
                [
                  0.010968921389396709,
                  0.0,
                  6,
                  0
                ],
                [
                  0.012797074954296161,
                  0.0,
                  7,
                  0
                ],
                [
                  0.014625228519195612,
                  0.0,
                  8,
                  0
                ],
                [
                  0.016453382084095063,
                  0.0,
                  9,
                  0
                ],
                [
                  0.018281535648994516,
                  0.0,
                  10,
                  0
                ],
                [
                  0.02010968921389397,
                  0.0,
                  11,
                  0
                ],
                [
                  0.021937842778793418,
                  0.0,
                  12,
                  0
                ],
                [
                  0.02376599634369287,
                  0.0,
                  13,
                  0
                ],
                [
                  0.025594149908592323,
                  0.0,
                  14,
                  0
                ],
                [
                  0.027422303473491772,
                  0.0,
                  15,
                  0
                ],
                [
                  0.029250457038391225,
                  0.0,
                  16,
                  0
                ],
                [
                  0.031078610603290677,
                  0.0,
                  17,
                  0
                ],
                [
                  0.03290676416819013,
                  0.0,
                  18,
                  0
                ],
                [
                  0.03473491773308958,
                  0.0,
                  19,
                  0
                ],
                [
                  0.03656307129798903,
                  0.0,
                  20,
                  0
                ],
                [
                  0.038391224862888484,
                  0.0,
                  21,
                  0
                ],
                [
                  0.04021937842778794,
                  0.0,
                  22,
                  0
                ],
                [
                  0.04204753199268738,
                  0.0,
                  23,
                  0
                ],
                [
                  0.043875685557586835,
                  0.0,
                  24,
                  0
                ],
                [
                  0.04570383912248629,
                  0.0,
                  25,
                  0
                ],
                [
                  0.04753199268738574,
                  0.0,
                  26,
                  0
                ],
                [
                  0.04936014625228519,
                  0.0,
                  27,
                  0
                ],
                [
                  0.051188299817184646,
                  0.0,
                  28,
                  0
                ],
                [
                  0.05301645338208409,
                  0.0,
                  29,
                  0
                ],
                [
                  0.054844606946983544,
                  0.0,
                  30,
                  0
                ],
                [
                  0.056672760511883,
                  0.0,
                  31,
                  0
                ],
                [
                  0.05850091407678245,
                  0.0,
                  32,
                  0
                ],
                [
                  0.0603290676416819,
                  0.0,
                  33,
                  0
                ],
                [
                  0.062157221206581355,
                  0.0,
                  34,
                  0
                ],
                [
                  0.06398537477148081,
                  0.0,
                  35,
                  0
                ],
                [
                  0.06581352833638025,
                  0.0,
                  36,
                  0
                ],
                [
                  0.06764168190127971,
                  0.0,
                  37,
                  0
                ],
                [
                  0.06946983546617916,
                  0.0,
                  38,
                  0
                ],
                [
                  0.0712979890310786,
                  0.0,
                  39,
                  0
                ],
                [
                  0.07312614259597806,
                  0.0,
                  40,
                  0
                ],
                [
                  0.07495429616087751,
                  0.0,
                  41,
                  0
                ],
                [
                  0.07678244972577697,
                  0.0,
                  42,
                  0
                ],
                [
                  0.07861060329067641,
                  0.0,
                  43,
                  0
                ],
                [
                  0.08043875685557587,
                  0.0,
                  44,
                  0
                ],
                [
                  0.08226691042047532,
                  0.0,
                  45,
                  0
                ],
                [
                  0.08409506398537477,
                  0.0,
                  46,
                  0
                ],
                [
                  0.08592321755027423,
                  0.0,
                  47,
                  0
                ],
                [
                  0.08775137111517367,
                  0.0,
                  48,
                  0
                ],
                [
                  0.08957952468007313,
                  0.02040816326530612,
                  49,
                  1
                ],
                [
                  0.09140767824497258,
                  0.02,
                  50,
                  1
                ],
                [
                  0.09323583180987204,
                  0.0196078431372549,
                  51,
                  1
                ],
                [
                  0.09506398537477148,
                  0.019230769230769232,
                  52,
                  1
                ],
                [
                  0.09689213893967093,
                  0.018867924528301886,
                  53,
                  1
                ],
                [
                  0.09872029250457039,
                  0.018518518518518517,
                  54,
                  1
                ],
                [
                  0.10054844606946983,
                  0.01818181818181818,
                  55,
                  1
                ],
                [
                  0.10237659963436929,
                  0.017857142857142856,
                  56,
                  1
                ],
                [
                  0.10420475319926874,
                  0.017543859649122806,
                  57,
                  1
                ],
                [
                  0.10603290676416818,
                  0.017241379310344827,
                  58,
                  1
                ],
                [
                  0.10786106032906764,
                  0.01694915254237288,
                  59,
                  1
                ],
                [
                  0.10968921389396709,
                  0.016666666666666666,
                  60,
                  1
                ],
                [
                  0.11151736745886655,
                  0.01639344262295082,
                  61,
                  1
                ],
                [
                  0.113345521023766,
                  0.016129032258064516,
                  62,
                  1
                ],
                [
                  0.11517367458866545,
                  0.015873015873015872,
                  63,
                  1
                ],
                [
                  0.1170018281535649,
                  0.015625,
                  64,
                  1
                ],
                [
                  0.11882998171846434,
                  0.015384615384615385,
                  65,
                  1
                ],
                [
                  0.1206581352833638,
                  0.015151515151515152,
                  66,
                  1
                ],
                [
                  0.12248628884826325,
                  0.014925373134328358,
                  67,
                  1
                ],
                [
                  0.12431444241316271,
                  0.014705882352941176,
                  68,
                  1
                ],
                [
                  0.12614259597806216,
                  0.014492753623188406,
                  69,
                  1
                ],
                [
                  0.12797074954296161,
                  0.014285714285714285,
                  70,
                  1
                ],
                [
                  0.12979890310786105,
                  0.014084507042253521,
                  71,
                  1
                ],
                [
                  0.1316270566727605,
                  0.013888888888888888,
                  72,
                  1
                ],
                [
                  0.13345521023765997,
                  0.0136986301369863,
                  73,
                  1
                ],
                [
                  0.13528336380255943,
                  0.013513513513513514,
                  74,
                  1
                ],
                [
                  0.13711151736745886,
                  0.013333333333333334,
                  75,
                  1
                ],
                [
                  0.13893967093235832,
                  0.013157894736842105,
                  76,
                  1
                ],
                [
                  0.14076782449725778,
                  0.012987012987012988,
                  77,
                  1
                ],
                [
                  0.1425959780621572,
                  0.01282051282051282,
                  78,
                  1
                ],
                [
                  0.14442413162705667,
                  0.012658227848101266,
                  79,
                  1
                ],
                [
                  0.14625228519195613,
                  0.0125,
                  80,
                  1
                ],
                [
                  0.1480804387568556,
                  0.012345679012345678,
                  81,
                  1
                ],
                [
                  0.14990859232175502,
                  0.012195121951219513,
                  82,
                  1
                ],
                [
                  0.15173674588665448,
                  0.012048192771084338,
                  83,
                  1
                ],
                [
                  0.15356489945155394,
                  0.011904761904761904,
                  84,
                  1
                ],
                [
                  0.15539305301645337,
                  0.011764705882352941,
                  85,
                  1
                ],
                [
                  0.15722120658135283,
                  0.011627906976744186,
                  86,
                  1
                ],
                [
                  0.1590493601462523,
                  0.011494252873563218,
                  87,
                  1
                ],
                [
                  0.16087751371115175,
                  0.011363636363636364,
                  88,
                  1
                ],
                [
                  0.16270566727605118,
                  0.011235955056179775,
                  89,
                  1
                ],
                [
                  0.16453382084095064,
                  0.011111111111111112,
                  90,
                  1
                ],
                [
                  0.1663619744058501,
                  0.01098901098901099,
                  91,
                  1
                ],
                [
                  0.16819012797074953,
                  0.010869565217391304,
                  92,
                  1
                ],
                [
                  0.170018281535649,
                  0.010752688172043012,
                  93,
                  1
                ],
                [
                  0.17184643510054845,
                  0.010638297872340425,
                  94,
                  1
                ],
                [
                  0.1736745886654479,
                  0.010526315789473684,
                  95,
                  1
                ],
                [
                  0.17550274223034734,
                  0.010416666666666666,
                  96,
                  1
                ],
                [
                  0.1773308957952468,
                  0.010309278350515464,
                  97,
                  1
                ],
                [
                  0.17915904936014626,
                  0.01020408163265306,
                  98,
                  1
                ],
                [
                  0.1809872029250457,
                  0.010101010101010102,
                  99,
                  1
                ],
                [
                  0.18281535648994515,
                  0.01,
                  100,
                  1
                ],
                [
                  0.1846435100548446,
                  0.009900990099009901,
                  101,
                  1
                ],
                [
                  0.18647166361974407,
                  0.00980392156862745,
                  102,
                  1
                ],
                [
                  0.1882998171846435,
                  0.009708737864077669,
                  103,
                  1
                ],
                [
                  0.19012797074954296,
                  0.009615384615384616,
                  104,
                  1
                ],
                [
                  0.19195612431444242,
                  0.009523809523809525,
                  105,
                  1
                ],
                [
                  0.19378427787934185,
                  0.009433962264150943,
                  106,
                  1
                ],
                [
                  0.1956124314442413,
                  0.009345794392523364,
                  107,
                  1
                ],
                [
                  0.19744058500914077,
                  0.009259259259259259,
                  108,
                  1
                ],
                [
                  0.19926873857404023,
                  0.009174311926605505,
                  109,
                  1
                ],
                [
                  0.20109689213893966,
                  0.00909090909090909,
                  110,
                  1
                ],
                [
                  0.20292504570383912,
                  0.009009009009009009,
                  111,
                  1
                ],
                [
                  0.20475319926873858,
                  0.008928571428571428,
                  112,
                  1
                ],
                [
                  0.20658135283363802,
                  0.008849557522123894,
                  113,
                  1
                ],
                [
                  0.20840950639853748,
                  0.017543859649122806,
                  114,
                  2
                ],
                [
                  0.21023765996343693,
                  0.017391304347826087,
                  115,
                  2
                ],
                [
                  0.21206581352833637,
                  0.017241379310344827,
                  116,
                  2
                ],
                [
                  0.21389396709323583,
                  0.017094017094017096,
                  117,
                  2
                ],
                [
                  0.21572212065813529,
                  0.01694915254237288,
                  118,
                  2
                ],
                [
                  0.21755027422303475,
                  0.01680672268907563,
                  119,
                  2
                ],
                [
                  0.21937842778793418,
                  0.016666666666666666,
                  120,
                  2
                ],
                [
                  0.22120658135283364,
                  0.01652892561983471,
                  121,
                  2
                ],
                [
                  0.2230347349177331,
                  0.01639344262295082,
                  122,
                  2
                ],
                [
                  0.22486288848263253,
                  0.016260162601626018,
                  123,
                  2
                ],
                [
                  0.226691042047532,
                  0.016129032258064516,
                  124,
                  2
                ],
                [
                  0.22851919561243145,
                  0.016,
                  125,
                  2
                ],
                [
                  0.2303473491773309,
                  0.015873015873015872,
                  126,
                  2
                ],
                [
                  0.23217550274223034,
                  0.015748031496062992,
                  127,
                  2
                ],
                [
                  0.2340036563071298,
                  0.015625,
                  128,
                  2
                ],
                [
                  0.23583180987202926,
                  0.015503875968992248,
                  129,
                  2
                ],
                [
                  0.2376599634369287,
                  0.015384615384615385,
                  130,
                  2
                ],
                [
                  0.23948811700182815,
                  0.015267175572519083,
                  131,
                  2
                ],
                [
                  0.2413162705667276,
                  0.015151515151515152,
                  132,
                  2
                ],
                [
                  0.24314442413162707,
                  0.015037593984962405,
                  133,
                  2
                ],
                [
                  0.2449725776965265,
                  0.014925373134328358,
                  134,
                  2
                ],
                [
                  0.24680073126142596,
                  0.014814814814814815,
                  135,
                  2
                ],
                [
                  0.24862888482632542,
                  0.014705882352941176,
                  136,
                  2
                ],
                [
                  0.25045703839122485,
                  0.014598540145985401,
                  137,
                  2
                ],
                [
                  0.2522851919561243,
                  0.014492753623188406,
                  138,
                  2
                ],
                [
                  0.25411334552102377,
                  0.014388489208633094,
                  139,
                  2
                ],
                [
                  0.25594149908592323,
                  0.014285714285714285,
                  140,
                  2
                ],
                [
                  0.2577696526508227,
                  0.014184397163120567,
                  141,
                  2
                ],
                [
                  0.2595978062157221,
                  0.014084507042253521,
                  142,
                  2
                ],
                [
                  0.26142595978062155,
                  0.013986013986013986,
                  143,
                  2
                ],
                [
                  0.263254113345521,
                  0.013888888888888888,
                  144,
                  2
                ],
                [
                  0.26508226691042047,
                  0.013793103448275862,
                  145,
                  2
                ],
                [
                  0.26691042047531993,
                  0.0136986301369863,
                  146,
                  2
                ],
                [
                  0.2687385740402194,
                  0.013605442176870748,
                  147,
                  2
                ],
                [
                  0.27056672760511885,
                  0.013513513513513514,
                  148,
                  2
                ],
                [
                  0.27239488117001825,
                  0.013422818791946308,
                  149,
                  2
                ],
                [
                  0.2742230347349177,
                  0.013333333333333334,
                  150,
                  2
                ],
                [
                  0.2760511882998172,
                  0.013245033112582781,
                  151,
                  2
                ],
                [
                  0.27787934186471663,
                  0.013157894736842105,
                  152,
                  2
                ],
                [
                  0.2797074954296161,
                  0.013071895424836602,
                  153,
                  2
                ],
                [
                  0.28153564899451555,
                  0.012987012987012988,
                  154,
                  2
                ],
                [
                  0.283363802559415,
                  0.012903225806451613,
                  155,
                  2
                ],
                [
                  0.2851919561243144,
                  0.01282051282051282,
                  156,
                  2
                ],
                [
                  0.2870201096892139,
                  0.012738853503184714,
                  157,
                  2
                ],
                [
                  0.28884826325411334,
                  0.012658227848101266,
                  158,
                  2
                ],
                [
                  0.2906764168190128,
                  0.012578616352201259,
                  159,
                  2
                ],
                [
                  0.29250457038391225,
                  0.0125,
                  160,
                  2
                ],
                [
                  0.2943327239488117,
                  0.012422360248447204,
                  161,
                  2
                ],
                [
                  0.2961608775137112,
                  0.012345679012345678,
                  162,
                  2
                ],
                [
                  0.2979890310786106,
                  0.012269938650306749,
                  163,
                  2
                ],
                [
                  0.29981718464351004,
                  0.012195121951219513,
                  164,
                  2
                ],
                [
                  0.3016453382084095,
                  0.012121212121212121,
                  165,
                  2
                ],
                [
                  0.30347349177330896,
                  0.012048192771084338,
                  166,
                  2
                ],
                [
                  0.3053016453382084,
                  0.011976047904191617,
                  167,
                  2
                ],
                [
                  0.3071297989031079,
                  0.011904761904761904,
                  168,
                  2
                ],
                [
                  0.30895795246800734,
                  0.011834319526627219,
                  169,
                  2
                ],
                [
                  0.31078610603290674,
                  0.011764705882352941,
                  170,
                  2
                ],
                [
                  0.3126142595978062,
                  0.011695906432748537,
                  171,
                  2
                ],
                [
                  0.31444241316270566,
                  0.011627906976744186,
                  172,
                  2
                ],
                [
                  0.3162705667276051,
                  0.011560693641618497,
                  173,
                  2
                ],
                [
                  0.3180987202925046,
                  0.011494252873563218,
                  174,
                  2
                ],
                [
                  0.31992687385740404,
                  0.011428571428571429,
                  175,
                  2
                ],
                [
                  0.3217550274223035,
                  0.011363636363636364,
                  176,
                  2
                ],
                [
                  0.3235831809872029,
                  0.011299435028248588,
                  177,
                  2
                ],
                [
                  0.32541133455210236,
                  0.011235955056179775,
                  178,
                  2
                ],
                [
                  0.3272394881170018,
                  0.0111731843575419,
                  179,
                  2
                ],
                [
                  0.3290676416819013,
                  0.011111111111111112,
                  180,
                  2
                ],
                [
                  0.33089579524680074,
                  0.011049723756906077,
                  181,
                  2
                ],
                [
                  0.3327239488117002,
                  0.01098901098901099,
                  182,
                  2
                ],
                [
                  0.33455210237659966,
                  0.01092896174863388,
                  183,
                  2
                ],
                [
                  0.33638025594149906,
                  0.010869565217391304,
                  184,
                  2
                ],
                [
                  0.3382084095063985,
                  0.010810810810810811,
                  185,
                  2
                ],
                [
                  0.340036563071298,
                  0.010752688172043012,
                  186,
                  2
                ],
                [
                  0.34186471663619744,
                  0.0106951871657754,
                  187,
                  2
                ],
                [
                  0.3436928702010969,
                  0.010638297872340425,
                  188,
                  2
                ],
                [
                  0.34552102376599636,
                  0.010582010582010581,
                  189,
                  2
                ],
                [
                  0.3473491773308958,
                  0.010526315789473684,
                  190,
                  2
                ],
                [
                  0.3491773308957952,
                  0.010471204188481676,
                  191,
                  2
                ],
                [
                  0.3510054844606947,
                  0.010416666666666666,
                  192,
                  2
                ],
                [
                  0.35283363802559414,
                  0.010362694300518135,
                  193,
                  2
                ],
                [
                  0.3546617915904936,
                  0.010309278350515464,
                  194,
                  2
                ],
                [
                  0.35648994515539306,
                  0.010256410256410256,
                  195,
                  2
                ],
                [
                  0.3583180987202925,
                  0.01020408163265306,
                  196,
                  2
                ],
                [
                  0.360146252285192,
                  0.01015228426395939,
                  197,
                  2
                ],
                [
                  0.3619744058500914,
                  0.010101010101010102,
                  198,
                  2
                ],
                [
                  0.36380255941499084,
                  0.010050251256281407,
                  199,
                  2
                ],
                [
                  0.3656307129798903,
                  0.01,
                  200,
                  2
                ],
                [
                  0.36745886654478976,
                  0.009950248756218905,
                  201,
                  2
                ],
                [
                  0.3692870201096892,
                  0.009900990099009901,
                  202,
                  2
                ],
                [
                  0.3711151736745887,
                  0.009852216748768473,
                  203,
                  2
                ],
                [
                  0.37294332723948814,
                  0.00980392156862745,
                  204,
                  2
                ],
                [
                  0.37477148080438755,
                  0.00975609756097561,
                  205,
                  2
                ],
                [
                  0.376599634369287,
                  0.009708737864077669,
                  206,
                  2
                ],
                [
                  0.37842778793418647,
                  0.00966183574879227,
                  207,
                  2
                ],
                [
                  0.3802559414990859,
                  0.009615384615384616,
                  208,
                  2
                ],
                [
                  0.3820840950639854,
                  0.009569377990430622,
                  209,
                  2
                ],
                [
                  0.38391224862888484,
                  0.009523809523809525,
                  210,
                  2
                ],
                [
                  0.3857404021937843,
                  0.009478672985781991,
                  211,
                  2
                ],
                [
                  0.3875685557586837,
                  0.009433962264150943,
                  212,
                  2
                ],
                [
                  0.38939670932358317,
                  0.009389671361502348,
                  213,
                  2
                ],
                [
                  0.3912248628884826,
                  0.009345794392523364,
                  214,
                  2
                ],
                [
                  0.3930530164533821,
                  0.009302325581395349,
                  215,
                  2
                ],
                [
                  0.39488117001828155,
                  0.009259259259259259,
                  216,
                  2
                ],
                [
                  0.396709323583181,
                  0.009216589861751152,
                  217,
                  2
                ],
                [
                  0.39853747714808047,
                  0.013761467889908258,
                  218,
                  3
                ],
                [
                  0.40036563071297987,
                  0.0136986301369863,
                  219,
                  3
                ],
                [
                  0.40219378427787933,
                  0.013636363636363636,
                  220,
                  3
                ],
                [
                  0.4040219378427788,
                  0.013574660633484163,
                  221,
                  3
                ],
                [
                  0.40585009140767825,
                  0.013513513513513514,
                  222,
                  3
                ],
                [
                  0.4076782449725777,
                  0.013452914798206279,
                  223,
                  3
                ],
                [
                  0.40950639853747717,
                  0.013392857142857142,
                  224,
                  3
                ],
                [
                  0.4113345521023766,
                  0.013333333333333334,
                  225,
                  3
                ],
                [
                  0.41316270566727603,
                  0.01327433628318584,
                  226,
                  3
                ],
                [
                  0.4149908592321755,
                  0.013215859030837005,
                  227,
                  3
                ],
                [
                  0.41681901279707495,
                  0.013157894736842105,
                  228,
                  3
                ],
                [
                  0.4186471663619744,
                  0.013100436681222707,
                  229,
                  3
                ],
                [
                  0.42047531992687387,
                  0.013043478260869565,
                  230,
                  3
                ],
                [
                  0.42230347349177333,
                  0.012987012987012988,
                  231,
                  3
                ],
                [
                  0.42413162705667273,
                  0.01293103448275862,
                  232,
                  3
                ],
                [
                  0.4259597806215722,
                  0.012875536480686695,
                  233,
                  3
                ],
                [
                  0.42778793418647165,
                  0.01282051282051282,
                  234,
                  3
                ],
                [
                  0.4296160877513711,
                  0.01276595744680851,
                  235,
                  3
                ],
                [
                  0.43144424131627057,
                  0.012711864406779662,
                  236,
                  3
                ],
                [
                  0.43327239488117003,
                  0.012658227848101266,
                  237,
                  3
                ],
                [
                  0.4351005484460695,
                  0.012605042016806723,
                  238,
                  3
                ],
                [
                  0.4369287020109689,
                  0.012552301255230125,
                  239,
                  3
                ],
                [
                  0.43875685557586835,
                  0.0125,
                  240,
                  3
                ],
                [
                  0.4405850091407678,
                  0.012448132780082987,
                  241,
                  3
                ],
                [
                  0.4424131627056673,
                  0.012396694214876033,
                  242,
                  3
                ],
                [
                  0.44424131627056673,
                  0.012345679012345678,
                  243,
                  3
                ],
                [
                  0.4460694698354662,
                  0.012295081967213115,
                  244,
                  3
                ],
                [
                  0.44789762340036565,
                  0.012244897959183673,
                  245,
                  3
                ],
                [
                  0.44972577696526506,
                  0.012195121951219513,
                  246,
                  3
                ],
                [
                  0.4515539305301645,
                  0.012145748987854251,
                  247,
                  3
                ],
                [
                  0.453382084095064,
                  0.012096774193548387,
                  248,
                  3
                ],
                [
                  0.45521023765996343,
                  0.012048192771084338,
                  249,
                  3
                ],
                [
                  0.4570383912248629,
                  0.012,
                  250,
                  3
                ],
                [
                  0.45886654478976235,
                  0.01195219123505976,
                  251,
                  3
                ],
                [
                  0.4606946983546618,
                  0.011904761904761904,
                  252,
                  3
                ],
                [
                  0.4625228519195612,
                  0.011857707509881422,
                  253,
                  3
                ],
                [
                  0.4643510054844607,
                  0.011811023622047244,
                  254,
                  3
                ],
                [
                  0.46617915904936014,
                  0.011764705882352941,
                  255,
                  3
                ],
                [
                  0.4680073126142596,
                  0.01171875,
                  256,
                  3
                ],
                [
                  0.46983546617915906,
                  0.011673151750972763,
                  257,
                  3
                ],
                [
                  0.4716636197440585,
                  0.011627906976744186,
                  258,
                  3
                ],
                [
                  0.473491773308958,
                  0.011583011583011582,
                  259,
                  3
                ],
                [
                  0.4753199268738574,
                  0.011538461538461539,
                  260,
                  3
                ],
                [
                  0.47714808043875684,
                  0.011494252873563218,
                  261,
                  3
                ],
                [
                  0.4789762340036563,
                  0.011450381679389313,
                  262,
                  3
                ],
                [
                  0.48080438756855576,
                  0.011406844106463879,
                  263,
                  3
                ],
                [
                  0.4826325411334552,
                  0.011363636363636364,
                  264,
                  3
                ],
                [
                  0.4844606946983547,
                  0.011320754716981131,
                  265,
                  3
                ],
                [
                  0.48628884826325414,
                  0.011278195488721804,
                  266,
                  3
                ],
                [
                  0.48811700182815354,
                  0.011235955056179775,
                  267,
                  3
                ],
                [
                  0.489945155393053,
                  0.011194029850746268,
                  268,
                  3
                ],
                [
                  0.49177330895795246,
                  0.011152416356877323,
                  269,
                  3
                ],
                [
                  0.4936014625228519,
                  0.011111111111111112,
                  270,
                  3
                ],
                [
                  0.4954296160877514,
                  0.01107011070110701,
                  271,
                  3
                ],
                [
                  0.49725776965265084,
                  0.011029411764705883,
                  272,
                  3
                ],
                [
                  0.4990859232175503,
                  0.01098901098901099,
                  273,
                  3
                ],
                [
                  0.5009140767824497,
                  0.010948905109489052,
                  274,
                  3
                ],
                [
                  0.5027422303473492,
                  0.01090909090909091,
                  275,
                  3
                ],
                [
                  0.5045703839122486,
                  0.010869565217391304,
                  276,
                  3
                ],
                [
                  0.506398537477148,
                  0.010830324909747292,
                  277,
                  3
                ],
                [
                  0.5082266910420475,
                  0.01079136690647482,
                  278,
                  3
                ],
                [
                  0.5100548446069469,
                  0.014336917562724014,
                  279,
                  4
                ],
                [
                  0.5118829981718465,
                  0.014285714285714285,
                  280,
                  4
                ],
                [
                  0.5137111517367459,
                  0.014234875444839857,
                  281,
                  4
                ],
                [
                  0.5155393053016454,
                  0.014184397163120567,
                  282,
                  4
                ],
                [
                  0.5173674588665448,
                  0.014134275618374558,
                  283,
                  4
                ],
                [
                  0.5191956124314442,
                  0.017605633802816902,
                  284,
                  5
                ],
                [
                  0.5210237659963437,
                  0.017543859649122806,
                  285,
                  5
                ],
                [
                  0.5228519195612431,
                  0.017482517482517484,
                  286,
                  5
                ],
                [
                  0.5246800731261426,
                  0.017421602787456445,
                  287,
                  5
                ],
                [
                  0.526508226691042,
                  0.017361111111111112,
                  288,
                  5
                ],
                [
                  0.5283363802559415,
                  0.01730103806228374,
                  289,
                  5
                ],
                [
                  0.5301645338208409,
                  0.017241379310344827,
                  290,
                  5
                ],
                [
                  0.5319926873857403,
                  0.01718213058419244,
                  291,
                  5
                ],
                [
                  0.5338208409506399,
                  0.017123287671232876,
                  292,
                  5
                ],
                [
                  0.5356489945155393,
                  0.017064846416382253,
                  293,
                  5
                ],
                [
                  0.5374771480804388,
                  0.017006802721088437,
                  294,
                  5
                ],
                [
                  0.5393053016453382,
                  0.01694915254237288,
                  295,
                  5
                ],
                [
                  0.5411334552102377,
                  0.016891891891891893,
                  296,
                  5
                ],
                [
                  0.5429616087751371,
                  0.016835016835016835,
                  297,
                  5
                ],
                [
                  0.5447897623400365,
                  0.016778523489932886,
                  298,
                  5
                ],
                [
                  0.546617915904936,
                  0.016722408026755852,
                  299,
                  5
                ],
                [
                  0.5484460694698354,
                  0.016666666666666666,
                  300,
                  5
                ],
                [
                  0.5502742230347349,
                  0.016611295681063124,
                  301,
                  5
                ],
                [
                  0.5521023765996343,
                  0.016556291390728478,
                  302,
                  5
                ],
                [
                  0.5539305301645339,
                  0.0165016501650165,
                  303,
                  5
                ],
                [
                  0.5557586837294333,
                  0.01644736842105263,
                  304,
                  5
                ],
                [
                  0.5575868372943327,
                  0.01639344262295082,
                  305,
                  5
                ],
                [
                  0.5594149908592322,
                  0.016339869281045753,
                  306,
                  5
                ],
                [
                  0.5612431444241316,
                  0.016286644951140065,
                  307,
                  5
                ],
                [
                  0.5630712979890311,
                  0.016233766233766232,
                  308,
                  5
                ],
                [
                  0.5648994515539305,
                  0.016181229773462782,
                  309,
                  5
                ],
                [
                  0.56672760511883,
                  0.016129032258064516,
                  310,
                  5
                ],
                [
                  0.5685557586837294,
                  0.01929260450160772,
                  311,
                  6
                ],
                [
                  0.5703839122486288,
                  0.019230769230769232,
                  312,
                  6
                ],
                [
                  0.5722120658135283,
                  0.019169329073482427,
                  313,
                  6
                ],
                [
                  0.5740402193784278,
                  0.01910828025477707,
                  314,
                  6
                ],
                [
                  0.5758683729433273,
                  0.01904761904761905,
                  315,
                  6
                ],
                [
                  0.5776965265082267,
                  0.0189873417721519,
                  316,
                  6
                ],
                [
                  0.5795246800731262,
                  0.01892744479495268,
                  317,
                  6
                ],
                [
                  0.5813528336380256,
                  0.0220125786163522,
                  318,
                  7
                ],
                [
                  0.583180987202925,
                  0.0219435736677116,
                  319,
                  7
                ],
                [
                  0.5850091407678245,
                  0.021875,
                  320,
                  7
                ],
                [
                  0.5868372943327239,
                  0.021806853582554516,
                  321,
                  7
                ],
                [
                  0.5886654478976234,
                  0.021739130434782608,
                  322,
                  7
                ],
                [
                  0.5904936014625228,
                  0.021671826625386997,
                  323,
                  7
                ],
                [
                  0.5923217550274223,
                  0.021604938271604937,
                  324,
                  7
                ],
                [
                  0.5941499085923218,
                  0.021538461538461538,
                  325,
                  7
                ],
                [
                  0.5959780621572212,
                  0.02147239263803681,
                  326,
                  7
                ],
                [
                  0.5978062157221207,
                  0.021406727828746176,
                  327,
                  7
                ],
                [
                  0.5996343692870201,
                  0.021341463414634148,
                  328,
                  7
                ],
                [
                  0.6014625228519196,
                  0.02127659574468085,
                  329,
                  7
                ],
                [
                  0.603290676416819,
                  0.024242424242424242,
                  330,
                  8
                ],
                [
                  0.6051188299817185,
                  0.02416918429003021,
                  331,
                  8
                ],
                [
                  0.6069469835466179,
                  0.024096385542168676,
                  332,
                  8
                ],
                [
                  0.6087751371115173,
                  0.024024024024024024,
                  333,
                  8
                ],
                [
                  0.6106032906764168,
                  0.023952095808383235,
                  334,
                  8
                ],
                [
                  0.6124314442413162,
                  0.023880597014925373,
                  335,
                  8
                ],
                [
                  0.6142595978062158,
                  0.023809523809523808,
                  336,
                  8
                ],
                [
                  0.6160877513711152,
                  0.02373887240356083,
                  337,
                  8
                ],
                [
                  0.6179159049360147,
                  0.023668639053254437,
                  338,
                  8
                ],
                [
                  0.6197440585009141,
                  0.02359882005899705,
                  339,
                  8
                ],
                [
                  0.6215722120658135,
                  0.023529411764705882,
                  340,
                  8
                ],
                [
                  0.623400365630713,
                  0.02346041055718475,
                  341,
                  8
                ],
                [
                  0.6252285191956124,
                  0.023391812865497075,
                  342,
                  8
                ],
                [
                  0.6270566727605119,
                  0.026239067055393587,
                  343,
                  9
                ],
                [
                  0.6288848263254113,
                  0.02616279069767442,
                  344,
                  9
                ],
                [
                  0.6307129798903108,
                  0.02608695652173913,
                  345,
                  9
                ],
                [
                  0.6325411334552102,
                  0.02601156069364162,
                  346,
                  9
                ],
                [
                  0.6343692870201096,
                  0.025936599423631124,
                  347,
                  9
                ],
                [
                  0.6361974405850092,
                  0.02586206896551724,
                  348,
                  9
                ],
                [
                  0.6380255941499086,
                  0.025787965616045846,
                  349,
                  9
                ],
                [
                  0.6398537477148081,
                  0.025714285714285714,
                  350,
                  9
                ],
                [
                  0.6416819012797075,
                  0.02849002849002849,
                  351,
                  10
                ],
                [
                  0.643510054844607,
                  0.03125,
                  352,
                  11
                ],
                [
                  0.6453382084095064,
                  0.031161473087818695,
                  353,
                  11
                ],
                [
                  0.6471663619744058,
                  0.031073446327683617,
                  354,
                  11
                ],
                [
                  0.6489945155393053,
                  0.03380281690140845,
                  355,
                  12
                ],
                [
                  0.6508226691042047,
                  0.033707865168539325,
                  356,
                  12
                ],
                [
                  0.6526508226691042,
                  0.03361344537815126,
                  357,
                  12
                ],
                [
                  0.6544789762340036,
                  0.0335195530726257,
                  358,
                  12
                ],
                [
                  0.6563071297989032,
                  0.036211699164345405,
                  359,
                  13
                ],
                [
                  0.6581352833638026,
                  0.03611111111111111,
                  360,
                  13
                ],
                [
                  0.659963436928702,
                  0.036011080332409975,
                  361,
                  13
                ],
                [
                  0.6617915904936015,
                  0.03591160220994475,
                  362,
                  13
                ],
                [
                  0.6636197440585009,
                  0.03581267217630854,
                  363,
                  13
                ],
                [
                  0.6654478976234004,
                  0.03571428571428571,
                  364,
                  13
                ],
                [
                  0.6672760511882998,
                  0.03561643835616438,
                  365,
                  13
                ],
                [
                  0.6691042047531993,
                  0.03551912568306011,
                  366,
                  13
                ],
                [
                  0.6709323583180987,
                  0.03814713896457766,
                  367,
                  14
                ],
                [
                  0.6727605118829981,
                  0.03804347826086957,
                  368,
                  14
                ],
                [
                  0.6745886654478976,
                  0.037940379403794036,
                  369,
                  14
                ],
                [
                  0.676416819012797,
                  0.03783783783783784,
                  370,
                  14
                ],
                [
                  0.6782449725776966,
                  0.04043126684636118,
                  371,
                  15
                ],
                [
                  0.680073126142596,
                  0.043010752688172046,
                  372,
                  16
                ],
                [
                  0.6819012797074955,
                  0.04289544235924933,
                  373,
                  16
                ],
                [
                  0.6837294332723949,
                  0.0427807486631016,
                  374,
                  16
                ],
                [
                  0.6855575868372943,
                  0.042666666666666665,
                  375,
                  16
                ],
                [
                  0.6873857404021938,
                  0.0425531914893617,
                  376,
                  16
                ],
                [
                  0.6892138939670932,
                  0.042440318302387266,
                  377,
                  16
                ],
                [
                  0.6910420475319927,
                  0.042328042328042326,
                  378,
                  16
                ],
                [
                  0.6928702010968921,
                  0.04221635883905013,
                  379,
                  16
                ],
                [
                  0.6946983546617916,
                  0.042105263157894736,
                  380,
                  16
                ],
                [
                  0.696526508226691,
                  0.04199475065616798,
                  381,
                  16
                ],
                [
                  0.6983546617915904,
                  0.041884816753926704,
                  382,
                  16
                ],
                [
                  0.70018281535649,
                  0.044386422976501305,
                  383,
                  17
                ],
                [
                  0.7020109689213894,
                  0.044270833333333336,
                  384,
                  17
                ],
                [
                  0.7038391224862889,
                  0.046753246753246755,
                  385,
                  18
                ],
                [
                  0.7056672760511883,
                  0.046632124352331605,
                  386,
                  18
                ],
                [
                  0.7074954296160878,
                  0.04909560723514212,
                  387,
                  19
                ],
                [
                  0.7093235831809872,
                  0.05154639175257732,
                  388,
                  20
                ],
                [
                  0.7111517367458866,
                  0.05141388174807198,
                  389,
                  20
                ],
                [
                  0.7129798903107861,
                  0.05128205128205128,
                  390,
                  20
                ],
                [
                  0.7148080438756855,
                  0.05115089514066496,
                  391,
                  20
                ],
                [
                  0.716636197440585,
                  0.05102040816326531,
                  392,
                  20
                ],
                [
                  0.7184643510054844,
                  0.05089058524173028,
                  393,
                  20
                ],
                [
                  0.720292504570384,
                  0.0532994923857868,
                  394,
                  21
                ],
                [
                  0.7221206581352834,
                  0.053164556962025315,
                  395,
                  21
                ],
                [
                  0.7239488117001828,
                  0.05303030303030303,
                  396,
                  21
                ],
                [
                  0.7257769652650823,
                  0.05289672544080604,
                  397,
                  21
                ],
                [
                  0.7276051188299817,
                  0.052763819095477386,
                  398,
                  21
                ],
                [
                  0.7294332723948812,
                  0.05263157894736842,
                  399,
                  21
                ],
                [
                  0.7312614259597806,
                  0.055,
                  400,
                  22
                ],
                [
                  0.7330895795246801,
                  0.05486284289276808,
                  401,
                  22
                ],
                [
                  0.7349177330895795,
                  0.05721393034825871,
                  402,
                  23
                ],
                [
                  0.7367458866544789,
                  0.05707196029776675,
                  403,
                  23
                ],
                [
                  0.7385740402193784,
                  0.05693069306930693,
                  404,
                  23
                ],
                [
                  0.7404021937842779,
                  0.056790123456790124,
                  405,
                  23
                ],
                [
                  0.7422303473491774,
                  0.05665024630541872,
                  406,
                  23
                ],
                [
                  0.7440585009140768,
                  0.056511056511056514,
                  407,
                  23
                ],
                [
                  0.7458866544789763,
                  0.056372549019607844,
                  408,
                  23
                ],
                [
                  0.7477148080438757,
                  0.05867970660146699,
                  409,
                  24
                ],
                [
                  0.7495429616087751,
                  0.05853658536585366,
                  410,
                  24
                ],
                [
                  0.7513711151736746,
                  0.06082725060827251,
                  411,
                  25
                ],
                [
                  0.753199268738574,
                  0.06067961165048544,
                  412,
                  25
                ],
                [
                  0.7550274223034735,
                  0.06053268765133172,
                  413,
                  25
                ],
                [
                  0.7568555758683729,
                  0.06280193236714976,
                  414,
                  26
                ],
                [
                  0.7586837294332724,
                  0.06265060240963856,
                  415,
                  26
                ],
                [
                  0.7605118829981719,
                  0.06490384615384616,
                  416,
                  27
                ],
                [
                  0.7623400365630713,
                  0.06474820143884892,
                  417,
                  27
                ],
                [
                  0.7641681901279708,
                  0.0645933014354067,
                  418,
                  27
                ],
                [
                  0.7659963436928702,
                  0.06443914081145585,
                  419,
                  27
                ],
                [
                  0.7678244972577697,
                  0.06666666666666667,
                  420,
                  28
                ],
                [
                  0.7696526508226691,
                  0.0688836104513064,
                  421,
                  29
                ],
                [
                  0.7714808043875686,
                  0.06872037914691943,
                  422,
                  29
                ],
                [
                  0.773308957952468,
                  0.06855791962174941,
                  423,
                  29
                ],
                [
                  0.7751371115173674,
                  0.07075471698113207,
                  424,
                  30
                ],
                [
                  0.7769652650822669,
                  0.07294117647058823,
                  425,
                  31
                ],
                [
                  0.7787934186471663,
                  0.07276995305164319,
                  426,
                  31
                ],
                [
                  0.7806215722120659,
                  0.07259953161592506,
                  427,
                  31
                ],
                [
                  0.7824497257769653,
                  0.07476635514018691,
                  428,
                  32
                ],
                [
                  0.7842778793418648,
                  0.07692307692307693,
                  429,
                  33
                ],
                [
                  0.7861060329067642,
                  0.07906976744186046,
                  430,
                  34
                ],
                [
                  0.7879341864716636,
                  0.08120649651972157,
                  431,
                  35
                ],
                [
                  0.7897623400365631,
                  0.08101851851851852,
                  432,
                  35
                ],
                [
                  0.7915904936014625,
                  0.08083140877598152,
                  433,
                  35
                ],
                [
                  0.793418647166362,
                  0.08064516129032258,
                  434,
                  35
                ],
                [
                  0.7952468007312614,
                  0.08045977011494253,
                  435,
                  35
                ],
                [
                  0.7970749542961609,
                  0.08256880733944955,
                  436,
                  36
                ],
                [
                  0.7989031078610603,
                  0.08237986270022883,
                  437,
                  36
                ],
                [
                  0.8007312614259597,
                  0.08447488584474885,
                  438,
                  37
                ],
                [
                  0.8025594149908593,
                  0.08428246013667426,
                  439,
                  37
                ],
                [
                  0.8043875685557587,
                  0.08636363636363636,
                  440,
                  38
                ],
                [
                  0.8062157221206582,
                  0.08616780045351474,
                  441,
                  38
                ],
                [
                  0.8080438756855576,
                  0.08823529411764706,
                  442,
                  39
                ],
                [
                  0.8098720292504571,
                  0.09029345372460497,
                  443,
                  40
                ],
                [
                  0.8117001828153565,
                  0.09234234234234234,
                  444,
                  41
                ],
                [
                  0.8135283363802559,
                  0.09213483146067415,
                  445,
                  41
                ],
                [
                  0.8153564899451554,
                  0.09192825112107623,
                  446,
                  41
                ],
                [
                  0.8171846435100548,
                  0.09172259507829977,
                  447,
                  41
                ],
                [
                  0.8190127970749543,
                  0.09151785714285714,
                  448,
                  41
                ],
                [
                  0.8208409506398537,
                  0.09131403118040089,
                  449,
                  41
                ],
                [
                  0.8226691042047533,
                  0.09111111111111111,
                  450,
                  41
                ],
                [
                  0.8244972577696527,
                  0.09312638580931264,
                  451,
                  42
                ],
                [
                  0.8263254113345521,
                  0.09513274336283185,
                  452,
                  43
                ],
                [
                  0.8281535648994516,
                  0.09713024282560706,
                  453,
                  44
                ],
                [
                  0.829981718464351,
                  0.09691629955947137,
                  454,
                  44
                ],
                [
                  0.8318098720292505,
                  0.0989010989010989,
                  455,
                  45
                ],
                [
                  0.8336380255941499,
                  0.09868421052631579,
                  456,
                  45
                ],
                [
                  0.8354661791590493,
                  0.09846827133479212,
                  457,
                  45
                ],
                [
                  0.8372943327239488,
                  0.0982532751091703,
                  458,
                  45
                ],
                [
                  0.8391224862888482,
                  0.10021786492374728,
                  459,
                  46
                ],
                [
                  0.8409506398537477,
                  0.1,
                  460,
                  46
                ],
                [
                  0.8427787934186471,
                  0.09978308026030369,
                  461,
                  46
                ],
                [
                  0.8446069469835467,
                  0.09956709956709957,
                  462,
                  46
                ],
                [
                  0.8464351005484461,
                  0.10151187904967603,
                  463,
                  47
                ],
                [
                  0.8482632541133455,
                  0.10129310344827586,
                  464,
                  47
                ],
                [
                  0.850091407678245,
                  0.1032258064516129,
                  465,
                  48
                ],
                [
                  0.8519195612431444,
                  0.10515021459227468,
                  466,
                  49
                ],
                [
                  0.8537477148080439,
                  0.10706638115631692,
                  467,
                  50
                ],
                [
                  0.8555758683729433,
                  0.10683760683760683,
                  468,
                  50
                ],
                [
                  0.8574040219378428,
                  0.10660980810234541,
                  469,
                  50
                ],
                [
                  0.8592321755027422,
                  0.10851063829787234,
                  470,
                  51
                ],
                [
                  0.8610603290676416,
                  0.11040339702760085,
                  471,
                  52
                ],
                [
                  0.8628884826325411,
                  0.11228813559322035,
                  472,
                  53
                ],
                [
                  0.8647166361974405,
                  0.11205073995771671,
                  473,
                  53
                ],
                [
                  0.8665447897623401,
                  0.11392405063291139,
                  474,
                  54
                ],
                [
                  0.8683729433272395,
                  0.11578947368421053,
                  475,
                  55
                ],
                [
                  0.870201096892139,
                  0.11554621848739496,
                  476,
                  55
                ],
                [
                  0.8720292504570384,
                  0.11530398322851153,
                  477,
                  55
                ],
                [
                  0.8738574040219378,
                  0.11506276150627615,
                  478,
                  55
                ],
                [
                  0.8756855575868373,
                  0.11691022964509394,
                  479,
                  56
                ],
                [
                  0.8775137111517367,
                  0.11666666666666667,
                  480,
                  56
                ],
                [
                  0.8793418647166362,
                  0.11850311850311851,
                  481,
                  57
                ],
                [
                  0.8811700182815356,
                  0.12033195020746888,
                  482,
                  58
                ],
                [
                  0.8829981718464351,
                  0.12008281573498965,
                  483,
                  58
                ],
                [
                  0.8848263254113345,
                  0.11983471074380166,
                  484,
                  58
                ],
                [
                  0.886654478976234,
                  0.12164948453608247,
                  485,
                  59
                ],
                [
                  0.8884826325411335,
                  0.12139917695473251,
                  486,
                  59
                ],
                [
                  0.8903107861060329,
                  0.12114989733059549,
                  487,
                  59
                ],
                [
                  0.8921389396709324,
                  0.12090163934426229,
                  488,
                  59
                ],
                [
                  0.8939670932358318,
                  0.12065439672801637,
                  489,
                  59
                ],
                [
                  0.8957952468007313,
                  0.12040816326530612,
                  490,
                  59
                ],
                [
                  0.8976234003656307,
                  0.12219959266802444,
                  491,
                  60
                ],
                [
                  0.8994515539305301,
                  0.12195121951219512,
                  492,
                  60
                ],
                [
                  0.9012797074954296,
                  0.12170385395537525,
                  493,
                  60
                ],
                [
                  0.903107861060329,
                  0.12348178137651822,
                  494,
                  61
                ],
                [
                  0.9049360146252285,
                  0.12323232323232323,
                  495,
                  61
                ],
                [
                  0.906764168190128,
                  0.125,
                  496,
                  62
                ],
                [
                  0.9085923217550275,
                  0.1267605633802817,
                  497,
                  63
                ],
                [
                  0.9104204753199269,
                  0.1285140562248996,
                  498,
                  64
                ],
                [
                  0.9122486288848263,
                  0.1282565130260521,
                  499,
                  64
                ],
                [
                  0.9140767824497258,
                  0.13,
                  500,
                  65
                ],
                [
                  0.9159049360146252,
                  0.1317365269461078,
                  501,
                  66
                ],
                [
                  0.9177330895795247,
                  0.13346613545816732,
                  502,
                  67
                ],
                [
                  0.9195612431444241,
                  0.13320079522862824,
                  503,
                  67
                ],
                [
                  0.9213893967093236,
                  0.1349206349206349,
                  504,
                  68
                ],
                [
                  0.923217550274223,
                  0.13465346534653466,
                  505,
                  68
                ],
                [
                  0.9250457038391224,
                  0.13636363636363635,
                  506,
                  69
                ],
                [
                  0.926873857404022,
                  0.13806706114398423,
                  507,
                  70
                ],
                [
                  0.9287020109689214,
                  0.1377952755905512,
                  508,
                  70
                ],
                [
                  0.9305301645338209,
                  0.137524557956778,
                  509,
                  70
                ],
                [
                  0.9323583180987203,
                  0.1392156862745098,
                  510,
                  71
                ],
                [
                  0.9341864716636198,
                  0.14090019569471623,
                  511,
                  72
                ],
                [
                  0.9360146252285192,
                  0.142578125,
                  512,
                  73
                ],
                [
                  0.9378427787934186,
                  0.1442495126705653,
                  513,
                  74
                ],
                [
                  0.9396709323583181,
                  0.14591439688715954,
                  514,
                  75
                ],
                [
                  0.9414990859232175,
                  0.14757281553398058,
                  515,
                  76
                ],
                [
                  0.943327239488117,
                  0.14922480620155038,
                  516,
                  77
                ],
                [
                  0.9451553930530164,
                  0.15087040618955513,
                  517,
                  78
                ],
                [
                  0.946983546617916,
                  0.1525096525096525,
                  518,
                  79
                ],
                [
                  0.9488117001828154,
                  0.15414258188824662,
                  519,
                  80
                ],
                [
                  0.9506398537477148,
                  0.15384615384615385,
                  520,
                  80
                ],
                [
                  0.9524680073126143,
                  0.15355086372360843,
                  521,
                  80
                ],
                [
                  0.9542961608775137,
                  0.15517241379310345,
                  522,
                  81
                ],
                [
                  0.9561243144424132,
                  0.15487571701720843,
                  523,
                  81
                ],
                [
                  0.9579524680073126,
                  0.15648854961832062,
                  524,
                  82
                ],
                [
                  0.9597806215722121,
                  0.1580952380952381,
                  525,
                  83
                ],
                [
                  0.9616087751371115,
                  0.1596958174904943,
                  526,
                  84
                ],
                [
                  0.9634369287020109,
                  0.15939278937381404,
                  527,
                  84
                ],
                [
                  0.9652650822669104,
                  0.16098484848484848,
                  528,
                  85
                ],
                [
                  0.9670932358318098,
                  0.16257088846880907,
                  529,
                  86
                ],
                [
                  0.9689213893967094,
                  0.1641509433962264,
                  530,
                  87
                ],
                [
                  0.9707495429616088,
                  0.1657250470809793,
                  531,
                  88
                ],
                [
                  0.9725776965265083,
                  0.16541353383458646,
                  532,
                  88
                ],
                [
                  0.9744058500914077,
                  0.1669793621013133,
                  533,
                  89
                ],
                [
                  0.9762340036563071,
                  0.16853932584269662,
                  534,
                  90
                ],
                [
                  0.9780621572212066,
                  0.16822429906542055,
                  535,
                  90
                ],
                [
                  0.979890310786106,
                  0.16977611940298507,
                  536,
                  91
                ],
                [
                  0.9817184643510055,
                  0.1713221601489758,
                  537,
                  92
                ],
                [
                  0.9835466179159049,
                  0.17100371747211895,
                  538,
                  92
                ],
                [
                  0.9853747714808044,
                  0.1725417439703154,
                  539,
                  93
                ],
                [
                  0.9872029250457038,
                  0.17407407407407408,
                  540,
                  94
                ],
                [
                  0.9890310786106032,
                  0.1756007393715342,
                  541,
                  95
                ],
                [
                  0.9908592321755028,
                  0.17712177121771217,
                  542,
                  96
                ],
                [
                  0.9926873857404022,
                  0.17863720073664824,
                  543,
                  97
                ],
                [
                  0.9945155393053017,
                  0.17830882352941177,
                  544,
                  97
                ],
                [
                  0.9963436928702011,
                  0.1798165137614679,
                  545,
                  98
                ],
                [
                  0.9981718464351006,
                  0.1794871794871795,
                  546,
                  98
                ],
                [
                  1.0,
                  0.1809872029250457,
                  547,
                  99
                ]
              ]
            }
          ]
        }
      ]
    },
    "dont_know_word": {
      "questions": 547,
      "option": "I don't know",
      "sentence": "If you do not know the answer, choose I don't know.",
      "models": [
        {
          "name": "Liquid d1-ultra",
          "text_run": 199,
          "english": {
            "run": 449,
            "right": 457,
            "wrong": 39,
            "dont_know": 51,
            "invalid": 0,
            "accuracy": 0.8354661791590493,
            "attempt_rate": 0.906764168190128,
            "non_hallucination": 0.5666666666666667,
            "mean_dont_know_probability": 0.2527647853971477
          },
          "turkish": {
            "run": 511,
            "right": 492,
            "wrong": 28,
            "dont_know": 27,
            "invalid": 0,
            "accuracy": 0.8994515539305301,
            "attempt_rate": 0.9506398537477148,
            "non_hallucination": 0.4909090909090909,
            "mean_dont_know_probability": 0.2317269341077451
          },
          "paired": {
            "english_only_wrong": 18,
            "turkish_only_wrong": 7,
            "p": 0.043285250663757324,
            "p_holm": 0.25971150398254395
          }
        },
        {
          "name": "Fastino GLiDE v1.2",
          "text_run": 248,
          "english": {
            "run": 523,
            "right": 481,
            "wrong": 36,
            "dont_know": 30,
            "invalid": 0,
            "accuracy": 0.8793418647166362,
            "attempt_rate": 0.9451553930530164,
            "non_hallucination": 0.45454545454545453,
            "mean_dont_know_probability": 0.12485239376322405
          },
          "turkish": {
            "run": 525,
            "right": 491,
            "wrong": 47,
            "dont_know": 9,
            "invalid": 0,
            "accuracy": 0.8976234003656307,
            "attempt_rate": 0.9835466179159049,
            "non_hallucination": 0.16071428571428573,
            "mean_dont_know_probability": 0.09828773071658943
          },
          "paired": {
            "english_only_wrong": 16,
            "turkish_only_wrong": 27,
            "p": 0.12628947438543037,
            "p_holm": 0.4400623959405334
          }
        },
        {
          "name": "Mammoth-1",
          "text_run": 374,
          "english": {
            "run": 505,
            "right": 474,
            "wrong": 36,
            "dont_know": 37,
            "invalid": 0,
            "accuracy": 0.8665447897623401,
            "attempt_rate": 0.9323583180987203,
            "non_hallucination": 0.5068493150684932,
            "mean_dont_know_probability": 0.09803614442413162
          },
          "turkish": {
            "run": 519,
            "right": 490,
            "wrong": 32,
            "dont_know": 25,
            "invalid": 0,
            "accuracy": 0.8957952468007313,
            "attempt_rate": 0.9542961608775137,
            "non_hallucination": 0.43859649122807015,
            "mean_dont_know_probability": 0.07410430347349178
          },
          "paired": {
            "english_only_wrong": 19,
            "turkish_only_wrong": 15,
            "p": 0.6075913612730801,
            "p_holm": 0.7633867953264257
          }
        },
        {
          "name": "vLLM-SR d3",
          "text_run": 259,
          "english": null,
          "turkish": {
            "run": 544,
            "right": 495,
            "wrong": 52,
            "dont_know": 0,
            "invalid": 0,
            "accuracy": 0.9049360146252285,
            "attempt_rate": 1.0,
            "non_hallucination": 0.0,
            "mean_dont_know_probability": 0.013720586834477463
          },
          "paired": null
        },
        {
          "name": "Drex Max v1",
          "text_run": 322,
          "english": null,
          "turkish": {
            "run": 542,
            "right": 497,
            "wrong": 49,
            "dont_know": 1,
            "invalid": 0,
            "accuracy": 0.9085923217550275,
            "attempt_rate": 0.9981718464351006,
            "non_hallucination": 0.02,
            "mean_dont_know_probability": 0.009298918512398528
          },
          "paired": null
        },
        {
          "name": "vLLM-SR d3-ultra",
          "text_run": 269,
          "english": null,
          "turkish": {
            "run": 540,
            "right": 494,
            "wrong": 51,
            "dont_know": 2,
            "invalid": 0,
            "accuracy": 0.903107861060329,
            "attempt_rate": 0.9963436928702011,
            "non_hallucination": 0.03773584905660377,
            "mean_dont_know_probability": 0.013052245597998327
          },
          "paired": null
        },
        {
          "name": "Perplexity Decider 27B v1.2",
          "text_run": 217,
          "english": {
            "run": 529,
            "right": 481,
            "wrong": 64,
            "dont_know": 2,
            "invalid": 0,
            "accuracy": 0.8793418647166362,
            "attempt_rate": 0.9963436928702011,
            "non_hallucination": 0.030303030303030304,
            "mean_dont_know_probability": 0.027666686351686155
          },
          "turkish": {
            "run": 531,
            "right": 495,
            "wrong": 51,
            "dont_know": 1,
            "invalid": 0,
            "accuracy": 0.9049360146252285,
            "attempt_rate": 0.9981718464351006,
            "non_hallucination": 0.019230769230769232,
            "mean_dont_know_probability": 0.01355389901351757
          },
          "paired": {
            "english_only_wrong": 29,
            "turkish_only_wrong": 16,
            "p": 0.07245426016254441,
            "p_holm": 0.3622713008127221
          }
        },
        {
          "name": "Drex Max 1.1",
          "text_run": 326,
          "english": {
            "run": 447,
            "right": 463,
            "wrong": 41,
            "dont_know": 43,
            "invalid": 0,
            "accuracy": 0.8464351005484461,
            "attempt_rate": 0.9213893967093236,
            "non_hallucination": 0.5119047619047619,
            "mean_dont_know_probability": 0.1425561642833011
          },
          "turkish": {
            "run": 513,
            "right": 481,
            "wrong": 48,
            "dont_know": 18,
            "invalid": 0,
            "accuracy": 0.8793418647166362,
            "attempt_rate": 0.9670932358318098,
            "non_hallucination": 0.2727272727272727,
            "mean_dont_know_probability": 0.10160692670378244
          },
          "paired": {
            "english_only_wrong": 20,
            "turkish_only_wrong": 27,
            "p": 0.38169339766321286,
            "p_holm": 0.7633867953264257
          }
        },
        {
          "name": "Liquid d1",
          "text_run": 52,
          "english": {
            "run": 445,
            "right": 478,
            "wrong": 67,
            "dont_know": 2,
            "invalid": 0,
            "accuracy": 0.8738574040219378,
            "attempt_rate": 0.9963436928702011,
            "non_hallucination": 0.028985507246376812,
            "mean_dont_know_probability": 0.009200322658382633
          },
          "turkish": null,
          "paired": null
        },
        {
          "name": "Jev 1.14.0",
          "text_run": 194,
          "english": {
            "run": 446,
            "right": 472,
            "wrong": 72,
            "dont_know": 3,
            "invalid": 0,
            "accuracy": 0.8628884826325411,
            "attempt_rate": 0.9945155393053017,
            "non_hallucination": 0.04,
            "mean_dont_know_probability": 0.015521023765996345
          },
          "turkish": null,
          "paired": null
        },
        {
          "name": "SubQ Snap 1.1",
          "text_run": 346,
          "english": {
            "run": 448,
            "right": 463,
            "wrong": 63,
            "dont_know": 21,
            "invalid": 0,
            "accuracy": 0.8464351005484461,
            "attempt_rate": 0.9616087751371115,
            "non_hallucination": 0.25,
            "mean_dont_know_probability": 0.11709256155179708
          },
          "turkish": {
            "run": 512,
            "right": 483,
            "wrong": 45,
            "dont_know": 19,
            "invalid": 0,
            "accuracy": 0.8829981718464351,
            "attempt_rate": 0.9652650822669104,
            "non_hallucination": 0.296875,
            "mean_dont_know_probability": 0.0887675601039831
          },
          "paired": {
            "english_only_wrong": 40,
            "turkish_only_wrong": 22,
            "p": 0.030015864349067292,
            "p_holm": 0.21011105044347106
          }
        },
        {
          "name": "Inception Mercury Decide",
          "text_run": 204,
          "english": {
            "run": 450,
            "right": 465,
            "wrong": 71,
            "dont_know": 11,
            "invalid": 0,
            "accuracy": 0.850091407678245,
            "attempt_rate": 0.979890310786106,
            "non_hallucination": 0.13414634146341464,
            "mean_dont_know_probability": 0.031531572744804136
          },
          "turkish": null,
          "paired": null
        },
        {
          "name": "Microsoft-Decision-1",
          "text_run": 205,
          "english": {
            "run": 451,
            "right": 467,
            "wrong": 77,
            "dont_know": 3,
            "invalid": 0,
            "accuracy": 0.8537477148080439,
            "attempt_rate": 0.9945155393053017,
            "non_hallucination": 0.0375,
            "mean_dont_know_probability": 0.03592171404356558
          },
          "turkish": null,
          "paired": null
        },
        {
          "name": "Elephant-1",
          "text_run": 373,
          "english": {
            "run": 501,
            "right": 457,
            "wrong": 67,
            "dont_know": 23,
            "invalid": 0,
            "accuracy": 0.8354661791590493,
            "attempt_rate": 0.9579524680073126,
            "non_hallucination": 0.25555555555555554,
            "mean_dont_know_probability": 0.06769455393053016
          },
          "turkish": {
            "run": 515,
            "right": 441,
            "wrong": 82,
            "dont_know": 24,
            "invalid": 0,
            "accuracy": 0.8062157221206582,
            "attempt_rate": 0.9561243144424132,
            "non_hallucination": 0.22641509433962265,
            "mean_dont_know_probability": 0.07318174954296161
          },
          "paired": {
            "english_only_wrong": 31,
            "turkish_only_wrong": 46,
            "p": 0.11001559898513336,
            "p_holm": 0.4400623959405334
          }
        },
        {
          "name": "Celeris-1 Decision",
          "text_run": 150,
          "english": {
            "run": 548,
            "right": 418,
            "wrong": 37,
            "dont_know": 92,
            "invalid": 0,
            "accuracy": 0.7641681901279708,
            "attempt_rate": 0.8318098720292505,
            "non_hallucination": 0.7131782945736435,
            "mean_dont_know_probability": 0.12687385740402193
          },
          "turkish": null,
          "paired": null
        },
        {
          "name": "GPT-6 Luna",
          "text_run": 143,
          "english": {
            "run": 444,
            "right": 441,
            "wrong": 96,
            "dont_know": 10,
            "invalid": 0,
            "accuracy": 0.8062157221206582,
            "attempt_rate": 0.9817184643510055,
            "non_hallucination": 0.09433962264150944,
            "mean_dont_know_probability": 0.030073126142595975
          },
          "turkish": null,
          "paired": null
        },
        {
          "name": "Perplexity Decider Flash v1.2",
          "text_run": 223,
          "english": {
            "run": 535,
            "right": 442,
            "wrong": 100,
            "dont_know": 5,
            "invalid": 0,
            "accuracy": 0.8080438756855576,
            "attempt_rate": 0.9908592321755028,
            "non_hallucination": 0.047619047619047616,
            "mean_dont_know_probability": 0.05526566245284184
          },
          "turkish": null,
          "paired": null
        }
      ]
    },
    "against_reference": {
      "reference": "Liquid d1-ultra",
      "reference_run": 411,
      "questions": 547,
      "models": [
        {
          "name": "Fastino GLiDE v1.2",
          "run": 521,
          "reference_only_right": 27,
          "other_only_right": 25,
          "p": 0.8898839652765371,
          "p_holm": 1.0
        },
        {
          "name": "Mammoth-1",
          "run": 503,
          "reference_only_right": 23,
          "other_only_right": 20,
          "p": 0.7607916425613439,
          "p_holm": 1.0
        },
        {
          "name": "vLLM-SR d3",
          "run": 554,
          "reference_only_right": 23,
          "other_only_right": 18,
          "p": 0.5327092552361137,
          "p_holm": 1.0
        },
        {
          "name": "Drex Max v1",
          "run": 552,
          "reference_only_right": 27,
          "other_only_right": 21,
          "p": 0.4708790136217118,
          "p_holm": 1.0
        },
        {
          "name": "vLLM-SR d3-ultra",
          "run": 550,
          "reference_only_right": 31,
          "other_only_right": 24,
          "p": 0.41875418826901556,
          "p_holm": 1.0
        },
        {
          "name": "Perplexity Decider 27B v1.2",
          "run": 527,
          "reference_only_right": 29,
          "other_only_right": 18,
          "p": 0.14386508844054902,
          "p_holm": 1.0
        },
        {
          "name": "Drex Max 1.1",
          "run": 399,
          "reference_only_right": 36,
          "other_only_right": 24,
          "p": 0.1550019040032606,
          "p_holm": 1.0
        },
        {
          "name": "Liquid d1",
          "run": 430,
          "reference_only_right": 40,
          "other_only_right": 19,
          "p": 0.008641499683281734,
          "p_holm": 0.06913199746625387
        },
        {
          "name": "Jev 1.14.0",
          "run": 410,
          "reference_only_right": 44,
          "other_only_right": 19,
          "p": 0.0022275315109880345,
          "p_holm": 0.02004778359889231
        },
        {
          "name": "SubQ Snap 1.1",
          "run": 413,
          "reference_only_right": 42,
          "other_only_right": 16,
          "p": 0.0008617822405421263,
          "p_holm": 0.010341386886505516
        },
        {
          "name": "Inception Mercury Decide",
          "run": 398,
          "reference_only_right": 46,
          "other_only_right": 19,
          "p": 0.0010900838044475788,
          "p_holm": 0.011990921848923367
        },
        {
          "name": "Microsoft-Decision-1",
          "run": 412,
          "reference_only_right": 46,
          "other_only_right": 19,
          "p": 0.0010900838044475788,
          "p_holm": 0.011990921848923367
        },
        {
          "name": "Elephant-1",
          "run": 499,
          "reference_only_right": 49,
          "other_only_right": 20,
          "p": 0.0006361729964503618,
          "p_holm": 0.008270248953854704
        },
        {
          "name": "Celeris-1 Decision",
          "run": 546,
          "reference_only_right": 50,
          "other_only_right": 14,
          "p": 7.069487542545679e-06,
          "p_holm": 0.00010604231313818518
        },
        {
          "name": "GPT-6 Luna",
          "run": 397,
          "reference_only_right": 54,
          "other_only_right": 18,
          "p": 2.568720337924735e-05,
          "p_holm": 0.0003596208473094629
        },
        {
          "name": "Perplexity Decider Flash v1.2",
          "run": 533,
          "reference_only_right": 64,
          "other_only_right": 16,
          "p": 5.871330414596199e-08,
          "p_holm": 9.394128663353918e-07
        }
      ]
    }
  },
  "dont_know_english_option": {
    "source": "Ümit's follow-up of 2 November 2026, step 2: the English “I don't know” comparison for Liquid d1-ultra, SubQ Snap 1.1, Elephant-1 and Mammoth-1, with Holm's correction over these four",
    "questions": 547,
    "models": [
      {
        "name": "Liquid d1-ultra",
        "text_run": 199,
        "english_only_wrong": 20,
        "turkish_only_wrong": 7,
        "p": 0.0191572904586792,
        "p_holm": 0.0766291618347168
      },
      {
        "name": "SubQ Snap 1.1",
        "text_run": 346,
        "english_only_wrong": 39,
        "turkish_only_wrong": 27,
        "p": 0.1752857944430353,
        "p_holm": 0.3505715888860706
      },
      {
        "name": "Elephant-1",
        "text_run": 373,
        "english_only_wrong": 31,
        "turkish_only_wrong": 46,
        "p": 0.11001559898513336,
        "p_holm": 0.33004679695540007
      },
      {
        "name": "Mammoth-1",
        "text_run": 374,
        "english_only_wrong": 18,
        "turkish_only_wrong": 15,
        "p": 0.7283324808813632,
        "p_holm": 0.7283324808813632
      }
    ]
  },
  "dont_know_same_answers": {
    "reference": "Liquid d1-ultra",
    "resamples": 10000,
    "seed": 20261103,
    "rule": "Each model's five-option run (Turkish: every main run on the Text tab, all its setups, each its own Text tab run; English: its first English run) keeps exactly k answers, k being the reference's answered count (right + wrong) in its run with \"I don't know\". A run's scored answers (score and choice not null; any other answer is never kept) are ranked by score, highest first, equal scores by question id ascending (answers.json questions.id), never by correctness; the first k are kept, and wrong is the number of wrong answers among them, a whole number. wrong_of_answers = wrong / k; difference = a model's wrong_of_answers - the reference's. A curve has one point [answered, wrong] per k from 1 to the run's scored answers, under the same selection. Paired bootstrap over questions: per family a new random.Random(20261103), Turkish first, English second; 10,000 resamples, each rng.choices(range(n), k=n) over the family's n question positions in answers.json order, each question counting as often as it was drawn (k stays fixed). In a resample the answers are ordered by (score descending, question id ascending), the copies of a question together, and the first k copies are kept, only as many copies of the boundary question as k needs: wrong = the wrong copies before the boundary question + (k - the copies before it) * (1 if it is wrong, else 0). d = a model's wrong / k - the reference's. Two-sided p = min(1, 2 * min(1 + #{d <= 0}, 1 + #{d >= 0}) / 10,001); the 95% interval is the percentile interval, the 250th smallest and the 250th largest of the 10,000 resampled d. Holm's correction within each family over the models other than the reference (Turkish: the other main runs; English: the other English reading models), in the family's order.",
    "turkish": {
      "questions": 2198,
      "k": 1582,
      "k_run": 511,
      "k_wording": "I don't know",
      "models": [
        {
          "name": "Gemini 3.8 Flash",
          "text_run": 140,
          "five_option_run": 140,
          "wrong": 7,
          "wrong_of_answers": 0.004424778761061947,
          "difference": -0.1011378002528445,
          "interval": [
            -0.11820480404551201,
            -0.08596713021491784
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000001100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000100000000000000000000000101010000000000000000000000000000000000000000000001000000000001000000000000001010000000000000010001000000000100000000000010000000001000000000010000000000100000100010000000000000000000000000000000000000100010010111000000101111111"
        },
        {
          "name": "GPT-6 Astra",
          "text_run": 277,
          "five_option_run": 277,
          "wrong": 6,
          "wrong_of_answers": 0.0037926675094816687,
          "difference": -0.10176991150442478,
          "interval": [
            -0.11946902654867256,
            -0.08659924146649811
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000001000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000001000000000000000000000000000000001000000000000000000000100000000000000000000001000000000000000000000000010000000000000000000000000000000000000000100000000000000000000000000000000000000001000000000000010001000000000000000000000000010000000000000000000000001000000001100001000000000000000100000001000010000000110000000100000000000000000000000010000000100101000010001010100001111010000001010000011010"
        },
        {
          "name": "GPT-6.1 Sol",
          "text_run": 276,
          "five_option_run": 276,
          "wrong": 8,
          "wrong_of_answers": 0.0050568900126422255,
          "difference": -0.10050568900126422,
          "interval": [
            -0.11820480404551201,
            -0.08533501896333755
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000001000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000100000000000000000100000010000000000000000010000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000100000100000000000000000000000001000000000000000000000000100000000000010010000011000100101000000110000000000000000000000000011100100000000011000100001000010101001100100000000111000110011"
        },
        {
          "name": "GPT-6 Astra",
          "text_run": 48,
          "five_option_run": 48,
          "wrong": 4,
          "wrong_of_answers": 0.0025284450063211127,
          "difference": -0.10303413400758533,
          "interval": [
            -0.12073324905183312,
            -0.08786346396965866
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000001000000000000000000000000000000000010000000000000000100000000000000000000000000000100100000000000000000100000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000001000000001000000000000000000000000100000100010000000010000000100000000000000000000000000000001000000000010000000000100000010010000100001000010000010000010010000000010000010011000000000101101010001010101101110010011011"
        },
        {
          "name": "GPT-6 Astra",
          "text_run": 191,
          "five_option_run": 191,
          "wrong": 11,
          "wrong_of_answers": 0.00695322376738306,
          "difference": -0.0986093552465234,
          "interval": [
            -0.11630847029077117,
            -0.08343868520859671
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000010000000000000000000000000000000000000000000000000000000000000000010010000000000100001000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000110000000000010000000000000000000000000000000000000000000000000000001000000001000001000000000001000000000000000000000000000000010000000000000000000000000100000001000000000000000001000100000000100100000000000000010000000000010000100000000000000000001000000000000000001000000000010000000000000001010000000010000001001000000000000100111000010111100000001111111011010"
        },
        {
          "name": "Gemini 3.8 Flash",
          "text_run": 188,
          "five_option_run": 188,
          "wrong": 13,
          "wrong_of_answers": 0.008217446270543615,
          "difference": -0.09734513274336283,
          "interval": [
            -0.11504424778761062,
            -0.08217446270543616
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100100000000000000000000000010000000000000000000000000000000000000000000000100100000000000000000000010000000000001000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000100000000000000000000000010000000000000000000000000000001000000000000000000100000100000000000000100000000000000000000000000000000000000000000001000100000000000000100000000000000000000000000000000000000000000010000000000000000000000000000000000100000000000000010000000000000000000000000000000100000100000000000000000000100000000010000000000000000000000000000000000000000000000000000000000000000000000000000010000000100000100000001000000000100000010000000000000000001001000000001010001011001101000000010100110011110000001001011001111010"
        },
        {
          "name": "GPT-6.1 Sol",
          "text_run": 190,
          "five_option_run": 190,
          "wrong": 12,
          "wrong_of_answers": 0.007585335018963337,
          "difference": -0.0979772439949431,
          "interval": [
            -0.11504424778761062,
            -0.08280657395701643
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000010000000000000000000100100100000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000100000000000000000000000000000000000010001010000000000000000000000000000000000000001000010001000000000000000001000000000000000000100100000000000000000000000000000000000000000000000000000001000000000000100000000000000100000000100011010000001010000000000000000000000100000000100000000000000000001000000100000000101000000000010000000000000000000000000000010000000100010000000100001010011000000000100001010000000100011101001010100011011000011010010101101001"
        },
        {
          "name": "GPT-6.1 Sol",
          "text_run": 124,
          "five_option_run": 124,
          "wrong": 13,
          "wrong_of_answers": 0.008217446270543615,
          "difference": -0.09734513274336283,
          "interval": [
            -0.1150442477876106,
            -0.08217446270543616
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100100010001000000000000000000000010000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000010000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000001000000000000000000000000000000001000000000100000000000100000000000000000000000000000000000000000000000000000000000000000001000000000000000000001000000000000000000000000000000000000000100000000000000100000010000000000000000000000000000000000000000001000010000000000000000000000000000000000000000000000000000000000000000000000000000010000001000000110000000000000000010000000000000000010001000000100010000000000100100000000101010000000000010010001000100010000010000000101110100001000011100000000001010001110100010001111010000000100110001100101"
        },
        {
          "name": "Gemini 4 Argon",
          "text_run": 557,
          "five_option_run": 557,
          "wrong": 17,
          "wrong_of_answers": 0.010745891276864728,
          "difference": -0.09481668773704172,
          "interval": [
            -0.11188369152970923,
            -0.07964601769911506
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000010000000000000000000000011000010000001000000000000000001010000000000100000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010100000010000010000000000000000000000000000000000001000000000000000000000000000000000010000000000000000000000000001000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000010000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000010000000001000000000000000000000000000000001011000000000000000000001000000000000010000100100000000010010100110000000000001000000000010000000000000000010000010000000100000000000101000100000110100010000000001001110000100000000000010100001001001100100001001010010001111010001010101011111"
        },
        {
          "name": "Claude Fable 5.1",
          "text_run": 310,
          "five_option_run": 310,
          "wrong": 19,
          "wrong_of_answers": 0.012010113780025285,
          "difference": -0.09355246523388117,
          "interval": [
            -0.11125158027812894,
            -0.07964601769911504
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100001001010100000000000010000000000000000100000000000000000000000000000000000000000000000000000001000000000000000000000000000000010000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000100000001000000000000100000000000001000000000000000000000000000000000000000000000000000010000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000100000001000000000000010000010000000000000000000000000000000000000000010000000000000000000000000000000000000000000100000000000001000000000000000000000000000000000000000000000000000100000000000000001000100000000000000000000000000000000010100000101100000000010000000000100000000000010000000000000000010001001000000000001101000101000000000000100010000000101000010000000000100010000000010010000001000100101100001101000010010000000101001000000011100100000101100000101000100101101101"
        },
        {
          "name": "Claude Opus 5.5",
          "text_run": 278,
          "five_option_run": 278,
          "wrong": 23,
          "wrong_of_answers": 0.014538558786346398,
          "difference": -0.09102402022756005,
          "interval": [
            -0.10809102402022756,
            -0.07648546144121365
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000001100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000100000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000010000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000010000000000000000000010000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000010000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000100000000000001010000000000000000000000000000000000000000000000000000000000000000000010000100000000000000000000101000000000000000000000000000000000001000010000000000000000000000000000000000000000100000000000000010000000000000010000000000000000000000000000000000000000000000000000000000011000010010000000000000000000000000000000000000000000000000000000010000000000100000000000010000000100010100010100000000010010001100000000110000000100000010000000000100000000000000001000010000101000000000000000000001011000000010010001110000000100001000100100000000000000000000000101010011000000010000010000000100110100000010001110000010100101000000001000010100111000011101010111100111000111100"
        },
        {
          "name": "GPT-6.1 Terra",
          "text_run": 275,
          "five_option_run": 275,
          "wrong": 31,
          "wrong_of_answers": 0.019595448798988623,
          "difference": -0.08596713021491782,
          "interval": [
            -0.10303413400758535,
            -0.07142857142857142
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000101000000000000100000000000000000000000001000100000000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000011000000000000000000000000000001000000000000000000000000000000000000000000000000000000001000000100000000000000000000000000100000000000000000000000001010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000001000001001000000001000100000001000000000000000000000000000000000000100000000000010100000000000000000000000000000000010000010000000000000000000000000011000000000000000000000000000000000001010100100000000000000000000000010000000000000000000010000010000100000000000000000000001000000000000000001100000000001000000010001000000010000000000000010011000000000000000000000100000000000000000000000001000000010000000000000000100000001000001000000001100000100000000100000100000000000100000000001000101100000001000000100000001000000000000000000001000010000000000100000010000000000001101010010000000000000000000010000100000001001100000010000000000001101011010010010001000100000101000000100000000100011001100001001110010100100011101000000011110100111101101000"
        },
        {
          "name": "Claude Opus 5.5",
          "text_run": 192,
          "five_option_run": 192,
          "wrong": 23,
          "wrong_of_answers": 0.014538558786346398,
          "difference": -0.09102402022756005,
          "interval": [
            -0.10745891276864727,
            -0.07648546144121365
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000010000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000100000000000000000001000000000000100000100000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000100000000000000000000000000000000000000000000001000000000000000100000000000000000010000000000000000000000001000000000100000000000000001000000000000000000000000100000100000000011000000100000000010000100000000000000000000000000000100000100001000000001000000000000000000000000000100000000000000000000000000000000000000000000000000100000000000000010010000000000001001000100000000000000000000000010010000000001011100000000000000001000000000000000010001010000100100100000000000000000100000000000000000000000000000011100000011000100000000001000000000010000000100100110101110000001100000011000000000100000110101100100100000001010000000000001000000001000000110100010000010100010000110100000001011000001010010010000010100101010000011100011100111110001111010010101001111001001001111110111111110001"
        },
        {
          "name": "Gemini 3.8 Flash",
          "text_run": 123,
          "five_option_run": 123,
          "wrong": 50,
          "wrong_of_answers": 0.0316055625790139,
          "difference": -0.07395701643489255,
          "interval": [
            -0.09228824273072062,
            -0.05815423514538559
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000001000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000100000001000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000010000000000000000000000100010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000010000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000001000000000000000000000000000001000101000000001000000100000000000010000000000000000000000001010000000000000000000000000000000000000000000000000011000000101000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001001000010000000001000000000000000000000000000000000000000000000000000000100100000000000000000000000000100011100100100000010001010000000000000011000101000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000010000000000001100000000000001000000000000000000000100000000000000000100000000000000000000000000000000001100100000001010001000000010100000010000000000000000000000100000000000000011000000000000000001000000000101000100000100001000000110001000001001010000001100000000001000000000000000000000010010000000001100000011010010001001000001000000000100000000000000000000101010000101100010110000010010000000000000100000000000100000000011000000010001000111010100001000010011000001010000100111001000001000000000010000100011001010110001010000001110111110"
        },
        {
          "name": "Grok 4.7",
          "text_run": 311,
          "five_option_run": 311,
          "wrong": 31,
          "wrong_of_answers": 0.019595448798988623,
          "difference": -0.08596713021491782,
          "interval": [
            -0.10303413400758535,
            -0.07142857142857142
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010101000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000100000100000000000100000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000010000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000001000000000010000000000000100001101001000000000000100000010000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000101000000000000000000000000000000000000000100000000000000000000000000000000000100000001000000100000000000000001000000000000000000000000000100000000000000000000000010010000000000100100000000100000000000000000001000000000101000000000000000100000100000000011000000000010001000000010000000000000000000000000000101000000000010001000100010000000100001001100000000000100000000000000100000000000101000010001000000010010000010010000000110010000000000100001100001000001000110101000000100000000100100000000000110000000000001100100110001100000100000000000110000000110000000000000010000000000110001000000000010000010100111001011000000000000100000100001101000110010111111000110101010000101001000000000001001110001111111000000110100000010000010010000111110011001100100000010101"
        },
        {
          "name": "DeepSeek-V4.1-Flash, first release",
          "text_run": 121,
          "five_option_run": 121,
          "wrong": 113,
          "wrong_of_answers": 0.07142857142857142,
          "difference": -0.034134007585335024,
          "interval": [
            -0.054361567635903926,
            -0.018331226295828063
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000100000000000000000000000000000000000000000000000000000000100000110000000000000010000000010000000000000000010000000000000000000001000000000000000000000000000000000001000000010010001000000000000000000000000000000000000000000000000000000000000000000000000010000000000110010000000000000000010000000000000100100000000000000010100011000000000000000000000000100000010000000000000000001000100000000010000001000000000010010000000000000001000000100000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000010000001000000100000000000000000000000000000000000000100000000000000000000000000100001000000100001000000000000000000000010000000000000000000100001000000000000010000001000001010000000000000000000000000000000000000000001000000001000100100010000000000000000000000000000000000010000000001000000000000010000000010000000100000100000001000000000000000000000000000000000000000000000000000000010000000000000000100000000000000001000000000110001000010000000000000000000100000000000000000000000000100000100000001000000000000100000000000010000100000000000000000000000000000100000000000000000000000000000001000010000000000000000000000000000000000000000010000000100001000000000000100000000010000100000000101110101000000000010000000000000000000000000001000000000000000010000000000000000000000000000010000000000000000000100000000000001000000000001001100000000000000000000000000100000000000000000000001000000000000100000001000001000000000000000000000000000000000100000000000000000000000000010000100000010000100000110100000000000000010000000000000000100000000000000000100000000000010000000000000000100000000000000000000000000000000001000000000001000000000000000010000000000000000000000100000000000001001000000000100000110100000000100010000001000100000000000010000000000000000100000000001000100001000000000010000010000000010010110011000000010000000001010000000000001000010100000000010000001000000010000000000000100010000010000010000000100000000100000010000000000000000000000000011000000000000001000000100000000000000000000000000000000000100000000000000001000000000000000000100100000000000000000101000000000000000000000000010000000000101100"
        },
        {
          "name": "GPT-6.1 Terra",
          "text_run": 274,
          "five_option_run": 274,
          "wrong": 50,
          "wrong_of_answers": 0.0316055625790139,
          "difference": -0.07395701643489255,
          "interval": [
            -0.09228824273072062,
            -0.06005056890012643
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000001000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000001001100000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000101000000000000000000000000000000001001000000000000000000000000000000000000000001000000100000000000000000000000001010000001000000010000100000000000000000000000000000100000000000000000000000000000000100001000010000000000000000000000000000000000000000000000010000000000000000000000000001000000000000000000000000000000000000000010000001000000001000010000000000110000000000000100100000001010000000000000000000000000000000000000000001000000000000100000000000000000000000010000001000000000010000000000000000000000000000000011001010000110000100000100000000000000000000000000100000001000000000000010010000001000000000000000001000000000000000000000000000000000000000000001001000110010100000100010000100000010000010100000101101000010000101000010000000000011000010001101001001000000000000000000000100000000000000011010000000001011001110000000000000101100000000000000000001000010010110000010001000101000010000000100101000001101001001101001110111000100010001100000110101001100000001000000110100000100100100100000000100001111000101100010000000011000001110000111001001010100011010000000001110000010101110001001111000101010100100010010101000000010000111110110101101010"
        },
        {
          "name": "Claude Opus 5.5",
          "text_run": 47,
          "five_option_run": 47,
          "wrong": 51,
          "wrong_of_answers": 0.032237673830594185,
          "difference": -0.07332490518331226,
          "interval": [
            -0.09039190897597978,
            -0.058786346396965866
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000100000000000000000000000011110000000000001000000000000100001000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100001000010000000000000000000000000000000100000000000100000000000000000000000000000000000000000000000001000001000000000000000000000000000000100000000010000000001000010100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000001000000000000000000000000000000000100000001000000000000000000010000000000000000000000000000000000000001001000000000000000000010000100000000010000000000000000000000001000000000000000000000000000000000000000000001000000000000000000000010010000000010000000000010000100000000000000000000010000000100000010001000000100100001000000000000000000000000000000000000000000000000000000000000110100000000001000000000100010000000001000000000100010000010000110000000000010000110000001000000000000000001000000000000000000000000000000010010010001000000000000010000100000000000010000000000000000010000100010000011010000000001000000000000010000000000010001001001000001100000000000010001000000000000000000100010000000100110100101000000001000000000010100000000010010000010000000100000000011011000000110001000101100010000001000001001000000000001011100010100110010011101011110000010000111000000100000010000110011101110010011001110011101111000111000001110000011101110100011111100010100010100100010101001110101011101111011101011000110100101"
        },
        {
          "name": "Claude Sonnet 5.5",
          "text_run": 193,
          "five_option_run": 193,
          "wrong": 65,
          "wrong_of_answers": 0.04108723135271808,
          "difference": -0.06447534766118837,
          "interval": [
            -0.08091024020227561,
            -0.04867256637168142
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000110000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000001000000000000000100100000000000000000000010000000001000000000000000000000000001000000000000000000000000000000000000000000000000000000000001000000000010000000000000000000000000100000010000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000100000000000010000000000000000000000100000000000000000000000001000000000000000000001000000010001000000100000000000000000100000000000000000000000000000000000000000100000100000000001010001100001010001001010000000011010011010001010110001000110000000000000000000000000000100000000000000100000000000000000000000000000010100000001000010000000100100000000000000100000000000000000010001000000000001001000101000000001000000000001000000000000000000001010000000100010001000000000000001001100000000000000110001000000000110000010001000000000100000000100010000000010000000110001100000100100000000000000010000001000001000101001010000000000000000011010001001101010011100000001000000110000000000000000000000010000000000001000000100100000000000000101000000000100000110000110000011111010100001110010000100100100000000101010010010011011101000010000001001100010011110010100000010010001010011010000101101100001000101000011100100000011011000100010101000010001111100111111101111101001000111101101111100000011100111011100011010111101"
        },
        {
          "name": "Claude Sonnet 5.5",
          "text_run": 55,
          "five_option_run": 55,
          "wrong": 113,
          "wrong_of_answers": 0.07142857142857142,
          "difference": -0.034134007585335024,
          "interval": [
            -0.05309734513274336,
            -0.018331226295828063
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000000100000100000000000000100000100000000000000000000000000000000000000000000000000000000000000001000000100000100000000000000000000000000000000000000000000000000000000000000000000010000000000000101000000000001000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000011000000000000000000000000000000000001011000000000000000000000000100000000000000000000000000000000000100000000000000000000000000000000000100010000000000000000000000000000000000000000000000000000000000000000000000000000010000100000000000000000001010000000010000000000100000010000000010000000000001000000000000000000000000001000100000000100000000000000000000000000000000100000100000000000000100000010000001000000000000000000000000110101000010000000000000000001001000000000000000000000000000000000000000000000000000000000000000000000000000000000010000100000000010000000000000000000000000100000000000000000000000000001000000000000000000001000000000010000000000000000000010000000000000000011010001000000000000000100000000000000000100100010000000011000100000000000000000000000000000101000010111110000000000000000000100000000000000000000000000000100100000000000000000000100001000000001000100000000100001000000000000000000000000000000000000000010000010010000000111001010000000000000000001000000000000000000001000000000000000000001000100100100000100001000000000000101000000110000001001000000000000000100100110000000000000000000000000000000100000001000010000000000000100000000100010000000000000000000010001100000010000100000010100000001000000100000100000000000000000101100100001000000000001100000101000011100001010000011011010110100000010101000110010001010000110000001000100101000010010000011000000000110011110001000010110000000000000000001000001100001000000000000000110000010000000010001110000110101100111000000100000100000010110000000001000100010110110111101011100011011001000000000001100000110101100000111001111001011010110111000101001011000100100001100110011111001100011100111101000110110001000111010101001001101011000101111110000010111100011001001011111111010001100111111010110111001111011"
        },
        {
          "name": "Kimi K3.1 Pro",
          "text_run": 320,
          "five_option_run": 320,
          "wrong": 141,
          "wrong_of_answers": 0.08912768647281921,
          "difference": -0.016434892541087237,
          "interval": [
            -0.0347661188369153,
            0.0
          ],
          "p": 0.05119488051194881,
          "p_holm": 0.10238976102389762,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000100000000100000000000000000000000000000001000000000011000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001010000000000000000000000000000000000000000000000110000100000000000000000000000000010000000000000000000000000001000000010000000000000000000000000000000010000000000000000001000000001001000000000111001000010100000100001010001100000100000001000000001100011000000000000000001100000000000000000000000010001100000000000000000000000000000000000000000000000000000100000000000100000000000000000000000000010000000000000000000000000000000000100000000000000000000000000000000000000010000000000000100000000000000000000000000101000010000000000000000011001000000000000101000000010010000001000000000000001100000000000000000000000001000000000000000000000000000000000000001000000000000000000000000000000100000010000000100100000100000000000000000000000000001000000000001100001000010000000100000000001000100000000110000000000001001000000000000000010100000001000000000000000000000000101010010000101101000010100000000000010000000111000000010000100101100000011000010101000001000010100000000100001000000000000000000000000000000010000000000000000000000001100100000000000010000000000100110000100000000000000001101000100000000000001100010000100000110000001000000000000000010000110010000100110000010100100001000010100001000000000000000000001000000000100000000000100000000101100100110001001000010000000001100010110000010000010010110100110001000111010101010001110000000010100000001010000010000001101101000011011110111010000010000000001100000000101110001100010000010010010011110100001011001000000000010110100001001100001010001000000000000000000010001000000000011011010100001110000010101101000101100100001001101010110000000011101001010010010001010100000111001001101001011111111111110000110101110101010011000100110000011011101100101100001001011110010100100111000111011111011100011011011111111111011011011110111110111111111001011"
        },
        {
          "name": "GLM-5.3-Flash",
          "text_run": 160,
          "five_option_run": 160,
          "wrong": 163,
          "wrong_of_answers": 0.10303413400758533,
          "difference": -0.0025284450063211145,
          "interval": [
            -0.02212389380530974,
            0.013906447534766123
          ],
          "p": 0.7119288071192881,
          "p_holm": 0.7119288071192881,
          "wrong_by_rank": "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000010100000000000100000000000000000000100000000000000000000000000000000000000000000000000000000000000100100000000000000000000000000010000000000000000000010000100110000000000000000000000000000000000100000000100000011000100000000010010000000001000000000000000000100000000000000000000000001000000000001000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000001000000000000001000000100000000001001010000000000001010000101000000000100000000000001000001001001000000000000000000000000000000000000100000000000100001000001000000100000000000000001000000000000000000010000000100000001101110101000000000100000010011101100000110000000000100000000000000000000000000000000000000000000010100100010010000000011100010100000000000000000000000000000000100000000000000000010000000011000001000010000000000000001000100000010100100000010000000000000000001000100100001000000000000000001000110000100001001001000000110000100010000101000000000000001110000000010001010001000010000000000000000011010000001000000000000000000001000110010000000000100000000000000000000000000000000000000000010001001001000001101000000001000000000010000000000100000000011001000110010011001000010001000010000010100001100000000000001000011001000111110001000001010000000000100011000100010100000000010001000001000000000000000000000000001110010001000000010011101000001000100010010110100000000100000010001000110100111100010000000100011000000101110101000001001000100111001001101000010000010000000000000000000111110001010100000010000000011001001000101000100111001010000111100001010000001000010011100100110001011100010000000010100001111101001111110010011110010000000001100000011010010010111010100001100100010100101101110000101011110100111110010111111010011111110110000100111111110"
        },
        {
          "name": "Liquid d1-ultra",
          "text_run": 199,
          "five_option_run": 199,
          "wrong": 167,
          "wrong_of_answers": 0.10556257901390645,
          "difference": null,
          "interval": null,
          "p": null,
          "p_holm": null,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000001000000000000000000000000001000000000000000000000000001001000010001000000000000000000101000000000000000000001000000000000000001100000000000000000000011000000000000101000000000000000001000000000100000011010000000000000000000000010000001001100100000010000000000000010000001000000000000000100000000010000000000000011100000000000001101010000110010000010010000001000000000100000000001100000000000100000000010000000001100000000000010100000001010000010010001000000000100001001100010001000001000100001011001000000000000000000000010000010000000001000100000101011101000000000101010010100000011011001000100001010010000101010001100001000100000000000010000000000010000101001100001010011000000000000010001101000010000011000000001000000101000000111010001011011101000100000111000000010000110011000011001000110010001100110000100010001100000101000000000011101010001010001000100000100001101101000000101000000010000111110011010101111000110001110100110111010100001011001000001110111000000100100000111100100110110000011011110100001110111110001101000100111000001111000110011110110000111000010111010100000010110101011100111010011110010010001110000011101001011011110111011100000101001110011000111010111110111111101111001010111101111100010010110010111111110011010010000010000011111101111011111001001000101010101101110000101100111011011100100011010010111110011101111110001101111100111110100101101011011100001001110111110011111111101011011111010100011110101010001101011011000110111101110110010110011101011111101",
          "points": [
            {
              "wording": "Bilmiyorum",
              "run": 350,
              "answered": 1581,
              "wrong": 205
            },
            {
              "wording": "I don't know",
              "run": 511,
              "answered": 1582,
              "wrong": 216
            }
          ]
        },
        {
          "name": "GLM-5.3",
          "text_run": 314,
          "five_option_run": 314,
          "wrong": 260,
          "wrong_of_answers": 0.16434892541087232,
          "difference": 0.058786346396965866,
          "interval": [
            0.03792667509481669,
            0.07838179519595448
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000000000000000100001010000000000000000000000000000000000000000000000000000000100100000000000000000000000000000000000000000001000000000000000000000000000000000000001000000000000000000000000000000000100010000000001000000001000001111000001000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000010100000000000000000000000100011100000010000000001100011000000000000000001000000001000001011000000000000100000000000000010100000000000000000000001001000000000000000000000010011000000000000100000000000000000000100000000010000100000000000100010000001000000000000001011001110110010010001000000000101001000001000000010000000000000000000000000000001000000000000000010000000100000000000000001100001000000011001100000011000010000000000000000000000010000010100001101000000011001101010010111100111000110101001000101001000100100010001000101000000000000010001000000000000000001000000000000000010100000000000000100000010000000000010000100001100000001000000000000000011001100000010010000101010000010001000000000000000001000000000001000000000000010000000000000000000000000001100010100001000010001000000010010000100001010000011110100000101001100001000111000111000000001000001110100011010001101100001110000010010000001010000001100001000000000101010011000000000100000100010000000110000001110100000010010000100000000001000000110101010001001000100000011100000101000000000110010010101000000001000101001100010000000001001100000001001010010000001100010001000110010100010100000010001000010100100101000011000000000011010000000010000000001000100110100010101000100010100000110100000000000011000010100000000111000011000100010000011000000001001001001000001011111111101011110000110101101000100000100100000011000000001100001011010100000011110011101011101100000010011100001000010100001100000001000010011100011011000000000000011010010001110011010001000110110111011111011011010001010000010111110111101111111000110111110010000001111100110110110011111101011001000110000001111101110100010110011111001001000010111001111111010011100011001101111110001110001011001101100111111111011010110101101011011110111110111111111010101111111111101111110110011111110110110111"
        },
        {
          "name": "Qwen3.8-27B",
          "text_run": 318,
          "five_option_run": 318,
          "wrong": 306,
          "wrong_of_answers": 0.1934260429835651,
          "difference": 0.08786346396965866,
          "interval": [
            0.06573957016434892,
            0.10745891276864727
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "000000000001000000000000001000000000100001100000000000000000000100000000000100000000100000000000000000010100001000000001000000000000000000101000010000000000000010010000000001000000000000100000000000000000000000100000000000000000000000100011010000000000010000000001110000000000000000000100010010001001001000000001000001001001000000000001000000000000100001100100000000100000101000001000000000000000001000000000000100000000010100010100100111100000000000001000000100000000000000001001000000000110000000000000010000010000000100000010001110100000001010100100000001011000000000000010000000000010000100010000000000000000000000010000000000001011100000000110000110001001101100001100000000000101000010000000011000000111000000010000010000000000000000000000010110000000000000100000000000001000000101000001010000010000010000010001001100000000000110000000000000000001010000001000000010010000000000010000001111100001011000001000010000100100010000101100000000011000010000000000000001000001010000000100011000000000010000000000001110000000000100010011100000000000100100101000010001010100000000000011000000101000110000100100100001100000000001000001001000001000000010100101000000010100000100000000000010000000101000010010000010000000001000000101000010000001000001010100001010010001010001011000001011110000000000011001010000000100000001110000100000010000001000100010000001010100110100000010001101000000000110100000010000001000000001101001000010001010000100011010000000001110001011100100101010010011101000101000100000100100000000001000010001010001000001011111000101000101001011000100000001100000000100001000001000000001000100010001000010000001001010111001011011001000001101101110000101110000100010100111100011010000001100010001000000001100010101001101000101001000101100101000100100010110101001001001001000111000100111111011000100001100110011000010000010000100001001010001001000010110010111000100000010001010000000010000001010000110001000010000000000000001000000001100111000111010001110001010000010011000001010100000010001100010011001011011101001011000000010010001100101010001000011000000100111111010101001000001010011000000110010100011110110"
        },
        {
          "name": "Gemma 4 31B",
          "text_run": 119,
          "five_option_run": 119,
          "wrong": 239,
          "wrong_of_answers": 0.15107458912768648,
          "difference": 0.04551201011378003,
          "interval": [
            0.02465233881163084,
            0.06131479140328697
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000100000001001000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000010000000000000100000000000100001000000000000000000000000000000000000000101000000000000000000110000000000000000010010000010000000000000000001100000100100000001100100000000000000010000000001000000000000100000000000000000010000000000000000100000000000000010000000000000000000000000000000000000000001000000000010000000000000010000000000000000001001000100001000100000000000010001000010000000000100000010000000100000010010001011000000000000000001000000000000000110010000001001010000001000000010110010001000111100000000010101000010010000010000100010010110010000001000010011001000001001001000001011110110010001001010000100010100010101000000110110100000110000101000000010100101011001000000011000101001000010000000000101000000000001001000101010001101010110010011100111110000000000010000001010100111001110010001110001101010010110000100010000110111010001110110011010010001010010001100010011001110010001100010100000110000011111100000110010100110001100100000000111000011011001101110000100111111010010000000001001001011111010100001101000010000111100000010101000101110110010000110010011000000100001101010111010110110110000001101001011001011111111110110100011110110111001011000010111101010011010100100011000011111100101101010010111110111011011011101001011011001011110111010101111111011100111011101010111101101101111110110011001110000010000101000001110001111001010101101101111001101100101110001110111110011000111111111100000111110010111011111101111111110010011111100101111111111111011100001111001100000110011101010111110111011010110111001000011110111011100111000111101001011111001011110111111101011001001101100100111"
        },
        {
          "name": "Kimi K3.1 Pro",
          "text_run": 315,
          "five_option_run": 315,
          "wrong": 294,
          "wrong_of_answers": 0.18584070796460178,
          "difference": 0.08027812895069533,
          "interval": [
            0.05815423514538556,
            0.09860935524652338
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000010000000010000000000000000000000010000000000000010000000000000000000100010000000000000000000000000010000000000000000000000000000000000000000000000001101110000010010000000000000000000000000000000000011000101011101000110010000000100000000000000000000010001000100001000000000001000000001000000000000000000110000000000000000000000000000000000000000000000000000000100000000000000000001000011010010000010001000100000000010000000000000001011000000000010001000000000001000000000000000000000010000000000000000000000000001001000001001100101000000000011000110010000000000111010110100000000100000000000111010101000000010000010010011100000000001000000000100100000100000000000000000000000001010110100000110000000010001100000110000010000000000000000000000000001100000100101011000000001110100110111010000011000000011010100101011010010000000000000100000000000000000010101001001000000010110001000000000000001001110010000101100000001000111100000000000000000010001000000000000000010000100000000000000010010000000011100011000000000100001000010110001000000010000100110000110000000000000000000010010000000001100100010011000100000000010000000000000010110000000000000000010000000000111100100001000010000010101100001001000100001100100110011010000101111100000001110110000000000000001101001001000010010000000000000111110110100100000101001100000010011000101001111001111101000001101010010001001100011000000000100000110000000100010100011001011010100000100101100101110011010100110011000001010001000100001100101010100000000000001010000000001000101101001001001101001011100011110010101001100001010010011010100001000110010001000101110011010011111000001000010011010011110100011111111111101101001001000100101100010100000110001011011100000001110001101101100010000100001101110010000000001110111111101111101001100100000111101001110110101000010011011011110111001001001011000101000101000101111110001010000100111111011001110010011110101101101100101000011011001110011101011110011111110111001011101101110100111011011101110000110011110001011111101010101110101011111000000111001000011111111110101111111001111"
        },
        {
          "name": "Fastino GLiDE v1.2",
          "text_run": 248,
          "five_option_run": 248,
          "wrong": 289,
          "wrong_of_answers": 0.18268015170670038,
          "difference": 0.07711757269279393,
          "interval": [
            0.057522123893805316,
            0.09671302149178256
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010010000000000000000000000000000000000000000000010000000000000000100000000000010001000000000000000000001000000000000000000000000000000000000100000000000000001000000000000000000000000000000010000000000001000001000000000000000000010000000000000000000000000100100000000000000101000000010000001000100000000000000011000001000000000000000000100000000001010000001011010000000001000000000000000010000000001100010000010101010000110001000000010011100000000100110000000000010000000010000001100000000011100001001000011011000100001001000000010000000000000100011110110110000001000100010001100100001100010000110000010010000000000011000001000000010101000000010000100001000001100000000001100100110101000000010100110100100011100101100010000000001000110100100100011100001100110101111010000001111010011100011110011111000110011101110101001010100100010100110011000011110100001110100011001010111001010000011010110000000101000000110011000000000001000110110001001000010000010100011000110010100100101111001010100001011111000110001010110001100001100100110101100010100001100100011110010011000110110100001001010011110110110000010101100001011011101111110001101010010101100011011111011101110010001100010011010100111000111000011011110111000011100010111100100110111111111101001101111010011110111101011111001011011111011101010111000010010111000111110101011111111000111100101110111110111111111100010111110110011010111110101010110110111111010000101111011111110101011001000101110001110011101011111110011011111111100111111001001011010010111110111111010110111110001011111010111111010010111110111101110100111110011110001000011001111110111100001111110111111110100111100011011000101110101111111101011001110101011000111111011111110111101110101111111111010111111111101101011101",
          "points": [
            {
              "wording": "Bilmiyorum",
              "run": 369,
              "answered": 1877,
              "wrong": 491
            },
            {
              "wording": "I don't know",
              "run": 525,
              "answered": 1772,
              "wrong": 427
            }
          ]
        },
        {
          "name": "GPT-6 Luna",
          "text_run": 133,
          "five_option_run": 133,
          "wrong": 388,
          "wrong_of_answers": 0.24525916561314792,
          "difference": 0.13969658659924147,
          "interval": [
            0.11820480404551201,
            0.1624525916561315
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000010010000001001000010000000000000000000000010000100000100000101010010000010010000000111000000010010000001000100110000100000000000100000010010010110101000111011011010001001000000000000000010011010000000100000000100101000000000000000000000000100000001000000000000000100100000000000000000000000000000100000000000000000000000000000000000000010000000000000000000000001000000111010100010010000010000011010110110101101110100101100010010101000000000000000001010100010000001000001000101001001000000000100000100101100001100001001110010010000000100000101011010100101100110110111001000000000000100111000000000000000000000010000000001000000100000000010000010000000010000000000010100001000000000001011001010000100000011001000000100000000000100010101000001100100100011111111100000011100100000000001000100110011011000000000000000000000001100000001111000000000000001001110101101011111000011100000010111000000000000001010100000000010101111010011101110001000010010001000001001000011000100100000010000000100010011001010011010111100011001010001100001000100000000000110111100101001110101000010001110110101000010010100000001010010000000100100100000100010010000010011111011010010100000100010111100011001110011111010000001111010010101101001100001010100010010000000101101111110011100101100000010001000010000011000001111010011100100000001000000011000010000100100100000000010001010001100100011011010011101110110001000000000001101001010111111100100000101001101111101001011000101000010110000100010001011111011010100000000100001000010000010000000010001010010000000110111000100100100011100000111111101100001011111001101001000011011010101000110100001111110111000001101001100001011001100001100010001110010101100111001000100110110010101011100100101011101010100000101010111001011100100011110101000001111000101101100011110100011111000000100101011110011011001010000011110111110011111000000101100000101001011111110010101111000011111010111101110011101011100101100111101111111111010011010101010110100100010010111111111111110111011111001100100101011110110"
        },
        {
          "name": "Fastino GLiDE",
          "text_run": 129,
          "five_option_run": 129,
          "wrong": 325,
          "wrong_of_answers": 0.2054361567635904,
          "difference": 0.09987357774968394,
          "interval": [
            0.07901390644753478,
            0.12010113780025283
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000101000000000000000010000001000000000000000011000000000001000100000001100000000000000010000000110000000000100001000000000010000000000010000010000010000000000000010000001000000000000000000000000000000000000000000000000000100000000100000000000000000000010000000101001100111000100000100000010001000000000101001000010100001000000000000001101000110101000010100000000000000000000100000000000000000000000000000001000000000000000000000010000001000110001000100000101000110000110000000010100000001000010000000000100001001001000001000101001001100000000100110001011000001001011001010101001001001000100000000000000000100000110001100010110000100100010010010101100001001101110000100100000001001101101100010100001010000000001011100100000100001000000100000111000101000011101101001011011000001111110101000011010011010000000010010010010010011101000000000110000010000011001000100110101111111100010001011111000000100110110100010010000101100100011100010001111011001000000001111001111000100110000100011010010011001001010101101010011010010001101010011100110100110110101000110101101010001000000111101000010001110010010011011111011000111100101100110110110001010111010010101100001101110110001000010110101100101010001010110110111010110000001001110001011111110001111111101000111111001110001010110111001001011011010000110010011010010111111110110101001010111101111110001110101101111010100110100110101010011101111111110000100001011111010011101100111011111011001110110101011011011101011111110011110101011110000111010010110110111111111111101011100101001100010001111010111111011111101010111101001110010111001011111101011101101011111111011000101110111000111101100101010001010111011011011111110101011101001010111010010010111111111111111101111110011111110010011110100001011011011111110010101111110110001000011",
          "points": [
            {
              "wording": "Bilmiyorum",
              "run": 351,
              "answered": 1677,
              "wrong": 356
            },
            {
              "wording": "I don't know",
              "run": 510,
              "answered": 1617,
              "wrong": 335
            }
          ]
        },
        {
          "name": "vLLM-SR d3",
          "text_run": 259,
          "five_option_run": 259,
          "wrong": 314,
          "wrong_of_answers": 0.19848293299620734,
          "difference": 0.09292035398230089,
          "interval": [
            0.07332490518331225,
            0.1125158027812895
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000100000000000000000000000100010000000000000000000100000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000001000000000000000000010000000000100101000000000000000001000000000000100000000000000000011000000100010000000111000000000000000010000000000000110010011010000001000000000100001000001000000000001010100000010001000100000011001000000000000000100010100000100011000100000100000000100000010000000000011000000110000001000110001010101000100000000000101100000101101001100101011011000011010010010011110101101000000010100100000011100010111001000000010001100101101100101001101001100000000001100101101110010000001001000010000110011101111011000000111100000010001110100001000000001010110010010010010011001000100000001010000000111101110110010110010000000100111101100110000100010011010100110010110011010101110110011000010101011101111010000101111110110110001001000001001011000101111010111001000000000100001011101110110110000011001111001100001111100010011110111100001101111011100100111101111000100110010100101111000101111010001011101010101001110001110110101111111111100100000011000111110101001000111111010111111001100100011110101110001011011001111001111010101110111101111111100100011010011001101111000101111010110101011101000011101110111111110001101011110011110101001010110101111011100100100111010100101101010001011001011010111111111111011110100000110011010111101100100101001101111110010100111111101101011110101111011010111111010110011111111110111110010110101100110101000011001111110010101101111111001111101111011101011111111101000010110001111001101011111101110110100111111111100111111000110100110111101111101100111111110111100011011111101111",
          "points": [
            {
              "wording": "Bilmiyorum",
              "run": 358,
              "answered": 2198,
              "wrong": 715
            },
            {
              "wording": "I don't know",
              "run": 544,
              "answered": 2196,
              "wrong": 714
            }
          ]
        },
        {
          "name": "vLLM-SR d3-ultra",
          "text_run": 269,
          "five_option_run": 269,
          "wrong": 315,
          "wrong_of_answers": 0.19911504424778761,
          "difference": 0.09355246523388117,
          "interval": [
            0.0720606826801517,
            0.1125158027812895
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000001000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000100000000010000000000000000000100000000000000000000000000000000000010000000010000000000000000000000000000000000000100000010000001000000100010000001000000000000000000000000000000000000000000010000000000000000001001000010001000100000000100011000000100000010100000000000000011000001000000001010100000000000010000001111000000000001000010010000000000000000000100110000100100100000100100000001000000101001100000110000100010000001000101000001010100100000100100010010001001100100100001001010000011001001010100000001010000110000000001101110001110101111100000001000010000110000001000010110000001001011101000001000000000110000011011001000100011011001010100000011101101110001110000101101100100100000101000001000001111000100110001000010101010001000011000010000110111000010001001110010100010000010100101010011011100100000010011100010011010001111001011101001110100101110000111100000101100110101111100010011111101100001110010001000011100000110111111101111001111001001100111011011110010010011110111001111100111110100000010110011101010101111111010100011111101011001001100111101010110101000001101000110100111111111110011101001011011101111010111110010010110111101110111100010101011110100101110111111111011100110011101111010011101000101111010101010110111000111011101111111110101101110110011011100001001101101100010011101010111101111010110101000100011110001110111001111110101100011101011111001110101100110110101111111111000110101100101110111111101001110111111011100111111101100101111101110101011111110101110110101011111011111111100110111011110010111011110100111101111111010011001010000110111111101000011111001011001110100010011101011100001111111101111011001111100010",
          "points": [
            {
              "wording": "Bilmiyorum",
              "run": 363,
              "answered": 2188,
              "wrong": 704
            },
            {
              "wording": "I don't know",
              "run": 540,
              "answered": 2189,
              "wrong": 704
            }
          ]
        },
        {
          "name": "Drex Max v1",
          "text_run": 322,
          "five_option_run": 322,
          "wrong": 321,
          "wrong_of_answers": 0.20290771175726927,
          "difference": 0.09734513274336282,
          "interval": [
            0.07648546144121365,
            0.11694058154235146
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000010100000000000000000000000000000000000000000000000000000000100000000000000000000000100001010000000000000000001000000000000000000000000110100000000000000000000000000000000000100000000001000000000000010000001000001000000000000000000001000001000000000001000000100000001000001000001000001000000000000010000000101100000100001000000000000000000000110000100010101001000001001000000000000000000100000000100000100000000000000011010000100000001001000000100000101010000100000010000011100000010110000100100000010000100000100000010000000000000000011000000110010000100011100110100011011110000001000101010001010000010001101111110010011000111010111010011001000101101010010010100000111100010010000000000000011000001110001000001110000011111000110000000110011000100100000000001110000010010100100110011000000110100011100001100110100100111111001000000000010011100010110100010010000000110100011001100100011000001011011000001110101100000100110010001110011100000111110011101010110010100000110110111001101100101111110010100110111100111101001010101010010111110111001010111010100000000100111000111110110010101001000000110011110011011101001110001001000110011010111011011100010000001001111101010011111001010101010000111101010011000000010010001001101100010111101110011111100101010100111111001110101111110111100000001111001111010001001100111101010101101001101110110111011111111011001001100100110100111110111111100111111111011111111110100110110100110101111111001101011010110100111000101010100001110110101011010101111010011111011101111111111011111111110101110001111100111000111101001011111111110101111100101101110111000101111110001101110001111011111101100111111111100001001110001110000010100101111111111000001010110110110111111101111111111110010111110000111101111111111101111",
          "points": [
            {
              "wording": "Bilmiyorum",
              "run": 367,
              "answered": 2196,
              "wrong": 711
            },
            {
              "wording": "I don't know",
              "run": 542,
              "answered": 2196,
              "wrong": 712
            }
          ]
        },
        {
          "name": "Drex Max 1.1",
          "text_run": 326,
          "five_option_run": 326,
          "wrong": 321,
          "wrong_of_answers": 0.20290771175726927,
          "difference": 0.09734513274336282,
          "interval": [
            0.0777496839443742,
            0.11757269279393172
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000100100000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000001000000000000010000000000000000101000000000101000000010011000000010010000000000000000001000000000000000101000010101110000001000000000000000000000000000000000000000000000100000010001000101001000001000000001010010100000000000000110010010001000010001000100000001111011000000001000000101010010000010000000010000110100011000001010000110000101001000011011110001000000011010010100000100101011000100100000000100100101110100000110111101100010111001000000100000000110001000001010011010001101001001010000011000100000100000000100000000010000101101001000000000010001100000010110111101011110111011001000101100100101001011111100010111010101100101010011011001011011000101000101010101011101001000001101010111111001011101001001001000110001101111110011001101100101110110101010011101111011011100001010001010110101110110011011010111011011000010000110111110000101010000110001011000111001010110000000010101011001000100110000110001001000111101010011011011111000010110010111001110100011001110101100100001111011001111101111001101101011010111110110011111001001001010010010111111011100110100011101111001100111000100000111101001101100011011111101101011101011110110100110101101110110100101010010101011101010111111100010100001111110111010101110111111011100010101010111101001001101110011111010101111011111110010110110111010101000111111101111100111011001111101001011100111011111100110010101111101011101000101010010101011100111101001110110101101101111111001110101101101111111011111111111011100110010111111101110111111111100011010011111111110111101101111111111111110111",
          "points": [
            {
              "wording": "Bilmiyorum",
              "run": 352,
              "answered": 1878,
              "wrong": 517
            },
            {
              "wording": "I don't know",
              "run": 513,
              "answered": 1712,
              "wrong": 413
            }
          ]
        },
        {
          "name": "Mammoth-1",
          "text_run": 374,
          "five_option_run": 374,
          "wrong": 322,
          "wrong_of_answers": 0.20353982300884957,
          "difference": 0.09797724399494312,
          "interval": [
            0.07774968394437423,
            0.11820480404551201
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000010000000000010000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000010000000000000000000000000000000100010000000100000000000000001001010000000010001010100000000000000000000000000100001010000000000000000100010000010000001000100110000000000000000000000000001001000010000100110000000001100000011010000001111000010001000000001010000000000000010000011000100110010001000000010100000000101010000001011101100000101001100010001110011000011000010101001001000100001010100000011001110011000000100000100000110000010010000110000100001100000011001010101000010100000011001011000000000100110110010100001000010000000111001000001000001000010010000100110101011101010000001100001000110101101000100111000100010001000001000010100100011101100001010000001000000101001000000000000011011001010100000011010010111000111111011000100111110100101001000101011111100001011000011010111010110011000011000110111101101100011100100010000101111000001000110011010001011011100001001001110001011111111110111110000111011100111110111110101111100010001010100010000111001110001000100110111011110110011001100111111101100111011011100111011011100011011100101010011000011001111011100101000010100111001011100011111100011100111000111100101001111011010101111101110011100111101011111111110101110011100111100110111101010000011001001011011001001111110101011100101111010010111011011001111100101110111101010000110011111100111011001001001111101111001011101001100110011111111110011001110110111110101011100110111110011111111111101011111011101101011110011011001110100101111011011011110010001101100110111010111111111011111111111110110111111111001011101010001111111111100011011111100100010111100111111111111110011111111110",
          "points": [
            {
              "wording": "Bilmiyorum",
              "run": 386,
              "answered": 1508,
              "wrong": 314
            },
            {
              "wording": "I don't know",
              "run": 519,
              "answered": 1445,
              "wrong": 276
            }
          ]
        },
        {
          "name": "Perplexity Decider 27B v1.2",
          "text_run": 217,
          "five_option_run": 217,
          "wrong": 328,
          "wrong_of_answers": 0.20733249051833122,
          "difference": 0.10176991150442477,
          "interval": [
            0.08027812895069532,
            0.12136536030341341
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000010000010000000000000000000000000000000100000000000000000000000000100000000000000100000000000000000001000000000000100000000000000000000000000000000000000000000010010100000000000000000000000000000000000000000000000000000000000000000000000000000100010000000000000000000000000100000000000001000100000000000000000000000100000000000000000100000100001000000000010000000001001000000000000000001010000000000000010100000001101010000000010110010100100110011001000000000000001001000011000101011000010101000010000010000000000010000100000010100111100010010001000100000010001000100000101110001001000001001100010100000000001000110000001001101000000001000000001101001001111000001000011010010100100000000100111010010000001100001111000000001101000010010101000000111011000101000000010000010000101101110000111000001000001110111111001100101000101010010000011010001010100001101110001010000100000001011100110001000101110010001110100110010010001101001100101110000110101001101101111011100111001000101011100100011100111001001000101111111011100101110001001001000010111010101000001100001101010011101001110111011111100101010101111100111001011011110001110101001011110010010111101101111100101111110111100101010111001000101011010011110010001101000111111011011101000111110101010110100111111100110111100010101101100001111010101011101101010101111001101010000111011100111000010111100011101110101011001101110101011000011101100000111011101100101110111010101000011000001110011100111011111110011010011001011110100101011011110011001110100101101110111110110011111011111110110110001111001111010111111010111111101110111001010111101111101010101010111111110010110111001111111011001111100101000000101111110111111101001111100111111110110101100011111111101011111111101011110110110011111001010111111111011011111111110111111111111",
          "points": [
            {
              "wording": "Bilmiyorum",
              "run": 372,
              "answered": 2196,
              "wrong": 717
            },
            {
              "wording": "I don't know",
              "run": 531,
              "answered": 2194,
              "wrong": 728
            }
          ]
        },
        {
          "name": "Perplexity Decider 27B v1.1",
          "text_run": 145,
          "five_option_run": 145,
          "wrong": 318,
          "wrong_of_answers": 0.20101137800252844,
          "difference": 0.09544879898862199,
          "interval": [
            0.07585335018963336,
            0.1156763590391909
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000010100000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000100010000000000000000000000000000000000000101000000000000000000000000000000000000110000000000000010000000000000000000000000010010000001000000000000000000100100001100000001000001010000000010100000001100000000000000000110110000001000010001000000000100000000000000001010000000001000100000000001011001000000001110000000100010001000100000001001000000000010001000000000010000100010000000111000001100000000010001001000110101001000000000000100101000000000011011101100000000110011000000000000000001100100000001011100000010000100011100001111000100110101001000000000010010101001000100101010011000001010000010001100101110010111111110000001110111100010101101010010011010111000100000100011000000101100101111010000010100101001010001010000000011111101010001110010100110101011100001101011101011010001111000001001100101011001110010000100101001110110010010000101010111101101100011000001100011111100010101110100010110010100001000011100001110010100111011001110111100011001001111010111101000110000111111000000101100100111001000011101000011100011110010111111110100111101001001100001111011000000101101010001111101101011010011000010111001110011111111111111001010011110110110100011011111110011100101111111110101011101110101000001100110101111111001110011010111000101111011001111011101100011101100100110110111110111110111011101101101110000010101110010110101110101010001111010000111110101111111110011011100011111010111011111101111011100101101011011000111111110010101111111101101111011010001011110011011111101111111110010011110110111100001011111111100011101101100101111010111111111011101101111111110111111100011111111011101111011111111110101111111010110111111111011"
        },
        {
          "name": "Gemma 4 31B",
          "text_run": 189,
          "five_option_run": 189,
          "wrong": 346,
          "wrong_of_answers": 0.21871049304677623,
          "difference": 0.11314791403286978,
          "interval": [
            0.08975979772439949,
            0.1333754740834387
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000010100000000000000000000000000000000000000000001000000000000000000000000000000000000000010000000000000000001000000010000000000000000000000000000000000000010000000000000000000000000010000000000000001000000000000000000000010000000000000000000000000100000000000000000010000000000000000000000000000000000100000010010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000100000000010000000100000000000000000000010001000001000000000000000010000001000001000100000110000000101000010000000001000010000000000000001000010100000000001110000000010101010000010000000001000000000001101000001000011001010001001101000001010100000000010011000001000100010010110100010001000100001000111110101000111101000000001011011010101111000000000101101001001000110011011000000100111001000110110101010011001000011001100010001110100000110100010100110100100101000000001000110100100000001010001100110100011000000101001010101011001111000011100100010111011110010010100000001101100001101101111011001001000111101101101100001000100001001001011000000100100101000001100011001110011100011100011110010110101110010100010111000001110001100110100011110001111010011100011001010100111111011110010010111001011011101000100101100010011110111110111110111001101001111110001100011011110111011010011110011000101011101101000110110100011100111110101001111111111110110011001010001001010111000110011111111011010000100011111010000001000110001111001111110110101000101111110010111001111100101001100110101101111111001100001011101110110101011110011011101110000111110110101010101000110111001100001110010101101111110011011111001010110110101100111111101101100010011101100011001110000111110101101101110010011111011010110101010001101111010111110011001111111100110110110010010100110101111011111011100001101111011111111101111011111111011111100110111010111001111111111111100100010110111110110111110111110001110111111011111100110"
        },
        {
          "name": "DeepSeek-V4.1-Flash, first release",
          "text_run": 120,
          "five_option_run": 120,
          "wrong": 346,
          "wrong_of_answers": 0.21871049304677623,
          "difference": 0.11314791403286978,
          "interval": [
            0.09292035398230089,
            0.1359039190897598
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000100000010000000000001000011000010000001000000010000000000000000000000000000000000100000000000000100000000000010001011001000000100000100110000000000000000010000000010010000011000000000000100000000001000000000000100010000000000000000100010000000010110011000000000000000010001100001000000100101000001000100110011000010000000000000100000000000001001000010000000011010100010101001100010010101100000100010001110111100010000100111000010000101000100111100000100010101000010100100100101100000000011100011000001011011001001011011000000100110010111010100000110001100010000000001100000011001010001100110100000000000100100101011000011110111100000010010100100111011011010110000110001000100100000010110111001010010001100011000001101011011101111110010010000001110011011001011110101011111000101001000101000100001010101101101010110011101110000001010101001101010100001101010001000000100110100010111011010111111000011010111111110011000111000110110010100011000100110101011111001111110111100100010111110110110100001111101100010100101000000001101010111110011111011101100100011010101100111101110111100010110110110110110111110011111111010001111100010110110011111101101101011100011011011000101010010011011011001011110001001100110011010111100000110100010011110111101001110111001111111011011011110111111111101101111010111110111111010111111110111011110111101010100011101101010111101110010000101011000001101110111011001111101011110010001110111011111111011000100101100110111111111101111111111100111101111011111010011000000001101110011110111101110110111111111010101100001110001011111001101111101011100100110111111011111110110110010111110110111011111010111011111110110"
        },
        {
          "name": "SubQ Snap Preview",
          "text_run": 177,
          "five_option_run": 177,
          "wrong": 356,
          "wrong_of_answers": 0.225031605562579,
          "difference": 0.11946902654867256,
          "interval": [
            0.09734513274336284,
            0.1390644753476612
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000001000000000001000000000000010000000100000000000000100000000000000000000001000000000000000000000000000000000000000000000000000001000000110000000000000000000000000001000000000100000000000000100100000000010000000000100000000001000000000000000100001001000001000000100000000010000010000000000000000000000001000000100100000010011001000000000111000000000000000001000000010011000000000001000000000000000000000001010011100000000000000001000001010000000101100000000000000000000000100001100000010000100000000000000100011000000010000001000010100000001010001001100010000101000000000010100000001000001011100000010110000000101110000111000000000101000011100001000010100010011010100010001110011101011000100010101010010000001000001010100100101001000000101101101100110001101100100000000001110101100000001010000001011110001010000011000000000001100001010101001110100101011101001011011100011001001010100011000100001011001110011011000111010001011101011111100001010100001100000100010010100111101101000010000101011111101010100111101101011111011001110111110100011010000001010011110010101110110011110110001101100101010111111001100010101100111100110010110111101010110110011110000110011010001001000000010000110111010110111111111001101100011000001110011001111011111101110011111101111011110101001100001110110101111110111101111111111111111100101110010111110101010111011010010101001001001010101110001111111010111100110001000111011111100010011010100100001001011001111111100110110101110111011101110100100010101111111011011111011101001010011111111000101111100011111111111111101111101010100101101110100011100100110110001111111111111001101111101011111001011110111000001111100111110101011101110111000111000111111111110011011101101010101010111101111110100101001101000011111010100111110001110110111111001111110111101110111111100111011101111011111101111111111101101111"
        },
        {
          "name": "SubQ Snap 1.1",
          "text_run": 346,
          "five_option_run": 346,
          "wrong": 361,
          "wrong_of_answers": 0.2281921618204804,
          "difference": 0.12262958280657396,
          "interval": [
            0.10113780025284451,
            0.1447534766118837
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000010000100000000000000001010000010000000000000000100000000001000000000000000000000000000000000110000000000000000001010000000000000000000000001000100000000010000000100000000000000000000000000000000101100100000001000100000001001000000010000100000100000011000000000000000000000000011100000000100010000000000000000001000000000100001100001001011000001100000110100101000100000000000001000100000000111000011010000100000000101000000001100011100010000000000010000100001000110000110011110001010010010011010001000011000001000000011001110000000100000011001110001100010000100011011000010011001010000001101001000100001001000101100100010011010110101010010100010101010001010111000011010000010000000100100110110000001001001000000110100100010101111101000011101101000010010010000001000010101101101101110000111110110111000001100010101001111011101011011001010110110101001100010111011100000110011011100101001010111011001011100001000001101100000011110101011001111001011010000101111011101001110101010101010111011110000110011000010011010110000001111111001011000111001110011111011110111101000100011000100011000110100100101001100111001100111111111110010010001111110100111111110111110011101111010110111111111100111110110101101001011101111011011111110101100000011100100110111000011011111011100100011011011110111011101100011001111111110001011111100011101001110011111011110111111110011011011001111111011110010101111010111011100110111110110011101111111001001011110111111101110010101111111111011101111100011001010101011101000001111111101011111111001111001000001111110111111111100011110010110111001011011111011100111101011101111101001110101001111111011111111001011111101001011111100011111111111101111101111111111111111111111011100111111111011001111101111111111",
          "points": [
            {
              "wording": "Bilmiyorum",
              "run": 392,
              "answered": 1761,
              "wrong": 490
            },
            {
              "wording": "I don't know",
              "run": 512,
              "answered": 1700,
              "wrong": 454
            }
          ]
        },
        {
          "name": "GPT-6 Luna",
          "text_run": 143,
          "five_option_run": 143,
          "wrong": 372,
          "wrong_of_answers": 0.23514538558786346,
          "difference": 0.12958280657395702,
          "interval": [
            0.10682680151706701,
            0.15107458912768645
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000010000000000010000000100000000000000000000000000000000000000000000001000000000000000000100000000000001000000010000100000001000001000010100000000000000000000000000000000001110000000001000110000000000000000001000000000100000000000100001100100000000000010000000010000000000110000100101000010010000000000000000101110010000001000100010000100000000001010010100000100001000100000000100000001101000000001000000010100100101110010110100001001010000000100100000100001000000000010100010000000000000000100110000000100000101000100000001000100110111000010001000100100110001101010001110011000001000011000000000010010010011100010100000011001000001010001001100000111101000111000010010100110000100101000001010000000001011011100010010001011110110000011001101110100111010011000111000000101110001010010110110100001010000000100100100100110010010011001000110000110000111111110010111010100100010010010111101011011100000011011111011000010001000000000011011110000111001000100101011101001000001101000011000011001111000000100101111101101000000010111001010011011110000101010101110010100101010100001101101111111010101010101110101000001101110010001110010101110000100110011010010001110010010110011010101001101111001100110010011010111111011010001101011111000111011000110111010110000111010111001011111110101011111010111001110000101000110100101100100110110111101010011110110111110011000111101111001111111110110011111111110111010111011111101111000111111110111110111011111110010100011110110001101101011111110111001100010100101111111100011101000101011111110110110111111111111111111111011100011101111110111111011101110111001101111100010111011111111110111101101100011010011110101111101111101110111010010111101011101110110011111001111111111101111111001101111111110100111010111011101010111111011111101111111110001010111011110110111101101110111"
        },
        {
          "name": "Liquid d1.1",
          "text_run": 254,
          "five_option_run": 254,
          "wrong": 434,
          "wrong_of_answers": 0.2743362831858407,
          "difference": 0.16877370417193424,
          "interval": [
            0.1441213653603034,
            0.19026548672566373
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000010000000010000000000000000000000000000000000000000000000000000000000000000000000100001000000000000000000100000010000000000000000000000100000000000000001000001000000000110000000000000000000000000000000000000000100000000000001000000000001000100000010001000000000000000101000011101000100000000000000010001000000010000010000000011100100000000011000000000000110001000101110000100000000000000011001000000011000000001001000001100110000000100001100000000001101000110010011100111000000000010010101010101001110010110000000001000000000001010110000011100000001011010000101110000010001010000010000111100001100010000100101000100000110011000010010101010111001010000011000100100100010001000000101010010101100010011101000001011100011010110010001010001110010000100001010111111001101001100101110100001001001011011001000001100110011000010101101010100100001100101001001110111000111110001100100110011001110011011001000010001101100010010110101000100111111100001001011011101001100010000011001110010101110100110110101000110011011100010111010110101100000010001101110010001000110011110111101110011100101000011110000110001101010111000110111111000011111111010001111110101100011010111101101011011100001001011110111110011111110011101100111011011011101011111010011011111111111111001000111111111001111110111110111101110110110111011111101111110101110010111111110111100010101111011011111111101100100111111001010100111011000011011000011010110001000111110010100111111111111101110001111111001110101011011011010010111111111111111000111010111111111010111101100111010101010001010101101101101011101001100110100110111111011010110111101110011101111110111101110010100111000111011000100110111000111100001100111111010101101110110010000101110111011111001110111111001010111111101011011110110101000110110111111110100100100011101101011111011111111111011101011111101111101101111110111001110111111010011011111011011"
        },
        {
          "name": "Claude Sonnet 5.5",
          "text_run": 54,
          "five_option_run": 54,
          "wrong": 440,
          "wrong_of_answers": 0.2781289506953224,
          "difference": 0.17256637168141595,
          "interval": [
            0.1491782553729456,
            0.1946902654867257
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010010000000000000000000000000010000000000000000000000000000000000000000000000000000001000000010000000000000000000000000000001000000001000000000100000000100001010001100100000000000000000000100001000000000000000000000010000000000000101000000100000000010000001000000100000000010000000000000000000100001000000000000000000000000010000001000001000001000001100010010100100000000000000000000000000001100000000000000000000001010011000000011000000000001000000010001000000000001010100100000001000000011010000000000000100100000001001000000010000011000001101000000000000100000010000011100001011011101110001010001010010010000010000100000010000000000100000000000011100100010100000100000010000000000000000001010011100001000010010010011110100000110010001000100101000000001000001011011010000001100111010000011100101010001000010011100010000010100001000000100000100001100110100000100011000000010110011110110000010110010010000000010010000011011001100100001000000011111000100010000101100111010110011100001010111100100110100100110111101010010000101001111000001010100010011010100011000111000000100000111111111101110001110111000011000011111100000110101011111000000001111100011111000011111000111001110100110000110100110110110110101010110110111111100110001011000010111100101111011111011011101100001010101011111101111010011001100100111010010101011010001101100011001011101111111011001010101011101111000111001110001001100101100011010100110111101100011111110111011011110101101100000010111111111111100111010110001111111010010011110110001000000111111110100010110011111001011001111110111000001110100111111000111101010110011010000101001000001111011011111010101110010000101101111111111111111010010011111101011000110101001011111011101110111111001101111010111011110101110010001011010011100111110111011111111011110101101100000101111101111110110011111111111111111101011101010010101111111101110111111111110110011111001111111011111010011011011010111110100111111111001111111010011101111011001110011111111000001111011101100111111010111111110011111110111101011010001010111010011011101"
        },
        {
          "name": "Celeris-1 Decision",
          "text_run": 150,
          "five_option_run": 150,
          "wrong": 450,
          "wrong_of_answers": 0.28445006321112515,
          "difference": 0.1788874841972187,
          "interval": [
            0.1536030341340076,
            0.20037926675094817
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000100000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000001100000000000000000000000000000000000000000000001000100000000000000000001100000000000000000000001101000000000000000100000010000000001000010000100000010000010000001010001000000000000100001101100000001011100100010000000001000000000000000000000001000001000100000000100010100100100011000011000010100010010011000000000000010001110000101011000000111001010000110000110000100010000011001010000010100011001000001010100000011110011011001110010000001100100010000101000100001011000101111100111111010000001010111010111001001011100101000010101010001100001001100011100101011001111111000010010101001000001110100110010101011110010001010101100001011010100000111000000001111001100100000100110000101001101110001111001100000011110100101100111100100011010000000111001101110000000110100111001001000011100001010110100000101010000001011000110000101110011100011111000010111011010100000010101101000101010010111110011110111100000111101101111000001000111111000000010001000111111000011110011100110000011111110111010111110101010011110110101011111101001101111010101110110000001000111010100111100001001110011101011010110001111110011010110010001111000111110010011110111111111111010111011111000100101011011100111100111111101000011111101101100010000111000001010111111111101110111010110100111000111011100100001111111011010000100011110010101111100101101111111001011110010010111110101010101111111111110111101001101011110001110111111100111011111011101111010111110101010100111000010100011111110101101111011101000010101101111011111011011111000110011010101100110110100111010111001011101111111101010011011111011011011111011011111011101110101111111111111011111111110110011111011111110011011111111111101100101111111111001011101101101111111100011011111111101111011010101011111110111111101011011111101010110101111111"
        },
        {
          "name": "Perplexity Decider 27B v1",
          "text_run": 44,
          "five_option_run": 44,
          "wrong": 430,
          "wrong_of_answers": 0.2718078381795196,
          "difference": 0.16624525916561314,
          "interval": [
            0.14538558786346395,
            0.18900126422250318
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000100000000000000000100000000000000000100100000000000000000000000001000000000000000001000000000000010001000000010000000000000000000000000000100000100000000001000000000111000000010100000000010000000010010000010000001000000000010110010000000100000000101000000000100000000100001000101000111011000000100010100000000000100001010010010010010100000001000000000001001010100100110100000000001000100011001001000000010000100100000000110001000110001011100010000100000010010001010001001010100100100000000000100001100001010111111101000000111000010111010100001000000010000101000110111001011101011001100111111000011000000110111010100100000110110000000111000001001101000001001011011110010101101000101001011100111000110001101100011010100000110100110101001110100101000101101000100110000000100100100100111101001000100010100101100011111000000011011111011110101100001011010000000111100110010011111101111001110100100100010101001110101100011100101111111001101101101100010000100010111000011101011101111101101001001011111111000110111110011100110101101101011011111010111011011101000111100111111111110101000111000000001111111100101001100011111111110010101110010001111000111001011011000110100010000100111111001111101111011011011111010100011111100011011100011011110110110101010011010111111110101110111111010101101100101100010010111111110011110100011000010001011100011111000100101110001101010101111110010111001101010111101110110110111110111011011111101101111111001011001101111010011111011001111011011110011101101110111111110001111110011111011111101111111110111011011111101100111111111001101010101111011110111010101110101110110111111111101011101100011111110111111110111110111111111111011111011101111100111011110111111110101111110111111111111101001110111111111110111111111110111001010011111101110111111011101110111111110"
        },
        {
          "name": "OneJev 27B",
          "text_run": 264,
          "five_option_run": 264,
          "wrong": 455,
          "wrong_of_answers": 0.28761061946902655,
          "difference": 0.1820480404551201,
          "interval": [
            0.15929203539823006,
            0.20353982300884957
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000100000000000000000000000000001000000000000000000000001000100000000100000000000000010000001000000000001000100000000100000000000000000000000000000000000000000000000000000000000000010010000000100000000000110000001010100010000000000000000000000010000000001101001110000000001010000001010000100100000010001000000010000001010001000000001001001000000000101000001000101000000100100010000000000011000010001001000000000000101100001000000001000111100000000000011010011000000110000000000101101001110111110001100001001100000010111100001100000010101010100100000010101111000011000001000011000001000111100000010111010111010000000010001001011101000111111100000111000110101011001100000101111001011000011110110010101100001100111100000000011011001011011110111011101011010011011000111111110110010111010110010001001100101001101111111001010100110001000001100111011110000000011000001111110001000110100111101111101111111110111011110011100101110001011010100101111101111101000010111110111101010111101110001011110111110000100100001110000010010110001101111000100101101110100000001101010111101110011101110011011111011011100011011100100011100111111100011011011100010110010100111011011010110111100111010010011011001110100111101100011110100011101011000111101111011110011111111111100101101010000101001001101000101111001101101010100011011000111010101000101111110101010101111110010011001011111001101111110101110110101111001110111101011011111101110011011111111111111111011111011011111011101011011101011101111110101111111111111100110110111101101011110111111110001101011111111011101111111001011100111001101111011111111001000011111111111111110111110011001111011111011110101111111111010111111010111001110011111011111101010101111111111101111101101011101111010001100011111101110111101011110101011111010001101110011101101111110011100111011111101100110011111011111111"
        },
        {
          "name": "Jev 1.14.0",
          "text_run": 194,
          "five_option_run": 194,
          "wrong": 485,
          "wrong_of_answers": 0.3065739570164349,
          "difference": 0.20101137800252847,
          "interval": [
            0.17888748419721873,
            0.22376738305941846
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000100000000100000000000000001000000000000000000000000000010000010000000000010101000001000000000000000000100110000000000000100000000100100010100000001100000000000000000000000000100000100000000000101000001100000000000001000100010000001100000010100001001001101000010100001000000100001101001000101000001001000000010001001000011010101000001011000000100100000100111001101000110101110010011001000000001100001100011100000010000100010100010000000001110011100100001011011000000111010110000011110010000101001010001010001010001111000110100100010101001000001001100001100110101000010000100111100101100001010010011000101110010001101000011000001000101011010110101100000100101111110000000110011010010001001110000000011010100110100100010010111111110000011111110100101001011010111110110000101100110101100001100011101111111000101110101100101001011101001110001010111011000001111111011111000110111001000110011110101011000000011111011100000011110101111011101111100000111000101011000011001110010110010111100111111101110011101010111010100110000011010001101101111011111011100010001111111100101111111100000111111111001111011101001011111001111011000101101101010110111010001111011011000001111101101011110110010100000110101011110001111111000011011111011010111110011110100111110101001001100101100111011000011111100110000110101111000110111010111110111110001100111111010111111001010111011110101111000101111111010110111011011010000111100001110110111001111111011011011110111110111000111111001010000011011100111001111111011111110001011111111111101110011001111011111011111110010011111111001000110111100111010011111111101110100011100101111111101111101111111101101111100101101110011010101101101011011011101011000111111011010101010111100111001101011101110101111011101011111111110011011110111011111110101001100101111111111101111011111100011111001000001"
        },
        {
          "name": "Inception Mercury Decide",
          "text_run": 204,
          "five_option_run": 204,
          "wrong": 487,
          "wrong_of_answers": 0.30783817951959547,
          "difference": 0.20227560050568902,
          "interval": [
            0.17762326169405815,
            0.2262958280657396
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000100000000000000000000000000000000000000000000000000000100000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000110001000000001010000001000010000000000000000000000000000000000000001000100000000100000000000000000000000000000001101000000000000000000000000000000000110000001000000000000000001100000000001000000010100000100000000100001000100000000000000000100010000000111000000000000000010011000000010000010010000000111000000000110001111001100010000100000000000000000100001001100010000000010000000001101010111000100100110010101010101010000110000000010010000001000110010100001100000000000001100001110010001110010100001010001011010000000101010001101001001001000000110010011111100100110010100010110111001100100001011000101011100110010001010100110010000011001001010111111000010001111000001011011101001000011111100001101101001100100111001101000110111110100010110000110001111101010011000101001001011111001010110111011100011001111101100010010101001011001001110011010000111011010100010101110000010010101110111100111100111100011011110110000101111111111110001101111111011011101011100001011101101110000001011101011000011101011011111001111001000000010011010011110111010011011011111101110110111111001110000111111110111101111110011000011100101110110011011101010001101111011110110110100111111111010011101111110101111011111100011111011111111111101011111011111111100001111101011111111001011001111000111011111011111010110101010010111011111010111011101011001111110101110111111000110010100011111001011001110100111111111111010011111111011111010010000111011111101110001101001110111111101101100000000011111111111110001011111111110111010111111001100111111010100111111111101110011100111111011100010111011100111011111011111111010010101111010110110011100101110000111000101111111011111101011111101111111111101101111001100100101111101110110111101111101111101011110110110101010101111110111111101111011100111001111011111111111111110101111001101010101111011001001110"
        },
        {
          "name": "GLM-5.3-Flash",
          "text_run": 159,
          "five_option_run": 159,
          "wrong": 515,
          "wrong_of_answers": 0.3255372945638432,
          "difference": 0.21997471554993678,
          "interval": [
            0.19532237673830594,
            0.24336283185840707
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000010010000000000000000101000000000000000010000000000000000000000000000000000000000000000010001001000110001000001011000101000000100000011001000110100001000000000001001000000001000000101100000000000000000000000000000000000000110000000100001000010000000000000000000000000001000010010010110000001110100100000111000100100000110100010110000110010100000100100000001100000010010000100000000100000000000010000000000000010101000000000000000001000000000000010001110010100011110100010000000100001011110111110011010000010010101110110110110100100110000110101010100100011111000001011000100000000000000000100000011111000010010100000000000000000000000000101001010100000100000000000010000010001000000010100100111000010010011000011011000100101110011001000100111100001000011001101100111011100001010110000110010100110000010011000100001010001101110100000000001100001100000000000001110100000000000000000001011101100100000000110001100000101000101000000010100101000000010011000110011001110011101100001011101111010011110111010010111110101100001101111100111000101011000000100001000001110010010100010000000001001110101101011100000010011010001010101001011001010000010010010100001001000011101000111110001011110101111100011110010110110110100111111010000100010001001011110101111011110011000001110110001011010011101111010111101101001000011101010001000110101000100001001010010110011101111010101011110010001110000001101101011110111000101010111111101110011001111101010101110010110011111111110101011010000111111011111011001010111111000110111001001000111001100100111101001101111101011111111100101010001111101010111111001110011111011111010101111011101011101110001111111101010010101101100001000110001111011001111001001010101010110100111111010001010110101101101011000011110101111111011101110111011110001001110011001011000110111101101011011001100001110110111111111110110111111101010011110111111101111100100111111111001100110111111101101101111010010111111111101100110101100010111111000110110111111011110011111111111111111110011111110111111011011111110110110010100111101010011011011111110010101111111111111111000011110101011111111111011"
        },
        {
          "name": "Jev 1.13.0",
          "text_run": 49,
          "five_option_run": 49,
          "wrong": 498,
          "wrong_of_answers": 0.3147914032869785,
          "difference": 0.20922882427307207,
          "interval": [
            0.18520859671302153,
            0.23198482932996206
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000001000000000000000000000000000000000000000000000001000000000000001000000000000000100000100000010001000001000000100100001010001000001001000000100100011000000101000000000000000100100000000010000000001010000000001001000000000000000000000000001001011001000000000000100001100001000000010000100000001000010000001100001000000000010001000100000001001100001010000100010110010011001010010001000000110100000001101001100100100000101100000111000001000110100000100000010010010011100000000011000010100010111000100000001000010010100110100101011000001001100011010001001011101001000001010101000001101100010111100001010100000000000110110000001110100100101011010001100000110000010100100100000101001111000010101011000010001101011001100011101101010010110010010001001100010011111100001001011101001001100010111111111000000001001001100110111010101111110011010000111100000001010100100100101111110111000000011111011111111100111010101000111011111111010001111100001000100110011101111111011110111111111101011111001100100101111111010110111110010101000110101101111011010110111100010000010110101111101111011011011011110110001000010101001111001100011111010101011011010110111110010101111111000111111010110111011000100111101111111011011111111111110111100110010010111011010101111101111001011111111001110110110101101011011111011011011111110110111011111111111010111101101110111101100000100101110100111011010011111101011100111110111110111001111111110110011101010101000011001111110111100100110100010111111010011111111101110111011101101011111111111101111010010110011110111101111111111000011111111100110011101111011000111101110100100111011010111011111101110101111111101110101100110101111101110011100111101111111100010000111101101111110011011111001111111011011111101011010110100110110111110111101101011111100111111101110011101110111111001101111111011111011110101011101111001011111001111110111101011111111"
        },
        {
          "name": "Clef",
          "text_run": 125,
          "five_option_run": 125,
          "wrong": 512,
          "wrong_of_answers": 0.32364096080910243,
          "difference": 0.21807838179519598,
          "interval": [
            0.19532237673830594,
            0.24209860935524652
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000010000000000000000000000000000000001000000000010010000000000000000000100010000100000000000000000000000000000000000001100000000000000000001000001000000000000001000001000001001000100000010000000001000000000000000000000000000010000000000110010010001010001110000000011000000001110000110010011011001100011011000001000001001000001100010000010000001100000100011010000010000000011000011000000100100001011110111010010000011000000000011000000111010000100001001101000000000110100110100010111001001010000110100010110001000101100111001001000101100000110001101111000011000000100110101100111010001001001111001000001101101001101000000010000110110000110110111001001011111101000010100000000011110011011110100000101100010100011111010110100001101111110000101000100000011110001111001000011011100001011011110111111010110111110010000111000111111100110111010011100101000000010111010101101011010110100111000001011010101011011110010001000111100110101010100011100110001110100001101111111000100100000101101000110011101100001101110111101110111011100011110010001110011010010000100010110110001101111011111101010110001000011111101010110101011111011101110101011101001101111010100111111000111011001011111001010100001011101101111111110011010110010111101010001010001100110111111110000010111111100110010101010111111000111111111111111111111010110101111001010100011111101110000111101100110111011110000010010101000010111110111110011110101100101111111011101100010010111111111011111000111011111101111110110010111011001011111001111111011010100111111111000110101011011110001101000111111110011111101101011101111101011011111110111111011111111101011101011110010100111001001110101011101110101111011111100111110100110111110100010111010111111011110101001101111111111111011110101111010101100111101111111101011110100111110011001111110101001111110111111011011110111110010100011110110111110111111110110111111101111111110111010000010001011111"
        },
        {
          "name": "Liquid d1",
          "text_run": 52,
          "five_option_run": 52,
          "wrong": 516,
          "wrong_of_answers": 0.32616940581542353,
          "difference": 0.22060682680151708,
          "interval": [
            0.1965865992414665,
            0.24336283185840707
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000001000000000000000100000010000000000000110000000000000000000100001000000000000001001000000000010001000000000000010001000000000000100000001000000101000010000000000000010000000100100000011110000000000100000100000001000010000000100010000010000000100000001100010101100011000000100000000000110001001100000101100110010010010000010001011000100000000000111100001010000010010000000011001100101011011001100100100010001001000010101000110001001011000101011101010000001011011000001101100000010101110101000010101101011101100011001000100010000110010111000000011001110101000010001110010001101100011011011001000010010101111110101110001100111001001110100000101101100101110100101110010111110100110000001011100101100110010001000111101000100010100010011101011110000100110100000101100001001111100110010010001000001011101100100100100001001011000100100110111101011011101101010100101111101001101010110100101000110011101010110001001111101110111100110101111110100100100001011101101000111011110000110100111000010001101001101111100111111011010001100100101111111110111111001011001111101111000110111010010010001111101001011111101111010101110111011001111110100101111110011000011001101111111000101000100011111111110101111111011101000010011101101010111101111011100111111000111101100111101101001001111111100011111101110111101011101101100100111101100110100001001011111111110010101111011010100111001101110111111011001110001110101110110000111101100010010111111100111111111011111011001010001000111001011111011111111110111011110011111010111111110111111111111011111011111111011100011101111101011101111111111011110011111100111011011010001110111111101110111111101010001011110010001001111111111111110111110011100110011110111110110101001011111001111111011111101100111101111110010011110110101100110101101110111101110101101101001111110100111100111110111111111111101110110101001111001011111101111"
        },
        {
          "name": "Drex 2.0",
          "text_run": 334,
          "five_option_run": 334,
          "wrong": 528,
          "wrong_of_answers": 0.33375474083438683,
          "difference": 0.22819216182048038,
          "interval": [
            0.2035398230088496,
            0.2503160556257901
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000100000000000000000000000000000000000000000000000000000000000000000101000000000000000000000000010001000000000000000000000000000000000000000000000001010000010000000000000000000000000010000010000100000000000010010000000001100000000100000001100010100101001001000000000010110001010000000000001010100001010101001000000000010100000010101010001000110000000011110010000100001100000000001101101010000010001000000100001100000001100010001110100000001101010000000010100100000001000000111110001110100010011010101111010010010100110001001001101100000110011001100110000000101100000000111100001100000011111000011100011110111011100111001000100000000110100101100000000000101100010110100100011100000001010100111011110111001000010111101100011000111110010100000100101010011101001110011101100110000001001011101000101010101110011000000001000110110111000001011111101010010001110001010000001011101001100101010101011101111111100011000001011111101011011001111100011100001110111101001001010100001100101011010100100111001101101100000001111001001001111111111100000001001111111101100111101111100111011111110111000011111101111111110101101101010100011110011111010111001111010110100110111111001101100110100110111111111001111101101111110111000011111110011111111110110010111111101011110100111000111001111010000001011011111011001010101110100110000011010100101110111110111011111011010011110111011101010001111001111001001110011101111111001111111011111011110111011010111111111001010111110111111110111001100110111011011111110110010101101100111111101011111101111111111101100001111011110001100011111001110101111011111100110111111011111001101111100001101110001101110010011111000101111110111111011000101100111001011111110111101011111111001111111111101111110101101110111100111110100101011111111111011101111100101111111010110111111101100111111110111110100001111101011101111100111111111101101111111101111111111111101111100011101111110011111011011111111011111101111101111"
        },
        {
          "name": "Cygnet",
          "text_run": 127,
          "five_option_run": 127,
          "wrong": 530,
          "wrong_of_answers": 0.3350189633375474,
          "difference": 0.22945638432364093,
          "interval": [
            0.20353982300884954,
            0.2528445006321113
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100001001000000000000000000000000000000100010000000000000000000000000000100000000000001000010000000000000000000000100000110000001000100011000001000010000001010000000000100101000010000110000100000100000001100000000001000000000000100000110101100000010000000000110000001000000000001000001000010100010000000000100000101100100000100100001001000000101000000001011010000100000000010101101000001010010110110110000000000101001100010000000000001000011010100010001100001000000100000101001010110010100011111110110110101010011100001011010100100001110100110000001001111100010001101100100110000010110000110001111010000000010110001111100010111100100110110010101001101010001001010001010000101001100011000001010011000010110110111000111111000011011010001111000010100010010000101101100101111100011110110010100011110100110100011000011110111100110100001111100100111101011101101110011001011101111001101010011010111110101010001010101010000001101000101010111111011111111111100000010001110001100111010100111110100111000111001110110110111010100110101101110111011010000011111000000001101110010011110011011101101001000001111101111111110010111001110110111111101000111010011011011101011101001111111011111010011111011111111010111110101111000000001100101111111111011011100111110001111101101111111111011111010011110111101111100101111011001011111110011110001110100100100101110010110101111101111110101101000101111111001000101111111110011011000011011111010101010111111111101101111111111111111010101111000011110010111111111101100111010111110111011111011101111111111111111111101101110100011101111011001111110101010000111011010101000111110010011111111011111111110101111110110111010111101111111110111001101011110111011010111111111111011110010101111111101111110110101110111100100110011100111011111111111111101111111111001101111111111011001110111011011011101110011101101101111011001101111011110110011111111100011111101011111111"
        },
        {
          "name": "Open-Jev 27B v1.1",
          "text_run": 45,
          "five_option_run": 45,
          "wrong": 504,
          "wrong_of_answers": 0.3185840707964602,
          "difference": 0.21302149178255372,
          "interval": [
            0.18836915297092288,
            0.23640960809102401
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000001010000000000000000000000010000000000010000000100000000000100010001000000000000000000000000000100000100000000000010000100000001000000000000010100010000000000010000000000000000000000000001000000110100000000000100010010000000000000000010000011011110000000000001011011000100000100100000000000100001000001000001001010110000000000000001000100000100001000010000000001110000000010000000011001001100111000100001110001100000110001000100001001010110100000110100001001100100010010011101010001001000100000011011001100001010000010001010010010001000000010010111110101100001100100000011001110101000110010000111110101010111110100000001001010011001111000000010101000111010000100100010101010001010010000110000000110110111011000011111001111100011001110100111011111011011001011010111000110000101001001101100111111110010011101011101101010001001010001000010101100011001100111000110011110010010111110111110000011111111101011100111111100011000011010010110111011110001101110010101100101011000011010111011100011110011110110111111101010110110110001010111100110110111011111001011100001110000100100100011101110111100101000001111110011100110101111101011111110110111100101110110101111101111110001111101010101000001000111111110011111010001110101110110001010110110101101111101111001000111111011111111111111111111011110111101010111111100111111111000011010010111001111100111110110111111101111111111110110110101101111010010011010100011111010111111001111111111111110011001101011001111111111111001011110111101111101111110101111110110011101111111100001111000111011100111111111011011011101111011111101111111110110111101110111111111110010111001111111110111100111111111111011011101110111101111110001011111111111101111101110101110110110110111111110111101110111110111111101101111111111110011111100010100101111111111001110111101011111011010011111111011010111111100111111111011111110111111111011111010111111"
        },
        {
          "name": "Winnow-12B",
          "text_run": 128,
          "five_option_run": 128,
          "wrong": 540,
          "wrong_of_answers": 0.3413400758533502,
          "difference": 0.23577749683944374,
          "interval": [
            0.20986093552465235,
            0.25916561314791403
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000100000000000010000000000000000000000000000000010100000000000001000000000010010000000000000010000010000000000010000000000000000000010000010010010100000000000000011000000001010100000010100001011000000000000010000010111010000110001010000011000001010110010010001000000010001100010000001010010000000000000000000010011000000010000000101100111000110001001101100100111001110000100001011011011000011001000001000000000100000010101110101011000010011101101101000001100010110101100011000100100001011101011101011110111110100110100000100011100011111110101100110010001101000000001100000101001100100010010010111001100111011001010111000000100001000011010010101111101100000100000000101110011100110000100100011001110100111010111100001010000010001011100011011100101100111110001101101000010010101011010111010011001011001011111110101110001011010111110001000101101101111111111000010001100011101001000101100011111011110110110101000110111110001000101100001011101011110011110001111001111111100011011011111001000011111101101100001101101101010011100111110111101101111010011000111010100111100011011100111001001011011110100110110011001000100011011011010010110110111100101111110111101010110101010101100111000110111101100100010101001110101111101011011100101111011101111011111110110111110010101111100111011111001111010011011111111111010111111001111101111111110111100001100111011111111111011111110110100011111010011101111011101011111110001001010011111101111010111101111111001011111110111100111001110001111001010010100000110100100101010101100110111111111111110101011011101100100110001111111000111010010111101110000011101111101011010111111111101101101111111110111110011110110011101110101001110111111010111111111100100111111111100001110111111111011100111111111100011010111111111011111110111111101101111111111000011110111111111110110111111010101111111110001110111111010110111101111011111100011101111111"
        },
        {
          "name": "Qwen3.8-27B",
          "text_run": 83,
          "five_option_run": 83,
          "wrong": 559,
          "wrong_of_answers": 0.3533501896333755,
          "difference": 0.24778761061946905,
          "interval": [
            0.22187104930467763,
            0.26991150442477874
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000100000000000000000000000100000000000000000100001010010000000000000000000000000000010000000000000010000000101000010100010000000000000000000001000000000000000000000000000000000000000000100010000001000001000000000000000000000000000000000000000100000000000000000000000000000000000000000000100001100001000000000100011111000001100100000001010001000000011001101100100010100001010010000111001000000000000010001101000000110001000001000000000000010000000000000000000110000000000000100000000100000000000000000000000000000000000010000001000000000000000000000000000000001000000000000000000000000000000000000000000010010000000000000000000000000000000000000000100000001110100000010000100001000101010000110000001100000010010000000000000000000101110111011100100100110010011011000101111111101101000001001010000000010100000000010000000010001000000000000000010001010000010000011110011011101000111011111110110000101111100101101100100101010101010011111000111010010010111000001101101101100101001111011111010111000100010100010100100110000001010011100101111111100011101010111001110000001111101100010110111110111110010111010011011110101000011100001011111111001001101000111010000111000000110111000000101111101100001101010111011101111101011001011110010000101110010111101100011001111101001111100100011110011001110111101100111100011011111001111101011111111111110110110111011000110110001111010100100101010001001110111011101101111001111110101011111111100011001111110110101111010111011111100010111101001101110111100001111100101111110110011111110101011111111011111111111111011111101011101011101100101101110001011011000111011110111011101011010101111001111111010001110111000110111111110111110011011111011100010011000011111111011111111011111101001011011111011100110011011111111110111111111110111110010100111001110011111101111111110111111111100111111011011111101001101011110111101111101011111110111111110111110010001011111011011111000111010101111101101111111111101111100001101111110101110111010101010011100101110100110111101011101111011010111111110111111010011111110111110111011111101010011101011111101100111111111111000111101010001101011001111111111110111101111111110111101111110011111010111101010111110111101110111"
        },
        {
          "name": "Qwen3.8-27B",
          "text_run": 53,
          "five_option_run": 53,
          "wrong": 635,
          "wrong_of_answers": 0.40139064475347663,
          "difference": 0.2958280657395702,
          "interval": [
            0.270543615676359,
            0.3192161820480404
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000010000000000000000000000000000010010000000000000000100100100000000010000000000000101100001000000000000000010010010101010000000000000000000000000000000001000000000000000001101000001110000111000100000101000101100100000010010000000000100000000000001000000001000000000000000000001000100100000001100000110010010010000001000000000001000010000000011001000000010000000000000000000001000000000001000000000000101000111101101111001011101001011111000100100100001010000001100100111011011100111011111011001001011111111011001110011101000101001111011011011010010111111100111011111001010100111011000111110110010001101110111001010000000111011011001110101101010101001101001101010111111100110001111101101011111110011110111011011111111111111011001111011110111111110010110110011011010010111111100011111110010111011000101110101110001100010100000111111011100000001011101110000100011101000111000100111110100011100001110000000010011011011011100010010101100111000000001110000011110111011011011000010100000000100100011000100100011010000010000000010110000000010011100110001010111000100000100100000000010001000000000000000000010001000110000000000000001100100001000000000010000001011000000010010000000000000000000000000000010110001001001110000111000000000000000000000000100000000000000001011100010000111111111011010110111100011011110111101010000000101010001111111001011100110111101111111001110001111101100111101110111111011110111101101111111011001001000111100000111001101111011110111011101101110010011011011011010111011110100001000000011111100111101011010010001100110101011110001110000010110110100111010101011010100101100110100101001110101011110011011110111111110110101111111011101111110111011101110101011101100011110000111110011100101000001100101110001111111100010001100001101011011101010100111001011011001101011110110101001111110111111111110110111111110010111111001011110111101110001101111101011111011011111111111111110111111100110111110011111101001111001001011001001111010001101110111110010111101111111100101100001111001100111100001010001111110111010011011101100100011011101110111100110101000000110010001101011010100100101110100000111101000100111010100100111011000011101100101100000001000110110010111101000100010100001110"
        },
        {
          "name": "Elephant-1",
          "text_run": 373,
          "five_option_run": 373,
          "wrong": 586,
          "wrong_of_answers": 0.37041719342604296,
          "difference": 0.26485461441213654,
          "interval": [
            0.24083438685208597,
            0.2895069532237674
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000001000000000100001000000000000000000000000000000000000000000000010001001000000000000000000000000000000000100000000010000000000000110000010000100000000000000000000101010101000000100001000000000000000010000000100000101000010000010001000010000000000101000010000100000100001100000100010000001101000100000001001001000000100000111000001000001100000000001010001000101100111000000100101100101010000001000010000001001000010100000010110010000000110101001101100000000001111000101000111001001010011111010101000011100001100110101001001010110101001010001111110111110110011010010100010011101110011110001101101001101110001000101001001010100001010001000100101111101001110010101000011001111010100100101101010010100000000101001101111111110100100101010111001000100100011011111011010001110111010110110011010111100100011110110111011101101010111011101100101011110000111010100111101110011111001010110110111100011110110011001100110110111000010001101011010110001011010110011110100101001000001010100001110001100010100101111111110100111010110000101011111011001101100101010011101010101011001001001111111110111011101110110111110110110011110110110010011111111110001011011110101101001011000110011100110110110010000101001111101101111110110010010011010100101111111111110101111001111101101011101111011101110101101011011001110111110101010111100101101000111010111000111000111111010000111101100111010111101111110110010111111111101110111111110110111001001010011111101111101101110111011111101101111101001001110011101111011110111101110111111100110101101101000111100011111001110010101011111011111110111111111111101011101101101101110011110111111111001011111111110100111101111100110111111111111001111111100111111111111101111111110111111110011110011101100111001111000111011111101101111111101111111111110110111111111110111111101111101110000000111110111011111011111111111110011110100110111111110101110111111110011110000111001011110111110110111001101110110011110111011111101101111111011110111111111011110101110",
          "points": [
            {
              "wording": "Bilmiyorum",
              "run": 382,
              "answered": 1793,
              "wrong": 758
            },
            {
              "wording": "I don't know",
              "run": 515,
              "answered": 1594,
              "wrong": 622
            }
          ]
        },
        {
          "name": "Qwen3.8-27B",
          "text_run": 50,
          "five_option_run": 50,
          "wrong": 599,
          "wrong_of_answers": 0.3786346396965866,
          "difference": 0.2730720606826802,
          "interval": [
            0.24905183312262957,
            0.2970922882427307
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000011001001000000000000000000010110000010001000000000000100000000000000010000001001010100100010010000001000000000100010000000000001000000000000000000000100100000000110000110000000000011001000000000000000001011000000000010000011000100101010001001011101001001000000001000100000000001101010001010011000100000000001000110000000010000111001010001001111000100011101010000001000101000100101001001001010101000110001000001011010101000001000100100000000000001000111000111001111100100010101000100111100010011011111100100001111110010010101100110011100110010000111110110101100100001011010011111100001110110010001111011110010010010100100011010111010100101011001110010111110000101000110111101000111111110101111011010100001011011010111111101011001110011100110110111111100011011001110001111010110010001011110110101110111101111001111111011101001110101000111101111111100101110100011111111001100011010110111111111110001110110100000001100111010011001101110011111111010010111110101011111101011001111101011111010111000001010110111111111111111011001100111011011001111011111110110111100111100011010101110101011111011111100100101111111111111101100101111010101111010111011011101110111111100011111111111011111111101001111111001011011110011010010111001000101110011111101011011111111001110101011111001111111101011111111101111100011110100110111101110010111000100111110110011110111111111110111111111111110111101011111011110000111011101110111111011111110111101111101110011110111101111111001110110111111111011011110111011011101101010111111010110011111110101111101100111111111111101100111010111100111011101100111110101101111100111111111111011011011111110111111000111100111001100111111111101111011111010011001111110101111011011110101111111101111111110010110110111011011010001010111101110011100011111010010111101110101101110111111111111110111110101011011111011111011111011111101111101110101"
        },
        {
          "name": "Perplexity Decider Flash v1.2",
          "text_run": 223,
          "five_option_run": 223,
          "wrong": 691,
          "wrong_of_answers": 0.4367888748419722,
          "difference": 0.33122629582806573,
          "interval": [
            0.3059418457648546,
            0.3558786346396966
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000001000000000000000010000000000000101000010000000000000110000000001000000000000000000000100010000010000010010000000000000000000100011011010000000001000000001000001011011101000010010000010010000101000001000100010101000110001001001000000000000010010100010110000000000010000000000000000001010001110110001000100011010101000100000110110010000100001110010010100100100100101010010110100011100101001000110110000010100000000101001100011000111000010110011111011000010010010001001111011100000001011110000001011001000100011011101010110101001000111111100010010111100110100110011100111100100000000001110100001000000101010000111100110111000011101111010001111011011100011100101111010010100110111100001101111010110001001110111001101001101110111011101000111101011110000011111100011011111101101011111011100000110111110110111000111011111111011011010000111000011010011101101100111100101111101101100111000100011111010011111110101010011010110001110000110101111001100010101001111111010000001001111111000110110101011111000010100000001110010111111110010000110111011011011110000111100001111100110111011010110101011000011111001110011111110110111111111111100101001111101110110100111011111100101111101110011011111110011110001111111111010011111101011010101010111110101001001100001101011101111111110111111001010111001101001111001111111110110100111101101111001111111110111110101111101111100110101110110101000110101101010111111111010011101101101001011111011001000111011111101111011010110111010111011111101011001000001110100110011111101101111111111111011100110101111000011001111111011011101011101111111010011111111111011111001101000011111101100111100110111101000111001111111101110011101111100100011111011110101011101101111001011011101110111111101111111111011111111010111010011101111111101110101100111101111001011011110111101001010101011111111111111111110011111111100111111111011111111101111100110010111110110101111110111101101111111011110110111111111011111111111011101111111000101111111111111101011111111111110101011101111111011111010111111111110111111011110011110"
        },
        {
          "name": "Microsoft-Decision-1",
          "text_run": 205,
          "five_option_run": 205,
          "wrong": 726,
          "wrong_of_answers": 0.4589127686472819,
          "difference": 0.3533501896333755,
          "interval": [
            0.3255372945638432,
            0.3780025284450063
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000100000001000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000100010000000000000000000000100000100000000000010000100000000000001000000001000000000000000000000000100000000101100000000000000100000000000010000000001000000000000100000010001000010110001000010000000000000000000010101000000000000000000010010001000001111010111000100000000000000101000000010100001001010110100001010011010010000000001110000100010000010000000011000010011011000001100100000100010010010001000100101000110010011101000001111001100010000001000100101000100000100101100001010101001100101111110110011011010000100111110100110110110101110100001000011110101101111111001111001110001001111100100011110101011111010010111010110010111000001011110001010110010010011101110111111100101110111000011111011111110111110010010011111011010101110010111100100111101111101010101011000111111111011000010110001010111111111111101111111011010110011001111111011011111000011111111010101011111111110101111001101101101011111101000111001111101011111010011110110011111111110011001111000001101111111011110111110111011111010000110110111011100001010011011001011111111001110011101001111111000111010000111111111111001000011011110110111110100111100000100101111001111111100011000001011011111111111110110110110101111111111011011011101111111000010111110011101111101100110100101000111101100101111100110110100000101111100011111100100110010111000001111011101111111111011101100101111111011010110011100011111100111101101001111011111011101110010111111101110011110111101111111111010111110111111111011011111111111110011010010110100101100110010101001111011110111110011111011011100111111111110010111111101111111110111001111111010111110111111110001111111111110111111110100111101001111111110101111111110011011111101001001111001110111111010111101101000111011100110110000100110111101101110111111011110100111001011101100000011111110101110111011111110110011101101011101111101101011110001001110011111111010101111111110111111001000111011111011010111111111110111110100101111110101101111111111111110101110100101101111011101111101101111110111110110010111110011111110110110111111111111111110101111111111101111"
        },
        {
          "name": "Clef-flash",
          "text_run": 139,
          "five_option_run": 139,
          "wrong": 716,
          "wrong_of_answers": 0.4525916561314791,
          "difference": 0.3470290771175727,
          "interval": [
            0.320480404551201,
            0.37168141592920356
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000100000001000000000100001000000000000000000000010100000000000000000010010000000000000000000100000001000100000000000100000000000000000100000000100000000000100001000000000000100010000000011100000010100001000110010000010001010010100100000010000000001100000001101101001100001101100001110000110100100100010001011001001000101101000000000000010000000101000000001100011001110001110000010110011000001101000000001100000000011000001001010010110110100100001100000001010000011001110010001000100000011001010110100100100101001011001001100111000100010010101010000010100101011010100000100101110000001110111110001001010000001101000001100100000100010010010100010010000010101001111101001011001110100010010011010100011110010001001010001110001010011011111110101001110101001001011000000010110001011000101011111010010010110000101000100100010000101011101001001001110011100101011010010110101110010001000101100101100011001101111010001110101111101011110011101100101010001101100110100111110001101011011110111011110100100101000110010000111110010001111111100011100100010011101100101010011100110111111001011011110010010000011101111110011011011010000111010111001011000000111011101100101111111011111101111111101111001101110000101001101001100001010110101101011110001010010011100111111111011111010011010100101111011011101111100111111001111011111010101100100010011111111111000010011011011111110100101011110101011111011111001111011101101111011100111111111001111101011000010010111111100111110110000001110010111101101101010110111111111110001011111101111101111111001111101101111001111001111101010111110110001011110111001011011110111011110010010111111110101101110111110100101101011110111011111011100001111111111111011111111101111001111111011001110111111110110111111111111011111100101111110110101110011111111111110101100111101111110111101111101111111100111111111111111011111110010111111111011101110111000001111111110100011110111111111111101111111001111111111011010110001111111110111111111101111111111010011111011111110101111001111111011011111111111011011100110110110011111001101011110111001110111111111101011111111011111110111101111110011111011101111001111101011101110111111111111011111101000111111011111111010111111111110010111011"
        },
        {
          "name": "Qwen3-14B",
          "text_run": 131,
          "five_option_run": 131,
          "wrong": 745,
          "wrong_of_answers": 0.47092288242730723,
          "difference": 0.36536030341340076,
          "interval": [
            0.336283185840708,
            0.3887484197218711
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000001000000000000000000001000010000000000100000000000000000000000000000010000000000000000000000000000000000011000000000001000010000000000010000000000000000001000000000000000001001000000000000000000000100000000000010000000101000000000100000000000000100000001000000000110000000000000000000000000000000000000000011001110100010000000110011000010000011001010000000010100011100000100100100001010100001010000000000101101111000000000000000001101010100011101000100000000111101110100010100011100011011000000101001000011100101011000001010100010100010010101000101011100001001111111011011110010110111011111101010111001110110100011100000111110000000000101111001101111101011111110001011101011111100110011111010010011011110011011001110010101110011111100000011010101011101010011100100100100000100001111101000111000011010011111011110010111100110011010111100010101111000111010011111000110001101111110010101011111110111101011100100010111101001101010011011000010010110000001101111111111110010111000000111011011100100101101100100110010100110101100101110000111011011001010111001110001111111111111111011110001001111001101001111101111010111001111101101111101100111110011001001101111010111010010111111011101111100010110010101100100001111111101111110100100011011010110111011111101011110111111010111110111001100110100100011111110101110111010101111100011111110010001011011101101000111100110111110010011011111100011111111111011110000111010100110101111110111101110101111110101100101011011100111111100111001110111111100110111111101111111111111111110011011101110110110001101101011111111001111111111111011010000011011111111100010101111110010111111111010101111111110101100111011111011011111101111101110001101101111001011010011110111111111101111010110011111111010100110000111110111001011101110111111111011011111111111100111111100110101100111111110110111111111101111100010110111111111001011101101111001001111101110101111110111101111110110101110111011011110001011011111110111011110101001101111011111101110110110001111101111011111101111001111111111111101110111111111110101110010011111110010110111110011011111101011111100110101101100111001111101010000111111111110110111111111110010111011101110110110011011111111111101110001111"
        },
        {
          "name": "Erk-14B",
          "text_run": 132,
          "five_option_run": 132,
          "wrong": 735,
          "wrong_of_answers": 0.4646017699115044,
          "difference": 0.35903919089759795,
          "interval": [
            0.3305941845764855,
            0.3836915297092288
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000100000000000000000000000000100000100010000000000000000000000010000000000000100000000000000000000000000000000000001000000000000100000100000000100000000000000000001001000000000001000010000000000010000000100000000000000000000000011100000000100000000000110100000000000010001100001100000000001000010001000000100000000010001001100010000011000100100000100000001011001110111010011000100000111001001000001101001010111000100000001011011110000010010010010000010000100100010001111100000001001100010000101000000111100001001100011100001111001100010000100010000111100000001000011110000010011111111100100110001000010001011111000010101010110010101100101010000001000111010001011011111101011001110001000001111001011010110001001000111101101000011111111011010011101111010100100100101000100111011111101111000001100001100101011101110111001001100000001010111001110011101101010111000000101111110011111100111010010110101011111000101111100111110100100110111010100011010011011111011011101010011011111110101000101010001110111111011001011100010110101101010110111111011000011001101110010001101111110101111000010101010101110011100011111101110110111000011100111111101100111011110011011111011111011101110101111011111111110111100100111011101100011111100111101101100011011110100101111100010100100111111111111111111100111110100110001101111101111111001001100110110111111000010111111111111011111001011001100010101100111101100111101111101111010110111100110110011101101001011111101011101001111111111110110111001110111101011010111101111101011110111010011101110111111011101111110110101011111111011111001110111111111111101001111111001111110111111111111111100011011111010111110111111111101111111111001111011110011111110111111101001011111111001111111011111111011111101111111011101101101111110111111011111111011001100110011000100100111011111101011111110111111011111101111100111111011100111101101101111011111111110111011011011110111101101011101110101110111111111101011110100110111101010111100111111001111111101110111111101010111011111011111111101111000010011111111110111001111100110110110111011101111111111011111111110110111110111100111000111111111101101011111101100011010101011110111111110111101111110111110111111111110"
        },
        {
          "name": "Metask-Jev 4B",
          "text_run": 126,
          "five_option_run": 126,
          "wrong": 841,
          "wrong_of_answers": 0.5316055625790139,
          "difference": 0.4260429835651075,
          "interval": [
            0.39823008849557523,
            0.4513274336283186
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000100000001001000000000100000000000000000000001000000000100000000100000000000000001000010100010000001110000000001000000000000000001100000000110000000010000001001010000000001000000100010110000000110110000100100010001000111000001100010011110000010000000000000100011111100101110010001000011001000101001000010100010100000101110101001001001110000111100101110100111001010000000100011110111101000111000010111111101000011000001001001101000101101010001011110011011111101101001100011011101101001010110100111110000111111001110001101111110001001101000111111101111101110101011101000000000100010111100000010110010100110100011111011111101100001100101111011011101011111110001011110011011111110010110101000011011110011011011111010011010111101011110011100011111011010100110110101110001111011101011111110101101001111111001101110110011111111011110111111110001010001111110111111010000101111111111101111101100101011001101010100111101111111001100110111110101011110110011011101101111001111001110011011011010111111111011111100111111001011111011101011110111111101111001111111000101101111111110011110111110111111111010111111110101111000111011110100110000111011101110111111110110011111101101110011110101011010101111010011101101101110111101111110011000010111111111011101011001101111111101111110111101111111111111101111111111011110100111011010011001110111101011111111111011111111111111010111111111011010111111111110110001110101101111001111101010111101011101111111010110010010111111010101011101111000011111001111111101101101111011111111010011001010010100111110110100110000111111111111111010011110111111111111110111011111011010101001111111111111110010101111011101110011000101100111111111110010101000111111111011111111011011110101011111111111110111101101110111111101110001101111100011111111111101111011111011110101111111011101111111110110111101111011111011010011111111011110110111111011100001111111111111111110110111111110111011011110111011001000111100101111011101111001000101101111111110010101111111111101011111100001011111111011110111111111101111111101111000111110111111111111100101111101111101111100100011111011111111111100011011"
        },
        {
          "name": "Liquid d1-2B (open weights)",
          "text_run": 340,
          "five_option_run": 340,
          "wrong": 1060,
          "wrong_of_answers": 0.6700379266750948,
          "difference": 0.5644753476611883,
          "interval": [
            0.5366624525916561,
            0.5903919089759797
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000000100000100000000011010010101010011010010001001111001100111100101000100100100010101000001000000100111101010110011010010010001010000101110010010100001101111010011011000111011010001100001011110010000011000010011010111100000111011111111111110101110000011010010111001101101011100100111100010101101011011011011011111000111011111111100010110011010110111011011100001010000111100111111110110001110111111110111011101001110110010100111111111111101000110111100111010111101111111100110011000101110100111110110011001111101111111011110100101111010110111000000100111101011010101101111011111101111111100111001111111101001001111101110101111111110110101011111010111110111101101111110101111011110011101100011011011110111111101110001100110101111010101011000001110011111111101110100011111101110000011111011111110100000101110111110111111011111101111101110101101111000000111110111011111100010101011111111111101101011111011011001110111111111100110010101000011101111111110111111110101111001011111101110111111011011001110001110111110001111111101111101101110001011101000101111111111101111110111001111100111111111010111101001011110011111111111101101111111111111110110000011111010101111100010111101110011111101111111111101110111000011111101001110111011011011111111111111101111101111111011111110011111111111111011011111111111111110100100110111111101111111101111110111101101101100111111111100101111101110111111010111111101111111011111011111011101111111101111010111011111111110110110111101101111100110101111101101101110111101111011111111001111010011111100011110011101010110111111011111010110111111111110101111111001110111011111111111001110111111111111110111101111111111000011111111110011110101011011111001111100111111101111101111001101100111011111110110111111101111011111111101110111111011111011110111111111111111111111111110101111111111011101110110111101111101101001011111111111110111111110001111111110111101111111101110011101111100111011101111010111111111111101100100001100101111110111110011101111111111010101111111101101111110011111111101011011111111111011101111111011101111011111111101111111110111001111101011111101111101111110111111111010011111100111111001011111110111111011011101111110011111011011110011111111111111111"
        },
        {
          "name": "Strands Decider 2B",
          "text_run": 141,
          "five_option_run": 141,
          "wrong": 1096,
          "wrong_of_answers": 0.6927939317319848,
          "difference": 0.5872313527180784,
          "interval": [
            0.5587863463969659,
            0.6125158027812896
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00000001000000000000000100000010001000000001000100001000011010000000100101001000100101101000001010111100100110011100011001110110110010111011110000111100101011100111101101111001110110101000101100110110011010110110001111111110010111011001101001010000011100101010111010100001111101011011110111011111111001000010111010101101001011111011101101010010011110101011111011111111110111110001011010001001011100001111010110011000101111010111111100000111101111100100000111101110111011111101010101110110111111011111111100111011110111001101011011111110111111111011100111100011101101111011010110110101111001101110101111011111000100011010010110001111011110010110001111110110101111111111011101100011001110111111111111100111011001111001100111110111001011110101010111011110111110111111110111110100110111100111111011110101110111111110111110101110010111010110111111011011111011010111111110111111111111110011101111011111110111011001110111111111111101111111111110111001111011110100011000111011000111111111110111010111110111011110110011111110110111011101111110110111111101100111011011000001011111111011111110101111101101111101111110111111101111000111101111111011111000101111111111110111110011111011111111111110111111111110111000111101111010110101011111110101111111111111111111111111111111111111111111010011100100101110111101111110111111001111111101111111111111111111110101111111110011101111110111111010100011100011011111111111110011011111111011111011101111100010111111111110111011110111110111111111001111111111101111111111101101101101110111011111011111110111111011110111100101110110111111111111101011111111111011110111111110101111010011111111111011011101011011111111111110111111111111111111011001111100111011111110111110110111110001111100111111111111111011111111111111111111110011011111101111111110111111111011111100111011110001111111101011011111111111111111111110011111101111101111111101111111111101011111111111111111110110110111011111111101010111101111101111111111101111111001011101111001111111010010111111011011011110000011000110110010111111111111011110111111011111111111101111011010111111111111101111011111011101011111111001111011101011111110110111011111101101111011110001111111111111111101100011111011111111111110101111"
        },
        {
          "name": "meraGPT Decider 1",
          "text_run": 243,
          "five_option_run": 243,
          "wrong": 1120,
          "wrong_of_answers": 0.7079646017699115,
          "difference": 0.6024020227560051,
          "interval": [
            0.5752212389380531,
            0.6283185840707964
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "00001101100010001010000100101010101010001000001110100000010101100011000010111101111010110000010101001001111111100101100110100101110011111000110110011111111101011010100011011001101100101100111101001101000111101011000011111011010111110011111110100010001011111100101011101011001001110101100100101111111111111111111101111010101101111110111011110110011110110100110001111101111111110111011111101110111011011010010001001011111111011111111110001010101011000011111001110111001111110111000111111110011010101111111111111101110111101011101011111111110010101111100101111111101101011110011111011111101101010101111111110111110110101001110100111011111011110110101111110111011110011111101111101110110111011111111111011111100111101011110101110101010111011011111011010110111111111101111110111111011110111111101010111111111011110011110001100000110111111011111111111110101011111110011011011111011010011111100111110100111111110111111101001011100010011011110101111111001111110110101000111111101110111111011100111100101110110110111101010110111001111101111101101100111110100010011111110101101101101111010111010111111111111011111011111111111111010010110111101011011111111110101111101111110011011111111111101011110101010111111111111111111011111101111110101111111111100111111101110111111101111001001111111010101111111110111111111111111101111111110111101011100111101111110011110110101101110000110000110111000111111110100110110111011111110111110111111111111111110110110110110110110011100111111011000100111111101111111111111111111111111111111111111111110111100101111111110101111011111100101110101100101111100110110111111101111111001111101101101111111111111111010111101111011111011010011011111100111111011101100111110111111110110111111111111011011111110101101111111111001111111111010111011111101010010011000011111101011100011011101111111010110111110010001111100111111110111111111111101101000111111110110111001111111011111111111011111111101111111001011101110101111111111111111001110111111111110111011111111101010101101111110100111011111111011111011111111011010111111011101011111111111111111111111111101010101101100101101111111011110111111001101101111110011111111110111111111001101111110111111110101111111110101110101101111101011111"
        },
        {
          "name": "GLiNER2.5-multi-Decide",
          "text_run": 135,
          "five_option_run": 135,
          "wrong": 1248,
          "wrong_of_answers": 0.7888748419721872,
          "difference": 0.6833122629582807,
          "interval": [
            0.6561314791403288,
            0.7073324905183312
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "01111000111001110001101101010111101111011101111001111111111111101011111111110101110001111111110111110001010101111111011110111110111101111110110111101110111111011111110110011011111111110101111111111110101111111111101111010111011111111000101101111011101101110111110101010111011100111011110111111111011101111111101101110111011010110110111111110011110011111101111011111011111111111111101110101010111011111111110010111110100111110110101111111111111101111100111101111111111011110011111111110101101111110011111111111101101110101111011101111111111010111111111110111111010111011101110111111110011111101110111111110110101011100111111111111111001111010111110111101101111011101011111100111111111011101111000111110111111111111111111111110111110111111111111001010011101110100111011101111011110011111001110110111110111101111111111011111101010111111101111010111110110110100110111111110111111110111111111111100011011010011100001001011111011111101111101101111101111111111011111111001111101011011111111111111111111111111111110110110111111111111010101101111011111111111111110111001111101111011111111111010101111111101100101111111111111001010111000111111111110111011001111111111010111111011011111010111001111111111101111111101100111111111111111110111110111111101111011101110111101101001101111111111111111111110110111111111111111110101111101110111111011111101110111111111111001001101111111011111111110011110011111101111111101111011010101011111011011111110111110101110011111111100011111011111110101101110011110011111011110101111011101101111111011111111111111011111111001111111111111111110111010110101101011110011001111011011011110111111101011011111001101110011111111111110010100011111111111111011111111011111101111110100011101111111110011110111101111110111101111111111110111101001110111001010111111101011111110111101111110111111111110111011101111111101111111111111111111110111111111011101011111111111111111111101111100100011111011111111111011110111111101110011111101011011111111111111101011101111101011111111111111110111111111111111110111111111111100101101110011111011111111101011110110111111011111111111111110111111111111111010101111111111111111111101010111011111111010111111111101110011111111110011101110111111111111011"
        },
        {
          "name": "Laya",
          "text_run": 51,
          "five_option_run": 51,
          "wrong": 1273,
          "wrong_of_answers": 0.804677623261694,
          "difference": 0.6991150442477876,
          "interval": [
            0.6731984829329962,
            0.7243994943109987
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "11110111111111101111111011111111111110110111111111011110101111111111111111111011111001110111111111111111111110111111011111111111111101111111111111111111011011110111111111111011111101111111101111111111111111111111111110111001111111110111111111111011111110111111101010111011111011111111111101111101111111100111111101100110111111011111110011110111111111110111011111100110101011101110011111111110011111101110111111010111111111111111101111011100111010101111101110111111001101111111011111011011111111111011101111111111010100110111100111111011101111111111111111011111110111111111110111111100011101101011011000110110111101111011111011101101011111101011110101111111111101010111111110110110001100011111011111011011011111110111101111000111111010110101111101111111011110111111110111111011111101110111110111110111011010111101111111111111111101111111110011111111011111100010110010111110111011111111111111011111111111011000111111111111011011100111111100111011111011011111111110111111111111111111111110111111111111111111111110111010110101111111001111011011100011111011111111111111111111110111101001101111011111011101111111011110011111111100111101111101111010111011111101111110111001111110111011011110101111100111111100111011110110101111111110001111111011111111110100100001111111111001111011111101101100111111111111111111101111011101001110101111110111010101011100101111111111111110101111101111001001111111010111101111111011101111111001101111011110111100111111101110111111111011110110110111111110011000111111111111111111100111111111111010001011111101100101111011011101011101111101111101111101111011011111111111011111111111011111111100111111111011011111101111111111111111111111111110111111111110011101111111111111111111111011111111011011111111111001111111011111001111101101111111110101101011111111011111111101111111100111110011101110101111111111101111111011110110101100111111101101111111111111101011110001011110011011111000110101111011011111101111111110111011110111111011111111111111111101111111101100111111111011110111010110101101111011111111110111010101111010111001101110111111111111111010110011111110011101111010010011111111111101111111111111101111111111110111111011111111011111111111111111111111110101000111111011"
        },
        {
          "name": "Laya multilingual",
          "text_run": 46,
          "five_option_run": 46,
          "wrong": 1321,
          "wrong_of_answers": 0.8350189633375474,
          "difference": 0.729456384323641,
          "interval": [
            0.7041719342604298,
            0.7528445006321113
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "11100111110111111111101111011111010111101111111111111111111101101001111101111010011101111111111111111111011111111111111111111111111111101010111111101110111011111111111111011111111110111111011111111111111111001111011011111111111011011110110111101111111011111111011111111111111111110101011111111100111111011111111101001011111111101111111101101011011111011111111111110111111111111101100111111101111111011001111111111101111011101111111101111111111101011111111001010111101110111011011110111011011111111111110111101011111101011011011111111111111111111110101110111110111111111111111011111100101111111111011110100111001110011111111110011111111110110111101101111111111111111101101111111111111011111101100111101111111111111010111110011111101111111111111111111111111011111111111111111111111111110111111111110111111101111110101001111111011110101111110111101111111001111111111011111111111011101101110111101011111101111111011111010110111111111101111111111111111111010110111011111111111111110010110111110101101110011011111111100110111011111110110101111111111110111011101111111111111001111111111110110111111011111111110111101111111011011111010111111111100111111111111111111111110111110100110011111101111101111111011011111111111011011101111111111111011111101111111111101111010111111111111011101111111111011111111111101110111111110110001111111110110011111101111100111101101101111111100111011011111111111111110110110111011111111011111111010111011111101111101111111111100111001101111110111111011111111111111110111110111110111010101111011111111011110111111100111011101111101111111101111111110100111111011111011111111111110110111111111111101111110011111011110110011111111111110101110111111111111111011111111101101110101111111111111101101111101111111111111001110110010101111101111011100101111111111111110111011111111111111101011101010100011010101011111111001101010110110011111011101111111111110010111001111101011111110111110111111111000011111111111111111111111111011101111110111011110101101111011111011111101111111011100110101011011110101011110101111011111100011101111111110011010101110111110111111101011000101101100111110101010111111001110110110111100111110111111111110111111111111111111111011111111011111111011010011111"
        },
        {
          "name": "GLiNER2.5 Multi (base)",
          "text_run": 122,
          "five_option_run": 122,
          "wrong": 1321,
          "wrong_of_answers": 0.8350189633375474,
          "difference": 0.729456384323641,
          "interval": [
            0.7041719342604298,
            0.752212389380531
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.014598540145985403,
          "wrong_by_rank": "11110111011011111111110111101001101011100111111011111111101111111111111111111001011111001111111111111110111011111011111111111111111011111011111111111111111111111111011111111110101110111111111111111111110111110011011111100111111101111111111011111111111111110101111011101110010101011111110111011111111111011011111011111001111110111111101111111111111111011101111111111110111111111001111111111111111000111111111100111111111111101111110111001101111111111101111111111111011111111111111110111111011111111110111111111111111101111111111111111111100111111111011111011101111111111111111111101111111111111011111111111110111011101011111111111111111111111011111111110110110111111111111101111111111110111101111111011111101111111111111100111011110111111111110011111011111010110010111111111111111011111100111111110111111111011110011111001001101111011011111011111110111111111111100101111111001001111111111011111111111111011111111111011111110111111111111111100111111111101110111111100101011010111110110010100111101111101000110111011011111101111101111111111111111111111111111111111011111111111111111111110110101001111111111000101111111111101111110111101111101111111110111011111011111101101111111111101111111100100110111110111011010111111101011011110111111111111111110111011100111111111111110011101111011011110010111111110110111111001010111111110001111111111111011111001010001111111011111011110111111111110111010111011111111111111101111011111011111111111001110110101110011101111111111111110111111111111111011010111111101111011011111101111111101111111110110111100110011011111110111111111011000111101101111011011111001111100101111110111101110111111111111111101111011111110010011111111111011011111111110111111111011111111010111111111111111111111101111111110011111011101111001111111111110110010111111111000111011111111111111110111101111111111111101101100011111011011111011101101101101110101110011111111111111111110011110101101111111011101111011011101101101111011100110111000111111110100111110111001011111110111100111111110011111110111011110101111011111111111111110101101111110101110110111111111111001111110111011110111111111110110001111111111110111111111111100111101101111011111111111010101111111010110101111001111111111110"
        }
      ]
    },
    "english": {
      "questions": 547,
      "k": 496,
      "k_run": 449,
      "k_wording": "I don't know",
      "models": [
        {
          "name": "Liquid d1-ultra",
          "text_run": 199,
          "five_option_run": 411,
          "wrong": 29,
          "wrong_of_answers": 0.05846774193548387,
          "difference": null,
          "interval": null,
          "p": null,
          "p_holm": null,
          "wrong_by_rank": "0000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000100000010001111000000000000100010000010100000000000000000000001001010000110000000001101001000100010000011101000001010001100010010010110011110011110100101010001000100001",
          "points": [
            {
              "wording": "I don't know",
              "run": 449,
              "answered": 496,
              "wrong": 39
            }
          ]
        },
        {
          "name": "Fastino GLiDE v1.2",
          "text_run": 248,
          "five_option_run": 521,
          "wrong": 26,
          "wrong_of_answers": 0.05241935483870968,
          "difference": -0.006048387096774188,
          "interval": [
            -0.024193548387096774,
            0.016129032258064516
          ],
          "p": 0.7645235476452354,
          "p_holm": 1.0,
          "wrong_by_rank": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001100000000000000000000100000010000001100000000000000000010001000000000001011000001000000000000100001001000000111010000001100000100000100001100111001100001101011001111001000110011101111001010001",
          "points": [
            {
              "wording": "I don't know",
              "run": 523,
              "answered": 517,
              "wrong": 36
            }
          ]
        },
        {
          "name": "Mammoth-1",
          "text_run": 374,
          "five_option_run": 503,
          "wrong": 31,
          "wrong_of_answers": 0.0625,
          "difference": 0.004032258064516132,
          "interval": [
            -0.01814516129032258,
            0.024193548387096774
          ],
          "p": 0.8371162883711629,
          "p_holm": 1.0,
          "wrong_by_rank": "0000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000101000000000000000100000000001000000000000010001000100001000100010010001000000000000001000001000100000010000010001111001111011000001001001001000000101111010001011100000011110011111110000100",
          "points": [
            {
              "wording": "I don't know",
              "run": 505,
              "answered": 510,
              "wrong": 36
            }
          ]
        },
        {
          "name": "vLLM-SR d3",
          "text_run": 259,
          "five_option_run": 554,
          "wrong": 29,
          "wrong_of_answers": 0.05846774193548387,
          "difference": 0.0,
          "interval": [
            -0.018145161290322585,
            0.02217741935483871
          ],
          "p": 0.9345065493450655,
          "p_holm": 1.0,
          "wrong_by_rank": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000001000000000110000000000001000000000001100000000000010000001001000000011000101111000000011000010000100000101100100010000010101110000001010011000000011010101100110111111111110",
          "points": []
        },
        {
          "name": "Drex Max v1",
          "text_run": 322,
          "five_option_run": 552,
          "wrong": 28,
          "wrong_of_answers": 0.056451612903225805,
          "difference": -0.0020161290322580627,
          "interval": [
            -0.02217741935483871,
            0.02016129032258064
          ],
          "p": 0.9773022697730227,
          "p_holm": 1.0,
          "wrong_by_rank": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000001100000010000000000000000101000000000110100000000000000000000000100000000001010010100000000001101000000000101000010100011000001101110010110100111110010101100011111110101001000011011",
          "points": []
        },
        {
          "name": "vLLM-SR d3-ultra",
          "text_run": 269,
          "five_option_run": 550,
          "wrong": 31,
          "wrong_of_answers": 0.0625,
          "difference": 0.004032258064516132,
          "interval": [
            -0.016129032258064516,
            0.024193548387096774
          ],
          "p": 0.7661233876612339,
          "p_holm": 1.0,
          "wrong_by_rank": "0000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000010000000000000000000001000000000000000000000000000000000000000100000000000000000000000000000001001101000000000000010100001000010000001100000000001000010101011110000000001000100110000001000001001010100000101111001111001000110101000010111111010111",
          "points": []
        },
        {
          "name": "Perplexity Decider 27B v1.2",
          "text_run": 217,
          "five_option_run": 527,
          "wrong": 34,
          "wrong_of_answers": 0.06854838709677419,
          "difference": 0.01008064516129032,
          "interval": [
            -0.010080645161290314,
            0.03225806451612904
          ],
          "p": 0.3187681231876812,
          "p_holm": 1.0,
          "wrong_by_rank": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000110000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000001000000000000000000000001000000000000000000001000000000000001011001011001000000000100100001000101000111100100101110011000001001000000010000010000100010101110001010001111011101000100011111011000101111",
          "points": [
            {
              "wording": "I don't know",
              "run": 529,
              "answered": 545,
              "wrong": 64
            }
          ]
        },
        {
          "name": "Drex Max 1.1",
          "text_run": 326,
          "five_option_run": 399,
          "wrong": 32,
          "wrong_of_answers": 0.06451612903225806,
          "difference": 0.006048387096774195,
          "interval": [
            -0.016129032258064516,
            0.028225806451612906
          ],
          "p": 0.6425357464253575,
          "p_holm": 1.0,
          "wrong_by_rank": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000100000000000000000000010000000000010011000000111000000000100010000000010000101100000001011001100000001010000000100010000001000100010000101101111000100001001111011001110101110101111101100111110",
          "points": [
            {
              "wording": "I don't know",
              "run": 447,
              "answered": 504,
              "wrong": 41
            }
          ]
        },
        {
          "name": "Liquid d1",
          "text_run": 52,
          "five_option_run": 430,
          "wrong": 44,
          "wrong_of_answers": 0.08870967741935484,
          "difference": 0.030241935483870976,
          "interval": [
            0.008064516129032251,
            0.056451612903225805
          ],
          "p": 0.016798320167983202,
          "p_holm": 0.15118488151184883,
          "wrong_by_rank": "0000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010001000000010000000001000000000000000000000001000100000100000000100000100000000010000110000100110001001000010000010001010000111000000111100010000100010001110000100000000011101110000001011010010101101000111100100100110110100010111111110",
          "points": [
            {
              "wording": "I don't know",
              "run": 445,
              "answered": 545,
              "wrong": 67
            }
          ]
        },
        {
          "name": "Jev 1.14.0",
          "text_run": 194,
          "five_option_run": 410,
          "wrong": 44,
          "wrong_of_answers": 0.08870967741935484,
          "difference": 0.030241935483870976,
          "interval": [
            0.0040322580645161255,
            0.05241935483870967
          ],
          "p": 0.0235976402359764,
          "p_holm": 0.1887811218878112,
          "wrong_by_rank": "0000000000000000000000000000000000000000000100000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000010000000000000010000000000000000000000000000000000000000010000000001000100000000000000000000000000000100000001000000000000110000010010000011001000010010010000000101000010000010001000100110010100001010011010110111010111001000011101001011100111101111101011111011010111011",
          "points": [
            {
              "wording": "I don't know",
              "run": 446,
              "answered": 544,
              "wrong": 72
            }
          ]
        },
        {
          "name": "SubQ Snap 1.1",
          "text_run": 346,
          "five_option_run": 413,
          "wrong": 44,
          "wrong_of_answers": 0.08870967741935484,
          "difference": 0.030241935483870976,
          "interval": [
            0.01008064516129032,
            0.058467741935483875
          ],
          "p": 0.007999200079992,
          "p_holm": 0.07999200079992,
          "wrong_by_rank": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001010000000001100000000000000001000000001000000000000000000000100000000000000000000000000000000010000010000010100000000000000000000000000101101100000001000110001010001111000110000000001001000000100000110110100000001111110011000000100111110110111000001101110111100100011100111010111111",
          "points": [
            {
              "wording": "I don't know",
              "run": 448,
              "answered": 526,
              "wrong": 63
            }
          ]
        },
        {
          "name": "Inception Mercury Decide",
          "text_run": 204,
          "five_option_run": 398,
          "wrong": 50,
          "wrong_of_answers": 0.10080645161290322,
          "difference": 0.04233870967741935,
          "interval": [
            0.016129032258064516,
            0.06653225806451613
          ],
          "p": 0.0013998600139986002,
          "p_holm": 0.015398460153984602,
          "wrong_by_rank": "0000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000100000000000000100000001000000101000000000000000000000000000000001000000001000000101000110101001011010000010001000000000001010001000010000000001100111011000101100101100001110010001101101101010010011001011001000101111110011101011100011001111",
          "points": [
            {
              "wording": "I don't know",
              "run": 450,
              "answered": 536,
              "wrong": 71
            }
          ]
        },
        {
          "name": "Microsoft-Decision-1",
          "text_run": 205,
          "five_option_run": 412,
          "wrong": 52,
          "wrong_of_answers": 0.10483870967741936,
          "difference": 0.04637096774193549,
          "interval": [
            0.02217741935483871,
            0.07258064516129031
          ],
          "p": 0.0005999400059994001,
          "p_holm": 0.007199280071992801,
          "wrong_by_rank": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000100000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000001000000000000000000000000000000000000000000000000001010000000000000000000000000000000000100000100000000000000000000000000100011100100010000000000010001000000000010000000100010000000010000110000000110000000000000100000001001000101000110000110001000011001010110101011101010001000000010110010110010000101110000001111111100100110011000001111101",
          "points": [
            {
              "wording": "I don't know",
              "run": 451,
              "answered": 544,
              "wrong": 77
            }
          ]
        },
        {
          "name": "Elephant-1",
          "text_run": 373,
          "five_option_run": 499,
          "wrong": 55,
          "wrong_of_answers": 0.11088709677419355,
          "difference": 0.05241935483870968,
          "interval": [
            0.02620967741935483,
            0.07862903225806453
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.0031996800319968005,
          "wrong_by_rank": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000100000000000000000000000000000010000000000000100000000000011000110000101000001000010001000000001010000101100000011000000101101001010000000000000100100100100100110000001010100000100010111011001010101110000001101010101110000000010100001101010101011110111000011011101101",
          "points": [
            {
              "wording": "I don't know",
              "run": 501,
              "answered": 524,
              "wrong": 67
            }
          ]
        },
        {
          "name": "Celeris-1 Decision",
          "text_run": 150,
          "five_option_run": 546,
          "wrong": 56,
          "wrong_of_answers": 0.11290322580645161,
          "difference": 0.05443548387096774,
          "interval": [
            0.026209677419354843,
            0.08064516129032256
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.0031996800319968005,
          "wrong_by_rank": "0000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000001000000000010000000000000000000000000000010000000001000001000010000010000000000011000000000101000100010001000000011010110000011010010011001000101000001101000000110010000000001101001100100000100011000101111000100000101111000011111101111100001000011110010111010110111111110",
          "points": [
            {
              "wording": "I don't know",
              "run": 548,
              "answered": 455,
              "wrong": 37
            }
          ]
        },
        {
          "name": "GPT-6 Luna",
          "text_run": 143,
          "five_option_run": 397,
          "wrong": 58,
          "wrong_of_answers": 0.11693548387096774,
          "difference": 0.05846774193548387,
          "interval": [
            0.03225806451612903,
            0.08870967741935483
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.0031996800319968005,
          "wrong_by_rank": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000001000000000000000010000001100000000000100000100000000011010001000001010000000000000000000100000010000001000100000000000011001101010000000000000000010100000100110011001000001110001100001000000111000001001000101011110100100010000100011110101010111010111110011001110100010001010001011011101",
          "points": [
            {
              "wording": "I don't know",
              "run": 444,
              "answered": 537,
              "wrong": 96
            }
          ]
        },
        {
          "name": "Perplexity Decider Flash v1.2",
          "text_run": 223,
          "five_option_run": 533,
          "wrong": 62,
          "wrong_of_answers": 0.125,
          "difference": 0.06653225806451613,
          "interval": [
            0.04032258064516129,
            0.09879032258064516
          ],
          "p": 0.00019998000199980003,
          "p_holm": 0.0031996800319968005,
          "wrong_by_rank": "0000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000010000100000000000000000000000000100000010000000000010000000000001000000011001000100000001000110000000000101011000001000001010000001010010100011001100111100001010101110000001110100010001011100111011000101100100000100101110111010110011111111110010111011110110110111110101",
          "points": [
            {
              "wording": "I don't know",
              "run": 535,
              "answered": 542,
              "wrong": 100
            }
          ]
        }
      ]
    }
  },
  "answer_key": {
    "version": 2,
    "graded_sha256": "797ad6c868bc1c9485bc54203f58839ad995b0443b7149f6a9e36cf0d07df4d7",
    "fingerprint": "cd7f998de262e355cd28f596465a51f4158211646ee0cbfbc85f5d8cdeced20a",
    "excluded": [
      379,
      459,
      652,
      2473,
      3024,
      3087,
      3185,
      3223,
      3242,
      3640,
      4115,
      4206
    ]
  },
  "formats": [
    {
      "run": 188,
      "base": 123,
      "questions_per_request": 1,
      "base_questions_per_request": 16,
      "correct": 2135,
      "base_correct": 2050,
      "questions": 2198,
      "first_only": 101,
      "second_only": 16,
      "difference": 0.03867151956323931,
      "low": 0.029162553646116657,
      "high": 0.04818048548036197,
      "p": 2.8669598238307665e-16,
      "p_holm": 1.72017589429846e-15
    },
    {
      "run": 189,
      "base": 119,
      "questions_per_request": 1,
      "base_questions_per_request": 16,
      "correct": 1468,
      "base_correct": 1607,
      "questions": 2198,
      "first_only": 125,
      "second_only": 264,
      "difference": -0.0632393084622384,
      "low": -0.08062692285193232,
      "high": -0.04585169407254446,
      "p": 1.5244364559022056e-12,
      "p_holm": 7.622182279511029e-12
    },
    {
      "run": 190,
      "base": 124,
      "questions_per_request": 1,
      "base_questions_per_request": 16,
      "correct": 2126,
      "base_correct": 2125,
      "questions": 2198,
      "first_only": 23,
      "second_only": 22,
      "difference": 0.00045495905368516835,
      "low": -0.0055268486090378505,
      "high": 0.006436766716408188,
      "p": 1.0,
      "p_holm": 1.0
    },
    {
      "run": 191,
      "base": 48,
      "questions_per_request": 1,
      "base_questions_per_request": 16,
      "correct": 2141,
      "base_correct": 2145,
      "questions": 2198,
      "first_only": 14,
      "second_only": 18,
      "difference": -0.0018198362147406734,
      "low": -0.006863591071483977,
      "high": 0.003223918642002631,
      "p": 0.5966148958541453,
      "p_holm": 1.0
    },
    {
      "run": 192,
      "base": 47,
      "questions_per_request": 1,
      "base_questions_per_request": 16,
      "correct": 2051,
      "base_correct": 1988,
      "questions": 2198,
      "first_only": 123,
      "second_only": 60,
      "difference": 0.028662420382165606,
      "low": 0.016659121777909333,
      "high": 0.04066571898642188,
      "p": 3.7142246405723422e-06,
      "p_holm": 1.1142673921717027e-05
    },
    {
      "run": 193,
      "base": 55,
      "questions_per_request": 1,
      "base_questions_per_request": 16,
      "correct": 1960,
      "base_correct": 1869,
      "questions": 2198,
      "first_only": 165,
      "second_only": 74,
      "difference": 0.041401273885350316,
      "low": 0.027724708986329033,
      "high": 0.0550778387843716,
      "p": 3.813892550627433e-09,
      "p_holm": 1.5255570202509732e-08
    }
  ],
  "reasoning": {
    "reading_units": [
      1,
      2,
      3,
      4,
      5,
      6
    ],
    "seed": 20261013,
    "resamples": 1000,
    "pairs": [
      {
        "low": 54,
        "high": 55,
        "reading_gain": 0.022865853658536585,
        "grammar_gain": 0.32749675745784695,
        "difference": 0.30463090379931035,
        "interval": [
          0.26798075125225107,
          0.3433007179777833
        ]
      },
      {
        "low": 120,
        "high": 121,
        "reading_gain": 0.056402439024390245,
        "grammar_gain": 0.34824902723735407,
        "difference": 0.29184658821296383,
        "interval": [
          0.2564055299539171,
          0.32805987170896633
        ]
      },
      {
        "low": 123,
        "high": 140,
        "reading_gain": 0.019817073170731708,
        "grammar_gain": 0.06031128404669261,
        "difference": 0.040494210875960904,
        "interval": [
          0.022405170701763885,
          0.05799398583761762
        ]
      },
      {
        "low": 159,
        "high": 160,
        "reading_gain": 0.07164634146341463,
        "grammar_gain": 0.2691309987029831,
        "difference": 0.19748465723956848,
        "interval": [
          0.1587173100871731,
          0.23243029761818831
        ]
      }
    ]
  },
  "replay": {
    "decision": 145,
    "questions": 2198,
    "cutoffs": [
      {
        "cutoff": 0.999,
        "kept": 90,
        "wrong": 0
      },
      {
        "cutoff": 0.995,
        "kept": 268,
        "wrong": 0
      },
      {
        "cutoff": 0.99,
        "kept": 349,
        "wrong": 0
      },
      {
        "cutoff": 0.985,
        "kept": 402,
        "wrong": 0
      },
      {
        "cutoff": 0.98,
        "kept": 443,
        "wrong": 3
      },
      {
        "cutoff": 0.97,
        "kept": 505,
        "wrong": 4
      },
      {
        "cutoff": 0.95,
        "kept": 589,
        "wrong": 6
      },
      {
        "cutoff": 0.9,
        "kept": 767,
        "wrong": 25
      },
      {
        "cutoff": 0.8,
        "kept": 970,
        "wrong": 62
      }
    ],
    "share": {
      "share": 0.25,
      "cut": 0.9605055053763134,
      "kept": 550,
      "wrong": 4,
      "interval": [
        0.9814508624779456,
        0.9971682950573496
      ]
    },
    "frontiers": [
      {
        "frontier": 47,
        "sure": {
          "kept": 349,
          "first_only": 0,
          "second_only": 0,
          "difference": 0.0,
          "usd_per_1000": 4.155468782438992
        },
        "share": {
          "kept": 550,
          "first_only": 1,
          "second_only": 3,
          "difference": -0.0009099181073703367,
          "held_out": {
            "splits": 300,
            "answered": 0.2499574888999705,
            "difference": -0.0011096162611619414,
            "worse": 0.72,
            "better": 0.043333333333333335
          },
          "usd_per_1000": 3.704582622046368
        }
      },
      {
        "frontier": 55,
        "sure": {
          "kept": 349,
          "first_only": 6,
          "second_only": 0,
          "difference": 0.00272975432211101,
          "usd_per_1000": 3.0818606839002136
        },
        "share": {
          "kept": 550,
          "first_only": 14,
          "second_only": 1,
          "difference": 0.005914467697907188,
          "held_out": {
            "splits": 300,
            "answered": 0.2499574888999705,
            "difference": 0.006266133896030463,
            "worse": 0.0,
            "better": 1.0
          },
          "usd_per_1000": 2.747683678621865
        }
      },
      {
        "frontier": 48,
        "sure": {
          "kept": 349,
          "first_only": 0,
          "second_only": 0,
          "difference": 0.0,
          "usd_per_1000": 4.532032413918041
        },
        "share": {
          "kept": 550,
          "first_only": 1,
          "second_only": 4,
          "difference": -0.001364877161055505,
          "held_out": {
            "splits": 300,
            "answered": 0.2499574888999705,
            "difference": -0.0015703993274813867,
            "worse": 0.86,
            "better": 0.02
          },
          "usd_per_1000": 4.040210996669123
        }
      },
      {
        "frontier": 124,
        "sure": {
          "kept": 349,
          "first_only": 0,
          "second_only": 0,
          "difference": 0.0,
          "usd_per_1000": 0.9586563784431377
        },
        "share": {
          "kept": 550,
          "first_only": 1,
          "second_only": 4,
          "difference": -0.001364877161055505,
          "held_out": {
            "splits": 300,
            "answered": 0.2499574888999705,
            "difference": -0.0015703993274813867,
            "worse": 0.86,
            "better": 0.02
          },
          "usd_per_1000": 0.8552874128602311
        }
      },
      {
        "frontier": 123,
        "sure": {
          "kept": 349,
          "first_only": 0,
          "second_only": 0,
          "difference": 0.0,
          "usd_per_1000": 0.2337027411096696
        },
        "share": {
          "kept": 550,
          "first_only": 0,
          "second_only": 4,
          "difference": -0.0018198362147406734,
          "held_out": {
            "splits": 300,
            "answered": 0.2499574888999705,
            "difference": -0.002019039082961255,
            "worse": 0.9633333333333334,
            "better": 0.0
          },
          "usd_per_1000": 0.2091416073839979
        }
      },
      {
        "frontier": 192,
        "sure": {
          "kept": 349,
          "first_only": 0,
          "second_only": 0,
          "difference": 0.0,
          "usd_per_1000": 9.773955058730701
        },
        "share": {
          "kept": 550,
          "first_only": 1,
          "second_only": 4,
          "difference": -0.001364877161055505,
          "held_out": {
            "splits": 300,
            "answered": 0.2499574888999705,
            "difference": -0.0015643304444155316,
            "worse": 0.86,
            "better": 0.02
          },
          "usd_per_1000": 8.712297810434004
        }
      },
      {
        "frontier": 193,
        "sure": {
          "kept": 349,
          "first_only": 0,
          "second_only": 0,
          "difference": 0.0,
          "usd_per_1000": 6.525186903173618
        },
        "share": {
          "kept": 550,
          "first_only": 2,
          "second_only": 1,
          "difference": 0.00045495905368516835,
          "held_out": {
            "splits": 300,
            "answered": 0.2499574888999705,
            "difference": 0.0002546229359817252,
            "worse": 0.21333333333333335,
            "better": 0.43333333333333335
          },
          "usd_per_1000": 5.816694824842833
        }
      },
      {
        "frontier": 191,
        "sure": {
          "kept": 349,
          "first_only": 0,
          "second_only": 0,
          "difference": 0.0,
          "usd_per_1000": 22.500030475666104
        },
        "share": {
          "kept": 550,
          "first_only": 0,
          "second_only": 4,
          "difference": -0.0018198362147406734,
          "held_out": {
            "splits": 300,
            "answered": 0.2499574888999705,
            "difference": -0.002019039082961255,
            "worse": 0.9633333333333334,
            "better": 0.0
          },
          "usd_per_1000": 20.054954536831808
        }
      },
      {
        "frontier": 190,
        "sure": {
          "kept": 349,
          "first_only": 0,
          "second_only": 0,
          "difference": 0.0,
          "usd_per_1000": 4.692110365316803
        },
        "share": {
          "kept": 550,
          "first_only": 1,
          "second_only": 4,
          "difference": -0.001364877161055505,
          "held_out": {
            "splits": 300,
            "answered": 0.2499574888999705,
            "difference": -0.0015643304444155316,
            "worse": 0.86,
            "better": 0.02
          },
          "usd_per_1000": 4.182887288667587
        }
      },
      {
        "frontier": 188,
        "sure": {
          "kept": 349,
          "first_only": 1,
          "second_only": 0,
          "difference": 0.00045495905368516835,
          "usd_per_1000": 0.8182289211778264
        },
        "share": {
          "kept": 550,
          "first_only": 1,
          "second_only": 4,
          "difference": -0.001364877161055505,
          "held_out": {
            "splits": 300,
            "answered": 0.2499574888999705,
            "difference": -0.0015552587514272745,
            "worse": 0.85,
            "better": 0.013333333333333334
          },
          "usd_per_1000": 0.7301254606843346
        }
      },
      {
        "frontier": 278,
        "sure": {
          "kept": 349,
          "first_only": 0,
          "second_only": 0,
          "difference": 0.0,
          "usd_per_1000": 5.48311498747724
        },
        "share": {
          "kept": 550,
          "first_only": 0,
          "second_only": 4,
          "difference": -0.0018198362147406734,
          "held_out": {
            "splits": 300,
            "answered": 0.2499574888999705,
            "difference": -0.002019039082961255,
            "worse": 0.9633333333333334,
            "better": 0.0
          },
          "usd_per_1000": 4.887903847521238
        }
      },
      {
        "frontier": 277,
        "sure": {
          "kept": 349,
          "first_only": 0,
          "second_only": 0,
          "difference": 0.0,
          "usd_per_1000": 5.1102266640282625
        },
        "share": {
          "kept": 550,
          "first_only": 1,
          "second_only": 4,
          "difference": -0.001364877161055505,
          "held_out": {
            "splits": 300,
            "answered": 0.2499574888999705,
            "difference": -0.0015703993274813867,
            "worse": 0.86,
            "better": 0.02
          },
          "usd_per_1000": 4.55555124771382
        }
      },
      {
        "frontier": 276,
        "sure": {
          "kept": 349,
          "first_only": 0,
          "second_only": 0,
          "difference": 0.0,
          "usd_per_1000": 1.3723824156794038
        },
        "share": {
          "kept": 550,
          "first_only": 2,
          "second_only": 4,
          "difference": -0.0009099181073703367,
          "held_out": {
            "splits": 300,
            "answered": 0.2499574888999705,
            "difference": -0.0011671947860556657,
            "worse": 0.7366666666666667,
            "better": 0.08333333333333333
          },
          "usd_per_1000": 1.2240383643828738
        }
      },
      {
        "frontier": 275,
        "sure": {
          "kept": 349,
          "first_only": 0,
          "second_only": 0,
          "difference": 0.0,
          "usd_per_1000": 0.44058317350291976
        },
        "share": {
          "kept": 550,
          "first_only": 1,
          "second_only": 4,
          "difference": -0.001364877161055505,
          "held_out": {
            "splits": 300,
            "answered": 0.2499574888999705,
            "difference": -0.0015703993274813867,
            "worse": 0.86,
            "better": 0.02
          },
          "usd_per_1000": 0.3935326039140553
        }
      },
      {
        "frontier": 274,
        "sure": {
          "kept": 349,
          "first_only": 0,
          "second_only": 0,
          "difference": 0.0,
          "usd_per_1000": 0.3663308364043414
        },
        "share": {
          "kept": 550,
          "first_only": 2,
          "second_only": 3,
          "difference": -0.00045495905368516835,
          "held_out": {
            "splits": 300,
            "answered": 0.2499574888999705,
            "difference": -0.0005942213837353536,
            "worse": 0.5533333333333333,
            "better": 0.15333333333333332
          },
          "usd_per_1000": 0.32735204602413803
        }
      },
      {
        "frontier": 140,
        "sure": {
          "kept": 349,
          "first_only": 0,
          "second_only": 0,
          "difference": 0.0,
          "usd_per_1000": 2.042760994756172
        },
        "share": {
          "kept": 550,
          "first_only": 0,
          "second_only": 4,
          "difference": -0.0018198362147406734,
          "held_out": {
            "splits": 300,
            "answered": 0.2499574888999705,
            "difference": -0.002019039082961255,
            "worse": 0.9633333333333334,
            "better": 0.0
          },
          "usd_per_1000": 1.82154182480392
        }
      }
    ]
  },
  "high_confidence": {
    "threshold": 0.95,
    "rule": "An answer counts when its chosen option has probability at least the cutoff; an answer without a score or without a valid choice never counts; wrong is a counted answer that is not right.",
    "models": [
      {
        "name": "Gemini 3.8 Flash",
        "text_run": 140,
        "kept": 2170,
        "wrong": 28
      },
      {
        "name": "GPT-6 Astra",
        "text_run": 277,
        "kept": 2122,
        "wrong": 27
      },
      {
        "name": "GPT-6.1 Sol",
        "text_run": 276,
        "kept": 2120,
        "wrong": 28
      },
      {
        "name": "GPT-6 Astra",
        "text_run": 48,
        "kept": 2097,
        "wrong": 23
      },
      {
        "name": "GPT-6 Astra",
        "text_run": 191,
        "kept": 2134,
        "wrong": 35
      },
      {
        "name": "Gemini 3.8 Flash",
        "text_run": 188,
        "kept": 2134,
        "wrong": 38
      },
      {
        "name": "GPT-6.1 Sol",
        "text_run": 190,
        "kept": 2123,
        "wrong": 45
      },
      {
        "name": "GPT-6.1 Sol",
        "text_run": 124,
        "kept": 2078,
        "wrong": 36
      },
      {
        "name": "Gemini 4 Argon",
        "text_run": 557,
        "kept": 2130,
        "wrong": 55
      },
      {
        "name": "Claude Fable 5.1",
        "text_run": 310,
        "kept": 379,
        "wrong": 0
      },
      {
        "name": "Claude Opus 5.5",
        "text_run": 278,
        "kept": 129,
        "wrong": 1
      },
      {
        "name": "GPT-6.1 Terra",
        "text_run": 275,
        "kept": 1921,
        "wrong": 62
      },
      {
        "name": "Claude Opus 5.5",
        "text_run": 192,
        "kept": 297,
        "wrong": 0
      },
      {
        "name": "Gemini 3.8 Flash",
        "text_run": 123,
        "kept": 1954,
        "wrong": 86
      },
      {
        "name": "Grok 4.7",
        "text_run": 311,
        "kept": 210,
        "wrong": 0
      },
      {
        "name": "DeepSeek-V4.1-Flash, first release",
        "text_run": 121,
        "kept": 2198,
        "wrong": 175
      },
      {
        "name": "GPT-6.1 Terra",
        "text_run": 274,
        "kept": 1872,
        "wrong": 95
      },
      {
        "name": "Claude Opus 5.5",
        "text_run": 47,
        "kept": 89,
        "wrong": 1
      },
      {
        "name": "Claude Sonnet 5.5",
        "text_run": 193,
        "kept": 166,
        "wrong": 1
      },
      {
        "name": "Claude Sonnet 5.5",
        "text_run": 55,
        "kept": 38,
        "wrong": 0
      },
      {
        "name": "Kimi K3.1 Pro",
        "text_run": 320,
        "kept": 1973,
        "wrong": 253
      },
      {
        "name": "GLM-5.3-Flash",
        "text_run": 160,
        "kept": 332,
        "wrong": 5
      },
      {
        "name": "Liquid d1-ultra",
        "text_run": 199,
        "kept": 549,
        "wrong": 0
      },
      {
        "name": "GLM-5.3",
        "text_run": 314,
        "kept": 129,
        "wrong": 5
      },
      {
        "name": "Qwen3.8-27B",
        "text_run": 318,
        "kept": 2117,
        "wrong": 493
      },
      {
        "name": "Gemma 4 31B",
        "text_run": 119,
        "kept": 1956,
        "wrong": 435
      },
      {
        "name": "Kimi K3.1 Pro",
        "text_run": 315,
        "kept": 1402,
        "wrong": 228
      },
      {
        "name": "Fastino GLiDE v1.2",
        "text_run": 248,
        "kept": 485,
        "wrong": 2
      },
      {
        "name": "GPT-6 Luna",
        "text_run": 133,
        "kept": 1493,
        "wrong": 361
      },
      {
        "name": "Fastino GLiDE",
        "text_run": 129,
        "kept": 302,
        "wrong": 0
      },
      {
        "name": "vLLM-SR d3",
        "text_run": 259,
        "kept": 503,
        "wrong": 4
      },
      {
        "name": "vLLM-SR d3-ultra",
        "text_run": 269,
        "kept": 833,
        "wrong": 30
      },
      {
        "name": "Drex Max v1",
        "text_run": 322,
        "kept": 736,
        "wrong": 24
      },
      {
        "name": "Drex Max 1.1",
        "text_run": 326,
        "kept": 493,
        "wrong": 3
      },
      {
        "name": "Mammoth-1",
        "text_run": 374,
        "kept": 667,
        "wrong": 9
      },
      {
        "name": "Perplexity Decider 27B v1.2",
        "text_run": 217,
        "kept": 633,
        "wrong": 11
      },
      {
        "name": "Perplexity Decider 27B v1.1",
        "text_run": 145,
        "kept": 589,
        "wrong": 6
      },
      {
        "name": "Gemma 4 31B",
        "text_run": 189,
        "kept": 2006,
        "wrong": 593
      },
      {
        "name": "DeepSeek-V4.1-Flash, first release",
        "text_run": 120,
        "kept": 869,
        "wrong": 49
      },
      {
        "name": "SubQ Snap Preview",
        "text_run": 177,
        "kept": 339,
        "wrong": 1
      },
      {
        "name": "SubQ Snap 1.1",
        "text_run": 346,
        "kept": 665,
        "wrong": 18
      },
      {
        "name": "GPT-6 Luna",
        "text_run": 143,
        "kept": 542,
        "wrong": 15
      },
      {
        "name": "Liquid d1.1",
        "text_run": 254,
        "kept": 336,
        "wrong": 2
      },
      {
        "name": "Claude Sonnet 5.5",
        "text_run": 54,
        "kept": 9,
        "wrong": 0
      },
      {
        "name": "Celeris-1 Decision",
        "text_run": 150,
        "kept": 123,
        "wrong": 0
      },
      {
        "name": "Perplexity Decider 27B v1",
        "text_run": 44,
        "kept": 293,
        "wrong": 0
      },
      {
        "name": "OneJev 27B",
        "text_run": 264,
        "kept": 166,
        "wrong": 0
      },
      {
        "name": "Jev 1.14.0",
        "text_run": 194,
        "kept": 390,
        "wrong": 4
      },
      {
        "name": "Inception Mercury Decide",
        "text_run": 204,
        "kept": 775,
        "wrong": 67
      },
      {
        "name": "GLM-5.3-Flash",
        "text_run": 159,
        "kept": 268,
        "wrong": 25
      },
      {
        "name": "Jev 1.13.0",
        "text_run": 49,
        "kept": 440,
        "wrong": 8
      },
      {
        "name": "Clef",
        "text_run": 125,
        "kept": 348,
        "wrong": 5
      },
      {
        "name": "Liquid d1",
        "text_run": 52,
        "kept": 502,
        "wrong": 20
      },
      {
        "name": "Drex 2.0",
        "text_run": 334,
        "kept": 180,
        "wrong": 0
      },
      {
        "name": "Cygnet",
        "text_run": 127,
        "kept": 335,
        "wrong": 4
      },
      {
        "name": "Open-Jev 27B v1.1",
        "text_run": 45,
        "kept": 0,
        "wrong": 0
      },
      {
        "name": "Winnow-12B",
        "text_run": 128,
        "kept": 641,
        "wrong": 55
      },
      {
        "name": "Qwen3.8-27B",
        "text_run": 83,
        "kept": 875,
        "wrong": 129
      },
      {
        "name": "Qwen3.8-27B",
        "text_run": 53,
        "kept": 2194,
        "wrong": 1004
      },
      {
        "name": "Elephant-1",
        "text_run": 373,
        "kept": 393,
        "wrong": 13
      },
      {
        "name": "Qwen3.8-27B",
        "text_run": 50,
        "kept": 434,
        "wrong": 17
      },
      {
        "name": "Perplexity Decider Flash v1.2",
        "text_run": 223,
        "kept": 154,
        "wrong": 1
      },
      {
        "name": "Microsoft-Decision-1",
        "text_run": 205,
        "kept": 378,
        "wrong": 29
      },
      {
        "name": "Clef-flash",
        "text_run": 139,
        "kept": 56,
        "wrong": 4
      },
      {
        "name": "Qwen3-14B",
        "text_run": 131,
        "kept": 1236,
        "wrong": 505
      },
      {
        "name": "Erk-14B",
        "text_run": 132,
        "kept": 302,
        "wrong": 27
      },
      {
        "name": "Metask-Jev 4B",
        "text_run": 126,
        "kept": 32,
        "wrong": 1
      },
      {
        "name": "Liquid d1-2B (open weights)",
        "text_run": 340,
        "kept": 4,
        "wrong": 0
      },
      {
        "name": "Strands Decider 2B",
        "text_run": 141,
        "kept": 10,
        "wrong": 1
      },
      {
        "name": "meraGPT Decider 1",
        "text_run": 243,
        "kept": 1,
        "wrong": 0
      },
      {
        "name": "GLiNER2.5-multi-Decide",
        "text_run": 135,
        "kept": 0,
        "wrong": 0
      },
      {
        "name": "Laya",
        "text_run": 51,
        "kept": 0,
        "wrong": 0
      },
      {
        "name": "Laya multilingual",
        "text_run": 46,
        "kept": 14,
        "wrong": 11
      },
      {
        "name": "GLiNER2.5 Multi (base)",
        "text_run": 122,
        "kept": 1,
        "wrong": 1
      }
    ],
    "repeats": [
      {
        "name": "Liquid d1-ultra",
        "text_run": 199,
        "runs": [
          {
            "run": 200,
            "kept": 538,
            "wrong": 0
          },
          {
            "run": 201,
            "kept": 542,
            "wrong": 0
          },
          {
            "run": 202,
            "kept": 534,
            "wrong": 0
          },
          {
            "run": 203,
            "kept": 550,
            "wrong": 0
          }
        ]
      }
    ],
    "cutoffs": {
      "cutoffs": [
        0.5,
        0.7,
        0.8,
        0.9,
        0.95,
        0.99
      ],
      "models": [
        {
          "name": "Liquid d1-ultra",
          "text_run": 199,
          "runs": [
            {
              "run": 199,
              "kept": [
                1712,
                1335,
                1094,
                802,
                549,
                90
              ],
              "wrong": [
                224,
                99,
                46,
                8,
                0,
                0
              ]
            },
            {
              "run": 200,
              "kept": [
                1705,
                1320,
                1089,
                795,
                538,
                86
              ],
              "wrong": [
                229,
                93,
                39,
                6,
                0,
                0
              ]
            },
            {
              "run": 201,
              "kept": [
                1719,
                1313,
                1091,
                794,
                542,
                96
              ],
              "wrong": [
                241,
                96,
                47,
                10,
                0,
                0
              ]
            },
            {
              "run": 202,
              "kept": [
                1713,
                1315,
                1088,
                796,
                534,
                91
              ],
              "wrong": [
                228,
                84,
                49,
                8,
                0,
                0
              ]
            },
            {
              "run": 203,
              "kept": [
                1706,
                1324,
                1080,
                803,
                550,
                90
              ],
              "wrong": [
                233,
                97,
                43,
                9,
                0,
                0
              ]
            }
          ]
        },
        {
          "name": "SubQ Snap 1.1",
          "text_run": 346,
          "runs": [
            {
              "run": 346,
              "kept": [
                1791,
                1323,
                1117,
                839,
                665,
                299
              ],
              "wrong": [
                499,
                229,
                138,
                52,
                18,
                1
              ]
            },
            {
              "run": 387,
              "kept": [
                1791,
                1323,
                1117,
                839,
                665,
                299
              ],
              "wrong": [
                499,
                229,
                138,
                52,
                18,
                1
              ]
            },
            {
              "run": 388,
              "kept": [
                1791,
                1323,
                1117,
                839,
                665,
                299
              ],
              "wrong": [
                499,
                229,
                138,
                52,
                18,
                1
              ]
            },
            {
              "run": 389,
              "kept": [
                1791,
                1323,
                1117,
                839,
                665,
                299
              ],
              "wrong": [
                499,
                229,
                138,
                52,
                18,
                1
              ]
            },
            {
              "run": 390,
              "kept": [
                1791,
                1323,
                1117,
                839,
                665,
                299
              ],
              "wrong": [
                499,
                229,
                138,
                52,
                18,
                1
              ]
            }
          ]
        },
        {
          "name": "Elephant-1",
          "text_run": 373,
          "runs": [
            {
              "run": 373,
              "kept": [
                1370,
                917,
                726,
                510,
                393,
                155
              ],
              "wrong": [
                455,
                195,
                102,
                35,
                13,
                0
              ]
            },
            {
              "run": 375,
              "kept": [
                1370,
                917,
                726,
                510,
                393,
                155
              ],
              "wrong": [
                455,
                195,
                102,
                35,
                13,
                0
              ]
            },
            {
              "run": 376,
              "kept": [
                1370,
                917,
                726,
                510,
                393,
                155
              ],
              "wrong": [
                455,
                195,
                102,
                35,
                13,
                0
              ]
            },
            {
              "run": 378,
              "kept": [
                1370,
                917,
                726,
                510,
                393,
                155
              ],
              "wrong": [
                455,
                195,
                102,
                35,
                13,
                0
              ]
            },
            {
              "run": 379,
              "kept": [
                1370,
                917,
                726,
                510,
                393,
                155
              ],
              "wrong": [
                455,
                195,
                102,
                35,
                13,
                0
              ]
            }
          ]
        }
      ],
      "check": {
        "cutoffs": "every whole percent from 0% to 98%",
        "holds": true
      }
    },
    "english": {
      "name": "Liquid d1-ultra",
      "questions": 547,
      "runs": [
        {
          "run": 411,
          "kept": 234,
          "wrong": 1,
          "wrong_questions": [
            280
          ]
        },
        {
          "run": 418,
          "kept": 237,
          "wrong": 1,
          "wrong_questions": [
            280
          ]
        },
        {
          "run": 422,
          "kept": 235,
          "wrong": 1,
          "wrong_questions": [
            280
          ]
        },
        {
          "run": 426,
          "kept": 230,
          "wrong": 1,
          "wrong_questions": [
            280
          ]
        },
        {
          "run": 428,
          "kept": 235,
          "wrong": 1,
          "wrong_questions": [
            280
          ]
        }
      ]
    }
  },
  "baselines": {
    "options": 5,
    "letter": "E",
    "letter_share": 0.22656960873521384
  }
}