{
  "scope": "30 APIs with complete billing records, plus OpenAuditor alone; no hybrid points.",
  "excluded": [
    {
      "model": "mistralai/mistral-small-3.2-24b-instruct",
      "reason": "Incomplete billing records; retained in source data"
    },
    {
      "model": "qwen/qwen3.7-flash",
      "reason": "Incomplete billing records; retained in source data"
    },
    {
      "model": "qwen/qwen3.8-flash",
      "reason": "Incomplete billing records; retained in source data"
    }
  ],
  "conditions": {
    "original": "30 APIs; reasoning disabled where supported; 24 concurrent requests.",
    "leading": "Fable, Astra, DeepSeek Pro; low reasoning; 4 concurrent requests.",
    "local": "Shared warm sequential M4 Max MPS timings; no network or model loading."
  },
  "source_sha256": {
    "original": "09a227fe0eeb6c710bc83e101c33213b593cd493c08c219df7fc071250c0384f",
    "leading": "29ded260b5d1fa0a5d2dd907218847ccf171b7bd11afecccee2c06181dea8dd0"
  },
  "facts": {
    "api_count": 30,
    "faster_than": 28,
    "local_ms": 160.62327049439773,
    "local_accuracy": 0.972,
    "higher_accuracy_apis_all_slower": true
  },
  "entries": [
    {
      "model": "openauditor/v0.3.0",
      "name": "OpenAuditor 0.6B",
      "kind": "local",
      "measurement": {
        "n": 500,
        "correct": 486,
        "accuracy": 0.972,
        "accuracy_wilson_95": [
          0.9535535973250511,
          0.9832490252792436
        ],
        "total_seconds": 83.05259788176045,
        "median_seconds": 0.16062327049439773,
        "p95_seconds": 0.22153566702036187
      },
      "cost_per_1000": 0.023070166078266792,
      "cost_basis": "Scenario: $0.50 per allocated machine-hour, 100% utilization",
      "errors": {},
      "numeric_key_compliance": null,
      "run": "original"
    },
    {
      "model": "openai/gpt-4.1-nano",
      "name": "OpenAI: GPT-4.1 Nano",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 494,
        "accuracy": 0.988,
        "accuracy_wilson_95": [
          0.9740696388835527,
          0.9944890048259724
        ],
        "total_seconds": 330.5819375384017,
        "median_seconds": 0.6051433540415019,
        "p95_seconds": 1.0113971249666065
      },
      "cost_usd": 0.007021,
      "cost_per_1000": 0.014042,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {},
      "successful_labels": 500,
      "complete_transport": true,
      "numeric_key_compliance": 0.99,
      "providers": {
        "OpenAI": 500
      },
      "input_tokens": 66190,
      "output_tokens": 1005,
      "preflight_cost_usd": 4.32e-05,
      "run": "original"
    },
    {
      "model": "openai/gpt-4.1-mini",
      "name": "OpenAI: GPT-4.1 Mini",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 496,
        "accuracy": 0.992,
        "accuracy_wilson_95": [
          0.9796129641658948,
          0.9968846848199377
        ],
        "total_seconds": 360.4603449405404,
        "median_seconds": 0.6674674169917125,
        "p95_seconds": 1.064537999976892
      },
      "cost_usd": 0.028076,
      "cost_per_1000": 0.056152,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {
        "Invalid or incomplete label": 1
      },
      "successful_labels": 499,
      "complete_transport": true,
      "numeric_key_compliance": 0.998,
      "providers": {
        "OpenAI": 500
      },
      "input_tokens": 66190,
      "output_tokens": 1000,
      "preflight_cost_usd": 0.0001728,
      "run": "original"
    },
    {
      "model": "openai/gpt-4.1",
      "name": "OpenAI: GPT-4.1",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 495,
        "accuracy": 0.99,
        "accuracy_wilson_95": [
          0.9768069002442693,
          0.9957212461034096
        ],
        "total_seconds": 359.2503143767826,
        "median_seconds": 0.6558008959982544,
        "p95_seconds": 1.0276261669932865
      },
      "cost_usd": 0.14038,
      "cost_per_1000": 0.28076,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {},
      "successful_labels": 500,
      "complete_transport": true,
      "numeric_key_compliance": 1.0,
      "providers": {
        "OpenAI": 500
      },
      "input_tokens": 66190,
      "output_tokens": 1000,
      "preflight_cost_usd": 0.000864,
      "run": "original"
    },
    {
      "model": "openai/gpt-4o-mini",
      "name": "OpenAI: GPT-4o-mini",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 492,
        "accuracy": 0.984,
        "accuracy_wilson_95": [
          0.9687488914666058,
          0.9918707469666117
        ],
        "total_seconds": 263.8602219266468,
        "median_seconds": 0.5019848960218951,
        "p95_seconds": 0.6848381250165403
      },
      "cost_usd": 0.0102411,
      "cost_per_1000": 0.0204822,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {
        "Invalid or incomplete label": 2
      },
      "successful_labels": 498,
      "complete_transport": true,
      "numeric_key_compliance": 0.964,
      "providers": {
        "OpenAI": 500
      },
      "input_tokens": 66190,
      "output_tokens": 521,
      "preflight_cost_usd": 6.3e-05,
      "run": "original"
    },
    {
      "model": "openai/gpt-5.6-luna",
      "name": "OpenAI: GPT-5.6 Luna",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 491,
        "accuracy": 0.982,
        "accuracy_wilson_95": [
          0.9661483290912607,
          0.990501806703803
        ],
        "total_seconds": 858.9793787939707,
        "median_seconds": 1.1447858744941186,
        "p95_seconds": 3.4226961249951273
      },
      "cost_usd": 0.016138,
      "cost_per_1000": 0.032276,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {},
      "successful_labels": 500,
      "complete_transport": true,
      "numeric_key_compliance": 1.0,
      "providers": {
        "OpenAI": 500
      },
      "input_tokens": 65690,
      "output_tokens": 2500,
      "preflight_cost_usd": 9.9e-05,
      "run": "original"
    },
    {
      "model": "openai/gpt-5.6-sol",
      "name": "OpenAI: GPT-5.6 Sol",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 495,
        "accuracy": 0.99,
        "accuracy_wilson_95": [
          0.9768069002442693,
          0.9957212461034096
        ],
        "total_seconds": 1662.7701584976166,
        "median_seconds": 3.400876541476464,
        "p95_seconds": 6.904222582990769
      },
      "cost_usd": 0.15638,
      "cost_per_1000": 0.31276,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {},
      "successful_labels": 500,
      "complete_transport": true,
      "numeric_key_compliance": 0.998,
      "providers": {
        "OpenAI": 500
      },
      "input_tokens": 65690,
      "output_tokens": 2500,
      "preflight_cost_usd": 0.00096,
      "run": "original"
    },
    {
      "model": "anthropic/claude-haiku-4.5",
      "name": "Anthropic: Claude Haiku 4.5",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 494,
        "accuracy": 0.988,
        "accuracy_wilson_95": [
          0.9740696388835527,
          0.9944890048259724
        ],
        "total_seconds": 409.28693951509194,
        "median_seconds": 0.7780484375252854,
        "p95_seconds": 0.9811019590124488
      },
      "cost_usd": 0.080218,
      "cost_per_1000": 0.160436,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {},
      "successful_labels": 500,
      "complete_transport": true,
      "numeric_key_compliance": 1.0,
      "providers": {
        "Amazon Bedrock": 500
      },
      "input_tokens": 67718,
      "output_tokens": 2500,
      "preflight_cost_usd": 0.000493,
      "run": "original"
    },
    {
      "model": "anthropic/claude-sonnet-5",
      "name": "Anthropic: Claude Sonnet 5",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 493,
        "accuracy": 0.986,
        "accuracy_wilson_95": [
          0.9713869308622147,
          0.9932022102091564
        ],
        "total_seconds": 887.6002512009,
        "median_seconds": 1.718754583504051,
        "p95_seconds": 2.1111920829862356
      },
      "cost_usd": 0.19042,
      "cost_per_1000": 0.38084,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {},
      "successful_labels": 500,
      "complete_transport": true,
      "numeric_key_compliance": 1.0,
      "providers": {
        "Claude Platform on AWS": 500
      },
      "input_tokens": 87710,
      "output_tokens": 1500,
      "preflight_cost_usd": 0.001178,
      "run": "original"
    },
    {
      "model": "google/gemini-2.5-flash-lite",
      "name": "Google: Gemini 2.5 Flash Lite",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 494,
        "accuracy": 0.988,
        "accuracy_wilson_95": [
          0.9740696388835527,
          0.9944890048259724
        ],
        "total_seconds": 224.3929377865279,
        "median_seconds": 0.42386018752586097,
        "p95_seconds": 0.5583864590153098
      },
      "cost_usd": 0.0065048,
      "cost_per_1000": 0.0130096,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {},
      "successful_labels": 500,
      "complete_transport": true,
      "numeric_key_compliance": 0.732,
      "providers": {
        "Google": 500
      },
      "input_tokens": 62548,
      "output_tokens": 625,
      "preflight_cost_usd": 4.08e-05,
      "run": "original"
    },
    {
      "model": "google/gemini-2.5-flash",
      "name": "Google: Gemini 2.5 Flash",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 493,
        "accuracy": 0.986,
        "accuracy_wilson_95": [
          0.9713869308622147,
          0.9932022102091564
        ],
        "total_seconds": 248.76296508917585,
        "median_seconds": 0.46842281249701045,
        "p95_seconds": 0.6624899580492638
      },
      "cost_usd": 0.0202344,
      "cost_per_1000": 0.0404688,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {
        "Invalid or incomplete label": 1
      },
      "successful_labels": 499,
      "complete_transport": true,
      "numeric_key_compliance": 0.78,
      "providers": {
        "Google": 500
      },
      "input_tokens": 62548,
      "output_tokens": 588,
      "preflight_cost_usd": 0.0001264,
      "run": "original"
    },
    {
      "model": "google/gemini-3.1-flash-lite",
      "name": "Google: Gemini 3.1 Flash Lite",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 496,
        "accuracy": 0.992,
        "accuracy_wilson_95": [
          0.9796129641658948,
          0.9968846848199377
        ],
        "total_seconds": 341.9583707738784,
        "median_seconds": 0.6557884794892743,
        "p95_seconds": 0.9215647499659099
      },
      "cost_usd": 0.016387,
      "cost_per_1000": 0.032774,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {},
      "successful_labels": 500,
      "complete_transport": true,
      "numeric_key_compliance": 1.0,
      "providers": {
        "Google": 500
      },
      "input_tokens": 62548,
      "output_tokens": 500,
      "preflight_cost_usd": 0.0001015,
      "run": "original"
    },
    {
      "model": "qwen/qwen3-8b",
      "name": "Qwen: Qwen3 8B",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 496,
        "accuracy": 0.992,
        "accuracy_wilson_95": [
          0.9796129641658948,
          0.9968846848199377
        ],
        "total_seconds": 332.8775847758516,
        "median_seconds": 0.645228125504218,
        "p95_seconds": 0.8494925000122748
      },
      "cost_usd": 0.008338642,
      "cost_per_1000": 0.016677284,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {},
      "successful_labels": 500,
      "complete_transport": true,
      "numeric_key_compliance": 1.0,
      "providers": {
        "Alibaba": 500
      },
      "input_tokens": 69326,
      "output_tokens": 500,
      "preflight_cost_usd": 5.1324e-05,
      "run": "original"
    },
    {
      "model": "google/gemma-3-27b-it",
      "name": "Google: Gemma 3 27B",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 496,
        "accuracy": 0.992,
        "accuracy_wilson_95": [
          0.9796129641658948,
          0.9968846848199377
        ],
        "total_seconds": 161.43429854873102,
        "median_seconds": 0.29188295849598944,
        "p95_seconds": 0.5982790420530364
      },
      "cost_usd": 0.0070842,
      "cost_per_1000": 0.0141684,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {},
      "successful_labels": 500,
      "complete_transport": true,
      "numeric_key_compliance": 0.768,
      "providers": {
        "Nebius": 500
      },
      "input_tokens": 67548,
      "output_tokens": 1098,
      "preflight_cost_usd": 4.39e-05,
      "run": "original"
    },
    {
      "model": "meta-llama/llama-3.2-1b-instruct",
      "name": "Meta: Llama 3.2 1B Instruct",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 49,
        "accuracy": 0.098,
        "accuracy_wilson_95": [
          0.07492391886460623,
          0.12720605086648182
        ],
        "total_seconds": 1800.5567155783065,
        "median_seconds": 2.3418112919898704,
        "p95_seconds": 9.874993375036865
      },
      "cost_usd": 0.004288569,
      "cost_per_1000": 0.008577138,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {
        "Invalid or incomplete label": 398
      },
      "successful_labels": 102,
      "complete_transport": true,
      "numeric_key_compliance": 0.132,
      "providers": {
        "Cloudflare": 500
      },
      "input_tokens": 68706,
      "output_tokens": 12107,
      "preflight_cost_usd": 1.9086e-05,
      "run": "original"
    },
    {
      "model": "meta-llama/llama-3.2-3b-instruct",
      "name": "Meta: Llama 3.2 3B Instruct",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 439,
        "accuracy": 0.878,
        "accuracy_wilson_95": [
          0.8463952769261602,
          0.9038407216849063
        ],
        "total_seconds": 124.94482434284873,
        "median_seconds": 0.22435187551309355,
        "p95_seconds": 0.39086033403873444
      },
      "cost_usd": 0.0043843604,
      "cost_per_1000": 0.0087687208,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {},
      "successful_labels": 500,
      "complete_transport": true,
      "numeric_key_compliance": 0.788,
      "providers": {
        "Cloudflare": 500
      },
      "input_tokens": 78206,
      "output_tokens": 1205,
      "preflight_cost_usd": 2.68279e-05,
      "run": "original"
    },
    {
      "model": "meta-llama/llama-3.1-8b-instruct",
      "name": "Meta: Llama 3.1 8B Instruct",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 480,
        "accuracy": 0.96,
        "accuracy_wilson_95": [
          0.9390264041651492,
          0.9739592026102227
        ],
        "total_seconds": 59.26007748604752,
        "median_seconds": 0.09963452053489164,
        "p95_seconds": 0.15847958397353068
      },
      "cost_usd": 0.01534258,
      "cost_per_1000": 0.03068516,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {},
      "successful_labels": 500,
      "complete_transport": true,
      "numeric_key_compliance": 0.926,
      "providers": {
        "CoreWeave": 500
      },
      "input_tokens": 68706,
      "output_tokens": 1033,
      "preflight_cost_usd": 9.482e-05,
      "run": "original"
    },
    {
      "model": "meta-llama/llama-3.3-70b-instruct",
      "name": "Meta: Llama 3.3 70B Instruct",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 494,
        "accuracy": 0.988,
        "accuracy_wilson_95": [
          0.9740696388835527,
          0.9944890048259724
        ],
        "total_seconds": 317.2056205851841,
        "median_seconds": 0.4161451875115745,
        "p95_seconds": 1.668889874999877
      },
      "cost_usd": 0.01486035,
      "cost_per_1000": 0.0297207,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {},
      "successful_labels": 500,
      "complete_transport": true,
      "numeric_key_compliance": 0.994,
      "providers": {
        "AkashML": 474,
        "CoreWeave": 26
      },
      "input_tokens": 77712,
      "output_tokens": 1004,
      "preflight_cost_usd": 8.064e-05,
      "run": "original"
    },
    {
      "model": "meta-llama/llama-4-maverick",
      "name": "Meta: Llama 4 Maverick",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 494,
        "accuracy": 0.988,
        "accuracy_wilson_95": [
          0.9740696388835527,
          0.9944890048259724
        ],
        "total_seconds": 317.1911564466427,
        "median_seconds": 0.5745751454960555,
        "p95_seconds": 0.9321282090386376
      },
      "cost_usd": 0.013345125,
      "cost_per_1000": 0.02669025,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {},
      "successful_labels": 500,
      "complete_transport": true,
      "numeric_key_compliance": 1.0,
      "providers": {
        "DigitalOcean": 500
      },
      "input_tokens": 67694,
      "output_tokens": 1000,
      "preflight_cost_usd": 8.229e-05,
      "run": "original"
    },
    {
      "model": "mistralai/mistral-nemo",
      "name": "Mistral: Mistral Nemo",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 483,
        "accuracy": 0.966,
        "accuracy_wilson_95": [
          0.9462286344983655,
          0.9786654801914678
        ],
        "total_seconds": 191.0904435516568,
        "median_seconds": 0.3122280004899949,
        "p95_seconds": 0.5505917079863138
      },
      "cost_usd": 0.002757008,
      "cost_per_1000": 0.005514016,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {
        "Invalid or incomplete label": 2
      },
      "successful_labels": 498,
      "complete_transport": true,
      "numeric_key_compliance": 0.53,
      "providers": {
        "Io Net": 473,
        "DekaLLM": 27
      },
      "input_tokens": 65439,
      "output_tokens": 1285,
      "preflight_cost_usd": 1.6816e-05,
      "run": "original"
    },
    {
      "model": "mistralai/ministral-3b-2512",
      "name": "Mistral: Ministral 3 3B 2512",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 457,
        "accuracy": 0.914,
        "accuracy_wilson_95": [
          0.8861601885014425,
          0.9355268575963924
        ],
        "total_seconds": 207.9254797201138,
        "median_seconds": 0.3732963960210327,
        "p95_seconds": 0.6952591249719262
      },
      "cost_usd": 0.00606032,
      "cost_per_1000": 0.01212064,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {},
      "successful_labels": 500,
      "complete_transport": true,
      "numeric_key_compliance": 0.982,
      "providers": {
        "Mistral": 500
      },
      "input_tokens": 66385,
      "output_tokens": 1015,
      "preflight_cost_usd": 7.04e-06,
      "run": "original"
    },
    {
      "model": "mistralai/mistral-large-2512",
      "name": "Mistral: Mistral Large 3 2512",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 497,
        "accuracy": 0.994,
        "accuracy_wilson_95": [
          0.9825097478959467,
          0.9979574037280398
        ],
        "total_seconds": 349.18739466846455,
        "median_seconds": 0.628262458514655,
        "p95_seconds": 1.1614113749819808
      },
      "cost_usd": 0.0347195,
      "cost_per_1000": 0.069439,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {
        "Invalid or incomplete label": 1
      },
      "successful_labels": 499,
      "complete_transport": true,
      "numeric_key_compliance": 0.97,
      "providers": {
        "Mistral": 500
      },
      "input_tokens": 66385,
      "output_tokens": 1018,
      "preflight_cost_usd": 0.000217,
      "run": "original"
    },
    {
      "model": "qwen/qwen3-30b-a3b-instruct-2507",
      "name": "Qwen: Qwen3 30B A3B Instruct 2507",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 495,
        "accuracy": 0.99,
        "accuracy_wilson_95": [
          0.9768069002442693,
          0.9957212461034096
        ],
        "total_seconds": 972.880771335389,
        "median_seconds": 1.8299474790110253,
        "p95_seconds": 3.2622302920208313
      },
      "cost_usd": 0.00424426785,
      "cost_per_1000": 0.0084885357,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {},
      "successful_labels": 500,
      "complete_transport": true,
      "numeric_key_compliance": 0.994,
      "providers": {
        "SiliconFlow": 147,
        "StreamLake": 353
      },
      "input_tokens": 67326,
      "output_tokens": 831,
      "preflight_cost_usd": 2.114055e-05,
      "run": "original"
    },
    {
      "model": "deepseek/deepseek-v4-flash-0731",
      "name": "DeepSeek: DeepSeek V4 Flash 0731",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 491,
        "accuracy": 0.982,
        "accuracy_wilson_95": [
          0.9661483290912607,
          0.990501806703803
        ],
        "total_seconds": 726.8716526252683,
        "median_seconds": 0.5665040830208454,
        "p95_seconds": 5.947828042029869
      },
      "cost_usd": 0.0040104314,
      "cost_per_1000": 0.0080208628,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {},
      "successful_labels": 500,
      "complete_transport": true,
      "numeric_key_compliance": 1.0,
      "providers": {
        "Relace": 485,
        "Inceptron": 15
      },
      "input_tokens": 64924,
      "output_tokens": 1000,
      "preflight_cost_usd": 2.44538e-05,
      "run": "original"
    },
    {
      "model": "amazon/nova-micro-v1",
      "name": "Amazon: Nova Micro 1.0",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 491,
        "accuracy": 0.982,
        "accuracy_wilson_95": [
          0.9661483290912607,
          0.990501806703803
        ],
        "total_seconds": 210.47532196046086,
        "median_seconds": 0.4090766459994484,
        "p95_seconds": 0.5453609160031192
      },
      "cost_usd": 0.00250922,
      "cost_per_1000": 0.00501844,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {
        "Invalid or incomplete label": 1
      },
      "successful_labels": 499,
      "complete_transport": true,
      "numeric_key_compliance": 0.256,
      "providers": {
        "Amazon Bedrock": 500
      },
      "input_tokens": 65660,
      "output_tokens": 1508,
      "preflight_cost_usd": 1.498e-05,
      "run": "original"
    },
    {
      "model": "amazon/nova-lite-v1",
      "name": "Amazon: Nova Lite 1.0",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 493,
        "accuracy": 0.986,
        "accuracy_wilson_95": [
          0.9713869308622147,
          0.9932022102091564
        ],
        "total_seconds": 410.10159253078746,
        "median_seconds": 0.4755054999841377,
        "p95_seconds": 2.969197292055469
      },
      "cost_usd": 0.00429888,
      "cost_per_1000": 0.00859776,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {},
      "successful_labels": 500,
      "complete_transport": true,
      "numeric_key_compliance": 0.978,
      "providers": {
        "Amazon Bedrock": 500
      },
      "input_tokens": 65660,
      "output_tokens": 1497,
      "preflight_cost_usd": 2.568e-05,
      "run": "original"
    },
    {
      "model": "cohere/command-r7b-12-2024",
      "name": "Cohere: Command R7B (12-2024)",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 485,
        "accuracy": 0.97,
        "accuracy_wilson_95": [
          0.9510963331640543,
          0.9817367868020865
        ],
        "total_seconds": 121.57807346608024,
        "median_seconds": 0.22833731249556877,
        "p95_seconds": 0.31850850000046194
      },
      "cost_usd": 0.00241245,
      "cost_per_1000": 0.0048249,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {},
      "successful_labels": 500,
      "complete_transport": true,
      "numeric_key_compliance": 1.0,
      "providers": {
        "Cohere": 500
      },
      "input_tokens": 62332,
      "output_tokens": 500,
      "preflight_cost_usd": 1.48875e-05,
      "run": "original"
    },
    {
      "model": "nvidia/nemotron-3.5-lightning",
      "name": "NVIDIA: Nemotron 3.5 Lightning",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 480,
        "accuracy": 0.96,
        "accuracy_wilson_95": [
          0.9390264041651492,
          0.9739592026102227
        ],
        "total_seconds": 75.26812473102473,
        "median_seconds": 0.13728989599621855,
        "p95_seconds": 0.22562079096678644
      },
      "cost_usd": 0.00525255,
      "cost_per_1000": 0.0105051,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {},
      "successful_labels": 500,
      "complete_transport": true,
      "numeric_key_compliance": 0.778,
      "providers": {
        "CoreWeave": 500
      },
      "input_tokens": 71885,
      "output_tokens": 1103,
      "preflight_cost_usd": 3.261e-05,
      "run": "original"
    },
    {
      "model": "anthropic/claude-fable-5",
      "name": "Claude Fable 5",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 497,
        "accuracy": 0.994,
        "accuracy_wilson_95": [
          0.9825097478959467,
          0.9979574037280398
        ],
        "total_seconds": 1669.5903178428998,
        "median_seconds": 3.208765791991027,
        "p95_seconds": 4.232279624964576
      },
      "cost_per_1000": 1.9394,
      "errors": {},
      "run": "leading"
    },
    {
      "model": "openai/gpt-6-astra",
      "name": "GPT-6 Astra",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 498,
        "accuracy": 0.996,
        "accuracy_wilson_95": [
          0.9855342847848231,
          0.9989023694773173
        ],
        "total_seconds": 768.3140909158392,
        "median_seconds": 1.3488265209889505,
        "p95_seconds": 2.720605082984548
      },
      "cost_per_1000": 1.5839,
      "errors": {},
      "run": "leading"
    },
    {
      "model": "deepseek/deepseek-v4-pro-0813",
      "name": "DeepSeek V4 Pro (0813)",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 499,
        "accuracy": 0.998,
        "accuracy_wilson_95": [
          0.9887592932948532,
          0.9996468636054407
        ],
        "total_seconds": 2295.1692707003676,
        "median_seconds": 3.2944941879832186,
        "p95_seconds": 12.528947709011845
      },
      "cost_per_1000": 0.4478047752,
      "errors": {},
      "run": "leading"
    }
  ]
}