{
  "date_utc": "2026-09-17",
  "completed_at": "2026-09-17T02:42:10.022536+00:00",
  "checkpoint_sha256": "ec83247177a6919c17944093fdd02d3258906dc809c16b326e0dfd879275efeb",
  "dataset_manifest_sha256": "43fc867f7e773012cbf202877a5267d3e496947f96cdee549228b237cb9ff73b",
  "hosted_models": 30,
  "cases_per_model": 500,
  "actual_hosted_test_charge_usd": null,
  "known_hosted_test_charge_usd": 0.8250638674,
  "unverified_test_charges": 13,
  "transport_failures": 13,
  "actual_preflight_charge_usd": 0.00503429325,
  "local_hardware": "Apple M4 Max, MPS, torch 2.8.0; warm sequential inference",
  "local_seconds_per_1000": 166.1051957635209,
  "local_cost_assumptions": {
    "allocated_hourly_usd": 0.5,
    "utilization": 1
  },
  "local_cost_rank": 16,
  "local_median_latency_rank": 3,
  "rank_denominator": 28,
  "latency_rank_denominator": 31,
  "fastest_hosted": "meta-llama/llama-3.1-8b-instruct",
  "cheapest_hosted": "cohere/command-r7b-12-2024",
  "fastest_hosted_to_local_latency_ratio": 0.6202994138285007,
  "local_vs_cheapest_cost_ratio": 4.781480668670189,
  "entries": [
    {
      "model": "openauditor/v0.3.0",
      "name": "OpenAuditor 0.6B",
      "kind": "local",
      "measurement": {
        "n": 500,
        "correct": 486,
        "accuracy": 0.972,
        "accuracy_wilson_95": [
          0.9535535973250511,
          0.9832490252792436
        ],
        "total_seconds": 83.05259788176045,
        "median_seconds": 0.16062327049439773,
        "p95_seconds": 0.22153566702036187
      },
      "cost_per_1000": 0.023070166078266792,
      "cost_basis": "Scenario: $0.50 per allocated machine-hour, 100% utilization",
      "errors": {},
      "numeric_key_compliance": null
    },
    {
      "model": "openai/gpt-4.1-nano",
      "name": "OpenAI: GPT-4.1 Nano",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 494,
        "accuracy": 0.988,
        "accuracy_wilson_95": [
          0.9740696388835527,
          0.9944890048259724
        ],
        "total_seconds": 330.5819375384017,
        "median_seconds": 0.6051433540415019,
        "p95_seconds": 1.0113971249666065
      },
      "cost_usd": 0.007021,
      "cost_per_1000": 0.014042,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {},
      "successful_labels": 500,
      "complete_transport": true,
      "numeric_key_compliance": 0.99,
      "providers": {
        "OpenAI": 500
      },
      "input_tokens": 66190,
      "output_tokens": 1005,
      "preflight_cost_usd": 4.32e-05
    },
    {
      "model": "openai/gpt-4.1-mini",
      "name": "OpenAI: GPT-4.1 Mini",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 496,
        "accuracy": 0.992,
        "accuracy_wilson_95": [
          0.9796129641658948,
          0.9968846848199377
        ],
        "total_seconds": 360.4603449405404,
        "median_seconds": 0.6674674169917125,
        "p95_seconds": 1.064537999976892
      },
      "cost_usd": 0.028076,
      "cost_per_1000": 0.056152,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {
        "Invalid or incomplete label": 1
      },
      "successful_labels": 499,
      "complete_transport": true,
      "numeric_key_compliance": 0.998,
      "providers": {
        "OpenAI": 500
      },
      "input_tokens": 66190,
      "output_tokens": 1000,
      "preflight_cost_usd": 0.0001728
    },
    {
      "model": "openai/gpt-4.1",
      "name": "OpenAI: GPT-4.1",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 495,
        "accuracy": 0.99,
        "accuracy_wilson_95": [
          0.9768069002442693,
          0.9957212461034096
        ],
        "total_seconds": 359.2503143767826,
        "median_seconds": 0.6558008959982544,
        "p95_seconds": 1.0276261669932865
      },
      "cost_usd": 0.14038,
      "cost_per_1000": 0.28076,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {},
      "successful_labels": 500,
      "complete_transport": true,
      "numeric_key_compliance": 1.0,
      "providers": {
        "OpenAI": 500
      },
      "input_tokens": 66190,
      "output_tokens": 1000,
      "preflight_cost_usd": 0.000864
    },
    {
      "model": "openai/gpt-4o-mini",
      "name": "OpenAI: GPT-4o-mini",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 492,
        "accuracy": 0.984,
        "accuracy_wilson_95": [
          0.9687488914666058,
          0.9918707469666117
        ],
        "total_seconds": 263.8602219266468,
        "median_seconds": 0.5019848960218951,
        "p95_seconds": 0.6848381250165403
      },
      "cost_usd": 0.0102411,
      "cost_per_1000": 0.0204822,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {
        "Invalid or incomplete label": 2
      },
      "successful_labels": 498,
      "complete_transport": true,
      "numeric_key_compliance": 0.964,
      "providers": {
        "OpenAI": 500
      },
      "input_tokens": 66190,
      "output_tokens": 521,
      "preflight_cost_usd": 6.3e-05
    },
    {
      "model": "openai/gpt-5.6-luna",
      "name": "OpenAI: GPT-5.6 Luna",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 491,
        "accuracy": 0.982,
        "accuracy_wilson_95": [
          0.9661483290912607,
          0.990501806703803
        ],
        "total_seconds": 858.9793787939707,
        "median_seconds": 1.1447858744941186,
        "p95_seconds": 3.4226961249951273
      },
      "cost_usd": 0.016138,
      "cost_per_1000": 0.032276,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {},
      "successful_labels": 500,
      "complete_transport": true,
      "numeric_key_compliance": 1.0,
      "providers": {
        "OpenAI": 500
      },
      "input_tokens": 65690,
      "output_tokens": 2500,
      "preflight_cost_usd": 9.9e-05
    },
    {
      "model": "openai/gpt-5.6-sol",
      "name": "OpenAI: GPT-5.6 Sol",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 495,
        "accuracy": 0.99,
        "accuracy_wilson_95": [
          0.9768069002442693,
          0.9957212461034096
        ],
        "total_seconds": 1662.7701584976166,
        "median_seconds": 3.400876541476464,
        "p95_seconds": 6.904222582990769
      },
      "cost_usd": 0.15638,
      "cost_per_1000": 0.31276,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {},
      "successful_labels": 500,
      "complete_transport": true,
      "numeric_key_compliance": 0.998,
      "providers": {
        "OpenAI": 500
      },
      "input_tokens": 65690,
      "output_tokens": 2500,
      "preflight_cost_usd": 0.00096
    },
    {
      "model": "anthropic/claude-haiku-4.5",
      "name": "Anthropic: Claude Haiku 4.5",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 494,
        "accuracy": 0.988,
        "accuracy_wilson_95": [
          0.9740696388835527,
          0.9944890048259724
        ],
        "total_seconds": 409.28693951509194,
        "median_seconds": 0.7780484375252854,
        "p95_seconds": 0.9811019590124488
      },
      "cost_usd": 0.080218,
      "cost_per_1000": 0.160436,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {},
      "successful_labels": 500,
      "complete_transport": true,
      "numeric_key_compliance": 1.0,
      "providers": {
        "Amazon Bedrock": 500
      },
      "input_tokens": 67718,
      "output_tokens": 2500,
      "preflight_cost_usd": 0.000493
    },
    {
      "model": "anthropic/claude-sonnet-5",
      "name": "Anthropic: Claude Sonnet 5",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 493,
        "accuracy": 0.986,
        "accuracy_wilson_95": [
          0.9713869308622147,
          0.9932022102091564
        ],
        "total_seconds": 887.6002512009,
        "median_seconds": 1.718754583504051,
        "p95_seconds": 2.1111920829862356
      },
      "cost_usd": 0.19042,
      "cost_per_1000": 0.38084,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {},
      "successful_labels": 500,
      "complete_transport": true,
      "numeric_key_compliance": 1.0,
      "providers": {
        "Claude Platform on AWS": 500
      },
      "input_tokens": 87710,
      "output_tokens": 1500,
      "preflight_cost_usd": 0.001178
    },
    {
      "model": "google/gemini-2.5-flash-lite",
      "name": "Google: Gemini 2.5 Flash Lite",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 494,
        "accuracy": 0.988,
        "accuracy_wilson_95": [
          0.9740696388835527,
          0.9944890048259724
        ],
        "total_seconds": 224.3929377865279,
        "median_seconds": 0.42386018752586097,
        "p95_seconds": 0.5583864590153098
      },
      "cost_usd": 0.0065048,
      "cost_per_1000": 0.0130096,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {},
      "successful_labels": 500,
      "complete_transport": true,
      "numeric_key_compliance": 0.732,
      "providers": {
        "Google": 500
      },
      "input_tokens": 62548,
      "output_tokens": 625,
      "preflight_cost_usd": 4.08e-05
    },
    {
      "model": "google/gemini-2.5-flash",
      "name": "Google: Gemini 2.5 Flash",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 493,
        "accuracy": 0.986,
        "accuracy_wilson_95": [
          0.9713869308622147,
          0.9932022102091564
        ],
        "total_seconds": 248.76296508917585,
        "median_seconds": 0.46842281249701045,
        "p95_seconds": 0.6624899580492638
      },
      "cost_usd": 0.0202344,
      "cost_per_1000": 0.0404688,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {
        "Invalid or incomplete label": 1
      },
      "successful_labels": 499,
      "complete_transport": true,
      "numeric_key_compliance": 0.78,
      "providers": {
        "Google": 500
      },
      "input_tokens": 62548,
      "output_tokens": 588,
      "preflight_cost_usd": 0.0001264
    },
    {
      "model": "google/gemini-3.1-flash-lite",
      "name": "Google: Gemini 3.1 Flash Lite",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 496,
        "accuracy": 0.992,
        "accuracy_wilson_95": [
          0.9796129641658948,
          0.9968846848199377
        ],
        "total_seconds": 341.9583707738784,
        "median_seconds": 0.6557884794892743,
        "p95_seconds": 0.9215647499659099
      },
      "cost_usd": 0.016387,
      "cost_per_1000": 0.032774,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {},
      "successful_labels": 500,
      "complete_transport": true,
      "numeric_key_compliance": 1.0,
      "providers": {
        "Google": 500
      },
      "input_tokens": 62548,
      "output_tokens": 500,
      "preflight_cost_usd": 0.0001015
    },
    {
      "model": "qwen/qwen3-8b",
      "name": "Qwen: Qwen3 8B",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 496,
        "accuracy": 0.992,
        "accuracy_wilson_95": [
          0.9796129641658948,
          0.9968846848199377
        ],
        "total_seconds": 332.8775847758516,
        "median_seconds": 0.645228125504218,
        "p95_seconds": 0.8494925000122748
      },
      "cost_usd": 0.008338642,
      "cost_per_1000": 0.016677284,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {},
      "successful_labels": 500,
      "complete_transport": true,
      "numeric_key_compliance": 1.0,
      "providers": {
        "Alibaba": 500
      },
      "input_tokens": 69326,
      "output_tokens": 500,
      "preflight_cost_usd": 5.1324e-05
    },
    {
      "model": "google/gemma-3-27b-it",
      "name": "Google: Gemma 3 27B",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 496,
        "accuracy": 0.992,
        "accuracy_wilson_95": [
          0.9796129641658948,
          0.9968846848199377
        ],
        "total_seconds": 161.43429854873102,
        "median_seconds": 0.29188295849598944,
        "p95_seconds": 0.5982790420530364
      },
      "cost_usd": 0.0070842,
      "cost_per_1000": 0.0141684,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {},
      "successful_labels": 500,
      "complete_transport": true,
      "numeric_key_compliance": 0.768,
      "providers": {
        "Nebius": 500
      },
      "input_tokens": 67548,
      "output_tokens": 1098,
      "preflight_cost_usd": 4.39e-05
    },
    {
      "model": "meta-llama/llama-3.2-1b-instruct",
      "name": "Meta: Llama 3.2 1B Instruct",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 49,
        "accuracy": 0.098,
        "accuracy_wilson_95": [
          0.07492391886460623,
          0.12720605086648182
        ],
        "total_seconds": 1800.5567155783065,
        "median_seconds": 2.3418112919898704,
        "p95_seconds": 9.874993375036865
      },
      "cost_usd": 0.004288569,
      "cost_per_1000": 0.008577138,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {
        "Invalid or incomplete label": 398
      },
      "successful_labels": 102,
      "complete_transport": true,
      "numeric_key_compliance": 0.132,
      "providers": {
        "Cloudflare": 500
      },
      "input_tokens": 68706,
      "output_tokens": 12107,
      "preflight_cost_usd": 1.9086e-05
    },
    {
      "model": "meta-llama/llama-3.2-3b-instruct",
      "name": "Meta: Llama 3.2 3B Instruct",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 439,
        "accuracy": 0.878,
        "accuracy_wilson_95": [
          0.8463952769261602,
          0.9038407216849063
        ],
        "total_seconds": 124.94482434284873,
        "median_seconds": 0.22435187551309355,
        "p95_seconds": 0.39086033403873444
      },
      "cost_usd": 0.0043843604,
      "cost_per_1000": 0.0087687208,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {},
      "successful_labels": 500,
      "complete_transport": true,
      "numeric_key_compliance": 0.788,
      "providers": {
        "Cloudflare": 500
      },
      "input_tokens": 78206,
      "output_tokens": 1205,
      "preflight_cost_usd": 2.68279e-05
    },
    {
      "model": "meta-llama/llama-3.1-8b-instruct",
      "name": "Meta: Llama 3.1 8B Instruct",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 480,
        "accuracy": 0.96,
        "accuracy_wilson_95": [
          0.9390264041651492,
          0.9739592026102227
        ],
        "total_seconds": 59.26007748604752,
        "median_seconds": 0.09963452053489164,
        "p95_seconds": 0.15847958397353068
      },
      "cost_usd": 0.01534258,
      "cost_per_1000": 0.03068516,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {},
      "successful_labels": 500,
      "complete_transport": true,
      "numeric_key_compliance": 0.926,
      "providers": {
        "CoreWeave": 500
      },
      "input_tokens": 68706,
      "output_tokens": 1033,
      "preflight_cost_usd": 9.482e-05
    },
    {
      "model": "meta-llama/llama-3.3-70b-instruct",
      "name": "Meta: Llama 3.3 70B Instruct",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 494,
        "accuracy": 0.988,
        "accuracy_wilson_95": [
          0.9740696388835527,
          0.9944890048259724
        ],
        "total_seconds": 317.2056205851841,
        "median_seconds": 0.4161451875115745,
        "p95_seconds": 1.668889874999877
      },
      "cost_usd": 0.01486035,
      "cost_per_1000": 0.0297207,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {},
      "successful_labels": 500,
      "complete_transport": true,
      "numeric_key_compliance": 0.994,
      "providers": {
        "AkashML": 474,
        "CoreWeave": 26
      },
      "input_tokens": 77712,
      "output_tokens": 1004,
      "preflight_cost_usd": 8.064e-05
    },
    {
      "model": "meta-llama/llama-4-maverick",
      "name": "Meta: Llama 4 Maverick",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 494,
        "accuracy": 0.988,
        "accuracy_wilson_95": [
          0.9740696388835527,
          0.9944890048259724
        ],
        "total_seconds": 317.1911564466427,
        "median_seconds": 0.5745751454960555,
        "p95_seconds": 0.9321282090386376
      },
      "cost_usd": 0.013345125,
      "cost_per_1000": 0.02669025,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {},
      "successful_labels": 500,
      "complete_transport": true,
      "numeric_key_compliance": 1.0,
      "providers": {
        "DigitalOcean": 500
      },
      "input_tokens": 67694,
      "output_tokens": 1000,
      "preflight_cost_usd": 8.229e-05
    },
    {
      "model": "mistralai/mistral-nemo",
      "name": "Mistral: Mistral Nemo",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 483,
        "accuracy": 0.966,
        "accuracy_wilson_95": [
          0.9462286344983655,
          0.9786654801914678
        ],
        "total_seconds": 191.0904435516568,
        "median_seconds": 0.3122280004899949,
        "p95_seconds": 0.5505917079863138
      },
      "cost_usd": 0.002757008,
      "cost_per_1000": 0.005514016,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {
        "Invalid or incomplete label": 2
      },
      "successful_labels": 498,
      "complete_transport": true,
      "numeric_key_compliance": 0.53,
      "providers": {
        "Io Net": 473,
        "DekaLLM": 27
      },
      "input_tokens": 65439,
      "output_tokens": 1285,
      "preflight_cost_usd": 1.6816e-05
    },
    {
      "model": "mistralai/mistral-small-3.2-24b-instruct",
      "name": "Mistral: Mistral Small 3.2 24B",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 477,
        "accuracy": 0.954,
        "accuracy_wilson_95": [
          0.9319222072317267,
          0.9691548916291838
        ],
        "total_seconds": 493.9661440376076,
        "median_seconds": 0.6996111879998352,
        "p95_seconds": 2.9433915420086123
      },
      "cost_usd": null,
      "cost_per_1000": null,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {
        "Invalid or incomplete label": 8,
        "OpenRouter HTTP 429": 3
      },
      "successful_labels": 489,
      "complete_transport": false,
      "numeric_key_compliance": 0.924,
      "providers": {
        "Venice": 497,
        "unknown": 3
      },
      "input_tokens": 65985,
      "output_tokens": 1049,
      "preflight_cost_usd": 4.04375e-05
    },
    {
      "model": "mistralai/ministral-3b-2512",
      "name": "Mistral: Ministral 3 3B 2512",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 457,
        "accuracy": 0.914,
        "accuracy_wilson_95": [
          0.8861601885014425,
          0.9355268575963924
        ],
        "total_seconds": 207.9254797201138,
        "median_seconds": 0.3732963960210327,
        "p95_seconds": 0.6952591249719262
      },
      "cost_usd": 0.00606032,
      "cost_per_1000": 0.01212064,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {},
      "successful_labels": 500,
      "complete_transport": true,
      "numeric_key_compliance": 0.982,
      "providers": {
        "Mistral": 500
      },
      "input_tokens": 66385,
      "output_tokens": 1015,
      "preflight_cost_usd": 7.04e-06
    },
    {
      "model": "mistralai/mistral-large-2512",
      "name": "Mistral: Mistral Large 3 2512",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 497,
        "accuracy": 0.994,
        "accuracy_wilson_95": [
          0.9825097478959467,
          0.9979574037280398
        ],
        "total_seconds": 349.18739466846455,
        "median_seconds": 0.628262458514655,
        "p95_seconds": 1.1614113749819808
      },
      "cost_usd": 0.0347195,
      "cost_per_1000": 0.069439,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {
        "Invalid or incomplete label": 1
      },
      "successful_labels": 499,
      "complete_transport": true,
      "numeric_key_compliance": 0.97,
      "providers": {
        "Mistral": 500
      },
      "input_tokens": 66385,
      "output_tokens": 1018,
      "preflight_cost_usd": 0.000217
    },
    {
      "model": "qwen/qwen3.7-flash",
      "name": "Qwen: Qwen3.7 Flash",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 483,
        "accuracy": 0.966,
        "accuracy_wilson_95": [
          0.9462286344983655,
          0.9786654801914678
        ],
        "total_seconds": 427.76769975124625,
        "median_seconds": 0.6487369170063175,
        "p95_seconds": 1.1781788329826668
      },
      "cost_usd": null,
      "cost_per_1000": null,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {
        "OpenRouter HTTP 429": 9
      },
      "successful_labels": 491,
      "complete_transport": false,
      "numeric_key_compliance": 0.98,
      "providers": {
        "Alibaba": 491,
        "unknown": 9
      },
      "input_tokens": 67995,
      "output_tokens": 493,
      "preflight_cost_usd": 1.32e-05
    },
    {
      "model": "qwen/qwen3.8-flash",
      "name": "Qwen: Qwen3.8 Flash",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 495,
        "accuracy": 0.99,
        "accuracy_wilson_95": [
          0.9768069002442693,
          0.9957212461034096
        ],
        "total_seconds": 530.1021604596172,
        "median_seconds": 0.7433171665179543,
        "p95_seconds": 2.022013458015863
      },
      "cost_usd": null,
      "cost_per_1000": null,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {
        "OpenRouter HTTP 429": 1
      },
      "successful_labels": 499,
      "complete_transport": false,
      "numeric_key_compliance": 0.998,
      "providers": {
        "Alibaba": 499,
        "unknown": 1
      },
      "input_tokens": 69122,
      "output_tokens": 499,
      "preflight_cost_usd": 6.546e-05
    },
    {
      "model": "qwen/qwen3-30b-a3b-instruct-2507",
      "name": "Qwen: Qwen3 30B A3B Instruct 2507",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 495,
        "accuracy": 0.99,
        "accuracy_wilson_95": [
          0.9768069002442693,
          0.9957212461034096
        ],
        "total_seconds": 972.880771335389,
        "median_seconds": 1.8299474790110253,
        "p95_seconds": 3.2622302920208313
      },
      "cost_usd": 0.00424426785,
      "cost_per_1000": 0.0084885357,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {},
      "successful_labels": 500,
      "complete_transport": true,
      "numeric_key_compliance": 0.994,
      "providers": {
        "SiliconFlow": 147,
        "StreamLake": 353
      },
      "input_tokens": 67326,
      "output_tokens": 831,
      "preflight_cost_usd": 2.114055e-05
    },
    {
      "model": "deepseek/deepseek-v4-flash-0731",
      "name": "DeepSeek: DeepSeek V4 Flash 0731",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 491,
        "accuracy": 0.982,
        "accuracy_wilson_95": [
          0.9661483290912607,
          0.990501806703803
        ],
        "total_seconds": 726.8716526252683,
        "median_seconds": 0.5665040830208454,
        "p95_seconds": 5.947828042029869
      },
      "cost_usd": 0.0040104314,
      "cost_per_1000": 0.0080208628,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {},
      "successful_labels": 500,
      "complete_transport": true,
      "numeric_key_compliance": 1.0,
      "providers": {
        "Relace": 485,
        "Inceptron": 15
      },
      "input_tokens": 64924,
      "output_tokens": 1000,
      "preflight_cost_usd": 2.44538e-05
    },
    {
      "model": "amazon/nova-micro-v1",
      "name": "Amazon: Nova Micro 1.0",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 491,
        "accuracy": 0.982,
        "accuracy_wilson_95": [
          0.9661483290912607,
          0.990501806703803
        ],
        "total_seconds": 210.47532196046086,
        "median_seconds": 0.4090766459994484,
        "p95_seconds": 0.5453609160031192
      },
      "cost_usd": 0.00250922,
      "cost_per_1000": 0.00501844,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {
        "Invalid or incomplete label": 1
      },
      "successful_labels": 499,
      "complete_transport": true,
      "numeric_key_compliance": 0.256,
      "providers": {
        "Amazon Bedrock": 500
      },
      "input_tokens": 65660,
      "output_tokens": 1508,
      "preflight_cost_usd": 1.498e-05
    },
    {
      "model": "amazon/nova-lite-v1",
      "name": "Amazon: Nova Lite 1.0",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 493,
        "accuracy": 0.986,
        "accuracy_wilson_95": [
          0.9713869308622147,
          0.9932022102091564
        ],
        "total_seconds": 410.10159253078746,
        "median_seconds": 0.4755054999841377,
        "p95_seconds": 2.969197292055469
      },
      "cost_usd": 0.00429888,
      "cost_per_1000": 0.00859776,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {},
      "successful_labels": 500,
      "complete_transport": true,
      "numeric_key_compliance": 0.978,
      "providers": {
        "Amazon Bedrock": 500
      },
      "input_tokens": 65660,
      "output_tokens": 1497,
      "preflight_cost_usd": 2.568e-05
    },
    {
      "model": "cohere/command-r7b-12-2024",
      "name": "Cohere: Command R7B (12-2024)",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 485,
        "accuracy": 0.97,
        "accuracy_wilson_95": [
          0.9510963331640543,
          0.9817367868020865
        ],
        "total_seconds": 121.57807346608024,
        "median_seconds": 0.22833731249556877,
        "p95_seconds": 0.31850850000046194
      },
      "cost_usd": 0.00241245,
      "cost_per_1000": 0.0048249,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {},
      "successful_labels": 500,
      "complete_transport": true,
      "numeric_key_compliance": 1.0,
      "providers": {
        "Cohere": 500
      },
      "input_tokens": 62332,
      "output_tokens": 500,
      "preflight_cost_usd": 1.48875e-05
    },
    {
      "model": "nvidia/nemotron-3.5-lightning",
      "name": "NVIDIA: Nemotron 3.5 Lightning",
      "kind": "hosted",
      "measurement": {
        "n": 500,
        "correct": 480,
        "accuracy": 0.96,
        "accuracy_wilson_95": [
          0.9390264041651492,
          0.9739592026102227
        ],
        "total_seconds": 75.26812473102473,
        "median_seconds": 0.13728989599621855,
        "p95_seconds": 0.22562079096678644
      },
      "cost_usd": 0.00525255,
      "cost_per_1000": 0.0105051,
      "cost_basis": "Actual per-request API charges scaled to 1,000 cases",
      "errors": {},
      "successful_labels": 500,
      "complete_transport": true,
      "numeric_key_compliance": 0.778,
      "providers": {
        "CoreWeave": 500
      },
      "input_tokens": 71885,
      "output_tokens": 1103,
      "preflight_cost_usd": 3.261e-05
    }
  ],
  "hybrid_haiku_replay": {
    "n": 500,
    "local": {
      "n": 500,
      "correct": 486,
      "accuracy": 0.972,
      "accuracy_wilson_95": [
        0.9535535973250511,
        0.9832490252792436
      ],
      "total_seconds": 83.05259788176045,
      "median_seconds": 0.16062327049439773,
      "p95_seconds": 0.22153566702036187
    },
    "hosted": {
      "n": 500,
      "correct": 494,
      "accuracy": 0.988,
      "accuracy_wilson_95": [
        0.9740696388835527,
        0.9944890048259724
      ],
      "total_seconds": 409.28693951509194,
      "median_seconds": 0.7780484375252854,
      "p95_seconds": 0.9811019590124488
    },
    "hybrid_replay": {
      "n": 500,
      "correct": 492,
      "accuracy": 0.984,
      "accuracy_wilson_95": [
        0.9687488914666058,
        0.9918707469666117
      ],
      "total_seconds": 91.14177325787023,
      "median_seconds": 0.1622084794798866,
      "p95_seconds": 0.22851016698405147
    },
    "threshold": 0.8502035140991211,
    "local_acceptances": 490,
    "escalations": 10,
    "paid_calls_avoided_fraction": 0.98,
    "actual_hosted_test_cost_usd": 0.080218,
    "hosted_usd_per_1000": 0.160436,
    "replayed_fallback_cost_usd": 0.0016250000000000001,
    "paired_correctness": {
      "both_correct": 481,
      "only_local_correct": 5,
      "only_hosted_correct": 13,
      "both_wrong": 1
    },
    "break_even_hourly_rate_at_full_utilization": {
      "local": 3.4771314488095175,
      "hybrid": 3.406694157873375
    },
    "local_cost_scenarios": [
      {
        "hourly_rate": 0.1,
        "utilization": 1,
        "local_usd_per_1000": 0.004614033215653359,
        "hybrid_usd_per_1000": 0.00786403321565336,
        "local_savings_fraction": 0.9712406615993084,
        "hybrid_savings_fraction": 0.9509833627387035
      },
      {
        "hourly_rate": 0.5,
        "utilization": 1,
        "local_usd_per_1000": 0.023070166078266792,
        "hybrid_usd_per_1000": 0.02632016607826679,
        "local_savings_fraction": 0.856203307996542,
        "hybrid_savings_fraction": 0.8359460091359371
      },
      {
        "hourly_rate": 1,
        "utilization": 1,
        "local_usd_per_1000": 0.046140332156533584,
        "hybrid_usd_per_1000": 0.04939033215653359,
        "local_savings_fraction": 0.712406615993084,
        "hybrid_savings_fraction": 0.6921493171324791
      },
      {
        "hourly_rate": 0.5,
        "utilization": 0.1,
        "local_usd_per_1000": 0.2307016607826679,
        "hybrid_usd_per_1000": 0.2339516607826679,
        "local_savings_fraction": -0.43796692003458015,
        "hybrid_savings_fraction": -0.45822421889518505
      }
    ],
    "scope": "Hybrid is a replay of measured predictions and request charges; every test case was actually sent to the hosted model for comparison. Local hardware rates are assumptions, not measured electricity/depreciation or a vendor quote. Measured warm sequential Mac compute is used; idle allocation is represented by utilization. Training, setup, operations, loading, and error-remediation costs are excluded. Lower utilization can eliminate apparent savings."
  },
  "limitations": [
    "One frozen 500-case, eight-choice, in-scope CLINC intent-routing test, not production traffic.",
    "Local checkpoint trained on CLINC training data; upstream training text and release-evaluation cases excluded from this test. Foundation-model pretraining exposure is unknown.",
    "Purposive panel of available non-mandatory-reasoning models, not all APIs or frontier models.",
    "Local compute versus network-inclusive API latency; different execution paths, no server/GPU speed claim.",
    "Local warm cost scenario excludes training, setup, operations, loading, and error remediation. API charges exclude credit purchase fees/tax.",
    "Single run, default provider routing, global concurrency 24 and per-model concurrency 4. Rankings can change by host, geography, load and time.",
    "Exact intent-name normalization is allowed for every API; numeric-key compliance is reported separately.",
    "Four unavailable initial entries and the preflight-driven protocol amendment are disclosed in protocol.json. The aborted preliminary run is excluded from the fresh final comparison.",
    "Accuracy intervals describe sampling uncertainty only. Point estimates do not establish statistical equivalence or general superiority."
  ]
}