{
 "version": "0.3.0",
 "checkpoint_sha256": "f64a77638758fe0da5a8400b5142a1b3a1a2f481b8fb7f8099f5fb88265b9992",
 "tasks": [
  {
   "task": "clinc_8way",
   "label": "CLINC: 8-way routing",
   "split": "test",
   "n": 150,
   "trained": 0.9666666666666667,
   "reference": 0.54,
   "trained_nll": 0.16479856473442542,
   "reference_nll": 1.5749481987953187
  },
  {
   "task": "emotion_presence",
   "label": "Emotion presence",
   "split": "test",
   "n": 150,
   "trained": 0.8866666666666667,
   "reference": 0.5066666666666667,
   "trained_nll": 0.2896981178333711,
   "reference_nll": 0.6983972714344661
  },
  {
   "task": "commonsense_qa",
   "label": "CommonsenseQA",
   "split": "test",
   "n": 150,
   "trained": 0.56,
   "reference": 0.23333333333333334,
   "trained_nll": 1.0263930546321596,
   "reference_nll": 1.548451331059138
  },
  {
   "task": "threshold_policy",
   "label": "Synthetic threshold",
   "split": "test",
   "n": 150,
   "trained": 1.0,
   "reference": 0.5266666666666666,
   "trained_nll": 0.004553504743974675,
   "reference_nll": 0.690964260896047
  },
  {
   "task": "ordinal_rubric",
   "label": "Synthetic ordinal rubric",
   "split": "test",
   "n": 150,
   "trained": 1.0,
   "reference": 0.31333333333333335,
   "trained_nll": 0.0,
   "reference_nll": 1.0959128856658935
  },
  {
   "task": "heldout_winogrande",
   "label": "WinoGrande [unseen]",
   "split": "heldout",
   "n": 150,
   "trained": 0.5,
   "reference": 0.56,
   "trained_nll": 0.7782885433733463,
   "reference_nll": 0.6798250985145569
  },
  {
   "task": "heldout_agent_trace",
   "label": "Agent trace [synthetic, unseen]",
   "split": "heldout",
   "n": 150,
   "trained": 0.7733333333333333,
   "reference": 0.12,
   "trained_nll": 1.6637681728914269,
   "reference_nll": 0.9315960345665614
  },
  {
   "task": "heldout_security_policy",
   "label": "Security policy [synthetic, unseen]",
   "split": "heldout",
   "n": 150,
   "trained": 0.41333333333333333,
   "reference": 0.6866666666666666,
   "trained_nll": 1.0490681455532709,
   "reference_nll": 0.8461787496010462
  }
 ],
 "cost": [],
 "latency": {
  "cases": {
   "questions_1": {
    "median_ms": 67.67566700000316,
    "min_ms": 67.01616698410362,
    "max_ms": 69.39929199870676,
    "repeats": 10,
    "usage": {
     "input_tokens": 35,
     "output_tokens": 0,
     "candidate_token_evaluations": 352
    }
   },
   "questions_4": {
    "median_ms": 224.841187504353,
    "min_ms": 214.73154099658132,
    "max_ms": 234.07733399653807,
    "repeats": 10,
    "usage": {
     "input_tokens": 56,
     "output_tokens": 0,
     "candidate_token_evaluations": 1408
    }
   },
   "questions_16": {
    "median_ms": 868.5696880274918,
    "min_ms": 843.0807500262745,
    "max_ms": 932.4874579906464,
    "repeats": 10,
    "usage": {
     "input_tokens": 140,
     "output_tokens": 0,
     "candidate_token_evaluations": 5632
    }
   },
   "candidates_2": {
    "median_ms": 52.425062021939084,
    "min_ms": 50.35108304582536,
    "max_ms": 54.46933303028345,
    "repeats": 10,
    "usage": {
     "input_tokens": 39,
     "output_tokens": 0,
     "candidate_token_evaluations": 244
    }
   },
   "candidates_8": {
    "median_ms": 156.71789599582553,
    "min_ms": 155.58120899368078,
    "max_ms": 159.85900000669062,
    "repeats": 10,
    "usage": {
     "input_tokens": 39,
     "output_tokens": 0,
     "candidate_token_evaluations": 976
    }
   },
   "candidates_32": {
    "median_ms": 607.1348334953655,
    "min_ms": 604.7938329866156,
    "max_ms": 630.181334039662,
    "repeats": 10,
    "usage": {
     "input_tokens": 39,
     "output_tokens": 0,
     "candidate_token_evaluations": 3948
    }
   },
   "candidates_128": {
    "median_ms": 2438.300979003543,
    "min_ms": 2429.0778330178,
    "max_ms": 2457.5657079694793,
    "repeats": 10,
    "usage": {
     "input_tokens": 39,
     "output_tokens": 0,
     "candidate_token_evaluations": 15908
    }
   },
   "candidates_255": {
    "median_ms": 4918.976749991998,
    "min_ms": 4907.403832999989,
    "max_ms": 5007.831083028577,
    "repeats": 10,
    "usage": {
     "input_tokens": 39,
     "output_tokens": 0,
     "candidate_token_evaluations": 31910
    }
   },
   "mixed": {
    "median_ms": 149.07177051645704,
    "min_ms": 147.6271660067141,
    "max_ms": 149.98208300676197,
    "repeats": 10,
    "usage": {
     "input_tokens": 48,
     "output_tokens": 0,
     "candidate_token_evaluations": 935
    }
   }
  },
  "permutation_plus_extra_question_max_probability_drift": 0.0,
  "schema_checks_passed": true,
  "scope": "Warm local Python API: includes tokenization, forward and CPU normalization. Model load, cold start and HTTP/network excluded. No generated output tokens.",
  "hardware": "mps:0",
  "torch": "2.8.0",
  "method": "independent candidate prompts, repeated state prefill"
 },
 "scope": "Public-data checkpoint, frozen test and held-out splits; latency is the warm local Python API."
}