{
 "schema": 1,
 "suite": "latency-triage",
 "date": "2026-10-19T10:49:09Z",
 "ollaya": "0.14.0",
 "machine": "rtx-5090",
 "machine_detail": {
  "id": "rtx-5090",
  "cpu": "AMD Ryzen Threadripper 3960X 24-Core Processor",
  "threads": 48,
  "gpus": [
   {
    "name": "NVIDIA GeForce RTX 5090",
    "driver": "596.21",
    "memory": "32607 MiB"
   }
  ],
  "os": "Linux 6.6.87.2-microsoft-standard-WSL2"
 },
 "device": "cuda",
 "protocol": {
  "preset": "triage",
  "questions": 5,
  "states": 30,
  "warm": 5,
  "n": 20,
  "client": "one request at a time over HTTP to /api/decide, timed by the client; each request a different short message, in turn",
  "memory": "vram_mb: peak rise of the GPU's used memory over its idle level before the load (nvidia-smi, 100 ms); ram_peak_mb and ram_mb: the runner's VmHWM and VmRSS"
 },
 "note": "Ollaya 0.14.0. Release candidate (feat/fp16-decoders), measured before the release; one model loaded at a time.",
 "results": [
  {
   "model": "clef:flash",
   "questions": "triage",
   "load_ms": 45465.6,
   "first_request_ms": 45735.5,
   "answered_by": "clef:flash",
   "p50_ms": 268.4,
   "p90_ms": 276,
   "min_ms": 255.3,
   "eval_p50_ms": 266.9,
   "input_tokens": 633,
   "device": "cuda:0",
   "precision": "F32",
   "digest": "32a132a38b50ae33ad68bfbf6ba6b4e192bed9616c407ceadfecdd05ff33d0ea",
   "size_mb": 18211,
   "ram_peak_mb": 2970,
   "ram_mb": 1642,
   "vram_mb": 20076,
   "gpu_idle_mb": 1037
  },
  {
   "model": "clef:flash-fp16",
   "questions": "triage",
   "load_ms": 12948.7,
   "first_request_ms": 13115.2,
   "answered_by": "clef:flash-fp16",
   "p50_ms": 149.6,
   "p90_ms": 186.3,
   "min_ms": 140.2,
   "eval_p50_ms": 148.4,
   "input_tokens": 633,
   "device": "cuda:0",
   "precision": "F16",
   "digest": "d79bf9209e5294838f25a55fa1b5875273e435ea5f33f4095b8a48abb7337000",
   "size_mb": 18211,
   "ram_peak_mb": 2996,
   "ram_mb": 1890,
   "vram_mb": 19874,
   "gpu_idle_mb": 1037
  },
  {
   "model": "decider:0.8b",
   "questions": "triage",
   "load_ms": 22430.9,
   "first_request_ms": 22520.6,
   "answered_by": "decider:0.8b",
   "p50_ms": 60.1,
   "p90_ms": 62.3,
   "min_ms": 55.4,
   "eval_p50_ms": 59,
   "input_tokens": 552,
   "device": "cuda:0",
   "precision": "F32",
   "digest": "e09897e82c50b80794654b63ad46907df9ec8b2874c8d7bf00b24975f7372eab",
   "size_mb": 1462,
   "ram_peak_mb": 7024,
   "ram_mb": 5597,
   "vram_mb": 5086,
   "gpu_idle_mb": 1037
  },
  {
   "model": "decider:2b",
   "questions": "triage",
   "load_ms": 36590.2,
   "first_request_ms": 36688,
   "answered_by": "decider:2b",
   "p50_ms": 90.7,
   "p90_ms": 94.2,
   "min_ms": 86.7,
   "eval_p50_ms": 89.6,
   "input_tokens": 544,
   "device": "cuda:0",
   "precision": "F32",
   "digest": "a0c989e14cfe346bbdaa971b6fff47bf4a5aa5517f4abc55e91e0d04f1390d58",
   "size_mb": 3616,
   "ram_peak_mb": 11722,
   "ram_mb": 1381,
   "vram_mb": 12144,
   "gpu_idle_mb": 1037
  },
  {
   "model": "decider:4b",
   "questions": "triage",
   "load_ms": 23809.3,
   "first_request_ms": 24024.4,
   "answered_by": "decider:4b",
   "p50_ms": 206.9,
   "p90_ms": 211.1,
   "min_ms": 202.1,
   "eval_p50_ms": 205.9,
   "input_tokens": 560,
   "device": "cuda:0",
   "precision": "F32",
   "digest": "443372d40dd760feb805b661c5c0e61695df3adbab7ae32bbe8549f80a83e199",
   "size_mb": 8051,
   "ram_peak_mb": 1981,
   "ram_mb": 1332,
   "vram_mb": 11280,
   "gpu_idle_mb": 1037
  },
  {
   "model": "decision:eos",
   "questions": "triage",
   "load_ms": 14519.6,
   "first_request_ms": 14624.5,
   "answered_by": "decision:eos",
   "p50_ms": 68.7,
   "p90_ms": 71.3,
   "min_ms": 65,
   "eval_p50_ms": 67.6,
   "input_tokens": 675,
   "device": "cuda:0",
   "precision": "F32",
   "digest": "5b4d2a082a8ed512ce92ad68ea35a661944093360853fad5adcc6e01c97a6e13",
   "size_mb": 1466,
   "ram_peak_mb": 6095,
   "ram_mb": 5638,
   "vram_mb": 5908,
   "gpu_idle_mb": 1037
  },
  {
   "model": "jeeves:9b",
   "questions": "triage",
   "load_ms": 40201,
   "first_request_ms": 40588,
   "answered_by": "jeeves:9b",
   "p50_ms": 380.2,
   "p90_ms": 383.4,
   "min_ms": 377,
   "eval_p50_ms": 379.2,
   "input_tokens": 513,
   "device": "cuda:0",
   "precision": "F32",
   "digest": "737328feb5c781ed9d090705659a8539e028edf8950cd2b0394a022a64a1e509",
   "size_mb": 17116,
   "ram_peak_mb": 2697,
   "ram_mb": 1569,
   "vram_mb": 18388,
   "gpu_idle_mb": 1037
  },
  {
   "model": "kev:0.8b",
   "questions": "triage",
   "load_ms": 12963,
   "first_request_ms": 13054.6,
   "answered_by": "kev:0.8b",
   "p50_ms": 62.3,
   "p90_ms": 65,
   "min_ms": 60,
   "eval_p50_ms": 61.3,
   "input_tokens": 288,
   "device": "cuda:0",
   "precision": "F32",
   "digest": "e0d91e65fbdf3511b2778a170342d44d4f4c59bc6221453ed5737ad1940fd5a3",
   "size_mb": 1738,
   "ram_peak_mb": 5379,
   "ram_mb": 5379,
   "vram_mb": 4648,
   "gpu_idle_mb": 1037
  },
  {
   "model": "kev:0.8b-fp16",
   "questions": "triage",
   "load_ms": 6955.2,
   "first_request_ms": 7093.7,
   "answered_by": "kev:0.8b-fp16",
   "p50_ms": 66.6,
   "p90_ms": 67.7,
   "min_ms": 64.4,
   "eval_p50_ms": 65.6,
   "input_tokens": 278,
   "device": "cuda:0",
   "precision": "F16",
   "digest": "0fdd775fc30cc278669ce578a798134020aaac0293ea8cac97439fca84de8131",
   "size_mb": 1739,
   "ram_peak_mb": 3431,
   "ram_mb": 3431,
   "vram_mb": 3682,
   "gpu_idle_mb": 1037
  },
  {
   "model": "kev:4b",
   "questions": "triage",
   "load_ms": 24077.5,
   "first_request_ms": 24251.8,
   "answered_by": "kev:4b",
   "p50_ms": 171.3,
   "p90_ms": 173.9,
   "min_ms": 166.5,
   "eval_p50_ms": 170.2,
   "input_tokens": 288,
   "device": "cuda:0",
   "precision": "F32",
   "digest": "51a8a258a11de95d7b4192265547baa12fe3092d3defabea939e144c99159b48",
   "size_mb": 9049,
   "ram_peak_mb": 2002,
   "ram_mb": 1445,
   "vram_mb": 11188,
   "gpu_idle_mb": 1037
  },
  {
   "model": "kev:4b-fp16",
   "questions": "triage",
   "load_ms": 8451,
   "first_request_ms": 8607.8,
   "answered_by": "kev:4b-fp16",
   "p50_ms": 98.1,
   "p90_ms": 99.7,
   "min_ms": 92.2,
   "eval_p50_ms": 97.1,
   "input_tokens": 273,
   "device": "cuda:0",
   "precision": "F16",
   "digest": "904795c918ecae7ad2093323d722cae1ea2bc4bc50fa22bd022fe50f836b0fc3",
   "size_mb": 9050,
   "ram_peak_mb": 2008,
   "ram_mb": 1707,
   "vram_mb": 11048,
   "gpu_idle_mb": 1037
  },
  {
   "model": "kev:9b",
   "questions": "triage",
   "load_ms": 39089.8,
   "first_request_ms": 39342.6,
   "answered_by": "kev:9b",
   "p50_ms": 248.1,
   "p90_ms": 250,
   "min_ms": 243.7,
   "eval_p50_ms": 247,
   "input_tokens": 288,
   "device": "cuda:0",
   "precision": "F32",
   "digest": "3da298ce9bfc390822fd59c475ea7448ce3e1093210fbf2341709e90067d42dd",
   "size_mb": 18618,
   "ram_peak_mb": 2983,
   "ram_mb": 1722,
   "vram_mb": 18362,
   "gpu_idle_mb": 1037
  },
  {
   "model": "kev:9b-fp16",
   "questions": "triage",
   "load_ms": 15164,
   "first_request_ms": 15321.5,
   "answered_by": "kev:9b-fp16",
   "p50_ms": 125.1,
   "p90_ms": 126.1,
   "min_ms": 120.5,
   "eval_p50_ms": 124,
   "input_tokens": 273,
   "device": "cuda:0",
   "precision": "F16",
   "digest": "35a2c187f75c665bcee4d55da13ae88216f0ebb7f6db7fefc264f29c91f44ab0",
   "size_mb": 18619,
   "ram_peak_mb": 3061,
   "ram_mb": 1980,
   "vram_mb": 18048,
   "gpu_idle_mb": 1037
  },
  {
   "model": "nimble:9b",
   "questions": "triage",
   "load_ms": 43462.5,
   "first_request_ms": 44477.4,
   "answered_by": "nimble:9b",
   "p50_ms": 1015,
   "p90_ms": 1019.2,
   "min_ms": 1011.3,
   "eval_p50_ms": 1014.1,
   "input_tokens": 2978,
   "device": "cuda:0",
   "precision": "F32",
   "digest": "6081051a422e6852ed41810f89dc70119b963696aadc5c5832ab5336551a156d",
   "size_mb": 18610,
   "ram_peak_mb": 3178,
   "ram_mb": 1778,
   "vram_mb": 20616,
   "gpu_idle_mb": 1037
  },
  {
   "model": "nimble:9b-fp16",
   "questions": "triage",
   "load_ms": 34329.9,
   "first_request_ms": 34819.4,
   "answered_by": "nimble:9b-fp16",
   "p50_ms": 428.4,
   "p90_ms": 431.5,
   "min_ms": 421.7,
   "eval_p50_ms": 427.3,
   "input_tokens": 2968,
   "device": "cuda:0",
   "precision": "F16",
   "digest": "c6aa45c1955a8726fecafd74eaac920e9e5ab9631c733bd28f883466964be695",
   "size_mb": 18611,
   "ram_peak_mb": 2987,
   "ram_mb": 2030,
   "vram_mb": 20162,
   "gpu_idle_mb": 1037
  },
  {
   "model": "nimble:9b-v3",
   "questions": "triage",
   "load_ms": 11437.5,
   "first_request_ms": 12380.8,
   "answered_by": "nimble:9b-v3",
   "p50_ms": 942.3,
   "p90_ms": 946.5,
   "min_ms": 937.2,
   "eval_p50_ms": 941.3,
   "input_tokens": 2798,
   "device": "cuda:0",
   "precision": "F32",
   "digest": "01cb73cdc839e8eb25a6209976a2cd1e83048b6b893c809d7144e71b34ff2b31",
   "size_mb": 19105,
   "ram_peak_mb": 3181,
   "ram_mb": 1783,
   "vram_mb": 21188,
   "gpu_idle_mb": 1037
  },
  {
   "model": "pplx-decider:flash",
   "questions": "triage",
   "load_ms": 21985.3,
   "first_request_ms": 22113.8,
   "answered_by": "pplx-decider:flash",
   "p50_ms": 126.5,
   "p90_ms": 130.9,
   "min_ms": 121.3,
   "eval_p50_ms": 125.5,
   "input_tokens": 320,
   "device": "cuda:0",
   "precision": "F32",
   "digest": "e403f9b1f0ac01129919433e754f66b8718ff76a61055eb841ec4f969173fd40",
   "size_mb": 8689,
   "ram_peak_mb": 1959,
   "ram_mb": 1315,
   "vram_mb": 10622,
   "gpu_idle_mb": 1037
  },
  {
   "model": "pplx-decider:flash-fp16",
   "questions": "triage",
   "load_ms": 7386.4,
   "first_request_ms": 7517.2,
   "answered_by": "pplx-decider:flash-fp16",
   "p50_ms": 89.9,
   "p90_ms": 95,
   "min_ms": 86.1,
   "eval_p50_ms": 88.9,
   "input_tokens": 320,
   "device": "cuda:0",
   "precision": "F16",
   "digest": "2d518c01dae07bf0915c1e7bc8dcc0327f90ecec3047ec0db31365e75ad91e97",
   "size_mb": 8690,
   "ram_peak_mb": 1962,
   "ram_mb": 1567,
   "vram_mb": 10516,
   "gpu_idle_mb": 1037
  },
  {
   "model": "strands-decider:2b",
   "questions": "triage",
   "load_ms": 31160.2,
   "first_request_ms": 31280.9,
   "answered_by": "strands-decider:2b",
   "p50_ms": 97.6,
   "p90_ms": 100.3,
   "min_ms": 92.2,
   "eval_p50_ms": 96.5,
   "input_tokens": 557,
   "device": "cuda:0",
   "precision": "F32",
   "digest": "9aae566845cbf83cafbe804ba9f486fd856239f3d405d265704cb42385bea63a",
   "size_mb": 4571,
   "ram_peak_mb": 11704,
   "ram_mb": 3208,
   "vram_mb": 8672,
   "gpu_idle_mb": 1037
  },
  {
   "model": "strands-decider:2b-fp16",
   "questions": "triage",
   "load_ms": 19099.2,
   "first_request_ms": 19239.2,
   "answered_by": "strands-decider:2b-fp16",
   "p50_ms": 71.6,
   "p90_ms": 75.6,
   "min_ms": 70.8,
   "eval_p50_ms": 70.6,
   "input_tokens": 567,
   "device": "cuda:0",
   "precision": "F16",
   "digest": "d23105625e3c264e6e47fc8677db367e07ae2fbad225cc2117f309bf95023753",
   "size_mb": 4572,
   "ram_peak_mb": 10101,
   "ram_mb": 9428,
   "vram_mb": 5926,
   "gpu_idle_mb": 1037
  }
 ],
 "edition": "2026.10"
}