{
  "schema": "fak.gcp-vm-bench.v2",
  "gpus": "NVIDIA H100 80GB HBM3, 81559 MiB",
  "model": "qwen2.5-3b-instruct-q8_0.gguf",
  "arch": "sm_90",
  "engines": {
    "fak-cpu": {
      "engine": "fak-cpu",
      "ok": true,
      "backend": "fak-in-kernel Q8_0 (pure-Go, quantized weights+activations, int8\u00d7int8\u2192int32 dot)",
      "precision": "Q8_0",
      "prefill_tok_per_sec": 109.72316246703932,
      "decode_tok_per_sec": 15.70367812648336,
      "workers": 26,
      "load_ms": 9223.448947,
      "raw": {
        "app_version": "0.34.0",
        "backend": {
          "registered_backends": [
            "cpu-ref"
          ],
          "selected": "legacy"
        },
        "budget": "default(GOMAXPROCS)",
        "decode": {
          "prompt_tokens": 16,
          "decode_steps": 128,
          "reps": 5,
          "per_token_median_ms": 63.679349,
          "tok_per_sec": 15.70367812648336
        },
        "engine": "fak-in-kernel Q8_0 (pure-Go, quantized weights+activations, int8\u00d7int8\u2192int32 dot)",
        "go_threads": "GOMAXPROCS=26, matmul workers=26, q8 decode workers=26 (FAK_WORKERS / FAK_BUDGET / -budget to pin)",
        "lean": true,
        "load_ms": 9223.448947,
        "model": "qwen2.5-3b-instruct-q8_0.gguf [gguf-lean]",
        "model_config": {
          "architectures": null,
          "head_dim": 128,
          "hidden_size": 2048,
          "index_head_dim": 0,
          "index_n_heads": 0,
          "index_topk": 0,
          "indexer_types": null,
          "intermediate_size": 11008,
          "is_moe": false,
          "kv_lora_rank": 0,
          "max_position_embeddings": 32768,
          "model_type": "qwen2",
          "num_attention_heads": 16,
          "num_experts_per_tok": 0,
          "num_hidden_layers": 36,
          "num_key_value_heads": 2,
          "num_local_experts": 0,
          "q_lora_rank": 0,
          "qk_nope_head_dim": 0,
          "qk_rope_head_dim": 0,
          "v_head_dim": 0,
          "vocab_size": 151936
        },
        "precision": "Q8_0",
        "prefill": [
          {
            "tokens": 512,
            "reps": 5,
            "median_ms": 4666.289127,
            "tok_per_sec": 109.72316246703932
          }
        ],
        "q4k": false,
        "q8_decode_budget": "default(GOMAXPROCS)",
        "q8_decode_workers": 26,
        "quant_ms": 0.000183,
        "quantized_at_load": true,
        "source": "/root/.cache/huggingface/hub/models--Qwen--Qwen2.5-3B-Instruct-GGUF/snapshots/7dabda4d13d513e3e842b20f0d435c732f172cbe/qwen2.5-3b-instruct-q8_0.gguf",
        "workers": 26
      },
      "raw_artifact": "fak-cpu-report.json"
    },
    "fak-cuda": {
      "engine": "fak-cuda",
      "ok": true,
      "backend": "fak-in-kernel via compute HAL backend \"cuda\"",
      "precision": "f32",
      "prefill_tok_per_sec": 51.04064262057563,
      "decode_tok_per_sec": 96.28140976395552,
      "workers": 26,
      "load_ms": 38230.014536,
      "raw": {
        "app_version": "0.34.0",
        "backend": {
          "caps": {
            "Async": true,
            "FusedAttn": true,
            "FusedFFN": false,
            "GraphCompile": false,
            "UploadDtype": true,
            "DeviceMemory": true,
            "Collective": false,
            "CapacityProbe": true,
            "HostCapacityProbe": true
          },
          "class": "approx",
          "registered_backends": [
            "cpu-ref",
            "cuda"
          ],
          "selected": "cuda",
          "tier": "sm_90"
        },
        "budget": "default(GOMAXPROCS)",
        "decode": {
          "prompt_tokens": 16,
          "decode_steps": 128,
          "reps": 5,
          "per_token_median_ms": 10.386221,
          "tok_per_sec": 96.28140976395552
        },
        "engine": "fak-in-kernel via compute HAL backend \"cuda\"",
        "go_threads": "GOMAXPROCS=26, matmul workers=26, q8 decode workers=26 (FAK_WORKERS / FAK_BUDGET / -budget to pin)",
        "lean": false,
        "load_ms": 38230.014536,
        "model": "qwen2.5-3b-instruct-q8_0.gguf [gguf]",
        "model_config": {
          "architectures": null,
          "head_dim": 128,
          "hidden_size": 2048,
          "index_head_dim": 0,
          "index_n_heads": 0,
          "index_topk": 0,
          "indexer_types": null,
          "intermediate_size": 11008,
          "is_moe": false,
          "kv_lora_rank": 0,
          "max_position_embeddings": 32768,
          "model_type": "qwen2",
          "num_attention_heads": 16,
          "num_experts_per_tok": 0,
          "num_hidden_layers": 36,
          "num_key_value_heads": 2,
          "num_local_experts": 0,
          "q_lora_rank": 0,
          "qk_nope_head_dim": 0,
          "qk_rope_head_dim": 0,
          "v_head_dim": 0,
          "vocab_size": 151936
        },
        "precision": "f32",
        "prefill": [
          {
            "tokens": 512,
            "reps": 5,
            "median_ms": 10031.221664,
            "tok_per_sec": 51.04064262057563
          }
        ],
        "q4k": false,
        "q8_decode_budget": "default(GOMAXPROCS)",
        "q8_decode_workers": 26,
        "quant_ms": 0,
        "quantized_at_load": false,
        "source": "/root/.cache/huggingface/hub/models--Qwen--Qwen2.5-3B-Instruct-GGUF/snapshots/7dabda4d13d513e3e842b20f0d435c732f172cbe/qwen2.5-3b-instruct-q8_0.gguf",
        "workers": 26
      },
      "raw_artifact": "fak-cuda-report.json"
    },
    "llama": {
      "engine": "llama",
      "ok": true,
      "backend": "llama.cpp CUDA",
      "precision": "Q8_0",
      "prefill_tok_per_sec": 19310.487752,
      "decode_tok_per_sec": 361.62486,
      "gpus": "NVIDIA H100 80GB HBM3, 81559 MiB",
      "model": "qwen2.5-3b-instruct-q8_0.gguf",
      "raw": [
        {
          "build_commit": "c1a1c8e",
          "build_number": 1,
          "cpu_info": "Intel(R) Xeon(R) Platinum 8481C CPU @ 2.70GHz",
          "gpu_info": "NVIDIA H100 80GB HBM3",
          "backends": "CUDA",
          "model_filename": "/root/.cache/huggingface/hub/models--Qwen--Qwen2.5-3B-Instruct-GGUF/snapshots/7dabda4d13d513e3e842b20f0d435c732f172cbe/qwen2.5-3b-instruct-q8_0.gguf",
          "model_type": "qwen2 3B Q8_0",
          "model_size": 3610132480,
          "model_n_params": 3397103616,
          "n_batch": 2048,
          "n_ubatch": 512,
          "n_threads": 13,
          "cpu_mask": "0x0",
          "cpu_strict": false,
          "poll": 50,
          "type_k": "f16",
          "type_v": "f16",
          "n_gpu_layers": 99,
          "n_cpu_moe": 0,
          "split_mode": "layer",
          "main_gpu": 0,
          "no_kv_offload": false,
          "flash_attn": -1,
          "devices": "auto",
          "tensor_split": "0.00",
          "tensor_buft_overrides": "none",
          "use_mmap": true,
          "use_direct_io": false,
          "embeddings": false,
          "no_op_offload": 0,
          "no_host": false,
          "fit_target": 0,
          "fit_min_ctx": 0,
          "n_prompt": 512,
          "n_gen": 0,
          "n_depth": 0,
          "test_time": "2026-06-28T10:51:42Z",
          "avg_ns": 27030538,
          "stddev_ns": 4629369,
          "avg_ts": 19310.487752,
          "stddev_ts": 2693.32978,
          "samples_ns": [
            35302507,
            25292089,
            25030761,
            24760769,
            24766565
          ],
          "samples_ts": [
            14503.2,
            20243.5,
            20454.8,
            20677.9,
            20673
          ]
        },
        {
          "build_commit": "c1a1c8e",
          "build_number": 1,
          "cpu_info": "Intel(R) Xeon(R) Platinum 8481C CPU @ 2.70GHz",
          "gpu_info": "NVIDIA H100 80GB HBM3",
          "backends": "CUDA",
          "model_filename": "/root/.cache/huggingface/hub/models--Qwen--Qwen2.5-3B-Instruct-GGUF/snapshots/7dabda4d13d513e3e842b20f0d435c732f172cbe/qwen2.5-3b-instruct-q8_0.gguf",
          "model_type": "qwen2 3B Q8_0",
          "model_size": 3610132480,
          "model_n_params": 3397103616,
          "n_batch": 2048,
          "n_ubatch": 512,
          "n_threads": 13,
          "cpu_mask": "0x0",
          "cpu_strict": false,
          "poll": 50,
          "type_k": "f16",
          "type_v": "f16",
          "n_gpu_layers": 99,
          "n_cpu_moe": 0,
          "split_mode": "layer",
          "main_gpu": 0,
          "no_kv_offload": false,
          "flash_attn": -1,
          "devices": "auto",
          "tensor_split": "0.00",
          "tensor_buft_overrides": "none",
          "use_mmap": true,
          "use_direct_io": false,
          "embeddings": false,
          "no_op_offload": 0,
          "no_host": false,
          "fit_target": 0,
          "fit_min_ctx": 0,
          "n_prompt": 0,
          "n_gen": 128,
          "n_depth": 0,
          "test_time": "2026-06-28T10:51:42Z",
          "avg_ns": 353958123,
          "stddev_ns": 263020,
          "avg_ts": 361.62486,
          "stddev_ts": 0.268526,
          "samples_ns": [
            354398955,
            353881213,
            353738151,
            353792011,
            353980285
          ],
          "samples_ts": [
            361.175,
            361.703,
            361.85,
            361.794,
            361.602
          ]
        }
      ],
      "raw_artifact": "llama-bench.json"
    }
  },
  "ok": true,
  "prefill_tok_per_sec": 19310.487752,
  "decode_tok_per_sec": 361.62486,
  "headline_engine": "llama"
}