{
  "_about": "SOTA-system registry. last_reviewed drives the industry-drift backlog (`--stale`).",
  "systems": [
    {
      "name": "AgentDojo",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "AMD MI300X/MI355X",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "AMD MI355X",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "AWQ",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "Baseten",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "Claude Opus 4.5 + agent scaffold",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "Container/microVM sandboxes",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "cuBLAS",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "DeepSeek",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "DeepSeek-V2/V3",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "DeepSeek-V3 MTP",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "DistServe",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "DynaServe",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "EAGLE-2",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "EAGLE-3",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "Equinox / FairBatching",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "FailSafe",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "FairBatching",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "FP8/FP4 quantization",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "Gemini 3.x",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "GenAI-Perf goodput mode",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "GPT-5.x",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "GPTQ",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "GQA-based models",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "group-128 INT4",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "H2O",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "Hosted frontier APIs",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "HuggingFace",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "hybrid real-time/best-effort schedulers",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "HydraServe",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "IFC / capability allow-lists",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "JSONSchemaBench",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "KevlarFlow",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "KIVI",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "KServe",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "KVQuant",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "KVTuner",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "Lakera",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "LiteLLM",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "llama.cpp",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "https://github.com/ggml-org/llama.cpp",
      "notes": "GGUF CPU/GPU inference engine (GGML). The portable single-stream baseline fak measures decode/prefill parity against on CPU (Q8_0) and on a consumer GPU; its hand-tuned int8 GEMM is the throughput reference in LLAMACPP-HEADTOHEAD-RESULTS.md, and it is a first-class selectable engine in the GCP head-to-head bench (tools/gcp_bench.py)."
    },
    {
      "name": "LLGuidance",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "LLGuidance/Guidance",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "LLM-42",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "llm-compressor",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "llm-d",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "LMCache",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "MagicDec",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "MangoBoost/AMD MI300X",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "Medusa",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "Medusa-2",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "MELON",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "Meta Llama Guard 3",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "MLC-LLM",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "MLPerf Inference",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "MLPerf Inference v6.0 submitters (DeepSeek-R1 671B offline)",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": "v6.0 (2026-03-30) supersedes v5.0/v5.1; headline model shifted to DeepSeek-R1 671B MoE."
    },
    {
      "name": "MLPerf Inference v6.0 server scenario (DeepSeek-R1, accuracy-constrained)",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": "v6.0 accuracy+latency-constrained server scenario; 8,064 tok/s/GPU on GB300 NVL72."
    },
    {
      "name": "MLPerf Inference v6.0",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": "v6.0 (2026-03-30) is the current round; supersedes v5.1."
    },
    {
      "name": "MLPerf Inference v6.0 interactive scenario",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": "v6.0 interactive floor: TPOT <= 15 ms, TTFT <= 1.5-2.0 s P99 (DeepSeek-R1 / GPT-OSS 120B)."
    },
    {
      "name": "ModelOpt accuracy harness",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "Mooncake",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "MOSS two-level microscaling",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "MuxWise",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "MX formats",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "Neural Magic / vLLM recovery-% methodology",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "NVFP4",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "NVIDIA",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "Ollama",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "OpenAI prompt caching",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "OpenAI Structured Outputs",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "Orca",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "P-EAGLE",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "PipeBoost",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "production SLO practice",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "QServe W4A8KV4",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "QuaRot",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "Quest",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "Ray Serve",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "Red Hat / Neural Magic llm-compressor",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "ReviveMoE",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "RunPod FlashBoot",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "Sarathi-Serve",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "ServerlessLLM",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "SGLang",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "https://arxiv.org/abs/2312.07104",
      "notes": "RadixAttention prefix-cache reuse + cache-aware router (Zheng et al., NeurIPS 2024). Published 50–99% prefix cache-hit band (the band fak's 86.7% radix hit-rate is measured inside); compressed-FSM structured-output decoding. The reuse-mechanism peer fak's split-reuse parity is measured against, and the only LIVE concurrent-serving head-to-head fak has run (fak-gateway 0.75× raw SGLang at peak; QWEN36-27B-GPU-SERVER-RESULTS.md)."
    },
    {
      "name": "SGLang-PD",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "SGLang-PD / vLLM disaggregated",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "sglang-router",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "SmoothQuant",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "SnapKV",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "speculative sampling",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "SpinQuant",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "Splitwise",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "StreamingLLM",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "TensorRT-LLM",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "https://github.com/NVIDIA/TensorRT-LLM",
      "notes": "NVIDIA's TensorRT-based LLM serving engine: in-flight (continuous) batching, paged KV, fused kernels, ModelOpt FP8 / NVFP4 quantization. The vendor SOTA throughput/latency bar on NVIDIA datacenter GPUs (a recurring MLPerf Inference submitter)."
    },
    {
      "name": "Thinking Machines batch-invariant-ops",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "TokenLake",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "Tuned warm per-agent KV cache",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "vAttention",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "vLLM",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "https://arxiv.org/abs/2309.06180",
      "notes": "PagedAttention LLM serving engine (Kwon et al., SOSP 2023). Continuous / iteration-level batching + paged KV cache (~96% KV utilization, <4% waste); Automatic Prefix Caching (block-hash within-instance KV reuse, docs.vllm.ai/en/latest/features/automatic_prefix_caching); FP8 E4M3/E5M2 KV-cache dtype; XGrammar guided decoding; OpenAI-compatible server. The de facto open SOTA baseline this scorecard positions fak against on continuous batching, prefix caching, and the OpenAI API surface; the live gateway head-to-head is fak-fronts-vLLM vs raw-vLLM (the adjudication tax), the analog of the measured SGLang row."
    },
    {
      "name": "vLLM/SGLang self-host on H100/H200",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "vLLM/SGLang sweeps",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "VTC",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "XGrammar",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    },
    {
      "name": "XGrammar-2",
      "kind": "serving-system",
      "last_reviewed": "2026-06-27",
      "url": "",
      "notes": ""
    }
  ]
}