RESULT {"label": "warmup16", "status": 200, "error": null, "max_tokens": 16, "ignore_eos": false, "ttft_s": 0.3767, "total_s": 1.5586, "usage": {"prompt_tokens": 14, "completion_tokens": 16, "total_tokens": 30}, "prompt_tok_s_at_ttft": 37.16, "post_first_decode_tok_s": 12.69, "keepalives": 0, "output": "彩虹之所以有多种颜色，是因为阳光（白光）被空气中的水滴折射、反射"}
RESULT {"label": "decode64", "status": 200, "error": null, "max_tokens": 64, "ignore_eos": true, "ttft_s": 0.7144, "total_s": 5.944, "usage": {"prompt_tokens": 35, "completion_tokens": 64, "total_tokens": 99}, "prompt_tok_s_at_ttft": 48.99, "post_first_decode_tok_s": 12.05, "keepalives": 0, "output": "alpha beta gamma delta epsilon zeta eta theta iota kappa lambda mu nu xi omicron pi rho sigma tau ups phi chi psi omega one two three four five six seven eight nine ten eleven twelve thirteen fourteen fifteen sixteen seventeen eighteen nineteen twenty twenty one twenty two twenty three twenty four twenty five twenty six twenty seven"}
RESULT {"label": "math", "status": 200, "error": null, "max_tokens": 16, "ignore_eos": false, "ttft_s": 0.4002, "total_s": 0.4796, "usage": {"prompt_tokens": 17, "completion_tokens": 1, "total_tokens": 18}, "prompt_tok_s_at_ttft": 42.48, "post_first_decode_tok_s": null, "keepalives": 0, "output": "410"}
RESULT {"label": "write_zh", "status": 200, "error": null, "max_tokens": 96, "ignore_eos": false, "ttft_s": 0.5978, "total_s": 6.9035, "usage": {"prompt_tokens": 35, "completion_tokens": 76, "total_tokens": 111}, "prompt_tok_s_at_ttft": 58.55, "post_first_decode_tok_s": 11.89, "keepalives": 0, "output": "本次更新显著优化了长上下文场景下的推理性能，吞吐能力提升达 30% 以上，有效应对高并发请求压力。  \n同时，流式输出响应延迟进一步降低，确保长文本生成过程更加流畅、实时。  \n此外，新增的 trace 可观测性功能，可精准追踪推理链路，便于快速定位性能瓶颈与异常。"}
SUMMARY {"env": {"ARLE_DSV4_COMBINE_REDUCE_SCATTER": "1"}, "results": [{"label": "warmup16", "status": 200, "error": null, "max_tokens": 16, "ignore_eos": false, "ttft_s": 0.3767, "total_s": 1.5586, "usage": {"prompt_tokens": 14, "completion_tokens": 16, "total_tokens": 30}, "prompt_tok_s_at_ttft": 37.16, "post_first_decode_tok_s": 12.69, "keepalives": 0, "output": "彩虹之所以有多种颜色，是因为阳光（白光）被空气中的水滴折射、反射"}, {"label": "decode64", "status": 200, "error": null, "max_tokens": 64, "ignore_eos": true, "ttft_s": 0.7144, "total_s": 5.944, "usage": {"prompt_tokens": 35, "completion_tokens": 64, "total_tokens": 99}, "prompt_tok_s_at_ttft": 48.99, "post_first_decode_tok_s": 12.05, "keepalives": 0, "output": "alpha beta gamma delta epsilon zeta eta theta iota kappa lambda mu nu xi omicron pi rho sigma tau ups phi chi psi omega one two three four five six seven eight nine ten eleven twelve thirteen fourteen fifteen sixteen seventeen eighteen nineteen twenty twenty one twenty two twenty three twenty four twenty five twenty six twenty seven"}, {"label": "math", "status": 200, "error": null, "max_tokens": 16, "ignore_eos": false, "ttft_s": 0.4002, "total_s": 0.4796, "usage": {"prompt_tokens": 17, "completion_tokens": 1, "total_tokens": 18}, "prompt_tok_s_at_ttft": 42.48, "post_first_decode_tok_s": null, "keepalives": 0, "output": "410"}, {"label": "write_zh", "status": 200, "error": null, "max_tokens": 96, "ignore_eos": false, "ttft_s": 0.5978, "total_s": 6.9035, "usage": {"prompt_tokens": 35, "completion_tokens": 76, "total_tokens": 111}, "prompt_tok_s_at_ttft": 58.55, "post_first_decode_tok_s": 11.89, "keepalives": 0, "output": "本次更新显著优化了长上下文场景下的推理性能，吞吐能力提升达 30% 以上，有效应对高并发请求压力。  \n同时，流式输出响应延迟进一步降低，确保长文本生成过程更加流畅、实时。  \n此外，新增的 trace 可观测性功能，可精准追踪推理链路，便于快速定位性能瓶颈与异常。"}], "request_traces": [{"chunks_seen": 17, "completion_tokens": 16, "error": null, "finish_reason": "length", "kv": {"fetch_backpressure": false, "fetch_queue_depth": 0, "fetch_waiters": 0, "gpu_util": 0.0, "store_backpressure": false, "store_queue_depth": 0}, "max_tokens": 16, "prefix": {"direct_gpu_attach": false, "hit_rate": 0.0, "lookup_latency_us": 0, "lookup_reusable_tokens": 0, "matched_tokens": 0, "prefetch": false, "ready_on_gpu": false, "recompute": false, "request_hit_rate": 0.0, "request_skip_rate": 0.0, "resume_prefill_tokens": 14, "skip_rate": 0.0, "staged": false}, "preprocess": {"queue_depth": 0, "tokenize_us": 265, "wait_us": 1}, "prompt_bytes": 123, "prompt_tokens": 14, "request_id": "chatcmpl-193b54f5-f32c-480d-b8de-d17684925267", "route": "/v1/chat/completions", "scheduler": {"active": 0, "batch_width": 0, "decode_tokens": 0, "loop_us": {"cleanup": 0, "total": 139835}, "phase_us": {"admission": 32, "decode": 70709, "emit": 4, "prefill": 69373, "total": 125912}, "pipeline_us": {"cpu_plan": 1, "gpu_command_queue_depth": 0, "gpu_completion_wait": 1034, "snapshot": 0}, "prefill_queue": 0, "prefill_tokens": 0, "running_batch": 0, "scheduled_decode_rows": 0, "scheduled_prefill_rows": 0, "scheduled_rows": 0, "step_last_s": 0.000966, "waiting": 0}, "stream": true, "terminal_seen": true, "text_bytes": 96, "throughput": {"completion_tokens_per_s_e2e": 10.27537351091888, "prompt_tokens_per_s_at_ttft": 37.300234398402985, "total_tokens_per_s_e2e": 19.2663253329729}, "token_ids_seen": 16, "total_ms": 1557.121012, "total_tokens": 30, "ttft_ms": 375.332762}, {"chunks_seen": 65, "completion_tokens": 64, "error": null, "finish_reason": "length", "kv": {"fetch_backpressure": false, "fetch_queue_depth": 0, "fetch_waiters": 0, "gpu_util": 0.0, "store_backpressure": false, "store_queue_depth": 0}, "max_tokens": 64, "prefix": {"direct_gpu_attach": false, "hit_rate": 0.0, "lookup_latency_us": 0, "lookup_reusable_tokens": 0, "matched_tokens": 0, "prefetch": false, "ready_on_gpu": false, "recompute": false, "request_hit_rate": 0.0, "request_skip_rate": 0.0, "resume_prefill_tokens": 35, "skip_rate": 0.0, "staged": false}, "preprocess": {"queue_depth": 0, "tokenize_us": 376, "wait_us": 0}, "prompt_bytes": 233, "prompt_tokens": 35, "request_id": "chatcmpl-a856a185-d819-4e7c-9a26-f5df0acd4e9b", "route": "/v1/chat/completions", "scheduler": {"active": 0, "batch_width": 0, "decode_tokens": 0, "loop_us": {"cleanup": 3, "total": 76844}, "phase_us": {"admission": 9, "decode": 76679, "emit": 3, "prefill": 150, "total": 76826}, "pipeline_us": {"cpu_plan": 1, "gpu_command_queue_depth": 0, "gpu_completion_wait": 1073, "snapshot": 0}, "prefill_queue": 0, "prefill_tokens": 0, "running_batch": 0, "scheduled_decode_rows": 0, "scheduled_prefill_rows": 0, "scheduled_rows": 0, "step_last_s": 0.001039, "waiting": 0}, "stream": true, "terminal_seen": true, "text_bytes": 334, "throughput": {"completion_tokens_per_s_e2e": 10.768030361113068, "prompt_tokens_per_s_at_ttft": 49.02239449849511, "total_tokens_per_s_e2e": 16.656796964846777}, "token_ids_seen": 64, "total_ms": 5943.519646000001, "total_tokens": 99, "ttft_ms": 713.959413}, {"chunks_seen": 2, "completion_tokens": 1, "error": null, "finish_reason": "stop", "kv": {"fetch_backpressure": false, "fetch_queue_depth": 0, "fetch_waiters": 0, "gpu_util": 0.0, "store_backpressure": false, "store_queue_depth": 0}, "max_tokens": 16, "prefix": {"direct_gpu_attach": false, "hit_rate": 0.0, "lookup_latency_us": 0, "lookup_reusable_tokens": 0, "matched_tokens": 0, "prefetch": false, "ready_on_gpu": false, "recompute": false, "request_hit_rate": 0.0, "request_skip_rate": 0.0, "resume_prefill_tokens": 17, "skip_rate": 0.0, "staged": false}, "preprocess": {"queue_depth": 0, "tokenize_us": 202, "wait_us": 0}, "prompt_bytes": 115, "prompt_tokens": 17, "request_id": "chatcmpl-633fd0b6-d999-40f4-a830-24e4ef606c26", "route": "/v1/chat/completions", "scheduler": {"active": 0, "batch_width": 0, "decode_tokens": 0, "loop_us": {"cleanup": 3, "total": 95500}, "phase_us": {"admission": 5292, "decode": 63041, "emit": 2, "prefill": 27160, "total": 95484}, "pipeline_us": {"cpu_plan": 1, "gpu_command_queue_depth": 0, "gpu_completion_wait": 912, "snapshot": 0}, "prefill_queue": 0, "prefill_tokens": 0, "running_batch": 0, "scheduled_decode_rows": 0, "scheduled_prefill_rows": 0, "scheduled_rows": 0, "step_last_s": 0.000886, "waiting": 0}, "stream": true, "terminal_seen": true, "text_bytes": 3, "throughput": {"completion_tokens_per_s_e2e": 2.0875302912387617, "prompt_tokens_per_s_at_ttft": 42.52616390339533, "total_tokens_per_s_e2e": 37.57554524229771}, "token_ids_seen": 1, "total_ms": 479.034965, "total_tokens": 18, "ttft_ms": 399.75390300000004}, {"chunks_seen": 77, "completion_tokens": 76, "error": null, "finish_reason": "stop", "kv": {"fetch_backpressure": false, "fetch_queue_depth": 0, "fetch_waiters": 0, "gpu_util": 0.0, "store_backpressure": false, "store_queue_depth": 0}, "max_tokens": 96, "prefix": {"direct_gpu_attach": false, "hit_rate": 0.0, "lookup_latency_us": 0, "lookup_reusable_tokens": 0, "matched_tokens": 0, "prefetch": false, "ready_on_gpu": false, "recompute": false, "request_hit_rate": 0.0, "request_skip_rate": 0.0, "resume_prefill_tokens": 35, "skip_rate": 0.0, "staged": false}, "preprocess": {"queue_depth": 0, "tokenize_us": 328, "wait_us": 0}, "prompt_bytes": 208, "prompt_tokens": 35, "request_id": "chatcmpl-e45566bd-364d-46b6-841e-1e8f2d967ba7", "route": "/v1/chat/completions", "scheduler": {"active": 0, "batch_width": 0, "decode_tokens": 0, "loop_us": {"cleanup": 6, "total": 77800}, "phase_us": {"admission": 4, "decode": 77762, "emit": 4, "prefill": 23, "total": 77793}, "pipeline_us": {"cpu_plan": 1, "gpu_command_queue_depth": 0, "gpu_completion_wait": 1135, "snapshot": 0}, "prefill_queue": 0, "prefill_tokens": 0, "running_batch": 0, "scheduled_decode_rows": 0, "scheduled_prefill_rows": 0, "scheduled_rows": 0, "step_last_s": 0.000883, "waiting": 0}, "stream": true, "terminal_seen": true, "text_bytes": 363, "throughput": {"completion_tokens_per_s_e2e": 11.0098142431587, "prompt_tokens_per_s_at_ttft": 58.590489334642655, "total_tokens_per_s_e2e": 16.08012343408705}, "token_ids_seen": 76, "total_ms": 6902.932086, "total_tokens": 111, "ttft_ms": 597.366576}]}
