{ "created_at_unix": 1781558112.3152022, "base_url": "http://127.0.0.1:18125", "model": "qwen36-35b-a3b-fp8", "tokenizer": "/mnt/fast-ai/llm-cache/hf/models--nameistoken--Qwen3.6-35B-A3B-Quark-W8A8-INT8/snapshots/cced56592e8c8935f8220836b4baa04dfd389118", "server_model_record": { "id": "qwen36-35b-a3b-fp8", "object": "model", "created": 1781558080, "owned_by": "vllm", "root": "/mnt/fast-ai/llm-cache/hf/models--nameistoken--Qwen3.6-35B-A3B-Quark-W8A8-INT8/snapshots/cced56592e8c8935f8220836b4baa04dfd389118", "parent": null, "max_model_len": 32768, "permission": [ { "id": "modelperm-ab87951b88a78437", "object": "model_permission", "created": 1781558080, "allow_create_engine": false, "allow_sampling": true, "allow_logprobs": true, "allow_search_indices": false, "allow_view": true, "allow_fine_tuning": false, "organization": "*", "group": null, "is_blocking": false } ] }, "prompt_kind": "preset", "prompt_preset": "natural-chat", "prompt_file": null, "endpoint": "completions", "seed": 0, "random_prefix_len": 0, "prompt_tokens_requested": 512, "prompt_tokens_actual": 498, "output_tokens_requested": 512, "mode": "stream", "ignore_eos": true, "skip_vram": true, "repeats": 1, "measurement_notes": [ "TTFT and e2e are measured both client-side and from vLLM Prometheus histogram deltas.", "tok_s_prefill_lower_bound_from_ttft is prompt_tokens / TTFT, so it is conservative and includes first-token scheduling/decode overhead.", "tok_s_out_client_after_first_chunk is generated-token-only throughput after the first streamed text chunk.", "tok_s_out_client_after_first_chunk_corrected subtracts the estimated first streamed chunk tokens from the numerator; this is more stable when --stream-interval > 1." ], "vram_mib_before_all": {}, "peak_vram_mib_observed": {}, "records": [ { "repeat": 1, "request_id": "cmpl-8dcd007292792118", "request_started_at_unix": 1781558106.0860207, "request_finished_at_unix": 1781558112.309814, "prompt_tokens_client": 498, "prompt_tokens_estimated_before_request": 498, "output_tokens_client": 512, "elapsed_s_client": 6.223707710858434, "ttft_ms_client": 272.79036887921393, "tok_s_out_client_after_first_chunk": 86.03715537909211, "tok_s_out_client_after_first_chunk_corrected": 85.86911405999231, "tok_s_out_client_e2e": 82.26607414527504, "tok_s_total_client": 162.28268532564022, "tok_s_prefill_lower_bound_from_ttft": 1825.5776479429314, "vllm_metric_deltas": { "prompt_tokens": 498.0, "generation_tokens": 512.0, "ttft_count": 1.0, "ttft_sum_s": 0.2602367401123047, "e2e_count": 1.0, "e2e_sum_s": 6.222656726837158 }, "vllm_histogram_deltas": { "vllm:request_queue_time_seconds": { "count": 1.0, "sum": 1.4377059414982796e-05, "mean": 1.4377059414982796e-05 }, "vllm:request_prefill_time_seconds": { "count": 1.0, "sum": 0.2550938909407705, "mean": 0.2550938909407705 }, "vllm:request_decode_time_seconds": { "count": 1.0, "sum": 5.962670223088935, "mean": 5.962670223088935 }, "vllm:request_inference_time_seconds": { "count": 1.0, "sum": 6.2177641140297055, "mean": 6.2177641140297055 }, "vllm:request_time_per_output_token_seconds": { "count": 1.0, "sum": 0.011668630573559574, "mean": 0.011668630573559574 }, "vllm:inter_token_latency_seconds": { "count": 511.0, "sum": 5.962670223088935, "mean": 0.01166863057355956 }, "vllm:iteration_tokens_total": { "count": 512.0, "sum": 1010.0, "mean": 1.97265625 } }, "ttft_ms_vllm_metrics": 260.2367401123047, "e2e_ms_vllm_metrics": 6222.656726837158, "queue_ms_vllm_histogram": 0.014377059414982796, "prefill_ms_vllm_histogram": 255.0938909407705, "decode_ms_vllm_histogram": 5962.670223088935, "decode_ms_per_generation_token_vllm_histogram": 11.645840279470576, "inference_ms_vllm_histogram": 6217.7641140297055, "time_per_output_token_ms_vllm_histogram": 11.668630573559573, "inter_token_ms_vllm_histogram": 11.668630573559561, "iteration_tokens_per_step_vllm_histogram": 1.97265625, "vram_mib_before": {}, "vram_mib_after": {}, "first_chunk_tokens_client_estimate": 1, "streamed_text_chunks": 510, "text_preview": "\n\nHere's a thinking process:\n\n1. **Analyze User Input:**\n - **Context:** Tuning an Intel XPU inference server.\n - **Key Observations (repeated many times, but essentially the same):**\n - Stable baseline decoding\n - Promp" } ], "summary": { "tok_s_out_client_after_first_chunk": { "mean": 86.03715537909211, "median": 86.03715537909211, "min": 86.03715537909211, "max": 86.03715537909211 }, "tok_s_out_client_after_first_chunk_corrected": { "mean": 85.86911405999231, "median": 85.86911405999231, "min": 85.86911405999231, "max": 85.86911405999231 }, "tok_s_out_client_e2e": { "mean": 82.26607414527504, "median": 82.26607414527504, "min": 82.26607414527504, "max": 82.26607414527504 }, "tok_s_total_client": { "mean": 162.28268532564022, "median": 162.28268532564022, "min": 162.28268532564022, "max": 162.28268532564022 }, "ttft_ms_client": { "mean": 272.79036887921393, "median": 272.79036887921393, "min": 272.79036887921393, "max": 272.79036887921393 }, "ttft_ms_vllm_metrics": { "mean": 260.2367401123047, "median": 260.2367401123047, "min": 260.2367401123047, "max": 260.2367401123047 }, "tok_s_prefill_lower_bound_from_ttft": { "mean": 1825.5776479429314, "median": 1825.5776479429314, "min": 1825.5776479429314, "max": 1825.5776479429314 }, "e2e_ms_vllm_metrics": { "mean": 6222.656726837158, "median": 6222.656726837158, "min": 6222.656726837158, "max": 6222.656726837158 }, "queue_ms_vllm_histogram": { "mean": 0.014377059414982796, "median": 0.014377059414982796, "min": 0.014377059414982796, "max": 0.014377059414982796 }, "prefill_ms_vllm_histogram": { "mean": 255.0938909407705, "median": 255.0938909407705, "min": 255.0938909407705, "max": 255.0938909407705 }, "decode_ms_vllm_histogram": { "mean": 5962.670223088935, "median": 5962.670223088935, "min": 5962.670223088935, "max": 5962.670223088935 }, "decode_ms_per_generation_token_vllm_histogram": { "mean": 11.645840279470576, "median": 11.645840279470576, "min": 11.645840279470576, "max": 11.645840279470576 }, "inference_ms_vllm_histogram": { "mean": 6217.7641140297055, "median": 6217.7641140297055, "min": 6217.7641140297055, "max": 6217.7641140297055 }, "time_per_output_token_ms_vllm_histogram": { "mean": 11.668630573559573, "median": 11.668630573559573, "min": 11.668630573559573, "max": 11.668630573559573 }, "inter_token_ms_vllm_histogram": { "mean": 11.668630573559561, "median": 11.668630573559561, "min": 11.668630573559561, "max": 11.668630573559561 }, "iteration_tokens_per_step_vllm_histogram": { "mean": 1.97265625, "median": 1.97265625, "min": 1.97265625, "max": 1.97265625 } } }