{ "label": "tp4-mtp-k1-replayssm-iter20-graph-restoresync", "stamp": "20260623023236", "return_codes": { "metrics": 0, "json_canary": 1, "color_canary": 1, "quality_suite": 0 }, "status": { "metrics": "passed", "json_canary": "failed", "color_canary": "failed", "quality_suite": "skipped" }, "artifacts": { "metrics": "/home/steve/llm-optimizations/data/qwen36-ablation-tp4-mtp-k1-replayssm-iter20-graph-restoresync-p512o512-20260623023236.json", "json_canary": "/home/steve/llm-optimizations/data/qwen36-ablation-tp4-mtp-k1-replayssm-iter20-graph-restoresync-json-repeat96-20260623023236.json", "color_canary": "/home/steve/llm-optimizations/data/qwen36-ablation-tp4-mtp-k1-replayssm-iter20-graph-restoresync-color-repeat96-20260623023236.json", "quality_suite": "/home/steve/llm-optimizations/data/qwen36-ablation-tp4-mtp-k1-replayssm-iter20-graph-restoresync-quality-suite-20260623023236.json", "log": "/home/steve/llm-optimizations/data/qwen36-ablation-tp4-mtp-k1-replayssm-iter20-graph-restoresync-20260623023236.log" }, "artifact_exists": { "metrics": true, "json_canary": true, "color_canary": true, "quality_suite": false, "log": true }, "env": { "MODEL_PATH": "/mnt/fast-ai/llm-cache/hf/models--nameistoken--Qwen3.6-35B-A3B-Quark-W8A8-INT8/snapshots/cced56592e8c8935f8220836b4baa04dfd389118", "SERVED_MODEL_NAME": "qwen36-35b-a3b-fp8", "SERVER_LAUNCHER": "/home/steve/llm-optimizations/scripts/launch-qwen36-quark-int8-hybrid-mtp.sh", "TP_SIZE": "4", "MAX_MODEL_LEN": "32768", "MAX_NUM_BATCHED_TOKENS": "8192", "MAX_NUM_SEQS": "48", "ONEAPI_DEVICE_SELECTOR": "level_zero:0,1,2,3", "ZE_AFFINITY_MASK": "0,1,2,3", "QWEN36_XPU_PREFLIGHT": "auto", "CACHE_LABEL": "qwen36-ablation-tp4-mtp-k1-replayssm-iter20-graph-restoresync", "TORCHINDUCTOR_CACHE_DIR": "/mnt/fast-ai/vllm-cache-exp/qwen36-ablation-tp4-mtp-k1-replayssm-iter20-graph-restoresync/torchinductor", "VLLM_CACHE_ROOT": "/mnt/fast-ai/vllm-cache-exp/qwen36-ablation-tp4-mtp-k1-replayssm-iter20-graph-restoresync/vllm", "BASE_URL": "http://127.0.0.1:18080", "PORT": "18080", "VLLM_XPU_GDN_NATIVE_FALLBACK": "decode,prefill", "VLLM_XPU_GDN_NATIVE_PREFILL_FALLBACK": null, "VLLM_XPU_GDN_PREFILL_RECURRENT_FALLBACK": "1", "VLLM_XPU_GDN_NATIVE_FALLBACK_INITIAL_DECODE_STEPS": null, "VLLM_XPU_GDN_NATIVE_FALLBACK_DECODE_LAYERS": null, "VLLM_XPU_GDN_NATIVE_FALLBACK_DECODE_MAX_LAYER": null, "VLLM_XPU_GDN_NATIVE_FALLBACK_DECODE_LAYER_REGEX": null, "VLLM_XPU_GDN_SYNC_BEFORE_NATIVE_DECODE": null, "VLLM_XPU_GDN_SYNC_BEFORE_NATIVE_DECODE_LAYERS": null, "VLLM_XPU_GREEDY_SAMPLE_TOPK_FALLBACK": null, "VLLM_XPU_GREEDY_SAMPLE_XPU_FALLBACK": null, "VLLM_XPU_GREEDY_SAMPLE_BF16_TOPK_FASTPATH": null, "VLLM_XPU_ASYNC_CLONE_SAMPLED_TOKEN_IDS": null, "VLLM_XPU_SANITIZE_LOGITS_NAN": null, "VLLM_XPU_LOCAL_ARGMAX_DECODE": null, "VLLM_XPU_LOCAL_ARGMAX_DIRECT_GATHER": null, "VLLM_XPU_LOCAL_ARGMAX_DIRECT_GATHER_REUSE": null, "VLLM_XPU_LOCAL_ARGMAX_PACKED_GATHER": null, "VLLM_XPU_LOCAL_ARGMAX_SYNC_TOKENS": null, "VLLM_XPU_DUMMY_LOGIT_INDICES_STATIC": null, "VLLM_XPU_ZERO_ALL_PREFILL_GDN_STATE": null, "VLLM_XPU_ZERO_FRESH_GDN_STATE": null, "VLLM_XPU_DISABLE_PREFILL_CUDAGRAPH_REPLAY": "0", "VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE": null, "XPU_GRAPH": "1", "VLLM_XPU_ENABLE_XPU_GRAPH": "1", "VLLM_XPU_FORCE_GRAPH_WITH_COMM": "1", "VLLM_XPU_GRAPH_NOOP_COMM_CAPTURE": "1", "VLLM_XPU_USE_CUSTOM_OP_COLLECTIVES": null, "VLLM_XPU_COMPILE_ALLREDUCE_CUSTOM_OP": null, "VLLM_XPU_CUSTOM_ALLREDUCE_GRAPH_CLONE_INPUT": null, "VLLM_XPU_CUSTOM_ALLREDUCE_CLONE_INPUT": null, "VLLM_XPU_CUDAGRAPH_REPLAY_MAX_PIECEWISE_INDEX": null, "VLLM_XPU_CUDAGRAPH_DISABLE_SUBMOD_REGEX": null, "VLLM_XPU_CUDAGRAPH_RECAPTURE_AFTER_N_REPLAYS": null, "VLLM_XPU_CUDAGRAPH_RECAPTURE_REGEX": null, "VLLM_XPU_CUDAGRAPH_STRONG_OUTPUT": null, "VLLM_XPU_CUDAGRAPH_PER_WRAPPER_POOL": null, "VLLM_XPU_CUDAGRAPH_NO_GLOBAL_POOL": null, "VLLM_XPU_DECODE_CUDAGRAPH_REPLAY_EAGER_EVERY_N_REQUESTS": null, "VLLM_XPU_CUDAGRAPH_ALLOW_RUNTIME_RECAPTURE": null, "VLLM_XPU_SYNC_CUDAGRAPH_REPLAY": null, "VLLM_XPU_CUDAGRAPH_SANITIZE_REPLAY_INPUTS": null, "VLLM_XPU_CUDAGRAPH_REPLAY_TRACE_FILE": null, "VLLM_XPU_CUDAGRAPH_REPLAY_TRACE_MAX_LINES": null, "VLLM_XPU_CUDAGRAPH_REPLAY_TRACE_RANK": null, "VLLM_XPU_CUDAGRAPH_REPLAY_TRACE_SUBMOD_REGEX": null, "VLLM_XPU_CUDAGRAPH_REPLAY_TRACE_REQ_REGEX": null, "VLLM_XPU_CUDAGRAPH_REPLAY_TRACE_INPUTS": null, "VLLM_XPU_CUDAGRAPH_REPLAY_TRACE_DIGEST": null, "VLLM_XPU_GDN_TRACE_FILE": null, "VLLM_XPU_GDN_TRACE_MAX_LINES": null, "VLLM_XPU_GDN_TRACE_RANK": null, "VLLM_XPU_GDN_TRACE_LAYER_REGEX": null, "VLLM_XPU_GDN_TRACE_REQ_REGEX": null, "VLLM_XPU_GDN_TRACE_PREFILL_ONLY": null, "VLLM_XPU_GDN_TRACE_DECODE_ONLY": null, "VLLM_XPU_GDN_TRACE_TENSOR_LIMIT": null, "VLLM_XPU_GDN_TRACE_STATE_LIMIT": null, "VLLM_XPU_CUDAGRAPH_ZERO_REPLAY_OUTPUT_REGEX": null, "VLLM_XPU_CUDAGRAPH_ZERO_REPLAY_OUTPUT_INDICES": null, "VLLM_XPU_CUDAGRAPH_CLONE_REPLAY_OUTPUT_REGEX": null, "VLLM_XPU_CUDAGRAPH_CLONE_REPLAY_OUTPUT_INDICES": null, "VLLM_XPU_CUDAGRAPH_STATIC_INPUT_REGEX": null, "VLLM_XPU_CUDAGRAPH_STATIC_INPUT_INDICES": null, "VLLM_XPU_CUDAGRAPH_STATIC_INPUT_MAX_NUMEL": null, "VLLM_XPU_CUDAGRAPH_COMPARE_DIRECT_REGEX": null, "VLLM_XPU_CUDAGRAPH_COMPARE_DIRECT_RETURN_DIRECT": null, "VLLM_XPU_W8A8_EXPERIMENTAL_ALLOW": null, "VLLM_XPU_W8A8_USE_OFFSETS": null, "VLLM_XPU_W8A8_OFFSETS_PREFIX_OP": null, "VLLM_XPU_MOE_W8A8_MIDDLE_LAYERLET": null, "VLLM_XPU_MOE_W8A8_FULL_LAYERLET": null, "VLLM_XPU_MOE_W8A8_FUSED_Q1": null, "VLLM_XPU_MOE_W8A8_FAST_GATHER": null, "VLLM_XPU_MOE_W8A8_DIRECT_GEMM2_GATHER": null, "VLLM_XPU_MOE_W8A8_DPAS_GEMM2_GATHER": null, "VLLM_XPU_MOE_W8A8_DPAS_GEMM2_GATHER_NTILE": null, "VLLM_XPU_MOE_W8A8_WORKSPACE_ATOMIC": null, "VLLM_XPU_MOE_W8A8_ROUTE_GEMM1": null, "VLLM_XPU_MOE_W8A8_ROUTE_GEMM1_MTILE": null, "VLLM_XPU_INT8_MOE_MIXED_WORKSPACE": null, "VLLM_XPU_INT8_MOE_PERSISTENT_SCRATCH": null, "VLLM_XPU_INT8_MOE_FUSED_PROLOGUE_OFFSET": null, "VLLM_XPU_INT8_MOE_FUSED_PROLOGUE_OFFSET_ALLOW_CAPTURE": null, "VLLM_XPU_INT8_MOE_FUSED_PROLOGUE_OFFSET_SKIP_LAYERS": null, "VLLM_XPU_INT8_MOE_FUSED_PROLOGUE_OFFSET_LAYER_REGEX": null, "VLLM_XPU_INT8_MOE_ACTIVE_OFFSET_GEMM": null, "VLLM_XPU_SHARED_EXPERTS_STREAM": null, "VLLM_XPU_SHARED_EXPERT_FUSED_ACT_QUANT": null, "VLLM_XPU_SHARED_EXPERT_FUSED_ACT_QUANT_OUT": null, "VLLM_XPU_SHARED_EXPERT_BOUNDARY_OUT": null, "VLLM_XPU_SHARED_EXPERT_FUSED_GATE_MUL": null, "VLLM_XPU_MOE_SHARED_ADD_ALLREDUCE_CUSTOM_OP": null, "VLLM_XPU_METADATA_COPY_ALLOW": null, "VLLM_XPU_BLOCK_TABLE_DIRTY_COMMIT": null, "VLLM_XPU_BLOCK_TABLE_DIRTY_COMMIT_LOG_EVERY": null, "GPU_MEMORY_UTILIZATION": "0.95", "COMPILATION_CONFIG": "{\"cudagraph_mode\":\"PIECEWISE\",\"max_cudagraph_capture_size\":4}", "SPEC_CONFIG": null, "ABLATION_FAST_GRAPH_AUTOCONFIG": "1", "VLLM_EXTRA_ARGS": "--no-async-scheduling --mamba-cache-mode align", "METRICS_PROMPT_KIND": null, "METRICS_PROMPT_PRESET": null, "METRICS_RANDOM_PREFIX_LEN": null, "METRICS_PROMPT_TOKENS": null, "METRICS_OUTPUT_TOKENS": null, "METRICS_WARMUP_OUTPUT_TOKENS": null, "JSON_LOGPROBS": null, "COLOR_LOGPROBS": null }, "metrics_summary": { "tok_s_out_client_after_first_chunk": { "mean": 73.59690880338317, "median": 73.59690880338317, "min": 73.02856201843642, "max": 74.16525558832993 }, "tok_s_out_client_after_first_chunk_corrected": { "mean": 73.45316484087655, "median": 73.45316484087655, "min": 72.88592810824416, "max": 74.02040157350896 }, "tok_s_out_client_e2e": { "mean": 71.63384923458285, "median": 71.63384923458285, "min": 71.10007655702755, "max": 72.16762191213816 }, "tok_s_total_client": { "mean": 141.30896040415757, "median": 141.30896040415757, "min": 140.25601039569887, "max": 142.36191041261628 }, "ttft_ms_client": { "mean": 190.6270420004148, "median": 190.6270420004148, "min": 190.1619020063663, "max": 191.09218199446332 }, "ttft_ms_vllm_metrics": { "mean": 175.69303512573242, "median": 175.69303512573242, "min": 175.54187774658203, "max": 175.8441925048828 }, "tok_s_prefill_lower_bound_from_ttft": { "mean": 2612.446585815486, "median": 2612.446585815486, "min": 2606.072078942659, "max": 2618.8210926883125 }, "e2e_ms_vllm_metrics": { "mean": 7146.629810333252, "median": 7146.629810333252, "min": 7093.356609344482, "max": 7199.9030113220215 }, "queue_ms_vllm_histogram": { "mean": 0.014862998796161264, "median": 0.014862998796161264, "min": 0.014015997294336557, "max": 0.01571000029798597 }, "prefill_ms_vllm_histogram": { "mean": 170.8975154979271, "median": 170.8975154979271, "min": 170.82530999323353, "max": 170.96972100262064 }, "decode_ms_vllm_histogram": { "mean": 6971.1847260041395, "median": 6971.1847260041395, "min": 6917.829890997382, "max": 7024.539561010897 }, "decode_ms_per_generation_token_vllm_histogram": { "mean": 13.615595167976835, "median": 13.615595167976835, "min": 13.511386505854261, "max": 13.719803830099409 }, "inference_ms_vllm_histogram": { "mean": 7142.082241502067, "median": 7142.082241502067, "min": 7088.7996120000025, "max": 7195.364871004131 }, "time_per_output_token_ms_vllm_histogram": { "mean": 13.64224016830555, "median": 13.64224016830555, "min": 13.537827575337335, "max": 13.746652761273765 }, "inter_token_ms_vllm_histogram": { "mean": 13.642240168305557, "median": 13.642240168305557, "min": 13.537827575337342, "max": 13.746652761273772 }, "iteration_tokens_per_step_vllm_histogram": { "mean": 1.97265625, "median": 1.97265625, "min": 1.97265625, "max": 1.97265625 } }, "speed": { "tok_s_out_corrected_mean": 73.45316484087655, "tok_s_out_after_first_chunk_mean": 73.59690880338317, "tok_s_out_e2e_mean": 71.63384923458285, "decode_ms_per_token_mean": 13.615595167976835, "ttft_ms_client_mean": 190.6270420004148 }, "json_canary": { "pass_all": false, "repeats_completed": 83, "mismatch_count": 1, "first_mismatch": { "index": 82, "request_id": null, "text": "{\"key\": \"12\", \"unit\": \"widgets\"}", "normalized": "{\"key\": \"12\", \"unit\": \"widgets\"}", "token_ids": [ 4754, 767, 763, 328, 16, 17, 487, 328, 3715, 763, 328, 28844, 8934 ], "finish_reason": "stop", "elapsed_s": 0.2709183970000595, "usage": { "prompt_tokens": 36, "total_tokens": 50, "completion_tokens": 14, "prompt_tokens_details": null }, "json_parsed": { "key": "12", "unit": "widgets" }, "pass": false } }, "color_canary": { "pass_all": false, "repeats_completed": 23, "mismatch_count": 1, "first_mismatch": { "index": 22, "request_id": null, "text": "blue, green, red, yellow", "normalized": "blue, green, red, yellow", "token_ids": [ 11855, 11, 5983, 11, 2438, 11, 13358 ], "finish_reason": "stop", "elapsed_s": 0.2019886799971573, "usage": { "prompt_tokens": 30, "total_tokens": 38, "completion_tokens": 8, "prompt_tokens_details": null }, "pass": false } }, "decision": { "accepted_by_requested_gates": false, "gate_failures": [ "json_canary", "color_canary" ] } }