{ "label": "tp4-mtp-k1-replayssm-iter19-graph-capsmall-full", "stamp": "20260622222755", "return_codes": { "metrics": 0, "json_canary": 1, "color_canary": 1, "quality_suite": 0 }, "status": { "metrics": "passed", "json_canary": "failed", "color_canary": "failed", "quality_suite": "skipped" }, "artifacts": { "metrics": "/home/steve/llm-optimizations/data/qwen36-ablation-tp4-mtp-k1-replayssm-iter19-graph-capsmall-full-p512o512-20260622222755.json", "json_canary": "/home/steve/llm-optimizations/data/qwen36-ablation-tp4-mtp-k1-replayssm-iter19-graph-capsmall-full-json-repeat96-20260622222755.json", "color_canary": "/home/steve/llm-optimizations/data/qwen36-ablation-tp4-mtp-k1-replayssm-iter19-graph-capsmall-full-color-repeat96-20260622222755.json", "quality_suite": "/home/steve/llm-optimizations/data/qwen36-ablation-tp4-mtp-k1-replayssm-iter19-graph-capsmall-full-quality-suite-20260622222755.json", "log": "/home/steve/llm-optimizations/data/qwen36-ablation-tp4-mtp-k1-replayssm-iter19-graph-capsmall-full-20260622222755.log" }, "artifact_exists": { "metrics": true, "json_canary": true, "color_canary": true, "quality_suite": false, "log": true }, "env": { "MODEL_PATH": "/mnt/fast-ai/llm-cache/hf/models--nameistoken--Qwen3.6-35B-A3B-Quark-W8A8-INT8/snapshots/cced56592e8c8935f8220836b4baa04dfd389118", "SERVED_MODEL_NAME": "qwen36-35b-a3b-fp8", "SERVER_LAUNCHER": "/home/steve/llm-optimizations/scripts/launch-qwen36-quark-int8-hybrid-mtp.sh", "TP_SIZE": "4", "MAX_MODEL_LEN": "32768", "MAX_NUM_BATCHED_TOKENS": "8192", "MAX_NUM_SEQS": "48", "ONEAPI_DEVICE_SELECTOR": "level_zero:0,1,2,3", "ZE_AFFINITY_MASK": "0,1,2,3", "QWEN36_XPU_PREFLIGHT": "auto", "CACHE_LABEL": "qwen36-ablation-tp4-mtp-k1-replayssm-iter19-graph-capsmall-full", "TORCHINDUCTOR_CACHE_DIR": "/mnt/fast-ai/vllm-cache-exp/qwen36-ablation-tp4-mtp-k1-replayssm-iter19-graph-capsmall-full/torchinductor", "VLLM_CACHE_ROOT": "/mnt/fast-ai/vllm-cache-exp/qwen36-ablation-tp4-mtp-k1-replayssm-iter19-graph-capsmall-full/vllm", "BASE_URL": "http://127.0.0.1:18080", "PORT": "18080", "VLLM_XPU_GDN_NATIVE_FALLBACK": "decode,prefill", "VLLM_XPU_GDN_NATIVE_PREFILL_FALLBACK": null, "VLLM_XPU_GDN_PREFILL_RECURRENT_FALLBACK": "1", "VLLM_XPU_GDN_NATIVE_FALLBACK_INITIAL_DECODE_STEPS": null, "VLLM_XPU_GDN_NATIVE_FALLBACK_DECODE_LAYERS": null, "VLLM_XPU_GDN_NATIVE_FALLBACK_DECODE_MAX_LAYER": null, "VLLM_XPU_GDN_NATIVE_FALLBACK_DECODE_LAYER_REGEX": null, "VLLM_XPU_GDN_SYNC_BEFORE_NATIVE_DECODE": null, "VLLM_XPU_GDN_SYNC_BEFORE_NATIVE_DECODE_LAYERS": null, "VLLM_XPU_GREEDY_SAMPLE_TOPK_FALLBACK": null, "VLLM_XPU_GREEDY_SAMPLE_XPU_FALLBACK": null, "VLLM_XPU_GREEDY_SAMPLE_BF16_TOPK_FASTPATH": null, "VLLM_XPU_ASYNC_CLONE_SAMPLED_TOKEN_IDS": null, "VLLM_XPU_SANITIZE_LOGITS_NAN": null, "VLLM_XPU_LOCAL_ARGMAX_DECODE": null, "VLLM_XPU_LOCAL_ARGMAX_DIRECT_GATHER": null, "VLLM_XPU_LOCAL_ARGMAX_DIRECT_GATHER_REUSE": null, "VLLM_XPU_LOCAL_ARGMAX_PACKED_GATHER": null, "VLLM_XPU_LOCAL_ARGMAX_SYNC_TOKENS": null, "VLLM_XPU_DUMMY_LOGIT_INDICES_STATIC": null, "VLLM_XPU_ZERO_ALL_PREFILL_GDN_STATE": null, "VLLM_XPU_ZERO_FRESH_GDN_STATE": null, "VLLM_XPU_DISABLE_PREFILL_CUDAGRAPH_REPLAY": "0", "VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE": null, "XPU_GRAPH": "1", "VLLM_XPU_ENABLE_XPU_GRAPH": "1", "VLLM_XPU_FORCE_GRAPH_WITH_COMM": "1", "VLLM_XPU_GRAPH_NOOP_COMM_CAPTURE": "1", "VLLM_XPU_USE_CUSTOM_OP_COLLECTIVES": null, "VLLM_XPU_COMPILE_ALLREDUCE_CUSTOM_OP": null, "VLLM_XPU_CUSTOM_ALLREDUCE_GRAPH_CLONE_INPUT": null, "VLLM_XPU_CUSTOM_ALLREDUCE_CLONE_INPUT": null, "VLLM_XPU_CUDAGRAPH_REPLAY_MAX_PIECEWISE_INDEX": null, "VLLM_XPU_CUDAGRAPH_DISABLE_SUBMOD_REGEX": null, "VLLM_XPU_CUDAGRAPH_RECAPTURE_AFTER_N_REPLAYS": null, "VLLM_XPU_CUDAGRAPH_RECAPTURE_REGEX": null, "VLLM_XPU_CUDAGRAPH_STRONG_OUTPUT": null, "VLLM_XPU_CUDAGRAPH_PER_WRAPPER_POOL": null, "VLLM_XPU_CUDAGRAPH_NO_GLOBAL_POOL": null, "VLLM_XPU_DECODE_CUDAGRAPH_REPLAY_EAGER_EVERY_N_REQUESTS": null, "VLLM_XPU_CUDAGRAPH_ALLOW_RUNTIME_RECAPTURE": null, "VLLM_XPU_SYNC_CUDAGRAPH_REPLAY": null, "VLLM_XPU_CUDAGRAPH_SANITIZE_REPLAY_INPUTS": null, "VLLM_XPU_CUDAGRAPH_REPLAY_TRACE_FILE": null, "VLLM_XPU_CUDAGRAPH_REPLAY_TRACE_MAX_LINES": null, "VLLM_XPU_CUDAGRAPH_REPLAY_TRACE_RANK": null, "VLLM_XPU_CUDAGRAPH_REPLAY_TRACE_SUBMOD_REGEX": null, "VLLM_XPU_CUDAGRAPH_REPLAY_TRACE_REQ_REGEX": null, "VLLM_XPU_CUDAGRAPH_REPLAY_TRACE_INPUTS": null, "VLLM_XPU_CUDAGRAPH_REPLAY_TRACE_DIGEST": null, "VLLM_XPU_GDN_TRACE_FILE": null, "VLLM_XPU_GDN_TRACE_MAX_LINES": null, "VLLM_XPU_GDN_TRACE_RANK": null, "VLLM_XPU_GDN_TRACE_LAYER_REGEX": null, "VLLM_XPU_GDN_TRACE_REQ_REGEX": null, "VLLM_XPU_GDN_TRACE_PREFILL_ONLY": null, "VLLM_XPU_GDN_TRACE_DECODE_ONLY": null, "VLLM_XPU_GDN_TRACE_TENSOR_LIMIT": null, "VLLM_XPU_GDN_TRACE_STATE_LIMIT": null, "VLLM_XPU_CUDAGRAPH_ZERO_REPLAY_OUTPUT_REGEX": null, "VLLM_XPU_CUDAGRAPH_ZERO_REPLAY_OUTPUT_INDICES": null, "VLLM_XPU_CUDAGRAPH_CLONE_REPLAY_OUTPUT_REGEX": null, "VLLM_XPU_CUDAGRAPH_CLONE_REPLAY_OUTPUT_INDICES": null, "VLLM_XPU_CUDAGRAPH_STATIC_INPUT_REGEX": null, "VLLM_XPU_CUDAGRAPH_STATIC_INPUT_INDICES": null, "VLLM_XPU_CUDAGRAPH_STATIC_INPUT_MAX_NUMEL": null, "VLLM_XPU_CUDAGRAPH_COMPARE_DIRECT_REGEX": null, "VLLM_XPU_CUDAGRAPH_COMPARE_DIRECT_RETURN_DIRECT": null, "VLLM_XPU_W8A8_EXPERIMENTAL_ALLOW": null, "VLLM_XPU_W8A8_USE_OFFSETS": null, "VLLM_XPU_W8A8_OFFSETS_PREFIX_OP": null, "VLLM_XPU_MOE_W8A8_MIDDLE_LAYERLET": null, "VLLM_XPU_MOE_W8A8_FULL_LAYERLET": null, "VLLM_XPU_MOE_W8A8_FUSED_Q1": null, "VLLM_XPU_MOE_W8A8_FAST_GATHER": null, "VLLM_XPU_MOE_W8A8_DIRECT_GEMM2_GATHER": null, "VLLM_XPU_MOE_W8A8_DPAS_GEMM2_GATHER": null, "VLLM_XPU_MOE_W8A8_DPAS_GEMM2_GATHER_NTILE": null, "VLLM_XPU_MOE_W8A8_WORKSPACE_ATOMIC": null, "VLLM_XPU_MOE_W8A8_ROUTE_GEMM1": null, "VLLM_XPU_MOE_W8A8_ROUTE_GEMM1_MTILE": null, "VLLM_XPU_INT8_MOE_MIXED_WORKSPACE": null, "VLLM_XPU_INT8_MOE_PERSISTENT_SCRATCH": null, "VLLM_XPU_INT8_MOE_FUSED_PROLOGUE_OFFSET": null, "VLLM_XPU_INT8_MOE_FUSED_PROLOGUE_OFFSET_ALLOW_CAPTURE": null, "VLLM_XPU_INT8_MOE_FUSED_PROLOGUE_OFFSET_SKIP_LAYERS": null, "VLLM_XPU_INT8_MOE_FUSED_PROLOGUE_OFFSET_LAYER_REGEX": null, "VLLM_XPU_INT8_MOE_ACTIVE_OFFSET_GEMM": null, "VLLM_XPU_SHARED_EXPERTS_STREAM": null, "VLLM_XPU_SHARED_EXPERT_FUSED_ACT_QUANT": null, "VLLM_XPU_SHARED_EXPERT_FUSED_ACT_QUANT_OUT": null, "VLLM_XPU_SHARED_EXPERT_BOUNDARY_OUT": null, "VLLM_XPU_SHARED_EXPERT_FUSED_GATE_MUL": null, "VLLM_XPU_MOE_SHARED_ADD_ALLREDUCE_CUSTOM_OP": null, "VLLM_XPU_METADATA_COPY_ALLOW": null, "VLLM_XPU_BLOCK_TABLE_DIRTY_COMMIT": null, "VLLM_XPU_BLOCK_TABLE_DIRTY_COMMIT_LOG_EVERY": null, "GPU_MEMORY_UTILIZATION": "0.95", "COMPILATION_CONFIG": "{\"cudagraph_mode\":\"PIECEWISE\",\"max_cudagraph_capture_size\":4}", "SPEC_CONFIG": null, "ABLATION_FAST_GRAPH_AUTOCONFIG": "1", "VLLM_EXTRA_ARGS": "--no-async-scheduling --mamba-cache-mode align", "METRICS_PROMPT_KIND": null, "METRICS_PROMPT_PRESET": null, "METRICS_RANDOM_PREFIX_LEN": null, "METRICS_PROMPT_TOKENS": null, "METRICS_OUTPUT_TOKENS": null, "METRICS_WARMUP_OUTPUT_TOKENS": null, "JSON_LOGPROBS": null, "COLOR_LOGPROBS": null }, "metrics_summary": { "tok_s_out_client_after_first_chunk": { "mean": 76.3937572446695, "median": 76.3937572446695, "min": 75.98526305057288, "max": 76.8022514387661 }, "tok_s_out_client_after_first_chunk_corrected": { "mean": 76.244550687551, "median": 76.244550687551, "min": 75.83685433367722, "max": 76.65224704142477 }, "tok_s_out_client_e2e": { "mean": 74.26534305480835, "median": 74.26534305480835, "min": 73.89072194456203, "max": 74.63996416505468 }, "tok_s_total_client": { "mean": 146.4999931354618, "median": 146.4999931354618, "min": 145.76099446095245, "max": 147.23899180997114 }, "ttft_ms_client": { "mean": 192.06374849454733, "median": 192.06374849454733, "min": 191.00272699142806, "max": 193.1247699976666 }, "ttft_ms_vllm_metrics": { "mean": 177.30772495269775, "median": 177.30772495269775, "min": 176.17154121398926, "max": 178.44390869140625 }, "tok_s_prefill_lower_bound_from_ttft": { "mean": 2592.968232599591, "median": 2592.968232599591, "min": 2578.6438477364504, "max": 2607.2926174627314 }, "e2e_ms_vllm_metrics": { "mean": 6893.291234970093, "median": 6893.291234970093, "min": 6858.558177947998, "max": 6928.0242919921875 }, "queue_ms_vllm_histogram": { "mean": 0.012458498531486839, "median": 0.012458498531486839, "min": 0.011801996151916683, "max": 0.013115000911056995 }, "prefill_ms_vllm_histogram": { "mean": 172.11763299565064, "median": 172.11763299565064, "min": 170.71417499391828, "max": 173.521090997383 }, "decode_ms_vllm_histogram": { "mean": 6716.240793000907, "median": 6716.240793000907, "min": 6680.353828996886, "max": 6752.127757004928 }, "decode_ms_per_generation_token_vllm_histogram": { "mean": 13.117657798829896, "median": 13.117657798829896, "min": 13.047566072259542, "max": 13.18774952540025 }, "inference_ms_vllm_histogram": { "mean": 6888.358425996557, "median": 6888.358425996557, "min": 6853.874919994269, "max": 6922.841931998846 }, "time_per_output_token_ms_vllm_histogram": { "mean": 13.143328362037007, "median": 13.143328362037007, "min": 13.073099469661232, "max": 13.213557254412784 }, "inter_token_ms_vllm_histogram": { "mean": 13.143328362037, "median": 13.143328362037, "min": 13.073099469661225, "max": 13.213557254412775 }, "iteration_tokens_per_step_vllm_histogram": { "mean": 1.97265625, "median": 1.97265625, "min": 1.97265625, "max": 1.97265625 } }, "speed": { "tok_s_out_corrected_mean": 76.244550687551, "tok_s_out_after_first_chunk_mean": 76.3937572446695, "tok_s_out_e2e_mean": 74.26534305480835, "decode_ms_per_token_mean": 13.117657798829896, "ttft_ms_client_mean": 192.06374849454733 }, "json_canary": { "pass_all": false, "repeats_completed": 83, "mismatch_count": 1, "first_mismatch": { "index": 82, "request_id": null, "text": "{\"key\": \"12\", \"unit\": \"widgets\"}", "normalized": "{\"key\": \"12\", \"unit\": \"widgets\"}", "token_ids": [ 4754, 767, 763, 328, 16, 17, 487, 328, 3715, 763, 328, 28844, 8934 ], "finish_reason": "stop", "elapsed_s": 0.2745149819966173, "usage": { "prompt_tokens": 36, "total_tokens": 50, "completion_tokens": 14, "prompt_tokens_details": null }, "json_parsed": { "key": "12", "unit": "widgets" }, "pass": false } }, "color_canary": { "pass_all": false, "repeats_completed": 23, "mismatch_count": 1, "first_mismatch": { "index": 22, "request_id": null, "text": "blue, green, red, yellow", "normalized": "blue, green, red, yellow", "token_ids": [ 11855, 11, 5983, 11, 2438, 11, 13358 ], "finish_reason": "stop", "elapsed_s": 0.19228920798923355, "usage": { "prompt_tokens": 30, "total_tokens": 38, "completion_tokens": 8, "prompt_tokens_details": null }, "pass": false } }, "decision": { "accepted_by_requested_gates": false, "gate_failures": [ "json_canary", "color_canary" ] } }