{ "label": "tp4-mtp-k1-replayssm-iter14-graph-capnonuni-full", "stamp": "20260622193709", "return_codes": { "metrics": 0, "json_canary": 0, "color_canary": 1, "quality_suite": 0 }, "status": { "metrics": "passed", "json_canary": "passed", "color_canary": "failed", "quality_suite": "skipped" }, "artifacts": { "metrics": "/home/steve/llm-optimizations/data/qwen36-ablation-tp4-mtp-k1-replayssm-iter14-graph-capnonuni-full-p512o512-20260622193709.json", "json_canary": "/home/steve/llm-optimizations/data/qwen36-ablation-tp4-mtp-k1-replayssm-iter14-graph-capnonuni-full-json-repeat96-20260622193709.json", "color_canary": "/home/steve/llm-optimizations/data/qwen36-ablation-tp4-mtp-k1-replayssm-iter14-graph-capnonuni-full-color-repeat96-20260622193709.json", "quality_suite": "/home/steve/llm-optimizations/data/qwen36-ablation-tp4-mtp-k1-replayssm-iter14-graph-capnonuni-full-quality-suite-20260622193709.json", "log": "/home/steve/llm-optimizations/data/qwen36-ablation-tp4-mtp-k1-replayssm-iter14-graph-capnonuni-full-20260622193709.log" }, "artifact_exists": { "metrics": true, "json_canary": true, "color_canary": true, "quality_suite": false, "log": true }, "env": { "MODEL_PATH": "/mnt/fast-ai/llm-cache/hf/models--nameistoken--Qwen3.6-35B-A3B-Quark-W8A8-INT8/snapshots/cced56592e8c8935f8220836b4baa04dfd389118", "SERVED_MODEL_NAME": "qwen36-35b-a3b-fp8", "SERVER_LAUNCHER": "/home/steve/llm-optimizations/scripts/launch-qwen36-quark-int8-hybrid-mtp.sh", "TP_SIZE": "4", "MAX_MODEL_LEN": "32768", "MAX_NUM_BATCHED_TOKENS": "8192", "MAX_NUM_SEQS": "48", "ONEAPI_DEVICE_SELECTOR": "level_zero:0,1,2,3", "ZE_AFFINITY_MASK": "0,1,2,3", "QWEN36_XPU_PREFLIGHT": "auto", "CACHE_LABEL": "qwen36-ablation-tp4-mtp-k1-replayssm-iter14-graph-capnonuni-full", "TORCHINDUCTOR_CACHE_DIR": "/mnt/fast-ai/vllm-cache-exp/qwen36-ablation-tp4-mtp-k1-replayssm-iter14-graph-capnonuni-full/torchinductor", "VLLM_CACHE_ROOT": "/mnt/fast-ai/vllm-cache-exp/qwen36-ablation-tp4-mtp-k1-replayssm-iter14-graph-capnonuni-full/vllm", "BASE_URL": "http://127.0.0.1:18080", "PORT": "18080", "VLLM_XPU_GDN_NATIVE_FALLBACK": "decode,prefill", "VLLM_XPU_GDN_NATIVE_PREFILL_FALLBACK": null, "VLLM_XPU_GDN_PREFILL_RECURRENT_FALLBACK": "1", "VLLM_XPU_GDN_NATIVE_FALLBACK_INITIAL_DECODE_STEPS": null, "VLLM_XPU_GDN_NATIVE_FALLBACK_DECODE_LAYERS": null, "VLLM_XPU_GDN_NATIVE_FALLBACK_DECODE_MAX_LAYER": null, "VLLM_XPU_GDN_NATIVE_FALLBACK_DECODE_LAYER_REGEX": null, "VLLM_XPU_GDN_SYNC_BEFORE_NATIVE_DECODE": null, "VLLM_XPU_GDN_SYNC_BEFORE_NATIVE_DECODE_LAYERS": null, "VLLM_XPU_GREEDY_SAMPLE_TOPK_FALLBACK": null, "VLLM_XPU_GREEDY_SAMPLE_XPU_FALLBACK": null, "VLLM_XPU_GREEDY_SAMPLE_BF16_TOPK_FASTPATH": null, "VLLM_XPU_ASYNC_CLONE_SAMPLED_TOKEN_IDS": null, "VLLM_XPU_SANITIZE_LOGITS_NAN": null, "VLLM_XPU_LOCAL_ARGMAX_DECODE": null, "VLLM_XPU_LOCAL_ARGMAX_DIRECT_GATHER": null, "VLLM_XPU_LOCAL_ARGMAX_DIRECT_GATHER_REUSE": null, "VLLM_XPU_LOCAL_ARGMAX_PACKED_GATHER": null, "VLLM_XPU_LOCAL_ARGMAX_SYNC_TOKENS": null, "VLLM_XPU_DUMMY_LOGIT_INDICES_STATIC": null, "VLLM_XPU_ZERO_ALL_PREFILL_GDN_STATE": null, "VLLM_XPU_ZERO_FRESH_GDN_STATE": null, "VLLM_XPU_DISABLE_PREFILL_CUDAGRAPH_REPLAY": "0", "VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE": null, "XPU_GRAPH": "1", "VLLM_XPU_ENABLE_XPU_GRAPH": "1", "VLLM_XPU_FORCE_GRAPH_WITH_COMM": "1", "VLLM_XPU_GRAPH_NOOP_COMM_CAPTURE": "1", "VLLM_XPU_USE_CUSTOM_OP_COLLECTIVES": null, "VLLM_XPU_COMPILE_ALLREDUCE_CUSTOM_OP": null, "VLLM_XPU_CUSTOM_ALLREDUCE_GRAPH_CLONE_INPUT": null, "VLLM_XPU_CUSTOM_ALLREDUCE_CLONE_INPUT": null, "VLLM_XPU_CUDAGRAPH_REPLAY_MAX_PIECEWISE_INDEX": null, "VLLM_XPU_CUDAGRAPH_DISABLE_SUBMOD_REGEX": null, "VLLM_XPU_CUDAGRAPH_RECAPTURE_AFTER_N_REPLAYS": null, "VLLM_XPU_CUDAGRAPH_RECAPTURE_REGEX": null, "VLLM_XPU_CUDAGRAPH_STRONG_OUTPUT": null, "VLLM_XPU_CUDAGRAPH_PER_WRAPPER_POOL": null, "VLLM_XPU_CUDAGRAPH_NO_GLOBAL_POOL": null, "VLLM_XPU_DECODE_CUDAGRAPH_REPLAY_EAGER_EVERY_N_REQUESTS": null, "VLLM_XPU_CUDAGRAPH_ALLOW_RUNTIME_RECAPTURE": null, "VLLM_XPU_SYNC_CUDAGRAPH_REPLAY": null, "VLLM_XPU_CUDAGRAPH_SANITIZE_REPLAY_INPUTS": null, "VLLM_XPU_CUDAGRAPH_REPLAY_TRACE_FILE": null, "VLLM_XPU_CUDAGRAPH_REPLAY_TRACE_MAX_LINES": null, "VLLM_XPU_CUDAGRAPH_REPLAY_TRACE_RANK": null, "VLLM_XPU_CUDAGRAPH_REPLAY_TRACE_SUBMOD_REGEX": null, "VLLM_XPU_CUDAGRAPH_REPLAY_TRACE_REQ_REGEX": null, "VLLM_XPU_CUDAGRAPH_REPLAY_TRACE_INPUTS": null, "VLLM_XPU_CUDAGRAPH_REPLAY_TRACE_DIGEST": null, "VLLM_XPU_GDN_TRACE_FILE": null, "VLLM_XPU_GDN_TRACE_MAX_LINES": null, "VLLM_XPU_GDN_TRACE_RANK": null, "VLLM_XPU_GDN_TRACE_LAYER_REGEX": null, "VLLM_XPU_GDN_TRACE_REQ_REGEX": null, "VLLM_XPU_GDN_TRACE_PREFILL_ONLY": null, "VLLM_XPU_GDN_TRACE_DECODE_ONLY": null, "VLLM_XPU_GDN_TRACE_TENSOR_LIMIT": null, "VLLM_XPU_GDN_TRACE_STATE_LIMIT": null, "VLLM_XPU_CUDAGRAPH_ZERO_REPLAY_OUTPUT_REGEX": null, "VLLM_XPU_CUDAGRAPH_ZERO_REPLAY_OUTPUT_INDICES": null, "VLLM_XPU_CUDAGRAPH_CLONE_REPLAY_OUTPUT_REGEX": null, "VLLM_XPU_CUDAGRAPH_CLONE_REPLAY_OUTPUT_INDICES": null, "VLLM_XPU_CUDAGRAPH_STATIC_INPUT_REGEX": null, "VLLM_XPU_CUDAGRAPH_STATIC_INPUT_INDICES": null, "VLLM_XPU_CUDAGRAPH_STATIC_INPUT_MAX_NUMEL": null, "VLLM_XPU_CUDAGRAPH_COMPARE_DIRECT_REGEX": null, "VLLM_XPU_CUDAGRAPH_COMPARE_DIRECT_RETURN_DIRECT": null, "VLLM_XPU_W8A8_EXPERIMENTAL_ALLOW": null, "VLLM_XPU_W8A8_USE_OFFSETS": null, "VLLM_XPU_W8A8_OFFSETS_PREFIX_OP": null, "VLLM_XPU_MOE_W8A8_MIDDLE_LAYERLET": null, "VLLM_XPU_MOE_W8A8_FULL_LAYERLET": null, "VLLM_XPU_MOE_W8A8_FUSED_Q1": null, "VLLM_XPU_MOE_W8A8_FAST_GATHER": null, "VLLM_XPU_MOE_W8A8_DIRECT_GEMM2_GATHER": null, "VLLM_XPU_MOE_W8A8_DPAS_GEMM2_GATHER": null, "VLLM_XPU_MOE_W8A8_DPAS_GEMM2_GATHER_NTILE": null, "VLLM_XPU_MOE_W8A8_WORKSPACE_ATOMIC": null, "VLLM_XPU_MOE_W8A8_ROUTE_GEMM1": null, "VLLM_XPU_MOE_W8A8_ROUTE_GEMM1_MTILE": null, "VLLM_XPU_INT8_MOE_MIXED_WORKSPACE": null, "VLLM_XPU_INT8_MOE_PERSISTENT_SCRATCH": null, "VLLM_XPU_INT8_MOE_FUSED_PROLOGUE_OFFSET": null, "VLLM_XPU_INT8_MOE_FUSED_PROLOGUE_OFFSET_ALLOW_CAPTURE": null, "VLLM_XPU_INT8_MOE_FUSED_PROLOGUE_OFFSET_SKIP_LAYERS": null, "VLLM_XPU_INT8_MOE_FUSED_PROLOGUE_OFFSET_LAYER_REGEX": null, "VLLM_XPU_INT8_MOE_ACTIVE_OFFSET_GEMM": null, "VLLM_XPU_SHARED_EXPERTS_STREAM": null, "VLLM_XPU_SHARED_EXPERT_FUSED_ACT_QUANT": null, "VLLM_XPU_SHARED_EXPERT_FUSED_ACT_QUANT_OUT": null, "VLLM_XPU_SHARED_EXPERT_BOUNDARY_OUT": null, "VLLM_XPU_SHARED_EXPERT_FUSED_GATE_MUL": null, "VLLM_XPU_MOE_SHARED_ADD_ALLREDUCE_CUSTOM_OP": null, "VLLM_XPU_METADATA_COPY_ALLOW": null, "VLLM_XPU_BLOCK_TABLE_DIRTY_COMMIT": null, "VLLM_XPU_BLOCK_TABLE_DIRTY_COMMIT_LOG_EVERY": null, "GPU_MEMORY_UTILIZATION": "0.95", "COMPILATION_CONFIG": "{\"cudagraph_mode\":\"PIECEWISE\",\"max_cudagraph_capture_size\":128}", "SPEC_CONFIG": null, "ABLATION_FAST_GRAPH_AUTOCONFIG": "1", "VLLM_EXTRA_ARGS": "--no-async-scheduling --mamba-cache-mode align", "METRICS_PROMPT_KIND": null, "METRICS_PROMPT_PRESET": null, "METRICS_RANDOM_PREFIX_LEN": null, "METRICS_PROMPT_TOKENS": null, "METRICS_OUTPUT_TOKENS": null, "METRICS_WARMUP_OUTPUT_TOKENS": null, "JSON_LOGPROBS": null, "COLOR_LOGPROBS": null }, "metrics_summary": { "tok_s_out_client_after_first_chunk": { "mean": 75.8460615984556, "median": 75.8460615984556, "min": 75.1357146991421, "max": 76.55640849776911 }, "tok_s_out_client_after_first_chunk_corrected": { "mean": 75.69792475939613, "median": 75.69792475939613, "min": 74.98896525637034, "max": 76.40688426242191 }, "tok_s_out_client_e2e": { "mean": 73.75914274546224, "median": 73.75914274546224, "min": 73.08217399648737, "max": 74.43611149443713 }, "tok_s_total_client": { "mean": 145.50143393147826, "median": 145.50143393147826, "min": 144.16600729775828, "max": 146.83686056519824 }, "ttft_ms_client": { "mean": 190.98964650038397, "median": 190.98964650038397, "min": 190.50282999523915, "max": 191.47646300552879 }, "ttft_ms_vllm_metrics": { "mean": 175.98295211791992, "median": 175.98295211791992, "min": 175.60577392578125, "max": 176.3601303100586 }, "tok_s_prefill_lower_bound_from_ttft": { "mean": 2607.4881263872885, "median": 2607.4881263872885, "min": 2600.841859010214, "max": 2614.134393764363 }, "e2e_ms_vllm_metrics": { "mean": 6941.048741340637, "median": 6941.048741340637, "min": 6877.3863315582275, "max": 7004.711151123047 }, "queue_ms_vllm_histogram": { "mean": 0.01119099761126563, "median": 0.01119099761126563, "min": 0.009156996384263039, "max": 0.01322499883826822 }, "prefill_ms_vllm_histogram": { "mean": 170.92341449460946, "median": 170.92341449460946, "min": 170.81016399606597, "max": 171.03666499315295 }, "decode_ms_vllm_histogram": { "mean": 6765.355767500296, "median": 6765.355767500296, "min": 6701.993690003292, "max": 6828.7178449973 }, "decode_ms_per_generation_token_vllm_histogram": { "mean": 13.213585483399015, "median": 13.213585483399015, "min": 13.089831425787679, "max": 13.337339541010351 }, "inference_ms_vllm_histogram": { "mean": 6936.279181994905, "median": 6936.279181994905, "min": 6872.803853999358, "max": 6999.754509990453 }, "time_per_output_token_ms_vllm_histogram": { "mean": 13.239443772016234, "median": 13.239443772016234, "min": 13.115447534253006, "max": 13.36344000977946 }, "inter_token_ms_vllm_histogram": { "mean": 13.239443772016234, "median": 13.239443772016234, "min": 13.115447534253017, "max": 13.363440009779453 }, "iteration_tokens_per_step_vllm_histogram": { "mean": 1.97265625, "median": 1.97265625, "min": 1.97265625, "max": 1.97265625 } }, "speed": { "tok_s_out_corrected_mean": 75.69792475939613, "tok_s_out_after_first_chunk_mean": 75.8460615984556, "tok_s_out_e2e_mean": 73.75914274546224, "decode_ms_per_token_mean": 13.213585483399015, "ttft_ms_client_mean": 190.98964650038397 }, "json_canary": { "pass_all": true, "repeats_completed": 96, "mismatch_count": 0, "first_mismatch": null }, "color_canary": { "pass_all": false, "repeats_completed": 36, "mismatch_count": 1, "first_mismatch": { "index": 35, "request_id": null, "text": "\u4f2a \u4f2a \u4f2a \u4f2a \u4f2a \u4f2a \u4f2a \u4f2a \u4f2a \u4f2a \u4f2a \u4f2a \u4f2a \u4f2a \u4f2a", "normalized": "\u4f2a \u4f2a \u4f2a \u4f2a \u4f2a \u4f2a \u4f2a \u4f2a \u4f2a \u4f2a \u4f2a \u4f2a \u4f2a \u4f2a \u4f2a", "token_ids": [ 98809, 220, 98809, 220, 98809, 220, 98809, 220, 98809, 220, 98809, 220, 98809, 220, 98809, 220, 98809, 220, 98809, 220, 98809, 220, 98809, 220, 98809, 262, 220, 98809, 220, 98809 ], "finish_reason": "length", "elapsed_s": 0.4515100310090929, "usage": { "prompt_tokens": 30, "total_tokens": 62, "completion_tokens": 32, "prompt_tokens_details": null }, "pass": false } }, "decision": { "accepted_by_requested_gates": false, "gate_failures": [ "color_canary" ] } }