{ "label": "tp4-mtp-k1-parity-fix-v2", "stamp": "20260620061440", "return_codes": { "metrics": 0, "json_canary": 1, "color_canary": 1, "quality_suite": 0 }, "status": { "metrics": "passed", "json_canary": "failed", "color_canary": "failed", "quality_suite": "skipped" }, "artifacts": { "metrics": "/home/steve/llm-optimizations/data/qwen36-ablation-tp4-mtp-k1-parity-fix-v2-p512o512-20260620061440.json", "json_canary": "/home/steve/llm-optimizations/data/qwen36-ablation-tp4-mtp-k1-parity-fix-v2-json-repeat96-20260620061440.json", "color_canary": "/home/steve/llm-optimizations/data/qwen36-ablation-tp4-mtp-k1-parity-fix-v2-color-repeat96-20260620061440.json", "quality_suite": "/home/steve/llm-optimizations/data/qwen36-ablation-tp4-mtp-k1-parity-fix-v2-quality-suite-20260620061440.json", "log": "/home/steve/llm-optimizations/data/qwen36-ablation-tp4-mtp-k1-parity-fix-v2-20260620061440.log" }, "artifact_exists": { "metrics": true, "json_canary": true, "color_canary": true, "quality_suite": false, "log": true }, "env": { "MODEL_PATH": "/mnt/fast-ai/qwen36-quark-int8-fp8-mtp-hybrid", "SERVED_MODEL_NAME": "qwen36-35b-a3b-fp8", "SERVER_LAUNCHER": "scripts/launch-qwen36-quark-int8-accepted.sh", "TP_SIZE": "4", "MAX_MODEL_LEN": "32768", "MAX_NUM_BATCHED_TOKENS": "8192", "MAX_NUM_SEQS": "48", "ONEAPI_DEVICE_SELECTOR": "level_zero:0,1,2,3", "ZE_AFFINITY_MASK": "0,1,2,3", "QWEN36_XPU_PREFLIGHT": "auto", "CACHE_LABEL": "qwen36-ablation-tp4-mtp-k1-parity-fix-v2", "TORCHINDUCTOR_CACHE_DIR": "/mnt/fast-ai/vllm-cache-exp/qwen36-ablation-tp4-mtp-k1-parity-fix-v2/torchinductor", "VLLM_CACHE_ROOT": "/mnt/fast-ai/vllm-cache-exp/qwen36-ablation-tp4-mtp-k1-parity-fix-v2/vllm", "BASE_URL": "http://127.0.0.1:18080", "PORT": "18080", "VLLM_XPU_GDN_NATIVE_FALLBACK": "prefill", "VLLM_XPU_GDN_NATIVE_PREFILL_FALLBACK": null, "VLLM_XPU_GDN_PREFILL_RECURRENT_FALLBACK": "1", "VLLM_XPU_GDN_NATIVE_FALLBACK_INITIAL_DECODE_STEPS": null, "VLLM_XPU_GDN_NATIVE_FALLBACK_DECODE_LAYERS": null, "VLLM_XPU_GDN_NATIVE_FALLBACK_DECODE_MAX_LAYER": null, "VLLM_XPU_GDN_NATIVE_FALLBACK_DECODE_LAYER_REGEX": null, "VLLM_XPU_GDN_SYNC_BEFORE_NATIVE_DECODE": null, "VLLM_XPU_GDN_SYNC_BEFORE_NATIVE_DECODE_LAYERS": null, "VLLM_XPU_GREEDY_SAMPLE_TOPK_FALLBACK": "1", "VLLM_XPU_GREEDY_SAMPLE_XPU_FALLBACK": null, "VLLM_XPU_GREEDY_SAMPLE_BF16_TOPK_FASTPATH": null, "VLLM_XPU_ASYNC_CLONE_SAMPLED_TOKEN_IDS": null, "VLLM_XPU_SANITIZE_LOGITS_NAN": null, "VLLM_XPU_LOCAL_ARGMAX_DECODE": null, "VLLM_XPU_LOCAL_ARGMAX_DIRECT_GATHER": null, "VLLM_XPU_LOCAL_ARGMAX_DIRECT_GATHER_REUSE": null, "VLLM_XPU_LOCAL_ARGMAX_PACKED_GATHER": null, "VLLM_XPU_LOCAL_ARGMAX_SYNC_TOKENS": null, "VLLM_XPU_DUMMY_LOGIT_INDICES_STATIC": null, "VLLM_XPU_ZERO_ALL_PREFILL_GDN_STATE": null, "VLLM_XPU_ZERO_FRESH_GDN_STATE": null, "VLLM_XPU_DISABLE_PREFILL_CUDAGRAPH_REPLAY": "1", "VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE": null, "XPU_GRAPH": "1", "VLLM_XPU_ENABLE_XPU_GRAPH": "1", "VLLM_XPU_FORCE_GRAPH_WITH_COMM": "1", "VLLM_XPU_GRAPH_NOOP_COMM_CAPTURE": "1", "VLLM_XPU_USE_CUSTOM_OP_COLLECTIVES": null, "VLLM_XPU_COMPILE_ALLREDUCE_CUSTOM_OP": null, "VLLM_XPU_CUSTOM_ALLREDUCE_GRAPH_CLONE_INPUT": null, "VLLM_XPU_CUSTOM_ALLREDUCE_CLONE_INPUT": null, "VLLM_XPU_CUDAGRAPH_REPLAY_MAX_PIECEWISE_INDEX": null, "VLLM_XPU_CUDAGRAPH_DISABLE_SUBMOD_REGEX": null, "VLLM_XPU_CUDAGRAPH_RECAPTURE_AFTER_N_REPLAYS": null, "VLLM_XPU_CUDAGRAPH_RECAPTURE_REGEX": null, "VLLM_XPU_CUDAGRAPH_STRONG_OUTPUT": null, "VLLM_XPU_CUDAGRAPH_PER_WRAPPER_POOL": null, "VLLM_XPU_CUDAGRAPH_NO_GLOBAL_POOL": null, "VLLM_XPU_DECODE_CUDAGRAPH_REPLAY_EAGER_EVERY_N_REQUESTS": null, "VLLM_XPU_CUDAGRAPH_ALLOW_RUNTIME_RECAPTURE": null, "VLLM_XPU_SYNC_CUDAGRAPH_REPLAY": null, "VLLM_XPU_CUDAGRAPH_SANITIZE_REPLAY_INPUTS": null, "VLLM_XPU_CUDAGRAPH_REPLAY_TRACE_FILE": null, "VLLM_XPU_CUDAGRAPH_REPLAY_TRACE_MAX_LINES": null, "VLLM_XPU_CUDAGRAPH_REPLAY_TRACE_RANK": null, "VLLM_XPU_CUDAGRAPH_REPLAY_TRACE_SUBMOD_REGEX": null, "VLLM_XPU_CUDAGRAPH_REPLAY_TRACE_REQ_REGEX": null, "VLLM_XPU_CUDAGRAPH_REPLAY_TRACE_INPUTS": null, "VLLM_XPU_CUDAGRAPH_REPLAY_TRACE_DIGEST": null, "VLLM_XPU_GDN_TRACE_FILE": null, "VLLM_XPU_GDN_TRACE_MAX_LINES": null, "VLLM_XPU_GDN_TRACE_RANK": null, "VLLM_XPU_GDN_TRACE_LAYER_REGEX": null, "VLLM_XPU_GDN_TRACE_REQ_REGEX": null, "VLLM_XPU_GDN_TRACE_PREFILL_ONLY": null, "VLLM_XPU_GDN_TRACE_DECODE_ONLY": null, "VLLM_XPU_GDN_TRACE_TENSOR_LIMIT": null, "VLLM_XPU_GDN_TRACE_STATE_LIMIT": null, "VLLM_XPU_CUDAGRAPH_ZERO_REPLAY_OUTPUT_REGEX": null, "VLLM_XPU_CUDAGRAPH_ZERO_REPLAY_OUTPUT_INDICES": null, "VLLM_XPU_CUDAGRAPH_CLONE_REPLAY_OUTPUT_REGEX": null, "VLLM_XPU_CUDAGRAPH_CLONE_REPLAY_OUTPUT_INDICES": null, "VLLM_XPU_CUDAGRAPH_STATIC_INPUT_REGEX": null, "VLLM_XPU_CUDAGRAPH_STATIC_INPUT_INDICES": null, "VLLM_XPU_CUDAGRAPH_STATIC_INPUT_MAX_NUMEL": null, "VLLM_XPU_CUDAGRAPH_COMPARE_DIRECT_REGEX": null, "VLLM_XPU_CUDAGRAPH_COMPARE_DIRECT_RETURN_DIRECT": null, "VLLM_XPU_W8A8_EXPERIMENTAL_ALLOW": null, "VLLM_XPU_W8A8_USE_OFFSETS": null, "VLLM_XPU_W8A8_OFFSETS_PREFIX_OP": null, "VLLM_XPU_MOE_W8A8_MIDDLE_LAYERLET": null, "VLLM_XPU_MOE_W8A8_FULL_LAYERLET": null, "VLLM_XPU_MOE_W8A8_FUSED_Q1": null, "VLLM_XPU_MOE_W8A8_FAST_GATHER": null, "VLLM_XPU_MOE_W8A8_DIRECT_GEMM2_GATHER": null, "VLLM_XPU_MOE_W8A8_DPAS_GEMM2_GATHER": null, "VLLM_XPU_MOE_W8A8_DPAS_GEMM2_GATHER_NTILE": null, "VLLM_XPU_MOE_W8A8_WORKSPACE_ATOMIC": null, "VLLM_XPU_MOE_W8A8_ROUTE_GEMM1": null, "VLLM_XPU_MOE_W8A8_ROUTE_GEMM1_MTILE": null, "VLLM_XPU_INT8_MOE_MIXED_WORKSPACE": "1", "VLLM_XPU_INT8_MOE_PERSISTENT_SCRATCH": null, "VLLM_XPU_INT8_MOE_FUSED_PROLOGUE_OFFSET": null, "VLLM_XPU_INT8_MOE_FUSED_PROLOGUE_OFFSET_ALLOW_CAPTURE": null, "VLLM_XPU_INT8_MOE_FUSED_PROLOGUE_OFFSET_SKIP_LAYERS": null, "VLLM_XPU_INT8_MOE_FUSED_PROLOGUE_OFFSET_LAYER_REGEX": null, "VLLM_XPU_INT8_MOE_ACTIVE_OFFSET_GEMM": null, "VLLM_XPU_SHARED_EXPERTS_STREAM": null, "VLLM_XPU_SHARED_EXPERT_FUSED_ACT_QUANT": null, "VLLM_XPU_SHARED_EXPERT_FUSED_ACT_QUANT_OUT": null, "VLLM_XPU_SHARED_EXPERT_BOUNDARY_OUT": null, "VLLM_XPU_SHARED_EXPERT_FUSED_GATE_MUL": null, "VLLM_XPU_MOE_SHARED_ADD_ALLREDUCE_CUSTOM_OP": null, "VLLM_XPU_METADATA_COPY_ALLOW": null, "VLLM_XPU_BLOCK_TABLE_DIRTY_COMMIT": null, "VLLM_XPU_BLOCK_TABLE_DIRTY_COMMIT_LOG_EVERY": null, "GPU_MEMORY_UTILIZATION": "0.95", "COMPILATION_CONFIG": "{\"cudagraph_mode\":\"PIECEWISE\",\"max_cudagraph_capture_size\":128}", "SPEC_CONFIG": null, "ABLATION_FAST_GRAPH_AUTOCONFIG": "0", "VLLM_EXTRA_ARGS": "--speculative-config {\"method\":\"mtp\",\"num_speculative_tokens\":1}", "METRICS_PROMPT_KIND": null, "METRICS_PROMPT_PRESET": null, "METRICS_RANDOM_PREFIX_LEN": null, "METRICS_PROMPT_TOKENS": null, "METRICS_OUTPUT_TOKENS": null, "METRICS_WARMUP_OUTPUT_TOKENS": null, "JSON_LOGPROBS": null, "COLOR_LOGPROBS": null }, "metrics_summary": { "tok_s_out_client_after_first_chunk": { "mean": 107.97654202272932, "median": 107.97654202272932, "min": 102.43259140781036, "max": 113.52049263764829 }, "tok_s_out_client_after_first_chunk_corrected": { "mean": 107.76565033909118, "median": 107.76565033909118, "min": 102.23252775271698, "max": 113.29877292546539 }, "tok_s_out_client_e2e": { "mean": 102.14490404563159, "median": 102.14490404563159, "min": 97.18693572438421, "max": 107.10287236687898 }, "tok_s_total_client": { "mean": 201.49678337126545, "median": 201.49678337126545, "min": 191.7164161750548, "max": 211.27715056747613 }, "ttft_ms_client": { "mean": 270.0202294990959, "median": 270.0202294990959, "min": 269.78866099852894, "max": 270.2517979996628 }, "ttft_ms_vllm_metrics": { "mean": 251.0848045349121, "median": 251.0848045349121, "min": 250.37050247192383, "max": 251.7991065979004 }, "tok_s_prefill_lower_bound_from_ttft": { "mean": 1844.3076179489658, "median": 1844.3076179489658, "min": 1842.7259455295884, "max": 1845.889290368343 }, "e2e_ms_vllm_metrics": { "mean": 5023.064613342285, "median": 5023.064613342285, "min": 4779.198169708252, "max": 5266.931056976318 }, "queue_ms_vllm_histogram": { "mean": 0.011001000530086458, "median": 0.011001000530086458, "min": 0.00902700048754923, "max": 0.012975000572623685 }, "prefill_ms_vllm_histogram": { "mean": 244.30397199921572, "median": 244.30397199921572, "min": 242.03861599926313, "max": 246.5693279991683 }, "decode_ms_vllm_histogram": { "mean": 4772.073827500208, "median": 4772.073827500208, "min": 4527.56539999973, "max": 5016.582255000685 }, "decode_ms_per_generation_token_vllm_histogram": { "mean": 9.320456694336343, "median": 9.320456694336343, "min": 8.842901171874473, "max": 9.798012216798213 }, "inference_ms_vllm_histogram": { "mean": 5016.377799499423, "median": 5016.377799499423, "min": 4774.134727998899, "max": 5258.620870999948 }, "time_per_output_token_ms_vllm_histogram": { "mean": 9.338696335616845, "median": 9.338696335616845, "min": 8.860206262230392, "max": 9.817186409003298 }, "inter_token_ms_vllm_histogram": { "mean": 16.65321474734497, "median": 16.65321474734497, "min": 16.584488644687656, "max": 16.721940850002284 }, "iteration_tokens_per_step_vllm_histogram": { "mean": 3.520806557218032, "median": 3.520806557218032, "min": 3.355481727574751, "max": 3.6861313868613137 } }, "speed": { "tok_s_out_corrected_mean": 107.76565033909118, "tok_s_out_after_first_chunk_mean": 107.97654202272932, "tok_s_out_e2e_mean": 102.14490404563159, "decode_ms_per_token_mean": 9.320456694336343, "ttft_ms_client_mean": 270.0202294990959 }, "json_canary": { "pass_all": false, "repeats_completed": 1, "mismatch_count": 1, "first_mismatch": { "index": 0, "request_id": null, "text": "{\"4404344.{\"44434040.{\"422}}{\"420}", "normalized": "{\"4404344.{\"44434040.{\"422}}{\"420}", "token_ids": [ 4754, 19, 19, 15, 19, 18, 19, 19, 13, 4754, 19, 19, 19, 18, 19, 15, 19, 15, 13, 4754, 19, 17, 17, 3307, 4754, 19, 17, 15, 92 ], "finish_reason": "stop", "elapsed_s": 0.5884255820001272, "usage": { "prompt_tokens": 36, "total_tokens": 66, "completion_tokens": 30, "prompt_tokens_details": null }, "json_parsed": null, "pass": false } }, "color_canary": { "pass_all": false, "repeats_completed": 1, "mismatch_count": 1, "first_mismatch": { "index": 0, "request_id": null, "text": "blue, green, red, yellow,", "normalized": "blue, green, red, yellow,", "token_ids": [ 11855, 11, 5983, 11, 2438, 11, 13358, 11 ], "finish_reason": "stop", "elapsed_s": 0.23539511099988886, "usage": { "prompt_tokens": 30, "total_tokens": 39, "completion_tokens": 9, "prompt_tokens_details": null }, "pass": false } }, "decision": { "accepted_by_requested_gates": false, "gate_failures": [ "json_canary", "color_canary" ] } }