[ { "label": "gemma4-26b-a4b-q8-b70-llamacpp-mtp-n7-q8target-q40draft-q8reorder-ub768-nmin3-pmin010-fresh-20260627T142318", "payload": { "batchSize": 1, "contextLength": 8192, "engineFlags": { "actualOutputTokens": 512, "actualPromptTokens": 588, "attentionBackend": "llama.cpp SYCL", "backendSampling": false, "benchmarkJson": "data/gemma4-q8-gpu0-mulmatid-fast-q8reorder-ub768-fullconfirm-20260627T142318Z/p512o512.json", "cachedTokensAllRows": [ 0, 0, 0, 0, 0, 0, 0, 0 ], "cachedTokensRow0": 0, "canaryRowsPassed": 6144, "chunkedPrefill": false, "commandSnippet": "ONEAPI_DEVICE_SELECTOR=level_zero:0 UR_L0_USE_IMMEDIATE_COMMANDLISTS=1 GGML_SYCL_DISABLE_GRAPH=0 GGML_SYCL_ENABLE_VMM=0 LLAMA_SERVER=/home/steve/src/llama.cpp-gemma-record-repro-c926/build-sycl-b70-aot-bmg-g31/bin/llama-server MODEL=/mnt/fast-ai/llm-models/gemma4-26b-a4b-it-q8-gguf/gemma-4-26B-A4B-it-UD-Q8_K_XL.gguf MTP_DRAFT_MODEL=/mnt/fast-ai/llm-models/gemma4-26b-a4b-it-q8-gguf/MTP/gemma-4-26B-A4B-it-Q4_0-MTP.gguf LLAMA_SYCL_MUL_MAT_ID_MULTI_TOKEN_FAST=1 LLAMA_SYCL_MUL_MAT_ID_Q8_0_REORDER=1 LLAMA_SPEC_VERIFY_BACKEND_ARGMAX_IDS=1 LLAMA_MTP_DEFER_TARGET_H_NEXTN=1 LLAMA_MTP_DRAFT_FAST_ARGMAX=1 LLAMA_MTP_DRAFT_DIRECT_ARGMAX_IDS=1 LLAMA_MTP_DRAFT_DIRECT_ARGMAX_UNROLL=7 LLAMA_GEMMA4_MTP_QONLY_ATTN_INPUTS=1 LLAMA_GEMMA4_MTP_FUSED_OUTPUT_ARGMAX=1 LLAMA_GEMMA4_MOE_SELECTED_SOFTMAX=1 LLAMA_GEMMA4_MOE_SELECTED_SOFTMAX_FUSED=1 LLAMA_GEMMA4_MOE_WEIGHTED_SUM=1 LLAMA_GEMMA4_MOE_REUSE_ATTN_RMS=1 LLAMA_SYCL_MUL_MAT_ID_ROUTE_CACHE=1 MTP_BACKEND_SAMPLING=0 MTP_N_MAX=7 MTP_N_MIN=3 MTP_P_MIN=0.10 MTP_DRAFT_DEVICE=SYCL0 MTP_DRAFT_THREADS=32 MTP_DRAFT_THREADS_BATCH=32 MTP_EXTRA_ARGS=\"--ctx-checkpoints 0\" CTX_SIZE=8192 BATCH_SIZE=1024 UBATCH_SIZE=768 POLL=100 THREADS=8 CANARY_REPEATS=1536 BENCH_REPEATS=8 PROMPT_TOKENS=512 MAX_TOKENS=512 BENCH_PROMPT_MODE=filled-long scripts/run-gemma4-26b-first-baseline.sh", "concurrency": 1, "contBatching": false, "draftModelFile": "gemma-4-26B-A4B-it-Q4_0-MTP.gguf", "draftModelFileBytes": 321126560, "draftModelRepo": "local quantized from unsloth/gemma-4-26B-A4B-it-GGUF F16 MTP draft via llama-quantize Q4_0", "extraFlags": "AOT BMG-G31 SYCL build; GGML_SYCL_DISABLE_GRAPH=0; VMM disabled; batch=1024; ubatch=768; poll=100; threads=8; draft threads/batch=32; ctx-checkpoints=0; fast argmax draft selection; no backend sampling; direct greedy sampled-token IDs; direct-unroll7; LLAMA_GEMMA4_MTP_QONLY_ATTN_INPUTS=1; LLAMA_GEMMA4_MTP_FUSED_OUTPUT_ARGMAX=1; LLAMA_SPEC_VERIFY_BACKEND_ARGMAX_IDS=1; LLAMA_MTP_DEFER_TARGET_H_NEXTN=1; LLAMA_GEMMA4_MOE_SELECTED_SOFTMAX=1; LLAMA_GEMMA4_MOE_SELECTED_SOFTMAX_FUSED=1; LLAMA_GEMMA4_MOE_WEIGHTED_SUM=1; LLAMA_GEMMA4_MOE_REUSE_ATTN_RMS=1; LLAMA_SYCL_MUL_MAT_ID_ROUTE_CACHE=1; LLAMA_SYCL_MUL_MAT_ID_MULTI_TOKEN_FAST=1; LLAMA_SYCL_MUL_MAT_ID_Q8_0_REORDER=1; MTP_N_MIN=3; MTP_P_MIN=0.10; reasoning off. Quantization file is Unsloth UD-Q8_K_XL GGUF; submitted quantization remains Q8_K_XL. The Q8_0 reorder flag refers to internal GGML Q8_0 expert tensors, not a lower-quality target model.", "firstRequestCachedTokens": 0, "firstRequestTokSOut": 169.9489959621758, "firstRequestTokSTotal": 135.18178237607523, "flashAttn": false, "freshResponseValidity": "headline tokSOut uses row 0 only: first measured request after canaries, cached_tokens=0. Rows 1-7 are repeated-prompt support/stability rows only and are not averaged into the headline. Speculative draft source is a Q4_0 Gemma MTP draft for the current request; no ngram/history or learned repeated-continuation reuse is used. Full validation: 6144/6144 canary rows, all benchmark rows cached_tokens=0. This result uses the Q8_0 MoE-ID reorder source patch and broad multi-token fast verifier path.", "kvCacheDtype": "f16", "llamaCppCommit": "c926ad098", "llamaGemma4MoeReuseAttnRms": true, "llamaGemma4MoeSelectedSoftmax": true, "llamaGemma4MoeSelectedSoftmaxFused": true, "llamaGemma4MoeWeightedSum": true, "llamaGemma4MtpFusedOutputArgmax": true, "llamaGemma4MtpQonlyAttnInputs": true, "llamaMtpDeferTargetHNextn": true, "llamaMtpDraftDirectArgmaxIds": true, "llamaMtpDraftDirectArgmaxUnroll": 7, "llamaSpecVerifyBackendArgmaxIds": true, "llamaSyclMulMatIdRouteCache": true, "llamaSyclMulMatIdRouteCacheDeviceMap": null, "maxRunningSeqs": 1, "pipelineParallel": 1, "prefixCaching": false, "promptMode": "filled-long", "promptTokensRequested": 512, "serverLog": "/mnt/fast-ai/bench-results/gemma4-26b-a4b-q8/servers/gemma4-q8-gpu0-mulmatid-fast-q8reorder-ub768-fullconfirm-20260627T142318Z.server.log", "specDecoding": true, "specDraftModel": "/mnt/fast-ai/llm-models/gemma4-26b-a4b-it-q8-gguf/MTP/gemma-4-26B-A4B-it-Q4_0-MTP.gguf", "specDraftNMin": 3, "specDraftPMin": 0.1, "specDraftQuantization": "Q4_0 draft only; target/verifier remains UD-Q8_K_XL", "specMethod": "draft-mtp", "specNumTokens": 7, "summaryJson": "data/gemma4-q8-gpu0-mulmatid-fast-q8reorder-ub768-fullconfirm-20260627T142318Z/summary.json", "temperature": 0, "tensorParallel": 1, "tokSOutMaxSupport": 170.34240479485297, "tokSOutMeanSupport": 169.5501066933547, "tokSOutMedianSupport": 169.3663572172759, "tokSOutMinSupport": 168.93763184706893, "tokSOutStdevSupport": 0.4705756045157824, "tokSTotalMaxSupport": 147.6961021331938, "tokSTotalMeanSupport": 145.61193653627558, "tokSTotalMinSupport": 135.18178237607523, "urL0UseImmediateCommandlists": 1, "llamaSyclMulMatIdMultiTokenFast": true, "llamaSyclMulMatIdQ80Reorder": true, "llamaSyclMulMatIdQ8_0Reorder": true, "freshResponseHeadlineValid": false, "historyAccelerated": false, "headlineUse": "diagnostic-pre-final-gate", "freshResponseNgramValidated": false, "localmaxxingSubmissionAllowedUnderCurrentPolicy": false, "realisticSuiteGatePassed": false, "primaryMetricName": "diagnostic_pre_final_gate_not_headline", "realisticSuiteCachedTokensAllZero": false, "realisticSuiteId": "", "currentPolicyCorrectionDate": "2026-06-27", "currentPolicyCorrection": "2026-06-27 correction: diagnostic/pre-final-gate artifact only under the current policy. Do not submit, promote, or advertise this payload as realistic headline throughput unless replaced by a fixed realistic-suite run with each prompt once, cached_tokens=0 on every request, no warmed/reused history, and primaryMetricName=median_tok_s_1_100_after_ttft." }, "engineName": "llama.cpp", "engineVersion": "c926ad098 local B70 SYCL/AOT Gemma patch stack + Q8_0 MoE-ID reorder verifier path; UBATCH_SIZE=768; n_min=3 p_min=0.10", "hardware": { "cpu": "AMD Ryzen Threadripper PRO 5955WX 16-Cores", "gpuCount": 1, "gpuName": "Intel Arc Pro B70", "hwClass": "DISCRETE_GPU", "os": "Ubuntu 24.04.4 LTS", "ramGb": 128, "vramGb": 32 }, "hfId": "unsloth/gemma-4-26B-A4B-it-GGUF", "modelRevision": "gemma-4-26B-A4B-it-UD-Q8_K_XL.gguf", "notes": "Gemma 4 26B A4B Q8/INT8-quality fresh-response single-B70 record. Target/verifier is Unsloth UD-Q8_K_XL GGUF; only the MTP draft is Q4_0. Headline uses row0 only because the benchmark prompt is repeated; rows 1-7 are support/stability only even though cached_tokens=0. Large chat canary passed 6144/6144 rows. This keeps the prior route-cache/fused-output-argmax/fused-selected-softmax/RMS-reuse recipe and adds the broad multi-token MoE-ID fast verifier path plus LLAMA_SYCL_MUL_MAT_ID_Q8_0_REORDER=1, which reorders Q8_0 expert slices per expert for the reorder MMVQ body. Fresh row0 improves the prior LocalMaxxing fresh headline cmqw1tgzx0366qr01g4lkv7f1 from 104.30919255569083 to 169.9489959621758 tok/s. Support mean is 169.5501066933547 tok/s. This is fresh-response speculative MTP, not warmed n-gram/history acceleration.\n\n2026-06-27 correction: diagnostic/pre-final-gate artifact only under the current policy. Do not submit, promote, or advertise this payload as realistic headline throughput unless replaced by a fixed realistic-suite run with each prompt once, cached_tokens=0 on every request, no warmed/reused history, and primaryMetricName=median_tok_s_1_100_after_ttft.", "outputTokens": 512, "promptTokens": 588, "quantization": "Q8_K_XL", "tokSOut": 169.9489959621758, "tokSTotal": 135.18178237607523, "ttftMs": 774.8240189976059 } } ]