{ "branch": "develop", "e2e_run": "local", "full_image_certification": false, "images": { "AVX2": { "id": "sha256:00607abbbb9a00d15c35225ca65496c709cff92b75da2ae4dfcf2728b0dc581a", "labels": { "org.llaminar.build_type": "Release", "org.llaminar.cpu_isa": "AVX2", "org.llaminar.cuda": "ON", "org.llaminar.image_role": "runtime", "org.llaminar.rocm": "ON", "org.llaminar.source_tree": "c241952773356d2d294843f3b281a3eed49f9897", "org.opencontainers.image.created": "", "org.opencontainers.image.description": "High-performance LLM inference engine (CUDA + ROCm)", "org.opencontainers.image.licenses": "AGPL-3.0-only", "org.opencontainers.image.revision": "557f0f97d15d447850b77b7ce4e91bb0663331b3", "org.opencontainers.image.source": "https://github.com/llaminar/llaminar", "org.opencontainers.image.title": "Llaminar", "org.opencontainers.image.version": "dev" }, "layers": [ "sha256:65e5d715ea720349e6475f91651a0956279fc295a45fd73aa04f1cec046281a1", "sha256:139eb73fad565f27dce03be4b5e2d86b389bf0b2eee309e819201e20695ab595", "sha256:3c75b0c1cc858895953de8d75ac64f8fa36032f06bb946c417571555db47eff1", "sha256:812c698b3f361211e5aa3c123942b21b36fe68cff8a46d1ce07b557e9d6d9e7d", "sha256:a9414d9628fbdf7696f687b6c50706008bb9dd2f690e1d6e3358bba7e3524951", "sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef", "sha256:559ed36805472f888b5e0e37c853e4b8a39cd348a248ee56256841a02f8167a9", "sha256:528725efcf41be3c0ba1b92125f8f21ac69a7377641ff9ab964754e146a89303", "sha256:b02630a6d04434941c3ea9ab22c4dbe0d49cdba5534b47776d6c97d7807b4b63", "sha256:437c9fef3e9c7e22b89348f30f7be4a80e4c51ef253657077299d9d68c6d4d88", "sha256:ed5e0f3e1867197ef2ab6498e04a0d27aaf50381409c7c68982e59658cae1204", "sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef" ], "registry_ref": "ghcr.io/llaminar/llaminar@sha256:e53daf9f4ca28deacafb921fd8b5f252f655ab8df48d762eee862866d51e312f", "tag": "ghcr.io/llaminar/llaminar:develop-557f0f97d15d447850b77b7ce4e91bb0663331b3-avx2" }, "AVX512": { "id": "sha256:872c2dd9c7bd2d3e1023bf297152d1a527ce9d4873842621f4f744ab3c2a7c78", "labels": { "org.llaminar.build_type": "Release", "org.llaminar.cpu_isa": "AVX512", "org.llaminar.cuda": "ON", "org.llaminar.image_role": "runtime", "org.llaminar.rocm": "ON", "org.llaminar.source_tree": "c241952773356d2d294843f3b281a3eed49f9897", "org.opencontainers.image.created": "", "org.opencontainers.image.description": "High-performance LLM inference engine (CUDA + ROCm)", "org.opencontainers.image.licenses": "AGPL-3.0-only", "org.opencontainers.image.revision": "557f0f97d15d447850b77b7ce4e91bb0663331b3", "org.opencontainers.image.source": "https://github.com/llaminar/llaminar", "org.opencontainers.image.title": "Llaminar", "org.opencontainers.image.version": "dev" }, "layers": [ "sha256:65e5d715ea720349e6475f91651a0956279fc295a45fd73aa04f1cec046281a1", "sha256:139eb73fad565f27dce03be4b5e2d86b389bf0b2eee309e819201e20695ab595", "sha256:3c75b0c1cc858895953de8d75ac64f8fa36032f06bb946c417571555db47eff1", "sha256:812c698b3f361211e5aa3c123942b21b36fe68cff8a46d1ce07b557e9d6d9e7d", "sha256:a9414d9628fbdf7696f687b6c50706008bb9dd2f690e1d6e3358bba7e3524951", "sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef", "sha256:7f5f5c09bb91904810a46d06a6d3adc96261b1b918a3d154cbdffa15a273b7ce", "sha256:fa7a6a197791a57da48d0c5df6304b72bbe6b8a2752e7b95c6bbb1a4813e6d0d", "sha256:930ebeb25fc9d9693ecafd48344f4d643a83c8884fd2a9c6480db6287f1d348c", "sha256:b2b6e33d921d283bd6e803a82d757af992ae1811a3153ad49df4c845fe851bcd", "sha256:d810a87a02219a253b55bd82a64b476e7b75c69f73e377b84d5d667719cfe11a", "sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef" ], "registry_ref": "ghcr.io/llaminar/llaminar@sha256:3c36fe3adb98e68ffe76ab7bf57c329fc04a4e5452c82bbd635b906c65623c37", "tag": "ghcr.io/llaminar/llaminar:develop-557f0f97d15d447850b77b7ce4e91bb0663331b3" } }, "passed": true, "recorded_at_utc": "2026-09-27T06:50:27.047059+00:00", "regression_threshold_pct": 10, "repository": "Llaminar/llaminar", "schema": 1, "scope": "full-http-e2e-and-benchmarks", "source": { "dirty": false, "revision": "557f0f97d15d447850b77b7ce4e91bb0663331b3", "tree": "c241952773356d2d294843f3b281a3eed49f9897" }, "variants": { "AVX2": { "cells": [ { "case": "Qwen36MoEExpertOverlay/Qwen36MoEExpertOverlayParityTest.ProductionParity/Qwen36MoE_35B_IQ3S_NodeTP_2xMPI_CPU_ExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "identity": { "case": "Qwen36MoEExpertOverlay/Qwen36MoEExpertOverlayParityTest.ProductionParity/Qwen36MoE_35B_IQ3S_NodeTP_2xMPI_CPU_ExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "configuration": { "benchmark": { "args": [ "--mpi-procs", "2", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--auto", "--only-backends", "cpu", "--auto-device-counts", "cpu=2", "--only-strategies", "tp", "--moe-routed-expert-owner-order", "ordinal", "--mtp", "--mtp-depth-policy", "dynamic", "--moe-residency-maintenance", "dynamic" ], "context_length": 8192, "policy": "production_defaults", "schema": 1 }, "cross_host_e2e": [], "e2e": { "cell_timeout_seconds": { "AVX2": 1200, "AVX512": 900 }, "context_length": 8192, "generation_tokens": 2048, "minimum_prompt_tokens": 4096, "movement_evidence": "required", "planning": { "device_counts": { "cpu": 2 }, "mode": "auto", "mpi_ranks": 2, "strategy": "tp" }, "readiness_timeout_seconds": 60, "request_timeout_seconds": 600, "server_args": [ "--mpi-procs", "2", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--tp-allreduce-precision", "schema", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--auto", "--only-backends", "cpu", "--auto-device-counts", "cpu=2", "--only-strategies", "tp", "--moe-routed-expert-owner-order", "ordinal", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "1", "--moe-residency-maintenance-max-window", "1", "--moe-residency-maintenance-window-growth", "1", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1000", "--moe-dynamic-min-improvement-permille", "0", "--moe-dynamic-max-swaps-per-layer", "20", "--moe-dynamic-max-plan-entries-per-wave", "20", "--moe-dynamic-min-window-activations", "0", "--moe-device-rebalance-maintenance-slack-tokens", "0", "--moe-device-rebalance-min-maintenance-period-tokens", "16", "--moe-device-rebalance-initial-maintenance-period-tokens", "2", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "0", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-max-post-wave-load-spread-permille", "1000", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ], "thinking_modes": "both", "tool_calling": "required" }, "id": "Qwen36MoE_35B_IQ3S_NodeTP_2xMPI_CPU_ExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "model": "Qwen3.6-35B-A3B-UD-IQ3_S.gguf", "model_parity_schema": 1, "runtime": { "context_length": 4096, "generation": { "max_tokens": 384, "minimum_completion_tokens": 384, "mtp_policy": "dynamic", "mtp_verify_mode": "speculative-sampling", "prefix_state": "hybrid_recurrent", "readiness_timeout_seconds": 60, "requests": [ { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "fresh_harbor", "prefix": "fresh" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_harbor", "prefix": "full" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "partial_mountain", "prefix": "partial" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_mountain", "prefix": "full" } ], "serial_control_id": "Qwen36MoE_35B_IQ3S_NodeTP_2xMPI_CPU_ExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPOff" }, "movement_evidence": "required", "server_args": [ "--mpi-procs", "2", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--backend", "mpi", "--tp-allreduce-precision", "schema", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--moe-routed-expert-placement", "single-domain", "--moe-routed-expert-continuation-domain", "qwen36_moe_cpu_node_tp", "--moe-routed-expert-base-model-domain", "qwen36_moe_cpu_node_tp", "--moe-routed-expert-shared-domain", "qwen36_moe_cpu_node_tp", "--moe-continuation-dense-policy", "tensor-parallel", "--moe-routed-expert-owner-order", "ordinal", "--moe-routed-expert-residency", "rebalanced", "--define-domain", "qwen36_moe_cpu_node_tp=localhost:0:cpu:0,localhost:1:cpu:0;scope=node_local;backend=mpi;routed_compute=apportioned;routed_phase=uniform;routed_decode_assignment=static-owner;routed_prefill_assignment=static-owner", "--moe-routed-expert-domain", "qwen36_moe_cpu_node_tp=localhost:0:cpu:0,localhost:1:cpu:0;scope=node_local;backend=mpi;routed_compute=apportioned;routed_phase=uniform;routed_decode_assignment=static-owner;routed_prefill_assignment=static-owner", "--moe-routed-expert-tier", "priority_0@qwen36_moe_cpu_node_tp;priority=0;max-experts-per-layer=0;memory-mb=auto", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "1", "--moe-residency-maintenance-max-window", "1", "--moe-residency-maintenance-window-growth", "1", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1000", "--moe-dynamic-min-improvement-permille", "0", "--moe-dynamic-max-swaps-per-layer", "20", "--moe-dynamic-max-plan-entries-per-wave", "20", "--moe-dynamic-min-window-activations", "0", "--moe-device-rebalance-maintenance-slack-tokens", "0", "--moe-device-rebalance-min-maintenance-period-tokens", "16", "--moe-device-rebalance-initial-maintenance-period-tokens", "2", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "0", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-max-post-wave-load-spread-permille", "1000", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ] } }, "cpu_isa": "AVX2", "hardware": { "accelerators": [ "0000:03:00.0 VGA compatible controller [0300]: ASPEED Technology, Inc. ASPEED Graphics Family [1a03:2000] (rev 41)", "0000:3f:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:42:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:8c:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:91:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:97:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:9a:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)" ], "cpu": { "Architecture:": "x86_64", "CPU(s):": "112", "Core(s) per socket:": "28", "Model name:": "Intel(R) Xeon(R) Gold 6238R CPU @ 2.20GHz", "NUMA node(s):": "2", "NUMA node0 CPU(s):": "0-27,56-83", "NUMA node1 CPU(s):": "28-55,84-111", "Socket(s):": "2", "Thread(s) per core:": "2" } }, "model_bytes": 15346432288, "model_files": { "Qwen3.6-35B-A3B-UD-IQ3_S.gguf": 15346432288 }, "prefill_tokens": 512, "workload": { "decode_tokens": 256, "measurement_iterations": 3, "prompt_repetitions": 512, "prompt_unit": " test", "schema": 1, "seed": 42, "temperature": 0, "top_k": 1, "top_p": 1, "warmup_iterations": 1 } }, "prefill_tokens": 512, "samples": { "decode": [ 27.970070921544718, 27.82492673118197, 27.963787981970885 ], "prefill": [ 110.69682212849453, 111.85195310440837, 111.6925154473672 ] }, "tokens_per_second": { "decode": 27.963787981970885, "prefill": 111.6925154473672 } }, { "case": "Qwen36MoEExpertOverlay/Qwen36MoEExpertOverlayParityTest.ProductionParity/Ornith15MoE_35B_Q4KM_NodeTP_2xMPI_CPU_ExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "identity": { "case": "Qwen36MoEExpertOverlay/Qwen36MoEExpertOverlayParityTest.ProductionParity/Ornith15MoE_35B_Q4KM_NodeTP_2xMPI_CPU_ExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "configuration": { "benchmark": { "args": [ "--mpi-procs", "2", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--auto", "--only-backends", "cpu", "--auto-device-counts", "cpu=2", "--only-strategies", "tp", "--moe-routed-expert-owner-order", "ordinal", "--mtp", "--mtp-depth-policy", "dynamic", "--moe-residency-maintenance", "dynamic" ], "context_length": 8192, "policy": "production_defaults", "schema": 1 }, "cross_host_e2e": [], "e2e": { "cell_timeout_seconds": { "AVX2": 1200, "AVX512": 900 }, "context_length": 8192, "generation_tokens": 2048, "minimum_prompt_tokens": 4096, "movement_evidence": "required", "planning": { "device_counts": { "cpu": 2 }, "mode": "auto", "mpi_ranks": 2, "strategy": "tp" }, "readiness_timeout_seconds": 60, "request_timeout_seconds": 600, "server_args": [ "--mpi-procs", "2", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--tp-allreduce-precision", "schema", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--auto", "--only-backends", "cpu", "--auto-device-counts", "cpu=2", "--only-strategies", "tp", "--moe-routed-expert-owner-order", "ordinal", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "1", "--moe-residency-maintenance-max-window", "1", "--moe-residency-maintenance-window-growth", "1", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1000", "--moe-dynamic-min-improvement-permille", "0", "--moe-dynamic-max-swaps-per-layer", "20", "--moe-dynamic-max-plan-entries-per-wave", "20", "--moe-dynamic-min-window-activations", "0", "--moe-device-rebalance-maintenance-slack-tokens", "0", "--moe-device-rebalance-min-maintenance-period-tokens", "16", "--moe-device-rebalance-initial-maintenance-period-tokens", "2", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "0", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-max-post-wave-load-spread-permille", "1000", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ], "thinking_modes": "both", "tool_calling": "required" }, "id": "Ornith15MoE_35B_Q4KM_NodeTP_2xMPI_CPU_ExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "model": "Ornith-1.5-35B-Q4_K_M.gguf", "model_parity_schema": 1, "runtime": { "context_length": 4096, "generation": { "max_tokens": 384, "minimum_completion_tokens": 384, "mtp_policy": "dynamic", "mtp_verify_mode": "speculative-sampling", "prefix_state": "hybrid_recurrent", "readiness_timeout_seconds": 60, "requests": [ { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "fresh_harbor", "prefix": "fresh" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_harbor", "prefix": "full" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "partial_mountain", "prefix": "partial" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_mountain", "prefix": "full" } ], "serial_control_id": "Ornith15MoE_35B_Q4KM_NodeTP_2xMPI_CPU_ExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPOff" }, "movement_evidence": "required", "server_args": [ "--mpi-procs", "2", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--backend", "mpi", "--tp-allreduce-precision", "schema", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--moe-routed-expert-placement", "single-domain", "--moe-routed-expert-continuation-domain", "qwen36_moe_cpu_node_tp", "--moe-routed-expert-base-model-domain", "qwen36_moe_cpu_node_tp", "--moe-routed-expert-shared-domain", "qwen36_moe_cpu_node_tp", "--moe-continuation-dense-policy", "tensor-parallel", "--moe-routed-expert-owner-order", "ordinal", "--moe-routed-expert-residency", "rebalanced", "--define-domain", "qwen36_moe_cpu_node_tp=localhost:0:cpu:0,localhost:1:cpu:0;scope=node_local;backend=mpi;routed_compute=apportioned;routed_phase=uniform;routed_decode_assignment=static-owner;routed_prefill_assignment=static-owner", "--moe-routed-expert-domain", "qwen36_moe_cpu_node_tp=localhost:0:cpu:0,localhost:1:cpu:0;scope=node_local;backend=mpi;routed_compute=apportioned;routed_phase=uniform;routed_decode_assignment=static-owner;routed_prefill_assignment=static-owner", "--moe-routed-expert-tier", "priority_0@qwen36_moe_cpu_node_tp;priority=0;max-experts-per-layer=0;memory-mb=auto", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "1", "--moe-residency-maintenance-max-window", "1", "--moe-residency-maintenance-window-growth", "1", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1000", "--moe-dynamic-min-improvement-permille", "0", "--moe-dynamic-max-swaps-per-layer", "20", "--moe-dynamic-max-plan-entries-per-wave", "20", "--moe-dynamic-min-window-activations", "0", "--moe-device-rebalance-maintenance-slack-tokens", "0", "--moe-device-rebalance-min-maintenance-period-tokens", "16", "--moe-device-rebalance-initial-maintenance-period-tokens", "2", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "0", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-max-post-wave-load-spread-permille", "1000", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ] } }, "cpu_isa": "AVX2", "hardware": { "accelerators": [ "0000:03:00.0 VGA compatible controller [0300]: ASPEED Technology, Inc. ASPEED Graphics Family [1a03:2000] (rev 41)", "0000:3f:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:42:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:8c:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:91:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:97:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:9a:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)" ], "cpu": { "Architecture:": "x86_64", "CPU(s):": "112", "Core(s) per socket:": "28", "Model name:": "Intel(R) Xeon(R) Gold 6238R CPU @ 2.20GHz", "NUMA node(s):": "2", "NUMA node0 CPU(s):": "0-27,56-83", "NUMA node1 CPU(s):": "28-55,84-111", "Socket(s):": "2", "Thread(s) per core:": "2" } }, "model_bytes": 21713463040, "model_files": { "Ornith-1.5-35B-Q4_K_M.gguf": 21713463040 }, "prefill_tokens": 512, "workload": { "decode_tokens": 256, "measurement_iterations": 3, "prompt_repetitions": 512, "prompt_unit": " test", "schema": 1, "seed": 42, "temperature": 0, "top_k": 1, "top_p": 1, "warmup_iterations": 1 } }, "prefill_tokens": 512, "samples": { "decode": [ 32.06036035979572, 32.19967995460208, 32.274360459265786 ], "prefill": [ 123.05934585384385, 124.34957576109406, 124.78734572862774 ] }, "tokens_per_second": { "decode": 32.19967995460208, "prefill": 124.34957576109406 } }, { "case": "Qwen35_122B_ExpertOverlay/Qwen35MoENodeExpertOverlayParityTest.ProductionParity/Qwen35_122B_CUDA2_CPU2_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "identity": { "case": "Qwen35_122B_ExpertOverlay/Qwen35MoENodeExpertOverlayParityTest.ProductionParity/Qwen35_122B_CUDA2_CPU2_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "configuration": { "benchmark": { "args": [ "--mpi-procs", "2", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--auto", "--only-backends", "cpu,cuda", "--auto-device-counts", "cpu=2,cuda=2", "--only-strategies", "expert-overlay", "--moe-routed-expert-owner-order", "ordinal", "--mtp", "--mtp-depth-policy", "dynamic", "--moe-residency-maintenance", "dynamic" ], "context_length": 8192, "policy": "production_defaults", "schema": 1 }, "cross_host_e2e": [], "e2e": { "cell_timeout_seconds": { "AVX2": 900, "AVX512": 900 }, "context_length": 8192, "generation_tokens": 2048, "minimum_prompt_tokens": 4096, "movement_evidence": "required", "planning": { "device_counts": { "cpu": 2, "cuda": 2 }, "mode": "auto", "mpi_ranks": 2, "strategy": "expert-overlay" }, "readiness_timeout_seconds": 180, "request_timeout_seconds": 600, "server_args": [ "--mpi-procs", "2", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--auto", "--only-backends", "cpu,cuda", "--auto-device-counts", "cpu=2,cuda=2", "--only-strategies", "expert-overlay", "--moe-routed-expert-owner-order", "ordinal", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "9", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "455.111115", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "49", "--moe-dynamic-imbalance-threshold-permille", "1000", "--moe-dynamic-min-improvement-permille", "0", "--moe-dynamic-max-swaps-per-layer", "2", "--moe-dynamic-max-plan-entries-per-wave", "196", "--moe-dynamic-min-window-activations", "0", "--moe-device-rebalance-maintenance-slack-tokens", "0", "--moe-device-rebalance-min-maintenance-period-tokens", "4096", "--moe-device-rebalance-initial-maintenance-period-tokens", "1", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "0", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-max-post-wave-load-spread-permille", "1000", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ], "thinking_modes": "both", "tool_calling": "required" }, "id": "Qwen35_122B_CUDA2_CPU2_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "model": "Qwen3.5-122B-A10B-UD-Q8_K_XL-00001-of-00004.gguf", "model_parity_schema": 1, "runtime": { "context_length": 4096, "generation": { "max_tokens": 384, "minimum_completion_tokens": 384, "mtp_policy": "dynamic", "mtp_verify_mode": "speculative-sampling", "prefix_state": "hybrid_recurrent", "readiness_timeout_seconds": 180, "requests": [ { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "fresh_harbor", "prefix": "fresh" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_harbor", "prefix": "full" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "partial_mountain", "prefix": "partial" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_mountain", "prefix": "full" } ], "serial_control_id": "Qwen35_122B_CUDA2_CPU2_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPOff" }, "movement_evidence": "required", "server_args": [ "--mpi-procs", "2", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--backend", "host", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--moe-routed-expert-placement", "tiered-overlay", "--moe-routed-expert-continuation-domain", "cuda_hot", "--moe-routed-expert-base-model-domain", "cuda_hot", "--moe-routed-expert-shared-domain", "cuda_hot", "--moe-continuation-dense-policy", "tensor-parallel-decode-mirrored-embedding", "--moe-routed-expert-owner-order", "ordinal", "--moe-routed-expert-residency", "rebalanced", "--moe-routed-expert-domain", "cuda_hot=localhost:-1:cuda:0,localhost:-1:cuda:1;scope=auto;backend=nccl;routed_compute=apportioned;routed_phase=uniform;routed_decode_assignment=static-owner;routed_prefill_assignment=static-owner", "--moe-routed-expert-domain", "cpu_cold=localhost:0:cpu:0,localhost:1:cpu:0;scope=auto;backend=upi;routed_compute=apportioned;routed_phase=uniform;routed_decode_assignment=static-owner;routed_prefill_assignment=static-owner", "--moe-routed-expert-tier", "priority0@cuda_hot;priority=0;max-experts-per-layer=0;memory-mb=auto", "--moe-routed-expert-tier", "priority1@cpu_cold;priority=1;max-experts-per-layer=0;memory-mb=auto", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "9", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "455.111115", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "49", "--moe-dynamic-imbalance-threshold-permille", "1000", "--moe-dynamic-min-improvement-permille", "0", "--moe-dynamic-max-swaps-per-layer", "2", "--moe-dynamic-max-plan-entries-per-wave", "196", "--moe-dynamic-min-window-activations", "0", "--moe-device-rebalance-maintenance-slack-tokens", "0", "--moe-device-rebalance-min-maintenance-period-tokens", "4096", "--moe-device-rebalance-initial-maintenance-period-tokens", "1", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "0", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-max-post-wave-load-spread-permille", "1000", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ] } }, "cpu_isa": "AVX2", "hardware": { "accelerators": [ "0000:03:00.0 VGA compatible controller [0300]: ASPEED Technology, Inc. ASPEED Graphics Family [1a03:2000] (rev 41)", "0000:3f:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:42:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:8c:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:91:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:97:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:9a:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)" ], "cpu": { "Architecture:": "x86_64", "CPU(s):": "112", "Core(s) per socket:": "28", "Model name:": "Intel(R) Xeon(R) Gold 6238R CPU @ 2.20GHz", "NUMA node(s):": "2", "NUMA node0 CPU(s):": "0-27,56-83", "NUMA node1 CPU(s):": "28-55,84-111", "Socket(s):": "2", "Thread(s) per core:": "2" } }, "model_bytes": 10943808, "model_files": { "Qwen3.5-122B-A10B-UD-Q8_K_XL-00001-of-00004.gguf": 10943808, "Qwen3.5-122B-A10B-UD-Q8_K_XL-00002-of-00004.gguf": 49799375552, "Qwen3.5-122B-A10B-UD-Q8_K_XL-00003-of-00004.gguf": 49820794656, "Qwen3.5-122B-A10B-UD-Q8_K_XL-00004-of-00004.gguf": 35297355104 }, "prefill_tokens": 512, "workload": { "decode_tokens": 256, "measurement_iterations": 3, "prompt_repetitions": 512, "prompt_unit": " test", "schema": 1, "seed": 42, "temperature": 0, "top_k": 1, "top_p": 1, "warmup_iterations": 1 } }, "prefill_tokens": 512, "samples": { "decode": [ 13.874989699574925, 13.893436997053376, 13.862548125871031 ], "prefill": [ 79.71676755798522, 79.63019896107099, 84.7881306598229 ] }, "tokens_per_second": { "decode": 13.874989699574925, "prefill": 79.71676755798522 } }, { "case": "Qwen35_122B_ExpertOverlay/Qwen35MoENodeExpertOverlayParityTest.ProductionParity/Qwen35_122B_CUDA2_ROCm4_CPU2_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "identity": { "case": "Qwen35_122B_ExpertOverlay/Qwen35MoENodeExpertOverlayParityTest.ProductionParity/Qwen35_122B_CUDA2_ROCm4_CPU2_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "configuration": { "benchmark": { "args": [ "--mpi-procs", "2", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--auto", "--only-backends", "cpu,cuda,rocm", "--auto-device-counts", "cpu=2,cuda=2,rocm=4", "--only-strategies", "expert-overlay", "--moe-routed-expert-owner-order", "ordinal", "--mtp", "--mtp-depth-policy", "dynamic", "--moe-residency-maintenance", "dynamic" ], "context_length": 8192, "policy": "production_defaults", "schema": 1 }, "cross_host_e2e": [], "e2e": { "cell_timeout_seconds": { "AVX2": 900, "AVX512": 900 }, "context_length": 8192, "generation_tokens": 2048, "minimum_prompt_tokens": 4096, "movement_evidence": "required", "planning": { "device_counts": { "cpu": 2, "cuda": 2, "rocm": 4 }, "mode": "auto", "mpi_ranks": 2, "strategy": "expert-overlay" }, "readiness_timeout_seconds": 180, "request_timeout_seconds": 600, "server_args": [ "--mpi-procs", "2", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--auto", "--only-backends", "cpu,cuda,rocm", "--auto-device-counts", "cpu=2,cuda=2,rocm=4", "--only-strategies", "expert-overlay", "--moe-routed-expert-owner-order", "ordinal", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "9", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "455.111115", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "49", "--moe-dynamic-imbalance-threshold-permille", "1000", "--moe-dynamic-min-improvement-permille", "0", "--moe-dynamic-max-swaps-per-layer", "2", "--moe-dynamic-max-plan-entries-per-wave", "392", "--moe-dynamic-min-window-activations", "0", "--moe-device-rebalance-maintenance-slack-tokens", "0", "--moe-device-rebalance-min-maintenance-period-tokens", "4096", "--moe-device-rebalance-initial-maintenance-period-tokens", "1", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "0", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-max-post-wave-load-spread-permille", "1000", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ], "thinking_modes": "both", "tool_calling": "required" }, "id": "Qwen35_122B_CUDA2_ROCm4_CPU2_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "model": "Qwen3.5-122B-A10B-UD-Q8_K_XL-00001-of-00004.gguf", "model_parity_schema": 1, "runtime": { "context_length": 4096, "generation": { "max_tokens": 384, "minimum_completion_tokens": 384, "mtp_policy": "dynamic", "mtp_verify_mode": "speculative-sampling", "prefix_state": "hybrid_recurrent", "readiness_timeout_seconds": 180, "requests": [ { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "fresh_harbor", "prefix": "fresh" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_harbor", "prefix": "full" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "partial_mountain", "prefix": "partial" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_mountain", "prefix": "full" } ], "serial_control_id": "Qwen35_122B_CUDA2_ROCm4_CPU2_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPOff" }, "movement_evidence": "required", "server_args": [ "--mpi-procs", "2", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--backend", "host", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--moe-routed-expert-placement", "tiered-overlay", "--moe-routed-expert-continuation-domain", "cuda_hot", "--moe-routed-expert-base-model-domain", "cuda_hot", "--moe-routed-expert-shared-domain", "cuda_hot", "--moe-continuation-dense-policy", "tensor-parallel-decode-mirrored-embedding", "--moe-routed-expert-owner-order", "ordinal", "--moe-routed-expert-residency", "rebalanced", "--moe-routed-expert-domain", "cuda_hot=localhost:-1:cuda:0,localhost:-1:cuda:1;scope=auto;backend=nccl;routed_compute=apportioned;routed_phase=uniform;routed_decode_assignment=static-owner;routed_prefill_assignment=static-owner", "--moe-routed-expert-domain", "rocm_warm=localhost:-1:rocm:0,localhost:-1:rocm:1,localhost:-1:rocm:2,localhost:-1:rocm:3;scope=auto;backend=rccl;routed_compute=apportioned;routed_phase=uniform;routed_decode_assignment=static-owner;routed_prefill_assignment=static-owner", "--moe-routed-expert-domain", "cpu_cold=localhost:0:cpu:0,localhost:1:cpu:0;scope=auto;backend=upi;routed_compute=apportioned;routed_phase=uniform;routed_decode_assignment=static-owner;routed_prefill_assignment=static-owner", "--moe-routed-expert-tier", "priority0@cuda_hot;priority=0;max-experts-per-layer=0;memory-mb=auto", "--moe-routed-expert-tier", "priority1@rocm_warm;priority=1;max-experts-per-layer=0;memory-mb=auto", "--moe-routed-expert-tier", "priority2@cpu_cold;priority=2;max-experts-per-layer=0;memory-mb=auto", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "9", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "455.111115", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "49", "--moe-dynamic-imbalance-threshold-permille", "1000", "--moe-dynamic-min-improvement-permille", "0", "--moe-dynamic-max-swaps-per-layer", "2", "--moe-dynamic-max-plan-entries-per-wave", "392", "--moe-dynamic-min-window-activations", "0", "--moe-device-rebalance-maintenance-slack-tokens", "0", "--moe-device-rebalance-min-maintenance-period-tokens", "4096", "--moe-device-rebalance-initial-maintenance-period-tokens", "1", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "0", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-max-post-wave-load-spread-permille", "1000", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ] } }, "cpu_isa": "AVX2", "hardware": { "accelerators": [ "0000:03:00.0 VGA compatible controller [0300]: ASPEED Technology, Inc. ASPEED Graphics Family [1a03:2000] (rev 41)", "0000:3f:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:42:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:8c:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:91:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:97:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:9a:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)" ], "cpu": { "Architecture:": "x86_64", "CPU(s):": "112", "Core(s) per socket:": "28", "Model name:": "Intel(R) Xeon(R) Gold 6238R CPU @ 2.20GHz", "NUMA node(s):": "2", "NUMA node0 CPU(s):": "0-27,56-83", "NUMA node1 CPU(s):": "28-55,84-111", "Socket(s):": "2", "Thread(s) per core:": "2" } }, "model_bytes": 10943808, "model_files": { "Qwen3.5-122B-A10B-UD-Q8_K_XL-00001-of-00004.gguf": 10943808, "Qwen3.5-122B-A10B-UD-Q8_K_XL-00002-of-00004.gguf": 49799375552, "Qwen3.5-122B-A10B-UD-Q8_K_XL-00003-of-00004.gguf": 49820794656, "Qwen3.5-122B-A10B-UD-Q8_K_XL-00004-of-00004.gguf": 35297355104 }, "prefill_tokens": 512, "workload": { "decode_tokens": 256, "measurement_iterations": 3, "prompt_repetitions": 512, "prompt_unit": " test", "schema": 1, "seed": 42, "temperature": 0, "top_k": 1, "top_p": 1, "warmup_iterations": 1 } }, "prefill_tokens": 512, "samples": { "decode": [ 19.49194227540058, 19.66726932725284, 19.74574674777924 ], "prefill": [ 122.65918766452351, 124.93735943990396, 124.61742732755283 ] }, "tokens_per_second": { "decode": 19.66726932725284, "prefill": 124.61742732755283 } }, { "case": "Qwen35_122B_ExpertOverlay/Qwen35MoENodeExpertOverlayParityTest.ProductionParity/Qwen35_122B_ROCm2_CPU2_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "identity": { "case": "Qwen35_122B_ExpertOverlay/Qwen35MoENodeExpertOverlayParityTest.ProductionParity/Qwen35_122B_ROCm2_CPU2_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "configuration": { "benchmark": { "args": [ "--mpi-procs", "2", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--auto", "--only-backends", "cpu,rocm", "--auto-device-counts", "cpu=2,rocm=2", "--only-strategies", "expert-overlay", "--moe-routed-expert-owner-order", "ordinal", "--mtp", "--mtp-depth-policy", "dynamic", "--moe-residency-maintenance", "dynamic" ], "context_length": 8192, "policy": "production_defaults", "schema": 1 }, "cross_host_e2e": [], "e2e": { "cell_timeout_seconds": { "AVX2": 900, "AVX512": 900 }, "context_length": 8192, "generation_tokens": 2048, "minimum_prompt_tokens": 4096, "movement_evidence": "required", "planning": { "device_counts": { "cpu": 2, "rocm": 2 }, "mode": "auto", "mpi_ranks": 2, "strategy": "expert-overlay" }, "readiness_timeout_seconds": 180, "request_timeout_seconds": 600, "server_args": [ "--mpi-procs", "2", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--auto", "--only-backends", "cpu,rocm", "--auto-device-counts", "cpu=2,rocm=2", "--only-strategies", "expert-overlay", "--moe-routed-expert-owner-order", "ordinal", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "9", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "455.111115", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "49", "--moe-dynamic-imbalance-threshold-permille", "1000", "--moe-dynamic-min-improvement-permille", "0", "--moe-dynamic-max-swaps-per-layer", "2", "--moe-dynamic-max-plan-entries-per-wave", "196", "--moe-dynamic-min-window-activations", "0", "--moe-device-rebalance-maintenance-slack-tokens", "0", "--moe-device-rebalance-min-maintenance-period-tokens", "4096", "--moe-device-rebalance-initial-maintenance-period-tokens", "1", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "0", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-max-post-wave-load-spread-permille", "1000", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ], "thinking_modes": "both", "tool_calling": "required" }, "id": "Qwen35_122B_ROCm2_CPU2_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "model": "Qwen3.5-122B-A10B-UD-Q8_K_XL-00001-of-00004.gguf", "model_parity_schema": 1, "runtime": { "context_length": 4096, "generation": { "max_tokens": 384, "minimum_completion_tokens": 384, "mtp_policy": "dynamic", "mtp_verify_mode": "speculative-sampling", "prefix_state": "hybrid_recurrent", "readiness_timeout_seconds": 180, "requests": [ { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "fresh_harbor", "prefix": "fresh" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_harbor", "prefix": "full" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "partial_mountain", "prefix": "partial" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_mountain", "prefix": "full" } ], "serial_control_id": "Qwen35_122B_ROCm2_CPU2_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPOff" }, "movement_evidence": "required", "server_args": [ "--mpi-procs", "2", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--backend", "host", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--moe-routed-expert-placement", "tiered-overlay", "--moe-routed-expert-continuation-domain", "rocm_hot", "--moe-routed-expert-base-model-domain", "rocm_hot", "--moe-routed-expert-shared-domain", "rocm_hot", "--moe-continuation-dense-policy", "tensor-parallel-decode-mirrored-embedding", "--moe-routed-expert-owner-order", "ordinal", "--moe-routed-expert-residency", "rebalanced", "--moe-routed-expert-domain", "rocm_hot=localhost:-1:rocm:0,localhost:-1:rocm:1;scope=auto;backend=rccl;routed_compute=apportioned;routed_phase=uniform;routed_decode_assignment=static-owner;routed_prefill_assignment=static-owner", "--moe-routed-expert-domain", "cpu_cold=localhost:0:cpu:0,localhost:1:cpu:0;scope=auto;backend=upi;routed_compute=apportioned;routed_phase=uniform;routed_decode_assignment=static-owner;routed_prefill_assignment=static-owner", "--moe-routed-expert-tier", "priority0@rocm_hot;priority=0;max-experts-per-layer=0;memory-mb=auto", "--moe-routed-expert-tier", "priority1@cpu_cold;priority=1;max-experts-per-layer=0;memory-mb=auto", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "9", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "455.111115", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "49", "--moe-dynamic-imbalance-threshold-permille", "1000", "--moe-dynamic-min-improvement-permille", "0", "--moe-dynamic-max-swaps-per-layer", "2", "--moe-dynamic-max-plan-entries-per-wave", "196", "--moe-dynamic-min-window-activations", "0", "--moe-device-rebalance-maintenance-slack-tokens", "0", "--moe-device-rebalance-min-maintenance-period-tokens", "4096", "--moe-device-rebalance-initial-maintenance-period-tokens", "1", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "0", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-max-post-wave-load-spread-permille", "1000", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ] } }, "cpu_isa": "AVX2", "hardware": { "accelerators": [ "0000:03:00.0 VGA compatible controller [0300]: ASPEED Technology, Inc. ASPEED Graphics Family [1a03:2000] (rev 41)", "0000:3f:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:42:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:8c:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:91:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:97:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:9a:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)" ], "cpu": { "Architecture:": "x86_64", "CPU(s):": "112", "Core(s) per socket:": "28", "Model name:": "Intel(R) Xeon(R) Gold 6238R CPU @ 2.20GHz", "NUMA node(s):": "2", "NUMA node0 CPU(s):": "0-27,56-83", "NUMA node1 CPU(s):": "28-55,84-111", "Socket(s):": "2", "Thread(s) per core:": "2" } }, "model_bytes": 10943808, "model_files": { "Qwen3.5-122B-A10B-UD-Q8_K_XL-00001-of-00004.gguf": 10943808, "Qwen3.5-122B-A10B-UD-Q8_K_XL-00002-of-00004.gguf": 49799375552, "Qwen3.5-122B-A10B-UD-Q8_K_XL-00003-of-00004.gguf": 49820794656, "Qwen3.5-122B-A10B-UD-Q8_K_XL-00004-of-00004.gguf": 35297355104 }, "prefill_tokens": 512, "workload": { "decode_tokens": 256, "measurement_iterations": 3, "prompt_repetitions": 512, "prompt_unit": " test", "schema": 1, "seed": 42, "temperature": 0, "top_k": 1, "top_p": 1, "warmup_iterations": 1 } }, "prefill_tokens": 512, "samples": { "decode": [ 20.511585826316395, 20.881290335133617, 21.229999682589856 ], "prefill": [ 99.01013664463989, 102.1496137270215, 106.725029760582 ] }, "tokens_per_second": { "decode": 20.881290335133617, "prefill": 102.1496137270215 } }, { "case": "Qwen35_122B_ExpertOverlay/Qwen35MoENodeExpertOverlayParityTest.ProductionParity/Qwen35_122B_ROCm4_CPU2_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "identity": { "case": "Qwen35_122B_ExpertOverlay/Qwen35MoENodeExpertOverlayParityTest.ProductionParity/Qwen35_122B_ROCm4_CPU2_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "configuration": { "benchmark": { "args": [ "--mpi-procs", "2", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--auto", "--only-backends", "cpu,rocm", "--auto-device-counts", "cpu=2,rocm=4", "--only-strategies", "expert-overlay", "--moe-routed-expert-owner-order", "ordinal", "--mtp", "--mtp-depth-policy", "dynamic", "--moe-residency-maintenance", "dynamic" ], "context_length": 8192, "policy": "production_defaults", "schema": 1 }, "cross_host_e2e": [], "e2e": { "cell_timeout_seconds": { "AVX2": 900, "AVX512": 900 }, "context_length": 8192, "generation_tokens": 2048, "minimum_prompt_tokens": 4096, "movement_evidence": "required", "planning": { "device_counts": { "cpu": 2, "rocm": 4 }, "mode": "auto", "mpi_ranks": 2, "strategy": "expert-overlay" }, "readiness_timeout_seconds": 180, "request_timeout_seconds": 600, "server_args": [ "--mpi-procs", "2", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--auto", "--only-backends", "cpu,rocm", "--auto-device-counts", "cpu=2,rocm=4", "--only-strategies", "expert-overlay", "--moe-routed-expert-owner-order", "ordinal", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "9", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "455.111115", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "49", "--moe-dynamic-imbalance-threshold-permille", "1000", "--moe-dynamic-min-improvement-permille", "0", "--moe-dynamic-max-swaps-per-layer", "2", "--moe-dynamic-max-plan-entries-per-wave", "294", "--moe-dynamic-min-window-activations", "0", "--moe-device-rebalance-maintenance-slack-tokens", "0", "--moe-device-rebalance-min-maintenance-period-tokens", "4096", "--moe-device-rebalance-initial-maintenance-period-tokens", "1", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "0", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-max-post-wave-load-spread-permille", "1000", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ], "thinking_modes": "both", "tool_calling": "required" }, "id": "Qwen35_122B_ROCm4_CPU2_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "model": "Qwen3.5-122B-A10B-UD-Q8_K_XL-00001-of-00004.gguf", "model_parity_schema": 1, "runtime": { "context_length": 4096, "generation": { "max_tokens": 384, "minimum_completion_tokens": 384, "mtp_policy": "dynamic", "mtp_verify_mode": "speculative-sampling", "prefix_state": "hybrid_recurrent", "readiness_timeout_seconds": 180, "requests": [ { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "fresh_harbor", "prefix": "fresh" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_harbor", "prefix": "full" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "partial_mountain", "prefix": "partial" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_mountain", "prefix": "full" } ], "serial_control_id": "Qwen35_122B_ROCm4_CPU2_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPOff" }, "movement_evidence": "required", "server_args": [ "--mpi-procs", "2", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--backend", "host", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--moe-routed-expert-placement", "tiered-overlay", "--moe-routed-expert-continuation-domain", "rocm_hot", "--moe-routed-expert-base-model-domain", "rocm_hot", "--moe-routed-expert-shared-domain", "rocm_hot", "--moe-continuation-dense-policy", "tensor-parallel-decode-mirrored-embedding", "--moe-routed-expert-owner-order", "ordinal", "--moe-routed-expert-residency", "rebalanced", "--moe-routed-expert-domain", "rocm_hot=localhost:-1:rocm:0,localhost:-1:rocm:1,localhost:-1:rocm:2,localhost:-1:rocm:3;scope=auto;backend=rccl;routed_compute=apportioned;routed_phase=uniform;routed_decode_assignment=static-owner;routed_prefill_assignment=static-owner", "--moe-routed-expert-domain", "cpu_cold=localhost:0:cpu:0,localhost:1:cpu:0;scope=auto;backend=upi;routed_compute=apportioned;routed_phase=uniform;routed_decode_assignment=static-owner;routed_prefill_assignment=static-owner", "--moe-routed-expert-tier", "priority0@rocm_hot;priority=0;max-experts-per-layer=0;memory-mb=auto", "--moe-routed-expert-tier", "priority1@cpu_cold;priority=1;max-experts-per-layer=0;memory-mb=auto", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "9", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "455.111115", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "49", "--moe-dynamic-imbalance-threshold-permille", "1000", "--moe-dynamic-min-improvement-permille", "0", "--moe-dynamic-max-swaps-per-layer", "2", "--moe-dynamic-max-plan-entries-per-wave", "294", "--moe-dynamic-min-window-activations", "0", "--moe-device-rebalance-maintenance-slack-tokens", "0", "--moe-device-rebalance-min-maintenance-period-tokens", "4096", "--moe-device-rebalance-initial-maintenance-period-tokens", "1", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "0", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-max-post-wave-load-spread-permille", "1000", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ] } }, "cpu_isa": "AVX2", "hardware": { "accelerators": [ "0000:03:00.0 VGA compatible controller [0300]: ASPEED Technology, Inc. ASPEED Graphics Family [1a03:2000] (rev 41)", "0000:3f:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:42:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:8c:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:91:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:97:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:9a:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)" ], "cpu": { "Architecture:": "x86_64", "CPU(s):": "112", "Core(s) per socket:": "28", "Model name:": "Intel(R) Xeon(R) Gold 6238R CPU @ 2.20GHz", "NUMA node(s):": "2", "NUMA node0 CPU(s):": "0-27,56-83", "NUMA node1 CPU(s):": "28-55,84-111", "Socket(s):": "2", "Thread(s) per core:": "2" } }, "model_bytes": 10943808, "model_files": { "Qwen3.5-122B-A10B-UD-Q8_K_XL-00001-of-00004.gguf": 10943808, "Qwen3.5-122B-A10B-UD-Q8_K_XL-00002-of-00004.gguf": 49799375552, "Qwen3.5-122B-A10B-UD-Q8_K_XL-00003-of-00004.gguf": 49820794656, "Qwen3.5-122B-A10B-UD-Q8_K_XL-00004-of-00004.gguf": 35297355104 }, "prefill_tokens": 512, "workload": { "decode_tokens": 256, "measurement_iterations": 3, "prompt_repetitions": 512, "prompt_unit": " test", "schema": 1, "seed": 42, "temperature": 0, "top_k": 1, "top_p": 1, "warmup_iterations": 1 } }, "prefill_tokens": 512, "samples": { "decode": [ 26.234970678210328, 26.83361584714674, 27.285694937863656 ], "prefill": [ 141.85599618268174, 146.9811945266633, 155.15006337005855 ] }, "tokens_per_second": { "decode": 26.83361584714674, "prefill": 146.9811945266633 } }, { "case": "Qwen36MoEExpertOverlay/Qwen36MoEExpertOverlayParityTest.ProductionParity/Ornith15MoE_35B_Q4KM_LocalTP_NCCL_2xCUDA_ExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "identity": { "case": "Qwen36MoEExpertOverlay/Qwen36MoEExpertOverlayParityTest.ProductionParity/Ornith15MoE_35B_Q4KM_LocalTP_NCCL_2xCUDA_ExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "configuration": { "benchmark": { "args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--auto", "--only-backends", "cuda", "--auto-device-counts", "cuda=2", "--only-strategies", "tp", "--moe-routed-expert-owner-order", "ordinal", "--mtp", "--mtp-depth-policy", "dynamic", "--moe-residency-maintenance", "dynamic" ], "context_length": 8192, "policy": "production_defaults", "schema": 1 }, "cross_host_e2e": [], "e2e": { "cell_timeout_seconds": { "AVX2": 900, "AVX512": 900 }, "context_length": 8192, "generation_tokens": 2048, "minimum_prompt_tokens": 4096, "movement_evidence": "required", "planning": { "device_counts": { "cuda": 2 }, "mode": "auto", "mpi_ranks": 1, "strategy": "tp" }, "readiness_timeout_seconds": 60, "request_timeout_seconds": 600, "server_args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--tp-allreduce-precision", "schema", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--auto", "--only-backends", "cuda", "--auto-device-counts", "cuda=2", "--only-strategies", "tp", "--moe-routed-expert-owner-order", "ordinal", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "1", "--moe-residency-maintenance-max-window", "1", "--moe-residency-maintenance-window-growth", "1", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1000", "--moe-dynamic-min-improvement-permille", "0", "--moe-dynamic-max-swaps-per-layer", "20", "--moe-dynamic-max-plan-entries-per-wave", "20", "--moe-dynamic-min-window-activations", "0", "--moe-device-rebalance-maintenance-slack-tokens", "0", "--moe-device-rebalance-min-maintenance-period-tokens", "16", "--moe-device-rebalance-initial-maintenance-period-tokens", "2", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "0", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-max-post-wave-load-spread-permille", "1000", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ], "thinking_modes": "both", "tool_calling": "required" }, "id": "Ornith15MoE_35B_Q4KM_LocalTP_NCCL_2xCUDA_ExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "model": "Ornith-1.5-35B-Q4_K_M.gguf", "model_parity_schema": 1, "runtime": { "context_length": 4096, "generation": { "max_tokens": 384, "minimum_completion_tokens": 384, "mtp_policy": "dynamic", "mtp_verify_mode": "speculative-sampling", "prefix_state": "hybrid_recurrent", "readiness_timeout_seconds": 60, "requests": [ { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "fresh_harbor", "prefix": "fresh" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_harbor", "prefix": "full" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "partial_mountain", "prefix": "partial" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_mountain", "prefix": "full" } ], "serial_control_id": "Ornith15MoE_35B_Q4KM_LocalTP_NCCL_2xCUDA_ExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPOff" }, "movement_evidence": "required", "server_args": [ "--mpi-procs", "1", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--backend", "nccl", "--tp-allreduce-precision", "schema", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--moe-routed-expert-placement", "single-domain", "--moe-routed-expert-continuation-domain", "qwen36_moe_cuda_local_tp", "--moe-routed-expert-base-model-domain", "qwen36_moe_cuda_local_tp", "--moe-routed-expert-shared-domain", "qwen36_moe_cuda_local_tp", "--moe-continuation-dense-policy", "tensor-parallel", "--moe-routed-expert-owner-order", "ordinal", "--moe-routed-expert-residency", "rebalanced", "--define-domain", "qwen36_moe_cuda_local_tp=localhost:-1:cuda:0,localhost:-1:cuda:1;scope=rank_local;backend=nccl;routed_compute=apportioned;routed_phase=uniform;routed_decode_assignment=static-owner;routed_prefill_assignment=static-owner", "--moe-routed-expert-domain", "qwen36_moe_cuda_local_tp=localhost:-1:cuda:0,localhost:-1:cuda:1;scope=rank_local;backend=nccl;routed_compute=apportioned;routed_phase=uniform;routed_decode_assignment=static-owner;routed_prefill_assignment=static-owner", "--moe-routed-expert-tier", "priority_0@qwen36_moe_cuda_local_tp;priority=0;max-experts-per-layer=0;memory-mb=auto", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "1", "--moe-residency-maintenance-max-window", "1", "--moe-residency-maintenance-window-growth", "1", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1000", "--moe-dynamic-min-improvement-permille", "0", "--moe-dynamic-max-swaps-per-layer", "20", "--moe-dynamic-max-plan-entries-per-wave", "20", "--moe-dynamic-min-window-activations", "0", "--moe-device-rebalance-maintenance-slack-tokens", "0", "--moe-device-rebalance-min-maintenance-period-tokens", "16", "--moe-device-rebalance-initial-maintenance-period-tokens", "2", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "0", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-max-post-wave-load-spread-permille", "1000", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ] } }, "cpu_isa": "AVX2", "hardware": { "accelerators": [ "0000:03:00.0 VGA compatible controller [0300]: ASPEED Technology, Inc. ASPEED Graphics Family [1a03:2000] (rev 41)", "0000:3f:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:42:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:8c:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:91:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:97:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:9a:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)" ], "cpu": { "Architecture:": "x86_64", "CPU(s):": "112", "Core(s) per socket:": "28", "Model name:": "Intel(R) Xeon(R) Gold 6238R CPU @ 2.20GHz", "NUMA node(s):": "2", "NUMA node0 CPU(s):": "0-27,56-83", "NUMA node1 CPU(s):": "28-55,84-111", "Socket(s):": "2", "Thread(s) per core:": "2" } }, "model_bytes": 21713463040, "model_files": { "Ornith-1.5-35B-Q4_K_M.gguf": 21713463040 }, "prefill_tokens": 512, "workload": { "decode_tokens": 256, "measurement_iterations": 3, "prompt_repetitions": 512, "prompt_unit": " test", "schema": 1, "seed": 42, "temperature": 0, "top_k": 1, "top_p": 1, "warmup_iterations": 1 } }, "prefill_tokens": 512, "samples": { "decode": [ 139.64509797454622, 139.29970176676784, 139.3162602596735 ], "prefill": [ 887.0883659405561, 887.8439089392327, 889.9658665585563 ] }, "tokens_per_second": { "decode": 139.3162602596735, "prefill": 887.8439089392327 } }, { "case": "Qwen36MoE/Qwen36MoESingleDeviceParityTest.ProductionParity/Qwen36MoE_35B_IQ3S_CUDA0_ActFP32_KVFP16_MTPDynamicDepth", "identity": { "case": "Qwen36MoE/Qwen36MoESingleDeviceParityTest.ProductionParity/Qwen36MoE_35B_IQ3S_CUDA0_ActFP32_KVFP16_MTPDynamicDepth", "configuration": { "benchmark": { "args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--auto", "--only-backends", "cuda", "--auto-device-counts", "cuda=1", "--only-strategies", "single", "--mtp", "--mtp-depth-policy", "dynamic" ], "context_length": 8192, "policy": "production_defaults", "schema": 1 }, "cross_host_e2e": [], "e2e": { "cell_timeout_seconds": { "AVX2": 900, "AVX512": 900 }, "context_length": 8192, "generation_tokens": 2048, "minimum_prompt_tokens": 4096, "movement_evidence": "not_applicable", "planning": { "device_counts": { "cuda": 1 }, "mode": "auto", "mpi_ranks": 1, "strategy": "single" }, "readiness_timeout_seconds": 60, "request_timeout_seconds": 600, "server_args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--tp-allreduce-precision", "schema", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--auto", "--only-backends", "cuda", "--auto-device-counts", "cuda=1", "--only-strategies", "single", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "256", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "1.5", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1300", "--moe-dynamic-min-improvement-permille", "50", "--moe-dynamic-max-swaps-per-layer", "4", "--moe-dynamic-max-plan-entries-per-wave", "16", "--moe-dynamic-min-window-activations", "64", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "15", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "256", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "128", "--moe-device-rebalance-max-post-wave-load-spread-permille", "100", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ], "thinking_modes": "both", "tool_calling": "required" }, "id": "Qwen36MoE_35B_IQ3S_CUDA0_ActFP32_KVFP16_MTPDynamicDepth", "model": "Qwen3.6-35B-A3B-UD-IQ3_S.gguf", "model_parity_schema": 1, "runtime": { "context_length": 4096, "generation": { "max_tokens": 384, "minimum_completion_tokens": 384, "mtp_policy": "dynamic", "mtp_verify_mode": "speculative-sampling", "prefix_state": "hybrid_recurrent", "readiness_timeout_seconds": 60, "requests": [ { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "fresh_harbor", "prefix": "fresh" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_harbor", "prefix": "full" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "partial_mountain", "prefix": "partial" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_mountain", "prefix": "full" } ], "serial_control_id": "Qwen36MoE_35B_IQ3S_CUDA0_ActFP32_KVFP16_MTPOff" }, "movement_evidence": "not_applicable", "server_args": [ "--mpi-procs", "1", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--backend", "host", "--tp-allreduce-precision", "schema", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--device", "localhost:-1:cuda:0", "--tensor-parallelism-degree", "1", "--tp-scope", "auto", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "256", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "1.5", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1300", "--moe-dynamic-min-improvement-permille", "50", "--moe-dynamic-max-swaps-per-layer", "4", "--moe-dynamic-max-plan-entries-per-wave", "16", "--moe-dynamic-min-window-activations", "64", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "15", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "256", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "128", "--moe-device-rebalance-max-post-wave-load-spread-permille", "100", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ] } }, "cpu_isa": "AVX2", "hardware": { "accelerators": [ "0000:03:00.0 VGA compatible controller [0300]: ASPEED Technology, Inc. ASPEED Graphics Family [1a03:2000] (rev 41)", "0000:3f:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:42:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:8c:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:91:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:97:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:9a:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)" ], "cpu": { "Architecture:": "x86_64", "CPU(s):": "112", "Core(s) per socket:": "28", "Model name:": "Intel(R) Xeon(R) Gold 6238R CPU @ 2.20GHz", "NUMA node(s):": "2", "NUMA node0 CPU(s):": "0-27,56-83", "NUMA node1 CPU(s):": "28-55,84-111", "Socket(s):": "2", "Thread(s) per core:": "2" } }, "model_bytes": 15346432288, "model_files": { "Qwen3.6-35B-A3B-UD-IQ3_S.gguf": 15346432288 }, "prefill_tokens": 512, "workload": { "decode_tokens": 256, "measurement_iterations": 3, "prompt_repetitions": 512, "prompt_unit": " test", "schema": 1, "seed": 42, "temperature": 0, "top_k": 1, "top_p": 1, "warmup_iterations": 1 } }, "prefill_tokens": 512, "samples": { "decode": [ 287.0802547386263, 287.0899166958844, 287.0160419073676 ], "prefill": [ 1752.7420185408955, 1760.9705550693034, 1763.2433748833598 ] }, "tokens_per_second": { "decode": 287.0802547386263, "prefill": 1760.9705550693034 } }, { "case": "Qwen38Dense/Qwen38DenseMultiDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_CUDA2_TP_ActFP32_KVFP16_MTPDynamicDepth", "identity": { "case": "Qwen38Dense/Qwen38DenseMultiDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_CUDA2_TP_ActFP32_KVFP16_MTPDynamicDepth", "configuration": { "benchmark": { "args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--auto", "--only-backends", "cuda", "--auto-device-counts", "cuda=2", "--only-strategies", "tp", "--mtp", "--mtp-depth-policy", "dynamic" ], "context_length": 8192, "policy": "production_defaults", "schema": 1 }, "cross_host_e2e": [], "e2e": { "cell_timeout_seconds": { "AVX2": 900, "AVX512": 900 }, "context_length": 8192, "generation_tokens": 2048, "minimum_prompt_tokens": 4096, "movement_evidence": "not_applicable", "planning": { "device_counts": { "cuda": 2 }, "mode": "auto", "mpi_ranks": 1, "strategy": "tp" }, "readiness_timeout_seconds": 60, "request_timeout_seconds": 600, "server_args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--auto", "--only-backends", "cuda", "--auto-device-counts", "cuda=2", "--only-strategies", "tp", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "256", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "1.5", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1300", "--moe-dynamic-min-improvement-permille", "50", "--moe-dynamic-max-swaps-per-layer", "4", "--moe-dynamic-max-plan-entries-per-wave", "16", "--moe-dynamic-min-window-activations", "64", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "15", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "256", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "128", "--moe-device-rebalance-max-post-wave-load-spread-permille", "100", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ], "thinking_modes": "both", "tool_calling": "required" }, "id": "Qwen38Dense_27B_IQ4XS_CUDA2_TP_ActFP32_KVFP16_MTPDynamicDepth", "model": "Qwen3.8-27B-IQ4_XS.gguf", "model_parity_schema": 1, "runtime": { "context_length": 4096, "generation": { "max_tokens": 384, "minimum_completion_tokens": 384, "mtp_policy": "dynamic", "mtp_verify_mode": "speculative-sampling", "prefix_state": "hybrid_recurrent", "readiness_timeout_seconds": 60, "requests": [ { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "fresh_harbor", "prefix": "fresh" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_harbor", "prefix": "full" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "partial_mountain", "prefix": "partial" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_mountain", "prefix": "full" } ], "serial_control_id": "Qwen38Dense_27B_IQ4XS_CUDA2_TP_ActFP32_KVFP16_MTPOff" }, "movement_evidence": "not_applicable", "server_args": [ "--mpi-procs", "1", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--backend", "nccl", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--tp-devices", "localhost:-1:cuda:0,localhost:-1:cuda:1", "--tensor-parallelism-degree", "2", "--tp-scope", "rank_local", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "256", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "1.5", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1300", "--moe-dynamic-min-improvement-permille", "50", "--moe-dynamic-max-swaps-per-layer", "4", "--moe-dynamic-max-plan-entries-per-wave", "16", "--moe-dynamic-min-window-activations", "64", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "15", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "256", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "128", "--moe-device-rebalance-max-post-wave-load-spread-permille", "100", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ] } }, "cpu_isa": "AVX2", "hardware": { "accelerators": [ "0000:03:00.0 VGA compatible controller [0300]: ASPEED Technology, Inc. ASPEED Graphics Family [1a03:2000] (rev 41)", "0000:3f:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:42:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:8c:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:91:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:97:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:9a:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)" ], "cpu": { "Architecture:": "x86_64", "CPU(s):": "112", "Core(s) per socket:": "28", "Model name:": "Intel(R) Xeon(R) Gold 6238R CPU @ 2.20GHz", "NUMA node(s):": "2", "NUMA node0 CPU(s):": "0-27,56-83", "NUMA node1 CPU(s):": "28-55,84-111", "Socket(s):": "2", "Thread(s) per core:": "2" } }, "model_bytes": 15705861088, "model_files": { "Qwen3.8-27B-IQ4_XS.gguf": 15705861088 }, "prefill_tokens": 512, "workload": { "decode_tokens": 256, "measurement_iterations": 3, "prompt_repetitions": 512, "prompt_unit": " test", "schema": 1, "seed": 42, "temperature": 0, "top_k": 1, "top_p": 1, "warmup_iterations": 1 } }, "prefill_tokens": 512, "samples": { "decode": [ 50.35706054916215, 50.327024035111855, 50.28606408785884 ], "prefill": [ 468.70927029727784, 470.268896779258, 469.42453704426146 ] }, "tokens_per_second": { "decode": 50.327024035111855, "prefill": 469.42453704426146 } }, { "case": "Qwen38Dense/Qwen38DenseMultiDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_CUDA2_PP_ActFP32_KVFP16_MTPDynamicDepth", "identity": { "case": "Qwen38Dense/Qwen38DenseMultiDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_CUDA2_PP_ActFP32_KVFP16_MTPDynamicDepth", "configuration": { "benchmark": { "args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--auto", "--only-backends", "cuda", "--auto-device-counts", "cuda=2", "--only-strategies", "pp", "--mtp", "--mtp-depth-policy", "dynamic" ], "context_length": 8192, "policy": "production_defaults", "schema": 1 }, "cross_host_e2e": [], "e2e": { "cell_timeout_seconds": { "AVX2": 900, "AVX512": 900 }, "context_length": 8192, "generation_tokens": 2048, "minimum_prompt_tokens": 4096, "movement_evidence": "not_applicable", "planning": { "device_counts": { "cuda": 2 }, "mode": "auto", "mpi_ranks": 1, "pipeline_domains": [ { "backend": "cuda", "devices": 1 }, { "backend": "cuda", "devices": 1 } ], "pipeline_layers": 64, "strategy": "pp" }, "readiness_timeout_seconds": 60, "request_timeout_seconds": 600, "server_args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--auto", "--only-backends", "cuda", "--auto-device-counts", "cuda=2", "--only-strategies", "pp", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "256", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "1.5", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1300", "--moe-dynamic-min-improvement-permille", "50", "--moe-dynamic-max-swaps-per-layer", "4", "--moe-dynamic-max-plan-entries-per-wave", "16", "--moe-dynamic-min-window-activations", "64", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "15", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "256", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "128", "--moe-device-rebalance-max-post-wave-load-spread-permille", "100", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ], "thinking_modes": "both", "tool_calling": "required" }, "id": "Qwen38Dense_27B_IQ4XS_CUDA2_PP_ActFP32_KVFP16_MTPDynamicDepth", "model": "Qwen3.8-27B-IQ4_XS.gguf", "model_parity_schema": 1, "runtime": { "context_length": 4096, "generation": { "max_tokens": 384, "minimum_completion_tokens": 384, "mtp_policy": "dynamic", "mtp_verify_mode": "speculative-sampling", "prefix_state": "hybrid_recurrent", "readiness_timeout_seconds": 60, "requests": [ { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "fresh_harbor", "prefix": "fresh" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_harbor", "prefix": "full" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "partial_mountain", "prefix": "partial" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_mountain", "prefix": "full" } ], "serial_control_id": "Qwen38Dense_27B_IQ4XS_CUDA2_PP_ActFP32_KVFP16_MTPOff" }, "movement_evidence": "not_applicable", "server_args": [ "--mpi-procs", "1", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--backend", "nccl", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--tensor-parallelism-degree", "1", "--tp-scope", "auto", "--define-domain", "parity_stage_0=localhost:-1:cuda:0;scope=rank_local;backend=auto;owner=0", "--define-domain", "parity_stage_1=localhost:-1:cuda:1;scope=rank_local;backend=auto;owner=0", "--pp-stage", "0=parity_stage_0:0-31", "--pp-stage", "1=parity_stage_1:32-63", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "256", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "1.5", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1300", "--moe-dynamic-min-improvement-permille", "50", "--moe-dynamic-max-swaps-per-layer", "4", "--moe-dynamic-max-plan-entries-per-wave", "16", "--moe-dynamic-min-window-activations", "64", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "15", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "256", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "128", "--moe-device-rebalance-max-post-wave-load-spread-permille", "100", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ] } }, "cpu_isa": "AVX2", "hardware": { "accelerators": [ "0000:03:00.0 VGA compatible controller [0300]: ASPEED Technology, Inc. ASPEED Graphics Family [1a03:2000] (rev 41)", "0000:3f:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:42:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:8c:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:91:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:97:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:9a:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)" ], "cpu": { "Architecture:": "x86_64", "CPU(s):": "112", "Core(s) per socket:": "28", "Model name:": "Intel(R) Xeon(R) Gold 6238R CPU @ 2.20GHz", "NUMA node(s):": "2", "NUMA node0 CPU(s):": "0-27,56-83", "NUMA node1 CPU(s):": "28-55,84-111", "Socket(s):": "2", "Thread(s) per core:": "2" } }, "model_bytes": 15705861088, "model_files": { "Qwen3.8-27B-IQ4_XS.gguf": 15705861088 }, "prefill_tokens": 512, "workload": { "decode_tokens": 256, "measurement_iterations": 3, "prompt_repetitions": 512, "prompt_unit": " test", "schema": 1, "seed": 42, "temperature": 0, "top_k": 1, "top_p": 1, "warmup_iterations": 1 } }, "prefill_tokens": 512, "samples": { "decode": [ 40.31915464549371, 39.8461318647918, 40.31356272301949 ], "prefill": [ 757.9576399710418, 762.826615166017, 760.6795019248906 ] }, "tokens_per_second": { "decode": 40.31356272301949, "prefill": 760.6795019248906 } }, { "case": "Qwen38Dense/Qwen38DenseSingleDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_CUDA0_ActFP32_KVFP16_MTPDynamicDepth", "identity": { "case": "Qwen38Dense/Qwen38DenseSingleDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_CUDA0_ActFP32_KVFP16_MTPDynamicDepth", "configuration": { "benchmark": { "args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--auto", "--only-backends", "cuda", "--auto-device-counts", "cuda=1", "--only-strategies", "single", "--mtp", "--mtp-depth-policy", "dynamic" ], "context_length": 8192, "policy": "production_defaults", "schema": 1 }, "cross_host_e2e": [], "e2e": { "cell_timeout_seconds": { "AVX2": 900, "AVX512": 900 }, "context_length": 8192, "generation_tokens": 2048, "minimum_prompt_tokens": 4096, "movement_evidence": "not_applicable", "planning": { "device_counts": { "cuda": 1 }, "mode": "auto", "mpi_ranks": 1, "strategy": "single" }, "readiness_timeout_seconds": 60, "request_timeout_seconds": 600, "server_args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--auto", "--only-backends", "cuda", "--auto-device-counts", "cuda=1", "--only-strategies", "single", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "256", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "1.5", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1300", "--moe-dynamic-min-improvement-permille", "50", "--moe-dynamic-max-swaps-per-layer", "4", "--moe-dynamic-max-plan-entries-per-wave", "16", "--moe-dynamic-min-window-activations", "64", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "15", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "256", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "128", "--moe-device-rebalance-max-post-wave-load-spread-permille", "100", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ], "thinking_modes": "both", "tool_calling": "required" }, "id": "Qwen38Dense_27B_IQ4XS_CUDA0_ActFP32_KVFP16_MTPDynamicDepth", "model": "Qwen3.8-27B-IQ4_XS.gguf", "model_parity_schema": 1, "runtime": { "context_length": 4096, "generation": { "max_tokens": 384, "minimum_completion_tokens": 384, "mtp_policy": "dynamic", "mtp_verify_mode": "speculative-sampling", "prefix_state": "hybrid_recurrent", "readiness_timeout_seconds": 60, "requests": [ { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "fresh_harbor", "prefix": "fresh" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_harbor", "prefix": "full" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "partial_mountain", "prefix": "partial" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_mountain", "prefix": "full" } ], "serial_control_id": "Qwen38Dense_27B_IQ4XS_CUDA0_ActFP32_KVFP16_MTPOff" }, "movement_evidence": "not_applicable", "server_args": [ "--mpi-procs", "1", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--backend", "host", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--device", "localhost:-1:cuda:0", "--tensor-parallelism-degree", "1", "--tp-scope", "auto", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "256", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "1.5", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1300", "--moe-dynamic-min-improvement-permille", "50", "--moe-dynamic-max-swaps-per-layer", "4", "--moe-dynamic-max-plan-entries-per-wave", "16", "--moe-dynamic-min-window-activations", "64", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "15", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "256", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "128", "--moe-device-rebalance-max-post-wave-load-spread-permille", "100", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ] } }, "cpu_isa": "AVX2", "hardware": { "accelerators": [ "0000:03:00.0 VGA compatible controller [0300]: ASPEED Technology, Inc. ASPEED Graphics Family [1a03:2000] (rev 41)", "0000:3f:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:42:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:8c:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:91:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:97:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:9a:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)" ], "cpu": { "Architecture:": "x86_64", "CPU(s):": "112", "Core(s) per socket:": "28", "Model name:": "Intel(R) Xeon(R) Gold 6238R CPU @ 2.20GHz", "NUMA node(s):": "2", "NUMA node0 CPU(s):": "0-27,56-83", "NUMA node1 CPU(s):": "28-55,84-111", "Socket(s):": "2", "Thread(s) per core:": "2" } }, "model_bytes": 15705861088, "model_files": { "Qwen3.8-27B-IQ4_XS.gguf": 15705861088 }, "prefill_tokens": 512, "workload": { "decode_tokens": 256, "measurement_iterations": 3, "prompt_repetitions": 512, "prompt_unit": " test", "schema": 1, "seed": 42, "temperature": 0, "top_k": 1, "top_p": 1, "warmup_iterations": 1 } }, "prefill_tokens": 512, "samples": { "decode": [ 62.23518302694091, 62.20860738139935, 62.18983807505286 ], "prefill": [ 703.5667598592601, 703.1167063856882, 703.430721342884 ] }, "tokens_per_second": { "decode": 62.20860738139935, "prefill": 703.430721342884 } }, { "case": "Qwen35_122B_ExpertOverlay/Qwen35MoENodeExpertOverlayParityTest.ProductionParity/Qwen35_122B_CUDA2_ROCm4_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "identity": { "case": "Qwen35_122B_ExpertOverlay/Qwen35MoENodeExpertOverlayParityTest.ProductionParity/Qwen35_122B_CUDA2_ROCm4_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "configuration": { "benchmark": { "args": [ "--mpi-procs", "2", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--auto", "--only-backends", "cuda,rocm", "--auto-device-counts", "cuda=2,rocm=4", "--only-strategies", "expert-overlay", "--moe-routed-expert-owner-order", "ordinal", "--mtp", "--mtp-depth-policy", "dynamic", "--moe-residency-maintenance", "dynamic" ], "context_length": 8192, "policy": "production_defaults", "schema": 1 }, "cross_host_e2e": [], "e2e": { "cell_timeout_seconds": { "AVX2": 900, "AVX512": 900 }, "context_length": 8192, "generation_tokens": 2048, "minimum_prompt_tokens": 4096, "movement_evidence": "required", "planning": { "device_counts": { "cuda": 2, "rocm": 4 }, "mode": "auto", "mpi_ranks": 2, "strategy": "expert-overlay" }, "readiness_timeout_seconds": 180, "request_timeout_seconds": 600, "server_args": [ "--mpi-procs", "2", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--auto", "--only-backends", "cuda,rocm", "--auto-device-counts", "cuda=2,rocm=4", "--only-strategies", "expert-overlay", "--moe-routed-expert-owner-order", "ordinal", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "9", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "455.111115", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "49", "--moe-dynamic-imbalance-threshold-permille", "1000", "--moe-dynamic-min-improvement-permille", "0", "--moe-dynamic-max-swaps-per-layer", "2", "--moe-dynamic-max-plan-entries-per-wave", "294", "--moe-dynamic-min-window-activations", "0", "--moe-device-rebalance-maintenance-slack-tokens", "0", "--moe-device-rebalance-min-maintenance-period-tokens", "4096", "--moe-device-rebalance-initial-maintenance-period-tokens", "1", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "0", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-max-post-wave-load-spread-permille", "1000", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ], "thinking_modes": "both", "tool_calling": "required" }, "id": "Qwen35_122B_CUDA2_ROCm4_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "model": "Qwen3.5-122B-A10B-UD-Q8_K_XL-00001-of-00004.gguf", "model_parity_schema": 1, "runtime": { "context_length": 4096, "generation": { "max_tokens": 384, "minimum_completion_tokens": 384, "mtp_policy": "dynamic", "mtp_verify_mode": "speculative-sampling", "prefix_state": "hybrid_recurrent", "readiness_timeout_seconds": 180, "requests": [ { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "fresh_harbor", "prefix": "fresh" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_harbor", "prefix": "full" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "partial_mountain", "prefix": "partial" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_mountain", "prefix": "full" } ], "serial_control_id": "Qwen35_122B_CUDA2_ROCm4_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPOff" }, "movement_evidence": "required", "server_args": [ "--mpi-procs", "2", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--backend", "host", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--moe-routed-expert-placement", "tiered-overlay", "--moe-routed-expert-continuation-domain", "cuda_hot", "--moe-routed-expert-base-model-domain", "cuda_hot", "--moe-routed-expert-shared-domain", "cuda_hot", "--moe-continuation-dense-policy", "tensor-parallel-decode-mirrored-embedding", "--moe-routed-expert-owner-order", "ordinal", "--moe-routed-expert-residency", "rebalanced", "--moe-routed-expert-domain", "cuda_hot=localhost:-1:cuda:0,localhost:-1:cuda:1;scope=auto;backend=nccl;routed_compute=apportioned;routed_phase=uniform;routed_decode_assignment=static-owner;routed_prefill_assignment=static-owner", "--moe-routed-expert-domain", "rocm_warm=localhost:-1:rocm:0,localhost:-1:rocm:1,localhost:-1:rocm:2,localhost:-1:rocm:3;scope=auto;backend=rccl;routed_compute=apportioned;routed_phase=uniform;routed_decode_assignment=static-owner;routed_prefill_assignment=static-owner", "--moe-routed-expert-tier", "priority0@cuda_hot;priority=0;max-experts-per-layer=0;memory-mb=auto", "--moe-routed-expert-tier", "priority1@rocm_warm;priority=1;max-experts-per-layer=0;memory-mb=auto", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "9", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "455.111115", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "49", "--moe-dynamic-imbalance-threshold-permille", "1000", "--moe-dynamic-min-improvement-permille", "0", "--moe-dynamic-max-swaps-per-layer", "2", "--moe-dynamic-max-plan-entries-per-wave", "294", "--moe-dynamic-min-window-activations", "0", "--moe-device-rebalance-maintenance-slack-tokens", "0", "--moe-device-rebalance-min-maintenance-period-tokens", "4096", "--moe-device-rebalance-initial-maintenance-period-tokens", "1", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "0", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-max-post-wave-load-spread-permille", "1000", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ] } }, "cpu_isa": "AVX2", "hardware": { "accelerators": [ "0000:03:00.0 VGA compatible controller [0300]: ASPEED Technology, Inc. ASPEED Graphics Family [1a03:2000] (rev 41)", "0000:3f:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:42:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:8c:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:91:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:97:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:9a:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)" ], "cpu": { "Architecture:": "x86_64", "CPU(s):": "112", "Core(s) per socket:": "28", "Model name:": "Intel(R) Xeon(R) Gold 6238R CPU @ 2.20GHz", "NUMA node(s):": "2", "NUMA node0 CPU(s):": "0-27,56-83", "NUMA node1 CPU(s):": "28-55,84-111", "Socket(s):": "2", "Thread(s) per core:": "2" } }, "model_bytes": 10943808, "model_files": { "Qwen3.5-122B-A10B-UD-Q8_K_XL-00001-of-00004.gguf": 10943808, "Qwen3.5-122B-A10B-UD-Q8_K_XL-00002-of-00004.gguf": 49799375552, "Qwen3.5-122B-A10B-UD-Q8_K_XL-00003-of-00004.gguf": 49820794656, "Qwen3.5-122B-A10B-UD-Q8_K_XL-00004-of-00004.gguf": 35297355104 }, "prefill_tokens": 512, "workload": { "decode_tokens": 256, "measurement_iterations": 3, "prompt_repetitions": 512, "prompt_unit": " test", "schema": 1, "seed": 42, "temperature": 0, "top_k": 1, "top_p": 1, "warmup_iterations": 1 } }, "prefill_tokens": 512, "samples": { "decode": [ 24.498790133013703, 25.582171777666233, 24.811728390907824 ], "prefill": [ 153.91657678489716, 155.22949920620863, 156.25732522972348 ] }, "tokens_per_second": { "decode": 24.811728390907824, "prefill": 155.22949920620863 } }, { "case": "Qwen38Dense/Qwen38DenseMultiDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_CUDA2_ROCm2_TPPP_ActFP32_KVFP16_MTPDynamicDepth", "identity": { "case": "Qwen38Dense/Qwen38DenseMultiDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_CUDA2_ROCm2_TPPP_ActFP32_KVFP16_MTPDynamicDepth", "configuration": { "benchmark": { "args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--auto", "--only-backends", "cuda,rocm", "--auto-device-counts", "cuda=2,rocm=2", "--only-strategies", "pp", "--mtp", "--mtp-depth-policy", "dynamic" ], "context_length": 8192, "policy": "production_defaults", "schema": 1 }, "cross_host_e2e": [], "e2e": { "cell_timeout_seconds": { "AVX2": 900, "AVX512": 900 }, "context_length": 8192, "generation_tokens": 2048, "minimum_prompt_tokens": 4096, "movement_evidence": "not_applicable", "planning": { "device_counts": { "cuda": 2, "rocm": 2 }, "mode": "auto", "mpi_ranks": 1, "pipeline_domains": [ { "backend": "cuda", "devices": 2 }, { "backend": "rocm", "devices": 2 } ], "pipeline_layers": 64, "strategy": "pp" }, "readiness_timeout_seconds": 60, "request_timeout_seconds": 600, "server_args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--auto", "--only-backends", "cuda,rocm", "--auto-device-counts", "cuda=2,rocm=2", "--only-strategies", "pp", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "256", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "1.5", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1300", "--moe-dynamic-min-improvement-permille", "50", "--moe-dynamic-max-swaps-per-layer", "4", "--moe-dynamic-max-plan-entries-per-wave", "16", "--moe-dynamic-min-window-activations", "64", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "15", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "256", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "128", "--moe-device-rebalance-max-post-wave-load-spread-permille", "100", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ], "thinking_modes": "both", "tool_calling": "required" }, "id": "Qwen38Dense_27B_IQ4XS_CUDA2_ROCm2_TPPP_ActFP32_KVFP16_MTPDynamicDepth", "model": "Qwen3.8-27B-IQ4_XS.gguf", "model_parity_schema": 1, "runtime": { "context_length": 4096, "generation": { "max_tokens": 384, "minimum_completion_tokens": 384, "mtp_policy": "dynamic", "mtp_verify_mode": "speculative-sampling", "prefix_state": "hybrid_recurrent", "readiness_timeout_seconds": 60, "requests": [ { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "fresh_harbor", "prefix": "fresh" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_harbor", "prefix": "full" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "partial_mountain", "prefix": "partial" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_mountain", "prefix": "full" } ], "serial_control_id": "Qwen38Dense_27B_IQ4XS_CUDA2_ROCm2_TPPP_ActFP32_KVFP16_MTPOff" }, "movement_evidence": "not_applicable", "server_args": [ "--mpi-procs", "1", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--backend", "heterogeneous", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--tensor-parallelism-degree", "1", "--tp-scope", "auto", "--define-domain", "parity_stage_0=localhost:-1:cuda:0,localhost:-1:cuda:1;scope=rank_local;backend=nccl;owner=0;weights=0.5,0.5", "--define-domain", "parity_stage_1=localhost:-1:rocm:0,localhost:-1:rocm:1;scope=rank_local;backend=rccl;owner=0;weights=0.5,0.5", "--pp-stage", "0=parity_stage_0:0-31", "--pp-stage", "1=parity_stage_1:32-63", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "256", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "1.5", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1300", "--moe-dynamic-min-improvement-permille", "50", "--moe-dynamic-max-swaps-per-layer", "4", "--moe-dynamic-max-plan-entries-per-wave", "16", "--moe-dynamic-min-window-activations", "64", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "15", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "256", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "128", "--moe-device-rebalance-max-post-wave-load-spread-permille", "100", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ] } }, "cpu_isa": "AVX2", "hardware": { "accelerators": [ "0000:03:00.0 VGA compatible controller [0300]: ASPEED Technology, Inc. ASPEED Graphics Family [1a03:2000] (rev 41)", "0000:3f:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:42:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:8c:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:91:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:97:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:9a:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)" ], "cpu": { "Architecture:": "x86_64", "CPU(s):": "112", "Core(s) per socket:": "28", "Model name:": "Intel(R) Xeon(R) Gold 6238R CPU @ 2.20GHz", "NUMA node(s):": "2", "NUMA node0 CPU(s):": "0-27,56-83", "NUMA node1 CPU(s):": "28-55,84-111", "Socket(s):": "2", "Thread(s) per core:": "2" } }, "model_bytes": 15705861088, "model_files": { "Qwen3.8-27B-IQ4_XS.gguf": 15705861088 }, "prefill_tokens": 512, "workload": { "decode_tokens": 256, "measurement_iterations": 3, "prompt_repetitions": 512, "prompt_unit": " test", "schema": 1, "seed": 42, "temperature": 0, "top_k": 1, "top_p": 1, "warmup_iterations": 1 } }, "prefill_tokens": 512, "samples": { "decode": [ 32.14287653932743, 32.261702905686136, 32.23586820221577 ], "prefill": [ 303.80561906981893, 302.80645726371966, 303.0929133510278 ] }, "tokens_per_second": { "decode": 32.23586820221577, "prefill": 303.0929133510278 } }, { "case": "Qwen36MoEExpertOverlay/Qwen36MoEExpertOverlayParityTest.ProductionParity/Ornith15MoE_35B_Q4KM_LocalTP_RCCL_2xROCm_ExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "identity": { "case": "Qwen36MoEExpertOverlay/Qwen36MoEExpertOverlayParityTest.ProductionParity/Ornith15MoE_35B_Q4KM_LocalTP_RCCL_2xROCm_ExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "configuration": { "benchmark": { "args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--auto", "--only-backends", "rocm", "--auto-device-counts", "rocm=2", "--only-strategies", "tp", "--moe-routed-expert-owner-order", "ordinal", "--mtp", "--mtp-depth-policy", "dynamic", "--moe-residency-maintenance", "dynamic" ], "context_length": 8192, "policy": "production_defaults", "schema": 1 }, "cross_host_e2e": [], "e2e": { "cell_timeout_seconds": { "AVX2": 900, "AVX512": 900 }, "context_length": 8192, "generation_tokens": 2048, "minimum_prompt_tokens": 4096, "movement_evidence": "required", "planning": { "device_counts": { "rocm": 2 }, "mode": "auto", "mpi_ranks": 1, "strategy": "tp" }, "readiness_timeout_seconds": 60, "request_timeout_seconds": 600, "server_args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--tp-allreduce-precision", "schema", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--auto", "--only-backends", "rocm", "--auto-device-counts", "rocm=2", "--only-strategies", "tp", "--moe-routed-expert-owner-order", "ordinal", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "1", "--moe-residency-maintenance-max-window", "1", "--moe-residency-maintenance-window-growth", "1", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1000", "--moe-dynamic-min-improvement-permille", "0", "--moe-dynamic-max-swaps-per-layer", "20", "--moe-dynamic-max-plan-entries-per-wave", "20", "--moe-dynamic-min-window-activations", "0", "--moe-device-rebalance-maintenance-slack-tokens", "0", "--moe-device-rebalance-min-maintenance-period-tokens", "16", "--moe-device-rebalance-initial-maintenance-period-tokens", "2", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "0", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-max-post-wave-load-spread-permille", "1000", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ], "thinking_modes": "both", "tool_calling": "required" }, "id": "Ornith15MoE_35B_Q4KM_LocalTP_RCCL_2xROCm_ExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "model": "Ornith-1.5-35B-Q4_K_M.gguf", "model_parity_schema": 1, "runtime": { "context_length": 4096, "generation": { "max_tokens": 384, "minimum_completion_tokens": 384, "mtp_policy": "dynamic", "mtp_verify_mode": "speculative-sampling", "prefix_state": "hybrid_recurrent", "readiness_timeout_seconds": 60, "requests": [ { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "fresh_harbor", "prefix": "fresh" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_harbor", "prefix": "full" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "partial_mountain", "prefix": "partial" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_mountain", "prefix": "full" } ], "serial_control_id": "Ornith15MoE_35B_Q4KM_LocalTP_RCCL_2xROCm_ExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPOff" }, "movement_evidence": "required", "server_args": [ "--mpi-procs", "1", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--backend", "rccl", "--tp-allreduce-precision", "schema", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--moe-routed-expert-placement", "single-domain", "--moe-routed-expert-continuation-domain", "qwen36_moe_rocm_local_tp", "--moe-routed-expert-base-model-domain", "qwen36_moe_rocm_local_tp", "--moe-routed-expert-shared-domain", "qwen36_moe_rocm_local_tp", "--moe-continuation-dense-policy", "tensor-parallel", "--moe-routed-expert-owner-order", "ordinal", "--moe-routed-expert-residency", "rebalanced", "--define-domain", "qwen36_moe_rocm_local_tp=localhost:-1:rocm:0,localhost:-1:rocm:1;scope=rank_local;backend=rccl;routed_compute=apportioned;routed_phase=uniform;routed_decode_assignment=static-owner;routed_prefill_assignment=static-owner", "--moe-routed-expert-domain", "qwen36_moe_rocm_local_tp=localhost:-1:rocm:0,localhost:-1:rocm:1;scope=rank_local;backend=rccl;routed_compute=apportioned;routed_phase=uniform;routed_decode_assignment=static-owner;routed_prefill_assignment=static-owner", "--moe-routed-expert-tier", "priority_0@qwen36_moe_rocm_local_tp;priority=0;max-experts-per-layer=0;memory-mb=auto", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "1", "--moe-residency-maintenance-max-window", "1", "--moe-residency-maintenance-window-growth", "1", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1000", "--moe-dynamic-min-improvement-permille", "0", "--moe-dynamic-max-swaps-per-layer", "20", "--moe-dynamic-max-plan-entries-per-wave", "20", "--moe-dynamic-min-window-activations", "0", "--moe-device-rebalance-maintenance-slack-tokens", "0", "--moe-device-rebalance-min-maintenance-period-tokens", "16", "--moe-device-rebalance-initial-maintenance-period-tokens", "2", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "0", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-max-post-wave-load-spread-permille", "1000", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ] } }, "cpu_isa": "AVX2", "hardware": { "accelerators": [ "0000:03:00.0 VGA compatible controller [0300]: ASPEED Technology, Inc. ASPEED Graphics Family [1a03:2000] (rev 41)", "0000:3f:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:42:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:8c:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:91:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:97:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:9a:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)" ], "cpu": { "Architecture:": "x86_64", "CPU(s):": "112", "Core(s) per socket:": "28", "Model name:": "Intel(R) Xeon(R) Gold 6238R CPU @ 2.20GHz", "NUMA node(s):": "2", "NUMA node0 CPU(s):": "0-27,56-83", "NUMA node1 CPU(s):": "28-55,84-111", "Socket(s):": "2", "Thread(s) per core:": "2" } }, "model_bytes": 21713463040, "model_files": { "Ornith-1.5-35B-Q4_K_M.gguf": 21713463040 }, "prefill_tokens": 512, "workload": { "decode_tokens": 256, "measurement_iterations": 3, "prompt_repetitions": 512, "prompt_unit": " test", "schema": 1, "seed": 42, "temperature": 0, "top_k": 1, "top_p": 1, "warmup_iterations": 1 } }, "prefill_tokens": 512, "samples": { "decode": [ 102.8145963040658, 102.1232806508951, 102.842005011795 ], "prefill": [ 777.6756030509387, 778.5301918973068, 777.3523872039368 ] }, "tokens_per_second": { "decode": 102.8145963040658, "prefill": 777.6756030509387 } }, { "case": "Qwen36MoE/Qwen36MoESingleDeviceParityTest.ProductionParity/Qwen36MoE_35B_IQ3S_ROCm0_ActFP32_KVFP16_MTPDynamicDepth", "identity": { "case": "Qwen36MoE/Qwen36MoESingleDeviceParityTest.ProductionParity/Qwen36MoE_35B_IQ3S_ROCm0_ActFP32_KVFP16_MTPDynamicDepth", "configuration": { "benchmark": { "args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--auto", "--only-backends", "rocm", "--auto-device-counts", "rocm=1", "--only-strategies", "single", "--mtp", "--mtp-depth-policy", "dynamic" ], "context_length": 8192, "policy": "production_defaults", "schema": 1 }, "cross_host_e2e": [ { "frontend": "plan-apply", "id": "Qwen36MoE_35B_IQ3S_ROCm0_ActFP32_KVFP16_MTPDynamicDepth_RemoteCPU1_plan-apply", "movement_evidence": "required", "owner_order": "ordinal", "planning": { "device_counts": { "cpu": 1, "rocm": 1 }, "mode": "auto", "strategy": "expert-overlay" }, "schema": 1, "server_policy_args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--tp-allreduce-precision", "schema", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--only-backends", "rocm,cpu", "--only-strategies", "expert-overlay", "--auto-hosts", "all", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "1", "--moe-residency-maintenance-max-window", "1", "--moe-residency-maintenance-window-growth", "1", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1000", "--moe-dynamic-min-improvement-permille", "0", "--moe-dynamic-max-swaps-per-layer", "20", "--moe-dynamic-max-plan-entries-per-wave", "20", "--moe-dynamic-min-window-activations", "0", "--moe-device-rebalance-maintenance-slack-tokens", "0", "--moe-device-rebalance-min-maintenance-period-tokens", "16", "--moe-device-rebalance-initial-maintenance-period-tokens", "2", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "0", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-max-post-wave-load-spread-permille", "1000", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10", "--auto-device-counts", "rocm=1,cpu=1" ], "topology": { "continuation_backend": "rocm", "continuation_devices": 1, "continuation_priority": 0, "cpu_ranks_per_host": 1, "execution_ranks": 2, "kind": "cross-host-expert-overlay", "remote_cpu_hosts": 1, "remote_priority": 1 } }, { "frontend": "auto-serve", "id": "Qwen36MoE_35B_IQ3S_ROCm0_ActFP32_KVFP16_MTPDynamicDepth_RemoteCPU1_auto-serve", "movement_evidence": "required", "owner_order": "ordinal", "planning": { "device_counts": { "cpu": 1, "rocm": 1 }, "mode": "auto", "strategy": "expert-overlay" }, "schema": 1, "server_policy_args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--tp-allreduce-precision", "schema", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--only-backends", "rocm,cpu", "--only-strategies", "expert-overlay", "--auto-hosts", "all", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "1", "--moe-residency-maintenance-max-window", "1", "--moe-residency-maintenance-window-growth", "1", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1000", "--moe-dynamic-min-improvement-permille", "0", "--moe-dynamic-max-swaps-per-layer", "20", "--moe-dynamic-max-plan-entries-per-wave", "20", "--moe-dynamic-min-window-activations", "0", "--moe-device-rebalance-maintenance-slack-tokens", "0", "--moe-device-rebalance-min-maintenance-period-tokens", "16", "--moe-device-rebalance-initial-maintenance-period-tokens", "2", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "0", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-max-post-wave-load-spread-permille", "1000", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10", "--auto-device-counts", "rocm=1,cpu=1" ], "topology": { "continuation_backend": "rocm", "continuation_devices": 1, "continuation_priority": 0, "cpu_ranks_per_host": 1, "execution_ranks": 2, "kind": "cross-host-expert-overlay", "remote_cpu_hosts": 1, "remote_priority": 1 } }, { "frontend": "plan-apply", "id": "Qwen36MoE_35B_IQ3S_ROCm0_ActFP32_KVFP16_MTPDynamicDepth_RemoteCPU2_plan-apply", "movement_evidence": "required", "owner_order": "ordinal", "planning": { "device_counts": { "cpu": 2, "rocm": 1 }, "mode": "auto", "strategy": "expert-overlay" }, "schema": 1, "server_policy_args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--tp-allreduce-precision", "schema", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--only-backends", "rocm,cpu", "--only-strategies", "expert-overlay", "--auto-hosts", "all", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "1", "--moe-residency-maintenance-max-window", "1", "--moe-residency-maintenance-window-growth", "1", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1000", "--moe-dynamic-min-improvement-permille", "0", "--moe-dynamic-max-swaps-per-layer", "20", "--moe-dynamic-max-plan-entries-per-wave", "20", "--moe-dynamic-min-window-activations", "0", "--moe-device-rebalance-maintenance-slack-tokens", "0", "--moe-device-rebalance-min-maintenance-period-tokens", "16", "--moe-device-rebalance-initial-maintenance-period-tokens", "2", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "0", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-max-post-wave-load-spread-permille", "1000", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10", "--auto-device-counts", "rocm=1,cpu=2" ], "topology": { "continuation_backend": "rocm", "continuation_devices": 1, "continuation_priority": 0, "cpu_ranks_per_host": 1, "execution_ranks": 3, "kind": "cross-host-expert-overlay", "remote_cpu_hosts": 2, "remote_priority": 1 } }, { "frontend": "auto-serve", "id": "Qwen36MoE_35B_IQ3S_ROCm0_ActFP32_KVFP16_MTPDynamicDepth_RemoteCPU2_auto-serve", "movement_evidence": "required", "owner_order": "ordinal", "planning": { "device_counts": { "cpu": 2, "rocm": 1 }, "mode": "auto", "strategy": "expert-overlay" }, "schema": 1, "server_policy_args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--tp-allreduce-precision", "schema", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--only-backends", "rocm,cpu", "--only-strategies", "expert-overlay", "--auto-hosts", "all", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "1", "--moe-residency-maintenance-max-window", "1", "--moe-residency-maintenance-window-growth", "1", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1000", "--moe-dynamic-min-improvement-permille", "0", "--moe-dynamic-max-swaps-per-layer", "20", "--moe-dynamic-max-plan-entries-per-wave", "20", "--moe-dynamic-min-window-activations", "0", "--moe-device-rebalance-maintenance-slack-tokens", "0", "--moe-device-rebalance-min-maintenance-period-tokens", "16", "--moe-device-rebalance-initial-maintenance-period-tokens", "2", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "0", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-max-post-wave-load-spread-permille", "1000", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10", "--auto-device-counts", "rocm=1,cpu=2" ], "topology": { "continuation_backend": "rocm", "continuation_devices": 1, "continuation_priority": 0, "cpu_ranks_per_host": 1, "execution_ranks": 3, "kind": "cross-host-expert-overlay", "remote_cpu_hosts": 2, "remote_priority": 1 } } ], "e2e": { "cell_timeout_seconds": { "AVX2": 900, "AVX512": 900 }, "context_length": 8192, "generation_tokens": 2048, "minimum_prompt_tokens": 4096, "movement_evidence": "not_applicable", "planning": { "device_counts": { "rocm": 1 }, "mode": "auto", "mpi_ranks": 1, "strategy": "single" }, "readiness_timeout_seconds": 60, "request_timeout_seconds": 600, "server_args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--tp-allreduce-precision", "schema", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--auto", "--only-backends", "rocm", "--auto-device-counts", "rocm=1", "--only-strategies", "single", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "256", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "1.5", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1300", "--moe-dynamic-min-improvement-permille", "50", "--moe-dynamic-max-swaps-per-layer", "4", "--moe-dynamic-max-plan-entries-per-wave", "16", "--moe-dynamic-min-window-activations", "64", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "15", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "256", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "128", "--moe-device-rebalance-max-post-wave-load-spread-permille", "100", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ], "thinking_modes": "both", "tool_calling": "required" }, "id": "Qwen36MoE_35B_IQ3S_ROCm0_ActFP32_KVFP16_MTPDynamicDepth", "model": "Qwen3.6-35B-A3B-UD-IQ3_S.gguf", "model_parity_schema": 1, "runtime": { "context_length": 4096, "generation": { "max_tokens": 384, "minimum_completion_tokens": 384, "mtp_policy": "dynamic", "mtp_verify_mode": "speculative-sampling", "prefix_state": "hybrid_recurrent", "readiness_timeout_seconds": 60, "requests": [ { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "fresh_harbor", "prefix": "fresh" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_harbor", "prefix": "full" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "partial_mountain", "prefix": "partial" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_mountain", "prefix": "full" } ], "serial_control_id": "Qwen36MoE_35B_IQ3S_ROCm0_ActFP32_KVFP16_MTPOff" }, "movement_evidence": "not_applicable", "server_args": [ "--mpi-procs", "1", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--backend", "host", "--tp-allreduce-precision", "schema", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--device", "localhost:-1:rocm:0", "--tensor-parallelism-degree", "1", "--tp-scope", "auto", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "256", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "1.5", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1300", "--moe-dynamic-min-improvement-permille", "50", "--moe-dynamic-max-swaps-per-layer", "4", "--moe-dynamic-max-plan-entries-per-wave", "16", "--moe-dynamic-min-window-activations", "64", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "15", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "256", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "128", "--moe-device-rebalance-max-post-wave-load-spread-permille", "100", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ] } }, "cpu_isa": "AVX2", "hardware": { "accelerators": [ "0000:03:00.0 VGA compatible controller [0300]: ASPEED Technology, Inc. ASPEED Graphics Family [1a03:2000] (rev 41)", "0000:3f:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:42:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:8c:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:91:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:97:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:9a:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)" ], "cpu": { "Architecture:": "x86_64", "CPU(s):": "112", "Core(s) per socket:": "28", "Model name:": "Intel(R) Xeon(R) Gold 6238R CPU @ 2.20GHz", "NUMA node(s):": "2", "NUMA node0 CPU(s):": "0-27,56-83", "NUMA node1 CPU(s):": "28-55,84-111", "Socket(s):": "2", "Thread(s) per core:": "2" } }, "model_bytes": 15346432288, "model_files": { "Qwen3.6-35B-A3B-UD-IQ3_S.gguf": 15346432288 }, "prefill_tokens": 512, "workload": { "decode_tokens": 256, "measurement_iterations": 3, "prompt_repetitions": 512, "prompt_unit": " test", "schema": 1, "seed": 42, "temperature": 0, "top_k": 1, "top_p": 1, "warmup_iterations": 1 } }, "prefill_tokens": 512, "samples": { "decode": [ 148.57787524307713, 148.55425573941247, 148.5711868756553 ], "prefill": [ 834.4446155436186, 837.81538381946, 835.3788081087642 ] }, "tokens_per_second": { "decode": 148.5711868756553, "prefill": 835.3788081087642 } }, { "case": "Qwen36MoE/Qwen36MoESingleDeviceParityTest.ProductionParity/Ornith15MoE_35B_Q4KM_ROCm0_ActFP32_KVFP16_MTPDynamicDepth", "identity": { "case": "Qwen36MoE/Qwen36MoESingleDeviceParityTest.ProductionParity/Ornith15MoE_35B_Q4KM_ROCm0_ActFP32_KVFP16_MTPDynamicDepth", "configuration": { "benchmark": { "args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--auto", "--only-backends", "rocm", "--auto-device-counts", "rocm=1", "--only-strategies", "single", "--mtp", "--mtp-depth-policy", "dynamic" ], "context_length": 8192, "policy": "production_defaults", "schema": 1 }, "cross_host_e2e": [], "e2e": { "cell_timeout_seconds": { "AVX2": 900, "AVX512": 900 }, "context_length": 8192, "generation_tokens": 2048, "minimum_prompt_tokens": 4096, "movement_evidence": "not_applicable", "planning": { "device_counts": { "rocm": 1 }, "mode": "auto", "mpi_ranks": 1, "strategy": "single" }, "readiness_timeout_seconds": 60, "request_timeout_seconds": 600, "server_args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--tp-allreduce-precision", "schema", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--auto", "--only-backends", "rocm", "--auto-device-counts", "rocm=1", "--only-strategies", "single", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "256", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "1.5", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1300", "--moe-dynamic-min-improvement-permille", "50", "--moe-dynamic-max-swaps-per-layer", "4", "--moe-dynamic-max-plan-entries-per-wave", "16", "--moe-dynamic-min-window-activations", "64", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "15", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "256", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "128", "--moe-device-rebalance-max-post-wave-load-spread-permille", "100", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ], "thinking_modes": "both", "tool_calling": "required" }, "id": "Ornith15MoE_35B_Q4KM_ROCm0_ActFP32_KVFP16_MTPDynamicDepth", "model": "Ornith-1.5-35B-Q4_K_M.gguf", "model_parity_schema": 1, "runtime": { "context_length": 4096, "generation": { "max_tokens": 384, "minimum_completion_tokens": 384, "mtp_policy": "dynamic", "mtp_verify_mode": "speculative-sampling", "prefix_state": "hybrid_recurrent", "readiness_timeout_seconds": 60, "requests": [ { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "fresh_harbor", "prefix": "fresh" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_harbor", "prefix": "full" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "partial_mountain", "prefix": "partial" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_mountain", "prefix": "full" } ], "serial_control_id": "Ornith15MoE_35B_Q4KM_ROCm0_ActFP32_KVFP16_MTPOff" }, "movement_evidence": "not_applicable", "server_args": [ "--mpi-procs", "1", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--backend", "host", "--tp-allreduce-precision", "schema", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--device", "localhost:-1:rocm:0", "--tensor-parallelism-degree", "1", "--tp-scope", "auto", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "256", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "1.5", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1300", "--moe-dynamic-min-improvement-permille", "50", "--moe-dynamic-max-swaps-per-layer", "4", "--moe-dynamic-max-plan-entries-per-wave", "16", "--moe-dynamic-min-window-activations", "64", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "15", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "256", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "128", "--moe-device-rebalance-max-post-wave-load-spread-permille", "100", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ] } }, "cpu_isa": "AVX2", "hardware": { "accelerators": [ "0000:03:00.0 VGA compatible controller [0300]: ASPEED Technology, Inc. ASPEED Graphics Family [1a03:2000] (rev 41)", "0000:3f:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:42:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:8c:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:91:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:97:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:9a:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)" ], "cpu": { "Architecture:": "x86_64", "CPU(s):": "112", "Core(s) per socket:": "28", "Model name:": "Intel(R) Xeon(R) Gold 6238R CPU @ 2.20GHz", "NUMA node(s):": "2", "NUMA node0 CPU(s):": "0-27,56-83", "NUMA node1 CPU(s):": "28-55,84-111", "Socket(s):": "2", "Thread(s) per core:": "2" } }, "model_bytes": 21713463040, "model_files": { "Ornith-1.5-35B-Q4_K_M.gguf": 21713463040 }, "prefill_tokens": 512, "workload": { "decode_tokens": 256, "measurement_iterations": 3, "prompt_repetitions": 512, "prompt_unit": " test", "schema": 1, "seed": 42, "temperature": 0, "top_k": 1, "top_p": 1, "warmup_iterations": 1 } }, "prefill_tokens": 512, "samples": { "decode": [ 165.95244778907076, 165.88333854535531, 166.0061188800792 ], "prefill": [ 817.941377981686, 815.2307666167885, 817.8227511571085 ] }, "tokens_per_second": { "decode": 165.95244778907076, "prefill": 817.8227511571085 } }, { "case": "Qwen38Dense/Qwen38DenseMultiDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_ROCm2_TP_ActFP32_KVFP16_MTPDynamicDepth", "identity": { "case": "Qwen38Dense/Qwen38DenseMultiDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_ROCm2_TP_ActFP32_KVFP16_MTPDynamicDepth", "configuration": { "benchmark": { "args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--auto", "--only-backends", "rocm", "--auto-device-counts", "rocm=2", "--only-strategies", "tp", "--mtp", "--mtp-depth-policy", "dynamic" ], "context_length": 8192, "policy": "production_defaults", "schema": 1 }, "cross_host_e2e": [], "e2e": { "cell_timeout_seconds": { "AVX2": 900, "AVX512": 900 }, "context_length": 8192, "generation_tokens": 2048, "minimum_prompt_tokens": 4096, "movement_evidence": "not_applicable", "planning": { "device_counts": { "rocm": 2 }, "mode": "auto", "mpi_ranks": 1, "strategy": "tp" }, "readiness_timeout_seconds": 60, "request_timeout_seconds": 600, "server_args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--auto", "--only-backends", "rocm", "--auto-device-counts", "rocm=2", "--only-strategies", "tp", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "256", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "1.5", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1300", "--moe-dynamic-min-improvement-permille", "50", "--moe-dynamic-max-swaps-per-layer", "4", "--moe-dynamic-max-plan-entries-per-wave", "16", "--moe-dynamic-min-window-activations", "64", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "15", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "256", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "128", "--moe-device-rebalance-max-post-wave-load-spread-permille", "100", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ], "thinking_modes": "both", "tool_calling": "required" }, "id": "Qwen38Dense_27B_IQ4XS_ROCm2_TP_ActFP32_KVFP16_MTPDynamicDepth", "model": "Qwen3.8-27B-IQ4_XS.gguf", "model_parity_schema": 1, "runtime": { "context_length": 4096, "generation": { "max_tokens": 384, "minimum_completion_tokens": 384, "mtp_policy": "dynamic", "mtp_verify_mode": "speculative-sampling", "prefix_state": "hybrid_recurrent", "readiness_timeout_seconds": 60, "requests": [ { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "fresh_harbor", "prefix": "fresh" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_harbor", "prefix": "full" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "partial_mountain", "prefix": "partial" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_mountain", "prefix": "full" } ], "serial_control_id": "Qwen38Dense_27B_IQ4XS_ROCm2_TP_ActFP32_KVFP16_MTPOff" }, "movement_evidence": "not_applicable", "server_args": [ "--mpi-procs", "1", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--backend", "rccl", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--tp-devices", "localhost:-1:rocm:0,localhost:-1:rocm:1", "--tensor-parallelism-degree", "2", "--tp-scope", "rank_local", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "256", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "1.5", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1300", "--moe-dynamic-min-improvement-permille", "50", "--moe-dynamic-max-swaps-per-layer", "4", "--moe-dynamic-max-plan-entries-per-wave", "16", "--moe-dynamic-min-window-activations", "64", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "15", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "256", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "128", "--moe-device-rebalance-max-post-wave-load-spread-permille", "100", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ] } }, "cpu_isa": "AVX2", "hardware": { "accelerators": [ "0000:03:00.0 VGA compatible controller [0300]: ASPEED Technology, Inc. ASPEED Graphics Family [1a03:2000] (rev 41)", "0000:3f:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:42:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:8c:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:91:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:97:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:9a:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)" ], "cpu": { "Architecture:": "x86_64", "CPU(s):": "112", "Core(s) per socket:": "28", "Model name:": "Intel(R) Xeon(R) Gold 6238R CPU @ 2.20GHz", "NUMA node(s):": "2", "NUMA node0 CPU(s):": "0-27,56-83", "NUMA node1 CPU(s):": "28-55,84-111", "Socket(s):": "2", "Thread(s) per core:": "2" } }, "model_bytes": 15705861088, "model_files": { "Qwen3.8-27B-IQ4_XS.gguf": 15705861088 }, "prefill_tokens": 512, "workload": { "decode_tokens": 256, "measurement_iterations": 3, "prompt_repetitions": 512, "prompt_unit": " test", "schema": 1, "seed": 42, "temperature": 0, "top_k": 1, "top_p": 1, "warmup_iterations": 1 } }, "prefill_tokens": 512, "samples": { "decode": [ 39.15675394770643, 39.08898115602566, 39.15493494068524 ], "prefill": [ 247.51709426897233, 247.37986313419174, 247.47950988805977 ] }, "tokens_per_second": { "decode": 39.15493494068524, "prefill": 247.47950988805977 } }, { "case": "Qwen38Dense/Qwen38DenseMultiDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_ROCm2_PP_ActFP32_KVFP16_MTPDynamicDepth", "identity": { "case": "Qwen38Dense/Qwen38DenseMultiDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_ROCm2_PP_ActFP32_KVFP16_MTPDynamicDepth", "configuration": { "benchmark": { "args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--auto", "--only-backends", "rocm", "--auto-device-counts", "rocm=2", "--only-strategies", "pp", "--mtp", "--mtp-depth-policy", "dynamic" ], "context_length": 8192, "policy": "production_defaults", "schema": 1 }, "cross_host_e2e": [], "e2e": { "cell_timeout_seconds": { "AVX2": 900, "AVX512": 900 }, "context_length": 8192, "generation_tokens": 2048, "minimum_prompt_tokens": 4096, "movement_evidence": "not_applicable", "planning": { "device_counts": { "rocm": 2 }, "mode": "auto", "mpi_ranks": 1, "pipeline_domains": [ { "backend": "rocm", "devices": 1 }, { "backend": "rocm", "devices": 1 } ], "pipeline_layers": 64, "strategy": "pp" }, "readiness_timeout_seconds": 60, "request_timeout_seconds": 600, "server_args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--auto", "--only-backends", "rocm", "--auto-device-counts", "rocm=2", "--only-strategies", "pp", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "256", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "1.5", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1300", "--moe-dynamic-min-improvement-permille", "50", "--moe-dynamic-max-swaps-per-layer", "4", "--moe-dynamic-max-plan-entries-per-wave", "16", "--moe-dynamic-min-window-activations", "64", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "15", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "256", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "128", "--moe-device-rebalance-max-post-wave-load-spread-permille", "100", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ], "thinking_modes": "both", "tool_calling": "required" }, "id": "Qwen38Dense_27B_IQ4XS_ROCm2_PP_ActFP32_KVFP16_MTPDynamicDepth", "model": "Qwen3.8-27B-IQ4_XS.gguf", "model_parity_schema": 1, "runtime": { "context_length": 4096, "generation": { "max_tokens": 384, "minimum_completion_tokens": 384, "mtp_policy": "dynamic", "mtp_verify_mode": "speculative-sampling", "prefix_state": "hybrid_recurrent", "readiness_timeout_seconds": 60, "requests": [ { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "fresh_harbor", "prefix": "fresh" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_harbor", "prefix": "full" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "partial_mountain", "prefix": "partial" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_mountain", "prefix": "full" } ], "serial_control_id": "Qwen38Dense_27B_IQ4XS_ROCm2_PP_ActFP32_KVFP16_MTPOff" }, "movement_evidence": "not_applicable", "server_args": [ "--mpi-procs", "1", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--backend", "rccl", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--tensor-parallelism-degree", "1", "--tp-scope", "auto", "--define-domain", "parity_stage_0=localhost:-1:rocm:0;scope=rank_local;backend=auto;owner=0", "--define-domain", "parity_stage_1=localhost:-1:rocm:1;scope=rank_local;backend=auto;owner=0", "--pp-stage", "0=parity_stage_0:0-31", "--pp-stage", "1=parity_stage_1:32-63", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "256", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "1.5", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1300", "--moe-dynamic-min-improvement-permille", "50", "--moe-dynamic-max-swaps-per-layer", "4", "--moe-dynamic-max-plan-entries-per-wave", "16", "--moe-dynamic-min-window-activations", "64", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "15", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "256", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "128", "--moe-device-rebalance-max-post-wave-load-spread-permille", "100", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ] } }, "cpu_isa": "AVX2", "hardware": { "accelerators": [ "0000:03:00.0 VGA compatible controller [0300]: ASPEED Technology, Inc. ASPEED Graphics Family [1a03:2000] (rev 41)", "0000:3f:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:42:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:8c:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:91:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:97:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:9a:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)" ], "cpu": { "Architecture:": "x86_64", "CPU(s):": "112", "Core(s) per socket:": "28", "Model name:": "Intel(R) Xeon(R) Gold 6238R CPU @ 2.20GHz", "NUMA node(s):": "2", "NUMA node0 CPU(s):": "0-27,56-83", "NUMA node1 CPU(s):": "28-55,84-111", "Socket(s):": "2", "Thread(s) per core:": "2" } }, "model_bytes": 15705861088, "model_files": { "Qwen3.8-27B-IQ4_XS.gguf": 15705861088 }, "prefill_tokens": 512, "workload": { "decode_tokens": 256, "measurement_iterations": 3, "prompt_repetitions": 512, "prompt_unit": " test", "schema": 1, "seed": 42, "temperature": 0, "top_k": 1, "top_p": 1, "warmup_iterations": 1 } }, "prefill_tokens": 512, "samples": { "decode": [ 20.163039813813167, 21.64080870437626, 21.63834653562228 ], "prefill": [ 186.4007281459019, 185.69590379386534, 185.24978783254966 ] }, "tokens_per_second": { "decode": 21.63834653562228, "prefill": 185.69590379386534 } }, { "case": "Qwen38Dense/Qwen38DenseSingleDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_ROCm0_ActFP32_KVFP16_MTPDynamicDepth", "identity": { "case": "Qwen38Dense/Qwen38DenseSingleDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_ROCm0_ActFP32_KVFP16_MTPDynamicDepth", "configuration": { "benchmark": { "args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--auto", "--only-backends", "rocm", "--auto-device-counts", "rocm=1", "--only-strategies", "single", "--mtp", "--mtp-depth-policy", "dynamic" ], "context_length": 32768, "policy": "production_defaults", "schema": 1 }, "cross_host_e2e": [], "e2e": { "cell_timeout_seconds": { "AVX2": 900, "AVX512": 900 }, "context_length": 32768, "generation_tokens": 2048, "minimum_prompt_tokens": 16384, "movement_evidence": "not_applicable", "planning": { "device_counts": { "rocm": 1 }, "mode": "auto", "mpi_ranks": 1, "strategy": "single" }, "readiness_timeout_seconds": 60, "request_timeout_seconds": 600, "server_args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--auto", "--only-backends", "rocm", "--auto-device-counts", "rocm=1", "--only-strategies", "single", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "256", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "1.5", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1300", "--moe-dynamic-min-improvement-permille", "50", "--moe-dynamic-max-swaps-per-layer", "4", "--moe-dynamic-max-plan-entries-per-wave", "16", "--moe-dynamic-min-window-activations", "64", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "15", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "256", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "128", "--moe-device-rebalance-max-post-wave-load-spread-permille", "100", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ], "thinking_modes": "both", "tool_calling": "required" }, "id": "Qwen38Dense_27B_IQ4XS_ROCm0_ActFP32_KVFP16_MTPDynamicDepth", "model": "Qwen3.8-27B-IQ4_XS.gguf", "model_parity_schema": 1, "runtime": { "context_length": 4096, "generation": { "max_tokens": 384, "minimum_completion_tokens": 384, "mtp_policy": "dynamic", "mtp_verify_mode": "speculative-sampling", "prefix_state": "hybrid_recurrent", "readiness_timeout_seconds": 60, "requests": [ { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "fresh_harbor", "prefix": "fresh" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_harbor", "prefix": "full" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "partial_mountain", "prefix": "partial" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_mountain", "prefix": "full" } ], "serial_control_id": "Qwen38Dense_27B_IQ4XS_ROCm0_ActFP32_KVFP16_MTPOff" }, "movement_evidence": "not_applicable", "server_args": [ "--mpi-procs", "1", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--backend", "host", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--device", "localhost:-1:rocm:0", "--tensor-parallelism-degree", "1", "--tp-scope", "auto", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "256", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "1.5", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1300", "--moe-dynamic-min-improvement-permille", "50", "--moe-dynamic-max-swaps-per-layer", "4", "--moe-dynamic-max-plan-entries-per-wave", "16", "--moe-dynamic-min-window-activations", "64", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "15", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "256", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "128", "--moe-device-rebalance-max-post-wave-load-spread-permille", "100", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ] } }, "cpu_isa": "AVX2", "hardware": { "accelerators": [ "0000:03:00.0 VGA compatible controller [0300]: ASPEED Technology, Inc. ASPEED Graphics Family [1a03:2000] (rev 41)", "0000:3f:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:42:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:8c:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:91:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:97:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:9a:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)" ], "cpu": { "Architecture:": "x86_64", "CPU(s):": "112", "Core(s) per socket:": "28", "Model name:": "Intel(R) Xeon(R) Gold 6238R CPU @ 2.20GHz", "NUMA node(s):": "2", "NUMA node0 CPU(s):": "0-27,56-83", "NUMA node1 CPU(s):": "28-55,84-111", "Socket(s):": "2", "Thread(s) per core:": "2" } }, "model_bytes": 15705861088, "model_files": { "Qwen3.8-27B-IQ4_XS.gguf": 15705861088 }, "prefill_tokens": 512, "workload": { "decode_tokens": 256, "measurement_iterations": 3, "prompt_repetitions": 512, "prompt_unit": " test", "schema": 1, "seed": 42, "temperature": 0, "top_k": 1, "top_p": 1, "warmup_iterations": 1 } }, "prefill_tokens": 512, "samples": { "decode": [ 39.21009094478446, 39.21991305022794, 39.21409296716583 ], "prefill": [ 182.33258211181794, 182.05432166006833, 181.82005276976886 ] }, "tokens_per_second": { "decode": 39.21409296716583, "prefill": 182.05432166006833 } } ], "comparisons": [ { "case": "Qwen36MoEExpertOverlay/Qwen36MoEExpertOverlayParityTest.ProductionParity/Qwen36MoE_35B_IQ3S_NodeTP_2xMPI_CPU_ExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "current": 111.6925154473672, "high_water": null, "passed": true, "phase": "prefill", "status": "new_baseline" }, { "case": "Qwen36MoEExpertOverlay/Qwen36MoEExpertOverlayParityTest.ProductionParity/Qwen36MoE_35B_IQ3S_NodeTP_2xMPI_CPU_ExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "current": 27.963787981970885, "high_water": null, "passed": true, "phase": "decode", "status": "new_baseline" }, { "case": "Qwen36MoEExpertOverlay/Qwen36MoEExpertOverlayParityTest.ProductionParity/Ornith15MoE_35B_Q4KM_NodeTP_2xMPI_CPU_ExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "current": 124.34957576109406, "high_water": null, "passed": true, "phase": "prefill", "status": "new_baseline" }, { "case": "Qwen36MoEExpertOverlay/Qwen36MoEExpertOverlayParityTest.ProductionParity/Ornith15MoE_35B_Q4KM_NodeTP_2xMPI_CPU_ExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "current": 32.19967995460208, "high_water": null, "passed": true, "phase": "decode", "status": "new_baseline" }, { "case": "Qwen35_122B_ExpertOverlay/Qwen35MoENodeExpertOverlayParityTest.ProductionParity/Qwen35_122B_CUDA2_CPU2_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "current": 79.71676755798522, "high_water": null, "passed": true, "phase": "prefill", "status": "new_baseline" }, { "case": "Qwen35_122B_ExpertOverlay/Qwen35MoENodeExpertOverlayParityTest.ProductionParity/Qwen35_122B_CUDA2_CPU2_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "current": 13.874989699574925, "high_water": null, "passed": true, "phase": "decode", "status": "new_baseline" }, { "case": "Qwen35_122B_ExpertOverlay/Qwen35MoENodeExpertOverlayParityTest.ProductionParity/Qwen35_122B_CUDA2_ROCm4_CPU2_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "current": 124.61742732755283, "high_water": null, "passed": true, "phase": "prefill", "status": "new_baseline" }, { "case": "Qwen35_122B_ExpertOverlay/Qwen35MoENodeExpertOverlayParityTest.ProductionParity/Qwen35_122B_CUDA2_ROCm4_CPU2_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "current": 19.66726932725284, "high_water": null, "passed": true, "phase": "decode", "status": "new_baseline" }, { "case": "Qwen35_122B_ExpertOverlay/Qwen35MoENodeExpertOverlayParityTest.ProductionParity/Qwen35_122B_ROCm2_CPU2_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "current": 102.1496137270215, "high_water": null, "passed": true, "phase": "prefill", "status": "new_baseline" }, { "case": "Qwen35_122B_ExpertOverlay/Qwen35MoENodeExpertOverlayParityTest.ProductionParity/Qwen35_122B_ROCm2_CPU2_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "current": 20.881290335133617, "high_water": null, "passed": true, "phase": "decode", "status": "new_baseline" }, { "case": "Qwen35_122B_ExpertOverlay/Qwen35MoENodeExpertOverlayParityTest.ProductionParity/Qwen35_122B_ROCm4_CPU2_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "current": 146.9811945266633, "high_water": null, "passed": true, "phase": "prefill", "status": "new_baseline" }, { "case": "Qwen35_122B_ExpertOverlay/Qwen35MoENodeExpertOverlayParityTest.ProductionParity/Qwen35_122B_ROCm4_CPU2_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "current": 26.83361584714674, "high_water": null, "passed": true, "phase": "decode", "status": "new_baseline" }, { "case": "Qwen36MoEExpertOverlay/Qwen36MoEExpertOverlayParityTest.ProductionParity/Ornith15MoE_35B_Q4KM_LocalTP_NCCL_2xCUDA_ExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "current": 887.8439089392327, "high_water": null, "passed": true, "phase": "prefill", "status": "new_baseline" }, { "case": "Qwen36MoEExpertOverlay/Qwen36MoEExpertOverlayParityTest.ProductionParity/Ornith15MoE_35B_Q4KM_LocalTP_NCCL_2xCUDA_ExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "current": 139.3162602596735, "high_water": null, "passed": true, "phase": "decode", "status": "new_baseline" }, { "case": "Qwen36MoE/Qwen36MoESingleDeviceParityTest.ProductionParity/Qwen36MoE_35B_IQ3S_CUDA0_ActFP32_KVFP16_MTPDynamicDepth", "current": 1760.9705550693034, "high_water": null, "passed": true, "phase": "prefill", "status": "new_baseline" }, { "case": "Qwen36MoE/Qwen36MoESingleDeviceParityTest.ProductionParity/Qwen36MoE_35B_IQ3S_CUDA0_ActFP32_KVFP16_MTPDynamicDepth", "current": 287.0802547386263, "high_water": null, "passed": true, "phase": "decode", "status": "new_baseline" }, { "case": "Qwen38Dense/Qwen38DenseMultiDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_CUDA2_TP_ActFP32_KVFP16_MTPDynamicDepth", "current": 469.42453704426146, "high_water": null, "passed": true, "phase": "prefill", "status": "new_baseline" }, { "case": "Qwen38Dense/Qwen38DenseMultiDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_CUDA2_TP_ActFP32_KVFP16_MTPDynamicDepth", "current": 50.327024035111855, "high_water": null, "passed": true, "phase": "decode", "status": "new_baseline" }, { "case": "Qwen38Dense/Qwen38DenseMultiDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_CUDA2_PP_ActFP32_KVFP16_MTPDynamicDepth", "current": 760.6795019248906, "high_water": null, "passed": true, "phase": "prefill", "status": "new_baseline" }, { "case": "Qwen38Dense/Qwen38DenseMultiDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_CUDA2_PP_ActFP32_KVFP16_MTPDynamicDepth", "current": 40.31356272301949, "high_water": null, "passed": true, "phase": "decode", "status": "new_baseline" }, { "case": "Qwen38Dense/Qwen38DenseSingleDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_CUDA0_ActFP32_KVFP16_MTPDynamicDepth", "current": 703.430721342884, "high_water": null, "passed": true, "phase": "prefill", "status": "new_baseline" }, { "case": "Qwen38Dense/Qwen38DenseSingleDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_CUDA0_ActFP32_KVFP16_MTPDynamicDepth", "current": 62.20860738139935, "high_water": null, "passed": true, "phase": "decode", "status": "new_baseline" }, { "case": "Qwen35_122B_ExpertOverlay/Qwen35MoENodeExpertOverlayParityTest.ProductionParity/Qwen35_122B_CUDA2_ROCm4_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "current": 155.22949920620863, "high_water": null, "passed": true, "phase": "prefill", "status": "new_baseline" }, { "case": "Qwen35_122B_ExpertOverlay/Qwen35MoENodeExpertOverlayParityTest.ProductionParity/Qwen35_122B_CUDA2_ROCm4_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "current": 24.811728390907824, "high_water": null, "passed": true, "phase": "decode", "status": "new_baseline" }, { "case": "Qwen38Dense/Qwen38DenseMultiDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_CUDA2_ROCm2_TPPP_ActFP32_KVFP16_MTPDynamicDepth", "current": 303.0929133510278, "high_water": null, "passed": true, "phase": "prefill", "status": "new_baseline" }, { "case": "Qwen38Dense/Qwen38DenseMultiDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_CUDA2_ROCm2_TPPP_ActFP32_KVFP16_MTPDynamicDepth", "current": 32.23586820221577, "high_water": null, "passed": true, "phase": "decode", "status": "new_baseline" }, { "case": "Qwen36MoEExpertOverlay/Qwen36MoEExpertOverlayParityTest.ProductionParity/Ornith15MoE_35B_Q4KM_LocalTP_RCCL_2xROCm_ExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "current": 777.6756030509387, "high_water": null, "passed": true, "phase": "prefill", "status": "new_baseline" }, { "case": "Qwen36MoEExpertOverlay/Qwen36MoEExpertOverlayParityTest.ProductionParity/Ornith15MoE_35B_Q4KM_LocalTP_RCCL_2xROCm_ExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "current": 102.8145963040658, "high_water": null, "passed": true, "phase": "decode", "status": "new_baseline" }, { "case": "Qwen36MoE/Qwen36MoESingleDeviceParityTest.ProductionParity/Qwen36MoE_35B_IQ3S_ROCm0_ActFP32_KVFP16_MTPDynamicDepth", "current": 835.3788081087642, "high_water": null, "passed": true, "phase": "prefill", "status": "new_baseline" }, { "case": "Qwen36MoE/Qwen36MoESingleDeviceParityTest.ProductionParity/Qwen36MoE_35B_IQ3S_ROCm0_ActFP32_KVFP16_MTPDynamicDepth", "current": 148.5711868756553, "high_water": null, "passed": true, "phase": "decode", "status": "new_baseline" }, { "case": "Qwen36MoE/Qwen36MoESingleDeviceParityTest.ProductionParity/Ornith15MoE_35B_Q4KM_ROCm0_ActFP32_KVFP16_MTPDynamicDepth", "current": 817.8227511571085, "high_water": null, "passed": true, "phase": "prefill", "status": "new_baseline" }, { "case": "Qwen36MoE/Qwen36MoESingleDeviceParityTest.ProductionParity/Ornith15MoE_35B_Q4KM_ROCm0_ActFP32_KVFP16_MTPDynamicDepth", "current": 165.95244778907076, "high_water": null, "passed": true, "phase": "decode", "status": "new_baseline" }, { "case": "Qwen38Dense/Qwen38DenseMultiDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_ROCm2_TP_ActFP32_KVFP16_MTPDynamicDepth", "current": 247.47950988805977, "high_water": null, "passed": true, "phase": "prefill", "status": "new_baseline" }, { "case": "Qwen38Dense/Qwen38DenseMultiDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_ROCm2_TP_ActFP32_KVFP16_MTPDynamicDepth", "current": 39.15493494068524, "high_water": null, "passed": true, "phase": "decode", "status": "new_baseline" }, { "case": "Qwen38Dense/Qwen38DenseMultiDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_ROCm2_PP_ActFP32_KVFP16_MTPDynamicDepth", "current": 185.69590379386534, "high_water": null, "passed": true, "phase": "prefill", "status": "new_baseline" }, { "case": "Qwen38Dense/Qwen38DenseMultiDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_ROCm2_PP_ActFP32_KVFP16_MTPDynamicDepth", "current": 21.63834653562228, "high_water": null, "passed": true, "phase": "decode", "status": "new_baseline" }, { "case": "Qwen38Dense/Qwen38DenseSingleDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_ROCm0_ActFP32_KVFP16_MTPDynamicDepth", "current": 182.05432166006833, "high_water": null, "passed": true, "phase": "prefill", "status": "new_baseline" }, { "case": "Qwen38Dense/Qwen38DenseSingleDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_ROCm0_ActFP32_KVFP16_MTPDynamicDepth", "current": 39.21409296716583, "high_water": null, "passed": true, "phase": "decode", "status": "new_baseline" } ], "e2e_report_digest": "9985848fb4f422fcd5ec99c363da25363e297e727f3dd4af223e0e8a61580aa3", "hardware": { "accelerators": [ "0000:03:00.0 VGA compatible controller [0300]: ASPEED Technology, Inc. ASPEED Graphics Family [1a03:2000] (rev 41)", "0000:3f:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:42:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:8c:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:91:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:97:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:9a:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)" ], "cpu": { "Architecture:": "x86_64", "CPU(s):": "112", "Core(s) per socket:": "28", "Model name:": "Intel(R) Xeon(R) Gold 6238R CPU @ 2.20GHz", "NUMA node(s):": "2", "NUMA node0 CPU(s):": "0-27,56-83", "NUMA node1 CPU(s):": "28-55,84-111", "Socket(s):": "2", "Thread(s) per core:": "2" } }, "report_digest": "a96e151804f1b61cb357d0bbb98879664f8534c4d4eb7e0535bee6c4caf6e648", "workload": { "decode_tokens": 256, "measurement_iterations": 3, "prompt_repetitions": 512, "prompt_unit": " test", "schema": 1, "seed": 42, "temperature": 0, "top_k": 1, "top_p": 1, "warmup_iterations": 1 } }, "AVX512": { "cells": [ { "case": "Qwen36MoEExpertOverlay/Qwen36MoEExpertOverlayParityTest.ProductionParity/Qwen36MoE_35B_IQ3S_NodeTP_2xMPI_CPU_ExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "identity": { "case": "Qwen36MoEExpertOverlay/Qwen36MoEExpertOverlayParityTest.ProductionParity/Qwen36MoE_35B_IQ3S_NodeTP_2xMPI_CPU_ExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "configuration": { "benchmark": { "args": [ "--mpi-procs", "2", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--auto", "--only-backends", "cpu", "--auto-device-counts", "cpu=2", "--only-strategies", "tp", "--moe-routed-expert-owner-order", "ordinal", "--mtp", "--mtp-depth-policy", "dynamic", "--moe-residency-maintenance", "dynamic" ], "context_length": 8192, "policy": "production_defaults", "schema": 1 }, "cross_host_e2e": [], "e2e": { "cell_timeout_seconds": { "AVX2": 1200, "AVX512": 900 }, "context_length": 8192, "generation_tokens": 2048, "minimum_prompt_tokens": 4096, "movement_evidence": "required", "planning": { "device_counts": { "cpu": 2 }, "mode": "auto", "mpi_ranks": 2, "strategy": "tp" }, "readiness_timeout_seconds": 60, "request_timeout_seconds": 600, "server_args": [ "--mpi-procs", "2", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--tp-allreduce-precision", "schema", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--auto", "--only-backends", "cpu", "--auto-device-counts", "cpu=2", "--only-strategies", "tp", "--moe-routed-expert-owner-order", "ordinal", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "1", "--moe-residency-maintenance-max-window", "1", "--moe-residency-maintenance-window-growth", "1", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1000", "--moe-dynamic-min-improvement-permille", "0", "--moe-dynamic-max-swaps-per-layer", "20", "--moe-dynamic-max-plan-entries-per-wave", "20", "--moe-dynamic-min-window-activations", "0", "--moe-device-rebalance-maintenance-slack-tokens", "0", "--moe-device-rebalance-min-maintenance-period-tokens", "16", "--moe-device-rebalance-initial-maintenance-period-tokens", "2", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "0", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-max-post-wave-load-spread-permille", "1000", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ], "thinking_modes": "both", "tool_calling": "required" }, "id": "Qwen36MoE_35B_IQ3S_NodeTP_2xMPI_CPU_ExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "model": "Qwen3.6-35B-A3B-UD-IQ3_S.gguf", "model_parity_schema": 1, "runtime": { "context_length": 4096, "generation": { "max_tokens": 384, "minimum_completion_tokens": 384, "mtp_policy": "dynamic", "mtp_verify_mode": "speculative-sampling", "prefix_state": "hybrid_recurrent", "readiness_timeout_seconds": 60, "requests": [ { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "fresh_harbor", "prefix": "fresh" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_harbor", "prefix": "full" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "partial_mountain", "prefix": "partial" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_mountain", "prefix": "full" } ], "serial_control_id": "Qwen36MoE_35B_IQ3S_NodeTP_2xMPI_CPU_ExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPOff" }, "movement_evidence": "required", "server_args": [ "--mpi-procs", "2", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--backend", "mpi", "--tp-allreduce-precision", "schema", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--moe-routed-expert-placement", "single-domain", "--moe-routed-expert-continuation-domain", "qwen36_moe_cpu_node_tp", "--moe-routed-expert-base-model-domain", "qwen36_moe_cpu_node_tp", "--moe-routed-expert-shared-domain", "qwen36_moe_cpu_node_tp", "--moe-continuation-dense-policy", "tensor-parallel", "--moe-routed-expert-owner-order", "ordinal", "--moe-routed-expert-residency", "rebalanced", "--define-domain", "qwen36_moe_cpu_node_tp=localhost:0:cpu:0,localhost:1:cpu:0;scope=node_local;backend=mpi;routed_compute=apportioned;routed_phase=uniform;routed_decode_assignment=static-owner;routed_prefill_assignment=static-owner", "--moe-routed-expert-domain", "qwen36_moe_cpu_node_tp=localhost:0:cpu:0,localhost:1:cpu:0;scope=node_local;backend=mpi;routed_compute=apportioned;routed_phase=uniform;routed_decode_assignment=static-owner;routed_prefill_assignment=static-owner", "--moe-routed-expert-tier", "priority_0@qwen36_moe_cpu_node_tp;priority=0;max-experts-per-layer=0;memory-mb=auto", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "1", "--moe-residency-maintenance-max-window", "1", "--moe-residency-maintenance-window-growth", "1", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1000", "--moe-dynamic-min-improvement-permille", "0", "--moe-dynamic-max-swaps-per-layer", "20", "--moe-dynamic-max-plan-entries-per-wave", "20", "--moe-dynamic-min-window-activations", "0", "--moe-device-rebalance-maintenance-slack-tokens", "0", "--moe-device-rebalance-min-maintenance-period-tokens", "16", "--moe-device-rebalance-initial-maintenance-period-tokens", "2", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "0", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-max-post-wave-load-spread-permille", "1000", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ] } }, "cpu_isa": "AVX512", "hardware": { "accelerators": [ "0000:03:00.0 VGA compatible controller [0300]: ASPEED Technology, Inc. ASPEED Graphics Family [1a03:2000] (rev 41)", "0000:3f:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:42:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:8c:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:91:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:97:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:9a:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)" ], "cpu": { "Architecture:": "x86_64", "CPU(s):": "112", "Core(s) per socket:": "28", "Model name:": "Intel(R) Xeon(R) Gold 6238R CPU @ 2.20GHz", "NUMA node(s):": "2", "NUMA node0 CPU(s):": "0-27,56-83", "NUMA node1 CPU(s):": "28-55,84-111", "Socket(s):": "2", "Thread(s) per core:": "2" } }, "model_bytes": 15346432288, "model_files": { "Qwen3.6-35B-A3B-UD-IQ3_S.gguf": 15346432288 }, "prefill_tokens": 512, "workload": { "decode_tokens": 256, "measurement_iterations": 3, "prompt_repetitions": 512, "prompt_unit": " test", "schema": 1, "seed": 42, "temperature": 0, "top_k": 1, "top_p": 1, "warmup_iterations": 1 } }, "prefill_tokens": 512, "samples": { "decode": [ 36.94747511771963, 37.00911391591542, 37.08819613386773 ], "prefill": [ 291.02381034552144, 294.6870673512179, 296.0125425787014 ] }, "tokens_per_second": { "decode": 37.00911391591542, "prefill": 294.6870673512179 } }, { "case": "Qwen36MoEExpertOverlay/Qwen36MoEExpertOverlayParityTest.ProductionParity/Ornith15MoE_35B_Q4KM_NodeTP_2xMPI_CPU_ExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "identity": { "case": "Qwen36MoEExpertOverlay/Qwen36MoEExpertOverlayParityTest.ProductionParity/Ornith15MoE_35B_Q4KM_NodeTP_2xMPI_CPU_ExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "configuration": { "benchmark": { "args": [ "--mpi-procs", "2", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--auto", "--only-backends", "cpu", "--auto-device-counts", "cpu=2", "--only-strategies", "tp", "--moe-routed-expert-owner-order", "ordinal", "--mtp", "--mtp-depth-policy", "dynamic", "--moe-residency-maintenance", "dynamic" ], "context_length": 8192, "policy": "production_defaults", "schema": 1 }, "cross_host_e2e": [], "e2e": { "cell_timeout_seconds": { "AVX2": 1200, "AVX512": 900 }, "context_length": 8192, "generation_tokens": 2048, "minimum_prompt_tokens": 4096, "movement_evidence": "required", "planning": { "device_counts": { "cpu": 2 }, "mode": "auto", "mpi_ranks": 2, "strategy": "tp" }, "readiness_timeout_seconds": 60, "request_timeout_seconds": 600, "server_args": [ "--mpi-procs", "2", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--tp-allreduce-precision", "schema", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--auto", "--only-backends", "cpu", "--auto-device-counts", "cpu=2", "--only-strategies", "tp", "--moe-routed-expert-owner-order", "ordinal", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "1", "--moe-residency-maintenance-max-window", "1", "--moe-residency-maintenance-window-growth", "1", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1000", "--moe-dynamic-min-improvement-permille", "0", "--moe-dynamic-max-swaps-per-layer", "20", "--moe-dynamic-max-plan-entries-per-wave", "20", "--moe-dynamic-min-window-activations", "0", "--moe-device-rebalance-maintenance-slack-tokens", "0", "--moe-device-rebalance-min-maintenance-period-tokens", "16", "--moe-device-rebalance-initial-maintenance-period-tokens", "2", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "0", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-max-post-wave-load-spread-permille", "1000", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ], "thinking_modes": "both", "tool_calling": "required" }, "id": "Ornith15MoE_35B_Q4KM_NodeTP_2xMPI_CPU_ExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "model": "Ornith-1.5-35B-Q4_K_M.gguf", "model_parity_schema": 1, "runtime": { "context_length": 4096, "generation": { "max_tokens": 384, "minimum_completion_tokens": 384, "mtp_policy": "dynamic", "mtp_verify_mode": "speculative-sampling", "prefix_state": "hybrid_recurrent", "readiness_timeout_seconds": 60, "requests": [ { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "fresh_harbor", "prefix": "fresh" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_harbor", "prefix": "full" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "partial_mountain", "prefix": "partial" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_mountain", "prefix": "full" } ], "serial_control_id": "Ornith15MoE_35B_Q4KM_NodeTP_2xMPI_CPU_ExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPOff" }, "movement_evidence": "required", "server_args": [ "--mpi-procs", "2", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--backend", "mpi", "--tp-allreduce-precision", "schema", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--moe-routed-expert-placement", "single-domain", "--moe-routed-expert-continuation-domain", "qwen36_moe_cpu_node_tp", "--moe-routed-expert-base-model-domain", "qwen36_moe_cpu_node_tp", "--moe-routed-expert-shared-domain", "qwen36_moe_cpu_node_tp", "--moe-continuation-dense-policy", "tensor-parallel", "--moe-routed-expert-owner-order", "ordinal", "--moe-routed-expert-residency", "rebalanced", "--define-domain", "qwen36_moe_cpu_node_tp=localhost:0:cpu:0,localhost:1:cpu:0;scope=node_local;backend=mpi;routed_compute=apportioned;routed_phase=uniform;routed_decode_assignment=static-owner;routed_prefill_assignment=static-owner", "--moe-routed-expert-domain", "qwen36_moe_cpu_node_tp=localhost:0:cpu:0,localhost:1:cpu:0;scope=node_local;backend=mpi;routed_compute=apportioned;routed_phase=uniform;routed_decode_assignment=static-owner;routed_prefill_assignment=static-owner", "--moe-routed-expert-tier", "priority_0@qwen36_moe_cpu_node_tp;priority=0;max-experts-per-layer=0;memory-mb=auto", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "1", "--moe-residency-maintenance-max-window", "1", "--moe-residency-maintenance-window-growth", "1", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1000", "--moe-dynamic-min-improvement-permille", "0", "--moe-dynamic-max-swaps-per-layer", "20", "--moe-dynamic-max-plan-entries-per-wave", "20", "--moe-dynamic-min-window-activations", "0", "--moe-device-rebalance-maintenance-slack-tokens", "0", "--moe-device-rebalance-min-maintenance-period-tokens", "16", "--moe-device-rebalance-initial-maintenance-period-tokens", "2", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "0", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-max-post-wave-load-spread-permille", "1000", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ] } }, "cpu_isa": "AVX512", "hardware": { "accelerators": [ "0000:03:00.0 VGA compatible controller [0300]: ASPEED Technology, Inc. ASPEED Graphics Family [1a03:2000] (rev 41)", "0000:3f:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:42:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:8c:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:91:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:97:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:9a:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)" ], "cpu": { "Architecture:": "x86_64", "CPU(s):": "112", "Core(s) per socket:": "28", "Model name:": "Intel(R) Xeon(R) Gold 6238R CPU @ 2.20GHz", "NUMA node(s):": "2", "NUMA node0 CPU(s):": "0-27,56-83", "NUMA node1 CPU(s):": "28-55,84-111", "Socket(s):": "2", "Thread(s) per core:": "2" } }, "model_bytes": 21713463040, "model_files": { "Ornith-1.5-35B-Q4_K_M.gguf": 21713463040 }, "prefill_tokens": 512, "workload": { "decode_tokens": 256, "measurement_iterations": 3, "prompt_repetitions": 512, "prompt_unit": " test", "schema": 1, "seed": 42, "temperature": 0, "top_k": 1, "top_p": 1, "warmup_iterations": 1 } }, "prefill_tokens": 512, "samples": { "decode": [ 41.844563275506765, 42.009006192223886, 42.09271361106392 ], "prefill": [ 351.50169191288893, 352.3774183487581, 356.70730548001336 ] }, "tokens_per_second": { "decode": 42.009006192223886, "prefill": 352.3774183487581 } }, { "case": "Qwen35_122B_ExpertOverlay/Qwen35MoENodeExpertOverlayParityTest.ProductionParity/Qwen35_122B_CUDA2_CPU2_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "identity": { "case": "Qwen35_122B_ExpertOverlay/Qwen35MoENodeExpertOverlayParityTest.ProductionParity/Qwen35_122B_CUDA2_CPU2_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "configuration": { "benchmark": { "args": [ "--mpi-procs", "2", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--auto", "--only-backends", "cpu,cuda", "--auto-device-counts", "cpu=2,cuda=2", "--only-strategies", "expert-overlay", "--moe-routed-expert-owner-order", "ordinal", "--mtp", "--mtp-depth-policy", "dynamic", "--moe-residency-maintenance", "dynamic" ], "context_length": 8192, "policy": "production_defaults", "schema": 1 }, "cross_host_e2e": [], "e2e": { "cell_timeout_seconds": { "AVX2": 900, "AVX512": 900 }, "context_length": 8192, "generation_tokens": 2048, "minimum_prompt_tokens": 4096, "movement_evidence": "required", "planning": { "device_counts": { "cpu": 2, "cuda": 2 }, "mode": "auto", "mpi_ranks": 2, "strategy": "expert-overlay" }, "readiness_timeout_seconds": 180, "request_timeout_seconds": 600, "server_args": [ "--mpi-procs", "2", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--auto", "--only-backends", "cpu,cuda", "--auto-device-counts", "cpu=2,cuda=2", "--only-strategies", "expert-overlay", "--moe-routed-expert-owner-order", "ordinal", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "9", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "455.111115", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "49", "--moe-dynamic-imbalance-threshold-permille", "1000", "--moe-dynamic-min-improvement-permille", "0", "--moe-dynamic-max-swaps-per-layer", "2", "--moe-dynamic-max-plan-entries-per-wave", "196", "--moe-dynamic-min-window-activations", "0", "--moe-device-rebalance-maintenance-slack-tokens", "0", "--moe-device-rebalance-min-maintenance-period-tokens", "4096", "--moe-device-rebalance-initial-maintenance-period-tokens", "1", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "0", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-max-post-wave-load-spread-permille", "1000", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ], "thinking_modes": "both", "tool_calling": "required" }, "id": "Qwen35_122B_CUDA2_CPU2_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "model": "Qwen3.5-122B-A10B-UD-Q8_K_XL-00001-of-00004.gguf", "model_parity_schema": 1, "runtime": { "context_length": 4096, "generation": { "max_tokens": 384, "minimum_completion_tokens": 384, "mtp_policy": "dynamic", "mtp_verify_mode": "speculative-sampling", "prefix_state": "hybrid_recurrent", "readiness_timeout_seconds": 180, "requests": [ { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "fresh_harbor", "prefix": "fresh" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_harbor", "prefix": "full" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "partial_mountain", "prefix": "partial" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_mountain", "prefix": "full" } ], "serial_control_id": "Qwen35_122B_CUDA2_CPU2_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPOff" }, "movement_evidence": "required", "server_args": [ "--mpi-procs", "2", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--backend", "host", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--moe-routed-expert-placement", "tiered-overlay", "--moe-routed-expert-continuation-domain", "cuda_hot", "--moe-routed-expert-base-model-domain", "cuda_hot", "--moe-routed-expert-shared-domain", "cuda_hot", "--moe-continuation-dense-policy", "tensor-parallel-decode-mirrored-embedding", "--moe-routed-expert-owner-order", "ordinal", "--moe-routed-expert-residency", "rebalanced", "--moe-routed-expert-domain", "cuda_hot=localhost:-1:cuda:0,localhost:-1:cuda:1;scope=auto;backend=nccl;routed_compute=apportioned;routed_phase=uniform;routed_decode_assignment=static-owner;routed_prefill_assignment=static-owner", "--moe-routed-expert-domain", "cpu_cold=localhost:0:cpu:0,localhost:1:cpu:0;scope=auto;backend=upi;routed_compute=apportioned;routed_phase=uniform;routed_decode_assignment=static-owner;routed_prefill_assignment=static-owner", "--moe-routed-expert-tier", "priority0@cuda_hot;priority=0;max-experts-per-layer=0;memory-mb=auto", "--moe-routed-expert-tier", "priority1@cpu_cold;priority=1;max-experts-per-layer=0;memory-mb=auto", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "9", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "455.111115", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "49", "--moe-dynamic-imbalance-threshold-permille", "1000", "--moe-dynamic-min-improvement-permille", "0", "--moe-dynamic-max-swaps-per-layer", "2", "--moe-dynamic-max-plan-entries-per-wave", "196", "--moe-dynamic-min-window-activations", "0", "--moe-device-rebalance-maintenance-slack-tokens", "0", "--moe-device-rebalance-min-maintenance-period-tokens", "4096", "--moe-device-rebalance-initial-maintenance-period-tokens", "1", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "0", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-max-post-wave-load-spread-permille", "1000", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ] } }, "cpu_isa": "AVX512", "hardware": { "accelerators": [ "0000:03:00.0 VGA compatible controller [0300]: ASPEED Technology, Inc. ASPEED Graphics Family [1a03:2000] (rev 41)", "0000:3f:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:42:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:8c:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:91:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:97:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:9a:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)" ], "cpu": { "Architecture:": "x86_64", "CPU(s):": "112", "Core(s) per socket:": "28", "Model name:": "Intel(R) Xeon(R) Gold 6238R CPU @ 2.20GHz", "NUMA node(s):": "2", "NUMA node0 CPU(s):": "0-27,56-83", "NUMA node1 CPU(s):": "28-55,84-111", "Socket(s):": "2", "Thread(s) per core:": "2" } }, "model_bytes": 10943808, "model_files": { "Qwen3.5-122B-A10B-UD-Q8_K_XL-00001-of-00004.gguf": 10943808, "Qwen3.5-122B-A10B-UD-Q8_K_XL-00002-of-00004.gguf": 49799375552, "Qwen3.5-122B-A10B-UD-Q8_K_XL-00003-of-00004.gguf": 49820794656, "Qwen3.5-122B-A10B-UD-Q8_K_XL-00004-of-00004.gguf": 35297355104 }, "prefill_tokens": 512, "workload": { "decode_tokens": 256, "measurement_iterations": 3, "prompt_repetitions": 512, "prompt_unit": " test", "schema": 1, "seed": 42, "temperature": 0, "top_k": 1, "top_p": 1, "warmup_iterations": 1 } }, "prefill_tokens": 512, "samples": { "decode": [ 13.868461365158778, 14.0495421473084, 14.19889636288354 ], "prefill": [ 123.83924792649192, 124.92697548586702, 139.41480347608703 ] }, "tokens_per_second": { "decode": 14.0495421473084, "prefill": 124.92697548586702 } }, { "case": "Qwen35_122B_ExpertOverlay/Qwen35MoENodeExpertOverlayParityTest.ProductionParity/Qwen35_122B_CUDA2_ROCm4_CPU2_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "identity": { "case": "Qwen35_122B_ExpertOverlay/Qwen35MoENodeExpertOverlayParityTest.ProductionParity/Qwen35_122B_CUDA2_ROCm4_CPU2_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "configuration": { "benchmark": { "args": [ "--mpi-procs", "2", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--auto", "--only-backends", "cpu,cuda,rocm", "--auto-device-counts", "cpu=2,cuda=2,rocm=4", "--only-strategies", "expert-overlay", "--moe-routed-expert-owner-order", "ordinal", "--mtp", "--mtp-depth-policy", "dynamic", "--moe-residency-maintenance", "dynamic" ], "context_length": 8192, "policy": "production_defaults", "schema": 1 }, "cross_host_e2e": [], "e2e": { "cell_timeout_seconds": { "AVX2": 900, "AVX512": 900 }, "context_length": 8192, "generation_tokens": 2048, "minimum_prompt_tokens": 4096, "movement_evidence": "required", "planning": { "device_counts": { "cpu": 2, "cuda": 2, "rocm": 4 }, "mode": "auto", "mpi_ranks": 2, "strategy": "expert-overlay" }, "readiness_timeout_seconds": 180, "request_timeout_seconds": 600, "server_args": [ "--mpi-procs", "2", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--auto", "--only-backends", "cpu,cuda,rocm", "--auto-device-counts", "cpu=2,cuda=2,rocm=4", "--only-strategies", "expert-overlay", "--moe-routed-expert-owner-order", "ordinal", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "9", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "455.111115", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "49", "--moe-dynamic-imbalance-threshold-permille", "1000", "--moe-dynamic-min-improvement-permille", "0", "--moe-dynamic-max-swaps-per-layer", "2", "--moe-dynamic-max-plan-entries-per-wave", "392", "--moe-dynamic-min-window-activations", "0", "--moe-device-rebalance-maintenance-slack-tokens", "0", "--moe-device-rebalance-min-maintenance-period-tokens", "4096", "--moe-device-rebalance-initial-maintenance-period-tokens", "1", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "0", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-max-post-wave-load-spread-permille", "1000", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ], "thinking_modes": "both", "tool_calling": "required" }, "id": "Qwen35_122B_CUDA2_ROCm4_CPU2_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "model": "Qwen3.5-122B-A10B-UD-Q8_K_XL-00001-of-00004.gguf", "model_parity_schema": 1, "runtime": { "context_length": 4096, "generation": { "max_tokens": 384, "minimum_completion_tokens": 384, "mtp_policy": "dynamic", "mtp_verify_mode": "speculative-sampling", "prefix_state": "hybrid_recurrent", "readiness_timeout_seconds": 180, "requests": [ { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "fresh_harbor", "prefix": "fresh" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_harbor", "prefix": "full" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "partial_mountain", "prefix": "partial" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_mountain", "prefix": "full" } ], "serial_control_id": "Qwen35_122B_CUDA2_ROCm4_CPU2_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPOff" }, "movement_evidence": "required", "server_args": [ "--mpi-procs", "2", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--backend", "host", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--moe-routed-expert-placement", "tiered-overlay", "--moe-routed-expert-continuation-domain", "cuda_hot", "--moe-routed-expert-base-model-domain", "cuda_hot", "--moe-routed-expert-shared-domain", "cuda_hot", "--moe-continuation-dense-policy", "tensor-parallel-decode-mirrored-embedding", "--moe-routed-expert-owner-order", "ordinal", "--moe-routed-expert-residency", "rebalanced", "--moe-routed-expert-domain", "cuda_hot=localhost:-1:cuda:0,localhost:-1:cuda:1;scope=auto;backend=nccl;routed_compute=apportioned;routed_phase=uniform;routed_decode_assignment=static-owner;routed_prefill_assignment=static-owner", "--moe-routed-expert-domain", "rocm_warm=localhost:-1:rocm:0,localhost:-1:rocm:1,localhost:-1:rocm:2,localhost:-1:rocm:3;scope=auto;backend=rccl;routed_compute=apportioned;routed_phase=uniform;routed_decode_assignment=static-owner;routed_prefill_assignment=static-owner", "--moe-routed-expert-domain", "cpu_cold=localhost:0:cpu:0,localhost:1:cpu:0;scope=auto;backend=upi;routed_compute=apportioned;routed_phase=uniform;routed_decode_assignment=static-owner;routed_prefill_assignment=static-owner", "--moe-routed-expert-tier", "priority0@cuda_hot;priority=0;max-experts-per-layer=0;memory-mb=auto", "--moe-routed-expert-tier", "priority1@rocm_warm;priority=1;max-experts-per-layer=0;memory-mb=auto", "--moe-routed-expert-tier", "priority2@cpu_cold;priority=2;max-experts-per-layer=0;memory-mb=auto", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "9", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "455.111115", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "49", "--moe-dynamic-imbalance-threshold-permille", "1000", "--moe-dynamic-min-improvement-permille", "0", "--moe-dynamic-max-swaps-per-layer", "2", "--moe-dynamic-max-plan-entries-per-wave", "392", "--moe-dynamic-min-window-activations", "0", "--moe-device-rebalance-maintenance-slack-tokens", "0", "--moe-device-rebalance-min-maintenance-period-tokens", "4096", "--moe-device-rebalance-initial-maintenance-period-tokens", "1", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "0", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-max-post-wave-load-spread-permille", "1000", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ] } }, "cpu_isa": "AVX512", "hardware": { "accelerators": [ "0000:03:00.0 VGA compatible controller [0300]: ASPEED Technology, Inc. ASPEED Graphics Family [1a03:2000] (rev 41)", "0000:3f:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:42:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:8c:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:91:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:97:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:9a:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)" ], "cpu": { "Architecture:": "x86_64", "CPU(s):": "112", "Core(s) per socket:": "28", "Model name:": "Intel(R) Xeon(R) Gold 6238R CPU @ 2.20GHz", "NUMA node(s):": "2", "NUMA node0 CPU(s):": "0-27,56-83", "NUMA node1 CPU(s):": "28-55,84-111", "Socket(s):": "2", "Thread(s) per core:": "2" } }, "model_bytes": 10943808, "model_files": { "Qwen3.5-122B-A10B-UD-Q8_K_XL-00001-of-00004.gguf": 10943808, "Qwen3.5-122B-A10B-UD-Q8_K_XL-00002-of-00004.gguf": 49799375552, "Qwen3.5-122B-A10B-UD-Q8_K_XL-00003-of-00004.gguf": 49820794656, "Qwen3.5-122B-A10B-UD-Q8_K_XL-00004-of-00004.gguf": 35297355104 }, "prefill_tokens": 512, "workload": { "decode_tokens": 256, "measurement_iterations": 3, "prompt_repetitions": 512, "prompt_unit": " test", "schema": 1, "seed": 42, "temperature": 0, "top_k": 1, "top_p": 1, "warmup_iterations": 1 } }, "prefill_tokens": 512, "samples": { "decode": [ 19.936387565610904, 20.119436376706723, 20.190882723957102 ], "prefill": [ 125.38222468146287, 124.55144778685616, 123.75526441419514 ] }, "tokens_per_second": { "decode": 20.119436376706723, "prefill": 124.55144778685616 } }, { "case": "Qwen35_122B_ExpertOverlay/Qwen35MoENodeExpertOverlayParityTest.ProductionParity/Qwen35_122B_ROCm2_CPU2_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "identity": { "case": "Qwen35_122B_ExpertOverlay/Qwen35MoENodeExpertOverlayParityTest.ProductionParity/Qwen35_122B_ROCm2_CPU2_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "configuration": { "benchmark": { "args": [ "--mpi-procs", "2", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--auto", "--only-backends", "cpu,rocm", "--auto-device-counts", "cpu=2,rocm=2", "--only-strategies", "expert-overlay", "--moe-routed-expert-owner-order", "ordinal", "--mtp", "--mtp-depth-policy", "dynamic", "--moe-residency-maintenance", "dynamic" ], "context_length": 8192, "policy": "production_defaults", "schema": 1 }, "cross_host_e2e": [], "e2e": { "cell_timeout_seconds": { "AVX2": 900, "AVX512": 900 }, "context_length": 8192, "generation_tokens": 2048, "minimum_prompt_tokens": 4096, "movement_evidence": "required", "planning": { "device_counts": { "cpu": 2, "rocm": 2 }, "mode": "auto", "mpi_ranks": 2, "strategy": "expert-overlay" }, "readiness_timeout_seconds": 180, "request_timeout_seconds": 600, "server_args": [ "--mpi-procs", "2", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--auto", "--only-backends", "cpu,rocm", "--auto-device-counts", "cpu=2,rocm=2", "--only-strategies", "expert-overlay", "--moe-routed-expert-owner-order", "ordinal", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "9", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "455.111115", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "49", "--moe-dynamic-imbalance-threshold-permille", "1000", "--moe-dynamic-min-improvement-permille", "0", "--moe-dynamic-max-swaps-per-layer", "2", "--moe-dynamic-max-plan-entries-per-wave", "196", "--moe-dynamic-min-window-activations", "0", "--moe-device-rebalance-maintenance-slack-tokens", "0", "--moe-device-rebalance-min-maintenance-period-tokens", "4096", "--moe-device-rebalance-initial-maintenance-period-tokens", "1", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "0", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-max-post-wave-load-spread-permille", "1000", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ], "thinking_modes": "both", "tool_calling": "required" }, "id": "Qwen35_122B_ROCm2_CPU2_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "model": "Qwen3.5-122B-A10B-UD-Q8_K_XL-00001-of-00004.gguf", "model_parity_schema": 1, "runtime": { "context_length": 4096, "generation": { "max_tokens": 384, "minimum_completion_tokens": 384, "mtp_policy": "dynamic", "mtp_verify_mode": "speculative-sampling", "prefix_state": "hybrid_recurrent", "readiness_timeout_seconds": 180, "requests": [ { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "fresh_harbor", "prefix": "fresh" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_harbor", "prefix": "full" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "partial_mountain", "prefix": "partial" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_mountain", "prefix": "full" } ], "serial_control_id": "Qwen35_122B_ROCm2_CPU2_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPOff" }, "movement_evidence": "required", "server_args": [ "--mpi-procs", "2", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--backend", "host", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--moe-routed-expert-placement", "tiered-overlay", "--moe-routed-expert-continuation-domain", "rocm_hot", "--moe-routed-expert-base-model-domain", "rocm_hot", "--moe-routed-expert-shared-domain", "rocm_hot", "--moe-continuation-dense-policy", "tensor-parallel-decode-mirrored-embedding", "--moe-routed-expert-owner-order", "ordinal", "--moe-routed-expert-residency", "rebalanced", "--moe-routed-expert-domain", "rocm_hot=localhost:-1:rocm:0,localhost:-1:rocm:1;scope=auto;backend=rccl;routed_compute=apportioned;routed_phase=uniform;routed_decode_assignment=static-owner;routed_prefill_assignment=static-owner", "--moe-routed-expert-domain", "cpu_cold=localhost:0:cpu:0,localhost:1:cpu:0;scope=auto;backend=upi;routed_compute=apportioned;routed_phase=uniform;routed_decode_assignment=static-owner;routed_prefill_assignment=static-owner", "--moe-routed-expert-tier", "priority0@rocm_hot;priority=0;max-experts-per-layer=0;memory-mb=auto", "--moe-routed-expert-tier", "priority1@cpu_cold;priority=1;max-experts-per-layer=0;memory-mb=auto", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "9", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "455.111115", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "49", "--moe-dynamic-imbalance-threshold-permille", "1000", "--moe-dynamic-min-improvement-permille", "0", "--moe-dynamic-max-swaps-per-layer", "2", "--moe-dynamic-max-plan-entries-per-wave", "196", "--moe-dynamic-min-window-activations", "0", "--moe-device-rebalance-maintenance-slack-tokens", "0", "--moe-device-rebalance-min-maintenance-period-tokens", "4096", "--moe-device-rebalance-initial-maintenance-period-tokens", "1", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "0", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-max-post-wave-load-spread-permille", "1000", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ] } }, "cpu_isa": "AVX512", "hardware": { "accelerators": [ "0000:03:00.0 VGA compatible controller [0300]: ASPEED Technology, Inc. ASPEED Graphics Family [1a03:2000] (rev 41)", "0000:3f:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:42:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:8c:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:91:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:97:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:9a:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)" ], "cpu": { "Architecture:": "x86_64", "CPU(s):": "112", "Core(s) per socket:": "28", "Model name:": "Intel(R) Xeon(R) Gold 6238R CPU @ 2.20GHz", "NUMA node(s):": "2", "NUMA node0 CPU(s):": "0-27,56-83", "NUMA node1 CPU(s):": "28-55,84-111", "Socket(s):": "2", "Thread(s) per core:": "2" } }, "model_bytes": 10943808, "model_files": { "Qwen3.5-122B-A10B-UD-Q8_K_XL-00001-of-00004.gguf": 10943808, "Qwen3.5-122B-A10B-UD-Q8_K_XL-00002-of-00004.gguf": 49799375552, "Qwen3.5-122B-A10B-UD-Q8_K_XL-00003-of-00004.gguf": 49820794656, "Qwen3.5-122B-A10B-UD-Q8_K_XL-00004-of-00004.gguf": 35297355104 }, "prefill_tokens": 512, "workload": { "decode_tokens": 256, "measurement_iterations": 3, "prompt_repetitions": 512, "prompt_unit": " test", "schema": 1, "seed": 42, "temperature": 0, "top_k": 1, "top_p": 1, "warmup_iterations": 1 } }, "prefill_tokens": 512, "samples": { "decode": [ 22.198659479782584, 22.61551433948939, 22.86258636287532 ], "prefill": [ 159.25788795368555, 164.72968452054434, 186.98197128743703 ] }, "tokens_per_second": { "decode": 22.61551433948939, "prefill": 164.72968452054434 } }, { "case": "Qwen35_122B_ExpertOverlay/Qwen35MoENodeExpertOverlayParityTest.ProductionParity/Qwen35_122B_ROCm4_CPU2_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "identity": { "case": "Qwen35_122B_ExpertOverlay/Qwen35MoENodeExpertOverlayParityTest.ProductionParity/Qwen35_122B_ROCm4_CPU2_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "configuration": { "benchmark": { "args": [ "--mpi-procs", "2", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--auto", "--only-backends", "cpu,rocm", "--auto-device-counts", "cpu=2,rocm=4", "--only-strategies", "expert-overlay", "--moe-routed-expert-owner-order", "ordinal", "--mtp", "--mtp-depth-policy", "dynamic", "--moe-residency-maintenance", "dynamic" ], "context_length": 8192, "policy": "production_defaults", "schema": 1 }, "cross_host_e2e": [], "e2e": { "cell_timeout_seconds": { "AVX2": 900, "AVX512": 900 }, "context_length": 8192, "generation_tokens": 2048, "minimum_prompt_tokens": 4096, "movement_evidence": "required", "planning": { "device_counts": { "cpu": 2, "rocm": 4 }, "mode": "auto", "mpi_ranks": 2, "strategy": "expert-overlay" }, "readiness_timeout_seconds": 180, "request_timeout_seconds": 600, "server_args": [ "--mpi-procs", "2", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--auto", "--only-backends", "cpu,rocm", "--auto-device-counts", "cpu=2,rocm=4", "--only-strategies", "expert-overlay", "--moe-routed-expert-owner-order", "ordinal", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "9", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "455.111115", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "49", "--moe-dynamic-imbalance-threshold-permille", "1000", "--moe-dynamic-min-improvement-permille", "0", "--moe-dynamic-max-swaps-per-layer", "2", "--moe-dynamic-max-plan-entries-per-wave", "294", "--moe-dynamic-min-window-activations", "0", "--moe-device-rebalance-maintenance-slack-tokens", "0", "--moe-device-rebalance-min-maintenance-period-tokens", "4096", "--moe-device-rebalance-initial-maintenance-period-tokens", "1", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "0", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-max-post-wave-load-spread-permille", "1000", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ], "thinking_modes": "both", "tool_calling": "required" }, "id": "Qwen35_122B_ROCm4_CPU2_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "model": "Qwen3.5-122B-A10B-UD-Q8_K_XL-00001-of-00004.gguf", "model_parity_schema": 1, "runtime": { "context_length": 4096, "generation": { "max_tokens": 384, "minimum_completion_tokens": 384, "mtp_policy": "dynamic", "mtp_verify_mode": "speculative-sampling", "prefix_state": "hybrid_recurrent", "readiness_timeout_seconds": 180, "requests": [ { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "fresh_harbor", "prefix": "fresh" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_harbor", "prefix": "full" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "partial_mountain", "prefix": "partial" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_mountain", "prefix": "full" } ], "serial_control_id": "Qwen35_122B_ROCm4_CPU2_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPOff" }, "movement_evidence": "required", "server_args": [ "--mpi-procs", "2", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--backend", "host", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--moe-routed-expert-placement", "tiered-overlay", "--moe-routed-expert-continuation-domain", "rocm_hot", "--moe-routed-expert-base-model-domain", "rocm_hot", "--moe-routed-expert-shared-domain", "rocm_hot", "--moe-continuation-dense-policy", "tensor-parallel-decode-mirrored-embedding", "--moe-routed-expert-owner-order", "ordinal", "--moe-routed-expert-residency", "rebalanced", "--moe-routed-expert-domain", "rocm_hot=localhost:-1:rocm:0,localhost:-1:rocm:1,localhost:-1:rocm:2,localhost:-1:rocm:3;scope=auto;backend=rccl;routed_compute=apportioned;routed_phase=uniform;routed_decode_assignment=static-owner;routed_prefill_assignment=static-owner", "--moe-routed-expert-domain", "cpu_cold=localhost:0:cpu:0,localhost:1:cpu:0;scope=auto;backend=upi;routed_compute=apportioned;routed_phase=uniform;routed_decode_assignment=static-owner;routed_prefill_assignment=static-owner", "--moe-routed-expert-tier", "priority0@rocm_hot;priority=0;max-experts-per-layer=0;memory-mb=auto", "--moe-routed-expert-tier", "priority1@cpu_cold;priority=1;max-experts-per-layer=0;memory-mb=auto", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "9", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "455.111115", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "49", "--moe-dynamic-imbalance-threshold-permille", "1000", "--moe-dynamic-min-improvement-permille", "0", "--moe-dynamic-max-swaps-per-layer", "2", "--moe-dynamic-max-plan-entries-per-wave", "294", "--moe-dynamic-min-window-activations", "0", "--moe-device-rebalance-maintenance-slack-tokens", "0", "--moe-device-rebalance-min-maintenance-period-tokens", "4096", "--moe-device-rebalance-initial-maintenance-period-tokens", "1", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "0", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-max-post-wave-load-spread-permille", "1000", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ] } }, "cpu_isa": "AVX512", "hardware": { "accelerators": [ "0000:03:00.0 VGA compatible controller [0300]: ASPEED Technology, Inc. ASPEED Graphics Family [1a03:2000] (rev 41)", "0000:3f:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:42:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:8c:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:91:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:97:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:9a:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)" ], "cpu": { "Architecture:": "x86_64", "CPU(s):": "112", "Core(s) per socket:": "28", "Model name:": "Intel(R) Xeon(R) Gold 6238R CPU @ 2.20GHz", "NUMA node(s):": "2", "NUMA node0 CPU(s):": "0-27,56-83", "NUMA node1 CPU(s):": "28-55,84-111", "Socket(s):": "2", "Thread(s) per core:": "2" } }, "model_bytes": 10943808, "model_files": { "Qwen3.5-122B-A10B-UD-Q8_K_XL-00001-of-00004.gguf": 10943808, "Qwen3.5-122B-A10B-UD-Q8_K_XL-00002-of-00004.gguf": 49799375552, "Qwen3.5-122B-A10B-UD-Q8_K_XL-00003-of-00004.gguf": 49820794656, "Qwen3.5-122B-A10B-UD-Q8_K_XL-00004-of-00004.gguf": 35297355104 }, "prefill_tokens": 512, "workload": { "decode_tokens": 256, "measurement_iterations": 3, "prompt_repetitions": 512, "prompt_unit": " test", "schema": 1, "seed": 42, "temperature": 0, "top_k": 1, "top_p": 1, "warmup_iterations": 1 } }, "prefill_tokens": 512, "samples": { "decode": [ 26.817618536046155, 27.70187307336983, 27.478068326425486 ], "prefill": [ 170.4709809819457, 177.17048901615635, 170.75167157290525 ] }, "tokens_per_second": { "decode": 27.478068326425486, "prefill": 170.75167157290525 } }, { "case": "Qwen36MoEExpertOverlay/Qwen36MoEExpertOverlayParityTest.ProductionParity/Ornith15MoE_35B_Q4KM_LocalTP_NCCL_2xCUDA_ExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "identity": { "case": "Qwen36MoEExpertOverlay/Qwen36MoEExpertOverlayParityTest.ProductionParity/Ornith15MoE_35B_Q4KM_LocalTP_NCCL_2xCUDA_ExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "configuration": { "benchmark": { "args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--auto", "--only-backends", "cuda", "--auto-device-counts", "cuda=2", "--only-strategies", "tp", "--moe-routed-expert-owner-order", "ordinal", "--mtp", "--mtp-depth-policy", "dynamic", "--moe-residency-maintenance", "dynamic" ], "context_length": 8192, "policy": "production_defaults", "schema": 1 }, "cross_host_e2e": [], "e2e": { "cell_timeout_seconds": { "AVX2": 900, "AVX512": 900 }, "context_length": 8192, "generation_tokens": 2048, "minimum_prompt_tokens": 4096, "movement_evidence": "required", "planning": { "device_counts": { "cuda": 2 }, "mode": "auto", "mpi_ranks": 1, "strategy": "tp" }, "readiness_timeout_seconds": 60, "request_timeout_seconds": 600, "server_args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--tp-allreduce-precision", "schema", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--auto", "--only-backends", "cuda", "--auto-device-counts", "cuda=2", "--only-strategies", "tp", "--moe-routed-expert-owner-order", "ordinal", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "1", "--moe-residency-maintenance-max-window", "1", "--moe-residency-maintenance-window-growth", "1", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1000", "--moe-dynamic-min-improvement-permille", "0", "--moe-dynamic-max-swaps-per-layer", "20", "--moe-dynamic-max-plan-entries-per-wave", "20", "--moe-dynamic-min-window-activations", "0", "--moe-device-rebalance-maintenance-slack-tokens", "0", "--moe-device-rebalance-min-maintenance-period-tokens", "16", "--moe-device-rebalance-initial-maintenance-period-tokens", "2", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "0", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-max-post-wave-load-spread-permille", "1000", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ], "thinking_modes": "both", "tool_calling": "required" }, "id": "Ornith15MoE_35B_Q4KM_LocalTP_NCCL_2xCUDA_ExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "model": "Ornith-1.5-35B-Q4_K_M.gguf", "model_parity_schema": 1, "runtime": { "context_length": 4096, "generation": { "max_tokens": 384, "minimum_completion_tokens": 384, "mtp_policy": "dynamic", "mtp_verify_mode": "speculative-sampling", "prefix_state": "hybrid_recurrent", "readiness_timeout_seconds": 60, "requests": [ { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "fresh_harbor", "prefix": "fresh" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_harbor", "prefix": "full" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "partial_mountain", "prefix": "partial" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_mountain", "prefix": "full" } ], "serial_control_id": "Ornith15MoE_35B_Q4KM_LocalTP_NCCL_2xCUDA_ExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPOff" }, "movement_evidence": "required", "server_args": [ "--mpi-procs", "1", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--backend", "nccl", "--tp-allreduce-precision", "schema", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--moe-routed-expert-placement", "single-domain", "--moe-routed-expert-continuation-domain", "qwen36_moe_cuda_local_tp", "--moe-routed-expert-base-model-domain", "qwen36_moe_cuda_local_tp", "--moe-routed-expert-shared-domain", "qwen36_moe_cuda_local_tp", "--moe-continuation-dense-policy", "tensor-parallel", "--moe-routed-expert-owner-order", "ordinal", "--moe-routed-expert-residency", "rebalanced", "--define-domain", "qwen36_moe_cuda_local_tp=localhost:-1:cuda:0,localhost:-1:cuda:1;scope=rank_local;backend=nccl;routed_compute=apportioned;routed_phase=uniform;routed_decode_assignment=static-owner;routed_prefill_assignment=static-owner", "--moe-routed-expert-domain", "qwen36_moe_cuda_local_tp=localhost:-1:cuda:0,localhost:-1:cuda:1;scope=rank_local;backend=nccl;routed_compute=apportioned;routed_phase=uniform;routed_decode_assignment=static-owner;routed_prefill_assignment=static-owner", "--moe-routed-expert-tier", "priority_0@qwen36_moe_cuda_local_tp;priority=0;max-experts-per-layer=0;memory-mb=auto", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "1", "--moe-residency-maintenance-max-window", "1", "--moe-residency-maintenance-window-growth", "1", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1000", "--moe-dynamic-min-improvement-permille", "0", "--moe-dynamic-max-swaps-per-layer", "20", "--moe-dynamic-max-plan-entries-per-wave", "20", "--moe-dynamic-min-window-activations", "0", "--moe-device-rebalance-maintenance-slack-tokens", "0", "--moe-device-rebalance-min-maintenance-period-tokens", "16", "--moe-device-rebalance-initial-maintenance-period-tokens", "2", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "0", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-max-post-wave-load-spread-permille", "1000", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ] } }, "cpu_isa": "AVX512", "hardware": { "accelerators": [ "0000:03:00.0 VGA compatible controller [0300]: ASPEED Technology, Inc. ASPEED Graphics Family [1a03:2000] (rev 41)", "0000:3f:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:42:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:8c:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:91:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:97:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:9a:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)" ], "cpu": { "Architecture:": "x86_64", "CPU(s):": "112", "Core(s) per socket:": "28", "Model name:": "Intel(R) Xeon(R) Gold 6238R CPU @ 2.20GHz", "NUMA node(s):": "2", "NUMA node0 CPU(s):": "0-27,56-83", "NUMA node1 CPU(s):": "28-55,84-111", "Socket(s):": "2", "Thread(s) per core:": "2" } }, "model_bytes": 21713463040, "model_files": { "Ornith-1.5-35B-Q4_K_M.gguf": 21713463040 }, "prefill_tokens": 512, "workload": { "decode_tokens": 256, "measurement_iterations": 3, "prompt_repetitions": 512, "prompt_unit": " test", "schema": 1, "seed": 42, "temperature": 0, "top_k": 1, "top_p": 1, "warmup_iterations": 1 } }, "prefill_tokens": 512, "samples": { "decode": [ 139.729164680686, 139.71622483036936, 139.59433601291687 ], "prefill": [ 888.4597844075287, 887.3892595832347, 885.9310189987751 ] }, "tokens_per_second": { "decode": 139.71622483036936, "prefill": 887.3892595832347 } }, { "case": "Qwen36MoE/Qwen36MoESingleDeviceParityTest.ProductionParity/Qwen36MoE_35B_IQ3S_CUDA0_ActFP32_KVFP16_MTPDynamicDepth", "identity": { "case": "Qwen36MoE/Qwen36MoESingleDeviceParityTest.ProductionParity/Qwen36MoE_35B_IQ3S_CUDA0_ActFP32_KVFP16_MTPDynamicDepth", "configuration": { "benchmark": { "args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--auto", "--only-backends", "cuda", "--auto-device-counts", "cuda=1", "--only-strategies", "single", "--mtp", "--mtp-depth-policy", "dynamic" ], "context_length": 8192, "policy": "production_defaults", "schema": 1 }, "cross_host_e2e": [], "e2e": { "cell_timeout_seconds": { "AVX2": 900, "AVX512": 900 }, "context_length": 8192, "generation_tokens": 2048, "minimum_prompt_tokens": 4096, "movement_evidence": "not_applicable", "planning": { "device_counts": { "cuda": 1 }, "mode": "auto", "mpi_ranks": 1, "strategy": "single" }, "readiness_timeout_seconds": 60, "request_timeout_seconds": 600, "server_args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--tp-allreduce-precision", "schema", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--auto", "--only-backends", "cuda", "--auto-device-counts", "cuda=1", "--only-strategies", "single", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "256", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "1.5", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1300", "--moe-dynamic-min-improvement-permille", "50", "--moe-dynamic-max-swaps-per-layer", "4", "--moe-dynamic-max-plan-entries-per-wave", "16", "--moe-dynamic-min-window-activations", "64", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "15", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "256", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "128", "--moe-device-rebalance-max-post-wave-load-spread-permille", "100", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ], "thinking_modes": "both", "tool_calling": "required" }, "id": "Qwen36MoE_35B_IQ3S_CUDA0_ActFP32_KVFP16_MTPDynamicDepth", "model": "Qwen3.6-35B-A3B-UD-IQ3_S.gguf", "model_parity_schema": 1, "runtime": { "context_length": 4096, "generation": { "max_tokens": 384, "minimum_completion_tokens": 384, "mtp_policy": "dynamic", "mtp_verify_mode": "speculative-sampling", "prefix_state": "hybrid_recurrent", "readiness_timeout_seconds": 60, "requests": [ { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "fresh_harbor", "prefix": "fresh" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_harbor", "prefix": "full" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "partial_mountain", "prefix": "partial" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_mountain", "prefix": "full" } ], "serial_control_id": "Qwen36MoE_35B_IQ3S_CUDA0_ActFP32_KVFP16_MTPOff" }, "movement_evidence": "not_applicable", "server_args": [ "--mpi-procs", "1", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--backend", "host", "--tp-allreduce-precision", "schema", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--device", "localhost:-1:cuda:0", "--tensor-parallelism-degree", "1", "--tp-scope", "auto", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "256", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "1.5", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1300", "--moe-dynamic-min-improvement-permille", "50", "--moe-dynamic-max-swaps-per-layer", "4", "--moe-dynamic-max-plan-entries-per-wave", "16", "--moe-dynamic-min-window-activations", "64", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "15", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "256", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "128", "--moe-device-rebalance-max-post-wave-load-spread-permille", "100", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ] } }, "cpu_isa": "AVX512", "hardware": { "accelerators": [ "0000:03:00.0 VGA compatible controller [0300]: ASPEED Technology, Inc. ASPEED Graphics Family [1a03:2000] (rev 41)", "0000:3f:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:42:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:8c:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:91:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:97:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:9a:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)" ], "cpu": { "Architecture:": "x86_64", "CPU(s):": "112", "Core(s) per socket:": "28", "Model name:": "Intel(R) Xeon(R) Gold 6238R CPU @ 2.20GHz", "NUMA node(s):": "2", "NUMA node0 CPU(s):": "0-27,56-83", "NUMA node1 CPU(s):": "28-55,84-111", "Socket(s):": "2", "Thread(s) per core:": "2" } }, "model_bytes": 15346432288, "model_files": { "Qwen3.6-35B-A3B-UD-IQ3_S.gguf": 15346432288 }, "prefill_tokens": 512, "workload": { "decode_tokens": 256, "measurement_iterations": 3, "prompt_repetitions": 512, "prompt_unit": " test", "schema": 1, "seed": 42, "temperature": 0, "top_k": 1, "top_p": 1, "warmup_iterations": 1 } }, "prefill_tokens": 512, "samples": { "decode": [ 283.04951141125537, 283.0466586455166, 283.0233576613219 ], "prefill": [ 1738.708065740389, 1726.2538010625199, 1737.5849757108697 ] }, "tokens_per_second": { "decode": 283.0466586455166, "prefill": 1737.5849757108697 } }, { "case": "Qwen38Dense/Qwen38DenseMultiDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_CUDA2_TP_ActFP32_KVFP16_MTPDynamicDepth", "identity": { "case": "Qwen38Dense/Qwen38DenseMultiDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_CUDA2_TP_ActFP32_KVFP16_MTPDynamicDepth", "configuration": { "benchmark": { "args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--auto", "--only-backends", "cuda", "--auto-device-counts", "cuda=2", "--only-strategies", "tp", "--mtp", "--mtp-depth-policy", "dynamic" ], "context_length": 8192, "policy": "production_defaults", "schema": 1 }, "cross_host_e2e": [], "e2e": { "cell_timeout_seconds": { "AVX2": 900, "AVX512": 900 }, "context_length": 8192, "generation_tokens": 2048, "minimum_prompt_tokens": 4096, "movement_evidence": "not_applicable", "planning": { "device_counts": { "cuda": 2 }, "mode": "auto", "mpi_ranks": 1, "strategy": "tp" }, "readiness_timeout_seconds": 60, "request_timeout_seconds": 600, "server_args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--auto", "--only-backends", "cuda", "--auto-device-counts", "cuda=2", "--only-strategies", "tp", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "256", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "1.5", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1300", "--moe-dynamic-min-improvement-permille", "50", "--moe-dynamic-max-swaps-per-layer", "4", "--moe-dynamic-max-plan-entries-per-wave", "16", "--moe-dynamic-min-window-activations", "64", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "15", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "256", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "128", "--moe-device-rebalance-max-post-wave-load-spread-permille", "100", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ], "thinking_modes": "both", "tool_calling": "required" }, "id": "Qwen38Dense_27B_IQ4XS_CUDA2_TP_ActFP32_KVFP16_MTPDynamicDepth", "model": "Qwen3.8-27B-IQ4_XS.gguf", "model_parity_schema": 1, "runtime": { "context_length": 4096, "generation": { "max_tokens": 384, "minimum_completion_tokens": 384, "mtp_policy": "dynamic", "mtp_verify_mode": "speculative-sampling", "prefix_state": "hybrid_recurrent", "readiness_timeout_seconds": 60, "requests": [ { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "fresh_harbor", "prefix": "fresh" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_harbor", "prefix": "full" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "partial_mountain", "prefix": "partial" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_mountain", "prefix": "full" } ], "serial_control_id": "Qwen38Dense_27B_IQ4XS_CUDA2_TP_ActFP32_KVFP16_MTPOff" }, "movement_evidence": "not_applicable", "server_args": [ "--mpi-procs", "1", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--backend", "nccl", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--tp-devices", "localhost:-1:cuda:0,localhost:-1:cuda:1", "--tensor-parallelism-degree", "2", "--tp-scope", "rank_local", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "256", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "1.5", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1300", "--moe-dynamic-min-improvement-permille", "50", "--moe-dynamic-max-swaps-per-layer", "4", "--moe-dynamic-max-plan-entries-per-wave", "16", "--moe-dynamic-min-window-activations", "64", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "15", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "256", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "128", "--moe-device-rebalance-max-post-wave-load-spread-permille", "100", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ] } }, "cpu_isa": "AVX512", "hardware": { "accelerators": [ "0000:03:00.0 VGA compatible controller [0300]: ASPEED Technology, Inc. ASPEED Graphics Family [1a03:2000] (rev 41)", "0000:3f:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:42:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:8c:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:91:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:97:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:9a:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)" ], "cpu": { "Architecture:": "x86_64", "CPU(s):": "112", "Core(s) per socket:": "28", "Model name:": "Intel(R) Xeon(R) Gold 6238R CPU @ 2.20GHz", "NUMA node(s):": "2", "NUMA node0 CPU(s):": "0-27,56-83", "NUMA node1 CPU(s):": "28-55,84-111", "Socket(s):": "2", "Thread(s) per core:": "2" } }, "model_bytes": 15705861088, "model_files": { "Qwen3.8-27B-IQ4_XS.gguf": 15705861088 }, "prefill_tokens": 512, "workload": { "decode_tokens": 256, "measurement_iterations": 3, "prompt_repetitions": 512, "prompt_unit": " test", "schema": 1, "seed": 42, "temperature": 0, "top_k": 1, "top_p": 1, "warmup_iterations": 1 } }, "prefill_tokens": 512, "samples": { "decode": [ 50.35203454147674, 50.37873541979683, 50.479057939312376 ], "prefill": [ 467.83973789395645, 468.26279619662506, 468.1480895375244 ] }, "tokens_per_second": { "decode": 50.37873541979683, "prefill": 468.1480895375244 } }, { "case": "Qwen38Dense/Qwen38DenseMultiDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_CUDA2_PP_ActFP32_KVFP16_MTPDynamicDepth", "identity": { "case": "Qwen38Dense/Qwen38DenseMultiDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_CUDA2_PP_ActFP32_KVFP16_MTPDynamicDepth", "configuration": { "benchmark": { "args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--auto", "--only-backends", "cuda", "--auto-device-counts", "cuda=2", "--only-strategies", "pp", "--mtp", "--mtp-depth-policy", "dynamic" ], "context_length": 8192, "policy": "production_defaults", "schema": 1 }, "cross_host_e2e": [], "e2e": { "cell_timeout_seconds": { "AVX2": 900, "AVX512": 900 }, "context_length": 8192, "generation_tokens": 2048, "minimum_prompt_tokens": 4096, "movement_evidence": "not_applicable", "planning": { "device_counts": { "cuda": 2 }, "mode": "auto", "mpi_ranks": 1, "pipeline_domains": [ { "backend": "cuda", "devices": 1 }, { "backend": "cuda", "devices": 1 } ], "pipeline_layers": 64, "strategy": "pp" }, "readiness_timeout_seconds": 60, "request_timeout_seconds": 600, "server_args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--auto", "--only-backends", "cuda", "--auto-device-counts", "cuda=2", "--only-strategies", "pp", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "256", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "1.5", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1300", "--moe-dynamic-min-improvement-permille", "50", "--moe-dynamic-max-swaps-per-layer", "4", "--moe-dynamic-max-plan-entries-per-wave", "16", "--moe-dynamic-min-window-activations", "64", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "15", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "256", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "128", "--moe-device-rebalance-max-post-wave-load-spread-permille", "100", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ], "thinking_modes": "both", "tool_calling": "required" }, "id": "Qwen38Dense_27B_IQ4XS_CUDA2_PP_ActFP32_KVFP16_MTPDynamicDepth", "model": "Qwen3.8-27B-IQ4_XS.gguf", "model_parity_schema": 1, "runtime": { "context_length": 4096, "generation": { "max_tokens": 384, "minimum_completion_tokens": 384, "mtp_policy": "dynamic", "mtp_verify_mode": "speculative-sampling", "prefix_state": "hybrid_recurrent", "readiness_timeout_seconds": 60, "requests": [ { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "fresh_harbor", "prefix": "fresh" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_harbor", "prefix": "full" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "partial_mountain", "prefix": "partial" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_mountain", "prefix": "full" } ], "serial_control_id": "Qwen38Dense_27B_IQ4XS_CUDA2_PP_ActFP32_KVFP16_MTPOff" }, "movement_evidence": "not_applicable", "server_args": [ "--mpi-procs", "1", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--backend", "nccl", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--tensor-parallelism-degree", "1", "--tp-scope", "auto", "--define-domain", "parity_stage_0=localhost:-1:cuda:0;scope=rank_local;backend=auto;owner=0", "--define-domain", "parity_stage_1=localhost:-1:cuda:1;scope=rank_local;backend=auto;owner=0", "--pp-stage", "0=parity_stage_0:0-31", "--pp-stage", "1=parity_stage_1:32-63", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "256", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "1.5", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1300", "--moe-dynamic-min-improvement-permille", "50", "--moe-dynamic-max-swaps-per-layer", "4", "--moe-dynamic-max-plan-entries-per-wave", "16", "--moe-dynamic-min-window-activations", "64", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "15", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "256", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "128", "--moe-device-rebalance-max-post-wave-load-spread-permille", "100", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ] } }, "cpu_isa": "AVX512", "hardware": { "accelerators": [ "0000:03:00.0 VGA compatible controller [0300]: ASPEED Technology, Inc. ASPEED Graphics Family [1a03:2000] (rev 41)", "0000:3f:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:42:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:8c:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:91:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:97:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:9a:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)" ], "cpu": { "Architecture:": "x86_64", "CPU(s):": "112", "Core(s) per socket:": "28", "Model name:": "Intel(R) Xeon(R) Gold 6238R CPU @ 2.20GHz", "NUMA node(s):": "2", "NUMA node0 CPU(s):": "0-27,56-83", "NUMA node1 CPU(s):": "28-55,84-111", "Socket(s):": "2", "Thread(s) per core:": "2" } }, "model_bytes": 15705861088, "model_files": { "Qwen3.8-27B-IQ4_XS.gguf": 15705861088 }, "prefill_tokens": 512, "workload": { "decode_tokens": 256, "measurement_iterations": 3, "prompt_repetitions": 512, "prompt_unit": " test", "schema": 1, "seed": 42, "temperature": 0, "top_k": 1, "top_p": 1, "warmup_iterations": 1 } }, "prefill_tokens": 512, "samples": { "decode": [ 40.104571547381795, 39.5760944123509, 40.045097658297664 ], "prefill": [ 743.7303214331563, 745.03230141383, 743.118349620533 ] }, "tokens_per_second": { "decode": 40.045097658297664, "prefill": 743.7303214331563 } }, { "case": "Qwen38Dense/Qwen38DenseSingleDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_CUDA0_ActFP32_KVFP16_MTPDynamicDepth", "identity": { "case": "Qwen38Dense/Qwen38DenseSingleDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_CUDA0_ActFP32_KVFP16_MTPDynamicDepth", "configuration": { "benchmark": { "args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--auto", "--only-backends", "cuda", "--auto-device-counts", "cuda=1", "--only-strategies", "single", "--mtp", "--mtp-depth-policy", "dynamic" ], "context_length": 8192, "policy": "production_defaults", "schema": 1 }, "cross_host_e2e": [], "e2e": { "cell_timeout_seconds": { "AVX2": 900, "AVX512": 900 }, "context_length": 8192, "generation_tokens": 2048, "minimum_prompt_tokens": 4096, "movement_evidence": "not_applicable", "planning": { "device_counts": { "cuda": 1 }, "mode": "auto", "mpi_ranks": 1, "strategy": "single" }, "readiness_timeout_seconds": 60, "request_timeout_seconds": 600, "server_args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--auto", "--only-backends", "cuda", "--auto-device-counts", "cuda=1", "--only-strategies", "single", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "256", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "1.5", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1300", "--moe-dynamic-min-improvement-permille", "50", "--moe-dynamic-max-swaps-per-layer", "4", "--moe-dynamic-max-plan-entries-per-wave", "16", "--moe-dynamic-min-window-activations", "64", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "15", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "256", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "128", "--moe-device-rebalance-max-post-wave-load-spread-permille", "100", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ], "thinking_modes": "both", "tool_calling": "required" }, "id": "Qwen38Dense_27B_IQ4XS_CUDA0_ActFP32_KVFP16_MTPDynamicDepth", "model": "Qwen3.8-27B-IQ4_XS.gguf", "model_parity_schema": 1, "runtime": { "context_length": 4096, "generation": { "max_tokens": 384, "minimum_completion_tokens": 384, "mtp_policy": "dynamic", "mtp_verify_mode": "speculative-sampling", "prefix_state": "hybrid_recurrent", "readiness_timeout_seconds": 60, "requests": [ { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "fresh_harbor", "prefix": "fresh" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_harbor", "prefix": "full" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "partial_mountain", "prefix": "partial" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_mountain", "prefix": "full" } ], "serial_control_id": "Qwen38Dense_27B_IQ4XS_CUDA0_ActFP32_KVFP16_MTPOff" }, "movement_evidence": "not_applicable", "server_args": [ "--mpi-procs", "1", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--backend", "host", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--device", "localhost:-1:cuda:0", "--tensor-parallelism-degree", "1", "--tp-scope", "auto", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "256", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "1.5", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1300", "--moe-dynamic-min-improvement-permille", "50", "--moe-dynamic-max-swaps-per-layer", "4", "--moe-dynamic-max-plan-entries-per-wave", "16", "--moe-dynamic-min-window-activations", "64", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "15", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "256", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "128", "--moe-device-rebalance-max-post-wave-load-spread-permille", "100", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ] } }, "cpu_isa": "AVX512", "hardware": { "accelerators": [ "0000:03:00.0 VGA compatible controller [0300]: ASPEED Technology, Inc. ASPEED Graphics Family [1a03:2000] (rev 41)", "0000:3f:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:42:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:8c:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:91:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:97:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:9a:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)" ], "cpu": { "Architecture:": "x86_64", "CPU(s):": "112", "Core(s) per socket:": "28", "Model name:": "Intel(R) Xeon(R) Gold 6238R CPU @ 2.20GHz", "NUMA node(s):": "2", "NUMA node0 CPU(s):": "0-27,56-83", "NUMA node1 CPU(s):": "28-55,84-111", "Socket(s):": "2", "Thread(s) per core:": "2" } }, "model_bytes": 15705861088, "model_files": { "Qwen3.8-27B-IQ4_XS.gguf": 15705861088 }, "prefill_tokens": 512, "workload": { "decode_tokens": 256, "measurement_iterations": 3, "prompt_repetitions": 512, "prompt_unit": " test", "schema": 1, "seed": 42, "temperature": 0, "top_k": 1, "top_p": 1, "warmup_iterations": 1 } }, "prefill_tokens": 512, "samples": { "decode": [ 62.74931544169534, 62.76201538530717, 62.73668388875897 ], "prefill": [ 718.360566494653, 717.7246164886172, 719.6645117456079 ] }, "tokens_per_second": { "decode": 62.74931544169534, "prefill": 718.360566494653 } }, { "case": "Qwen35_122B_ExpertOverlay/Qwen35MoENodeExpertOverlayParityTest.ProductionParity/Qwen35_122B_CUDA2_ROCm4_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "identity": { "case": "Qwen35_122B_ExpertOverlay/Qwen35MoENodeExpertOverlayParityTest.ProductionParity/Qwen35_122B_CUDA2_ROCm4_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "configuration": { "benchmark": { "args": [ "--mpi-procs", "2", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--auto", "--only-backends", "cuda,rocm", "--auto-device-counts", "cuda=2,rocm=4", "--only-strategies", "expert-overlay", "--moe-routed-expert-owner-order", "ordinal", "--mtp", "--mtp-depth-policy", "dynamic", "--moe-residency-maintenance", "dynamic" ], "context_length": 8192, "policy": "production_defaults", "schema": 1 }, "cross_host_e2e": [], "e2e": { "cell_timeout_seconds": { "AVX2": 900, "AVX512": 900 }, "context_length": 8192, "generation_tokens": 2048, "minimum_prompt_tokens": 4096, "movement_evidence": "required", "planning": { "device_counts": { "cuda": 2, "rocm": 4 }, "mode": "auto", "mpi_ranks": 2, "strategy": "expert-overlay" }, "readiness_timeout_seconds": 180, "request_timeout_seconds": 600, "server_args": [ "--mpi-procs", "2", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--auto", "--only-backends", "cuda,rocm", "--auto-device-counts", "cuda=2,rocm=4", "--only-strategies", "expert-overlay", "--moe-routed-expert-owner-order", "ordinal", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "9", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "455.111115", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "49", "--moe-dynamic-imbalance-threshold-permille", "1000", "--moe-dynamic-min-improvement-permille", "0", "--moe-dynamic-max-swaps-per-layer", "2", "--moe-dynamic-max-plan-entries-per-wave", "294", "--moe-dynamic-min-window-activations", "0", "--moe-device-rebalance-maintenance-slack-tokens", "0", "--moe-device-rebalance-min-maintenance-period-tokens", "4096", "--moe-device-rebalance-initial-maintenance-period-tokens", "1", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "0", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-max-post-wave-load-spread-permille", "1000", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ], "thinking_modes": "both", "tool_calling": "required" }, "id": "Qwen35_122B_CUDA2_ROCm4_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "model": "Qwen3.5-122B-A10B-UD-Q8_K_XL-00001-of-00004.gguf", "model_parity_schema": 1, "runtime": { "context_length": 4096, "generation": { "max_tokens": 384, "minimum_completion_tokens": 384, "mtp_policy": "dynamic", "mtp_verify_mode": "speculative-sampling", "prefix_state": "hybrid_recurrent", "readiness_timeout_seconds": 180, "requests": [ { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "fresh_harbor", "prefix": "fresh" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_harbor", "prefix": "full" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "partial_mountain", "prefix": "partial" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_mountain", "prefix": "full" } ], "serial_control_id": "Qwen35_122B_CUDA2_ROCm4_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPOff" }, "movement_evidence": "required", "server_args": [ "--mpi-procs", "2", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--backend", "host", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--moe-routed-expert-placement", "tiered-overlay", "--moe-routed-expert-continuation-domain", "cuda_hot", "--moe-routed-expert-base-model-domain", "cuda_hot", "--moe-routed-expert-shared-domain", "cuda_hot", "--moe-continuation-dense-policy", "tensor-parallel-decode-mirrored-embedding", "--moe-routed-expert-owner-order", "ordinal", "--moe-routed-expert-residency", "rebalanced", "--moe-routed-expert-domain", "cuda_hot=localhost:-1:cuda:0,localhost:-1:cuda:1;scope=auto;backend=nccl;routed_compute=apportioned;routed_phase=uniform;routed_decode_assignment=static-owner;routed_prefill_assignment=static-owner", "--moe-routed-expert-domain", "rocm_warm=localhost:-1:rocm:0,localhost:-1:rocm:1,localhost:-1:rocm:2,localhost:-1:rocm:3;scope=auto;backend=rccl;routed_compute=apportioned;routed_phase=uniform;routed_decode_assignment=static-owner;routed_prefill_assignment=static-owner", "--moe-routed-expert-tier", "priority0@cuda_hot;priority=0;max-experts-per-layer=0;memory-mb=auto", "--moe-routed-expert-tier", "priority1@rocm_warm;priority=1;max-experts-per-layer=0;memory-mb=auto", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "9", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "455.111115", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "49", "--moe-dynamic-imbalance-threshold-permille", "1000", "--moe-dynamic-min-improvement-permille", "0", "--moe-dynamic-max-swaps-per-layer", "2", "--moe-dynamic-max-plan-entries-per-wave", "294", "--moe-dynamic-min-window-activations", "0", "--moe-device-rebalance-maintenance-slack-tokens", "0", "--moe-device-rebalance-min-maintenance-period-tokens", "4096", "--moe-device-rebalance-initial-maintenance-period-tokens", "1", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "0", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-max-post-wave-load-spread-permille", "1000", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ] } }, "cpu_isa": "AVX512", "hardware": { "accelerators": [ "0000:03:00.0 VGA compatible controller [0300]: ASPEED Technology, Inc. ASPEED Graphics Family [1a03:2000] (rev 41)", "0000:3f:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:42:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:8c:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:91:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:97:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:9a:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)" ], "cpu": { "Architecture:": "x86_64", "CPU(s):": "112", "Core(s) per socket:": "28", "Model name:": "Intel(R) Xeon(R) Gold 6238R CPU @ 2.20GHz", "NUMA node(s):": "2", "NUMA node0 CPU(s):": "0-27,56-83", "NUMA node1 CPU(s):": "28-55,84-111", "Socket(s):": "2", "Thread(s) per core:": "2" } }, "model_bytes": 10943808, "model_files": { "Qwen3.5-122B-A10B-UD-Q8_K_XL-00001-of-00004.gguf": 10943808, "Qwen3.5-122B-A10B-UD-Q8_K_XL-00002-of-00004.gguf": 49799375552, "Qwen3.5-122B-A10B-UD-Q8_K_XL-00003-of-00004.gguf": 49820794656, "Qwen3.5-122B-A10B-UD-Q8_K_XL-00004-of-00004.gguf": 35297355104 }, "prefill_tokens": 512, "workload": { "decode_tokens": 256, "measurement_iterations": 3, "prompt_repetitions": 512, "prompt_unit": " test", "schema": 1, "seed": 42, "temperature": 0, "top_k": 1, "top_p": 1, "warmup_iterations": 1 } }, "prefill_tokens": 512, "samples": { "decode": [ 24.535875253499142, 25.721946133499852, 25.029439150278442 ], "prefill": [ 151.83135185269106, 153.59136216002435, 155.46959784544305 ] }, "tokens_per_second": { "decode": 25.029439150278442, "prefill": 153.59136216002435 } }, { "case": "Qwen38Dense/Qwen38DenseMultiDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_CUDA2_ROCm2_TPPP_ActFP32_KVFP16_MTPDynamicDepth", "identity": { "case": "Qwen38Dense/Qwen38DenseMultiDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_CUDA2_ROCm2_TPPP_ActFP32_KVFP16_MTPDynamicDepth", "configuration": { "benchmark": { "args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--auto", "--only-backends", "cuda,rocm", "--auto-device-counts", "cuda=2,rocm=2", "--only-strategies", "pp", "--mtp", "--mtp-depth-policy", "dynamic" ], "context_length": 8192, "policy": "production_defaults", "schema": 1 }, "cross_host_e2e": [], "e2e": { "cell_timeout_seconds": { "AVX2": 900, "AVX512": 900 }, "context_length": 8192, "generation_tokens": 2048, "minimum_prompt_tokens": 4096, "movement_evidence": "not_applicable", "planning": { "device_counts": { "cuda": 2, "rocm": 2 }, "mode": "auto", "mpi_ranks": 1, "pipeline_domains": [ { "backend": "cuda", "devices": 2 }, { "backend": "rocm", "devices": 2 } ], "pipeline_layers": 64, "strategy": "pp" }, "readiness_timeout_seconds": 60, "request_timeout_seconds": 600, "server_args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--auto", "--only-backends", "cuda,rocm", "--auto-device-counts", "cuda=2,rocm=2", "--only-strategies", "pp", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "256", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "1.5", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1300", "--moe-dynamic-min-improvement-permille", "50", "--moe-dynamic-max-swaps-per-layer", "4", "--moe-dynamic-max-plan-entries-per-wave", "16", "--moe-dynamic-min-window-activations", "64", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "15", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "256", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "128", "--moe-device-rebalance-max-post-wave-load-spread-permille", "100", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ], "thinking_modes": "both", "tool_calling": "required" }, "id": "Qwen38Dense_27B_IQ4XS_CUDA2_ROCm2_TPPP_ActFP32_KVFP16_MTPDynamicDepth", "model": "Qwen3.8-27B-IQ4_XS.gguf", "model_parity_schema": 1, "runtime": { "context_length": 4096, "generation": { "max_tokens": 384, "minimum_completion_tokens": 384, "mtp_policy": "dynamic", "mtp_verify_mode": "speculative-sampling", "prefix_state": "hybrid_recurrent", "readiness_timeout_seconds": 60, "requests": [ { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "fresh_harbor", "prefix": "fresh" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_harbor", "prefix": "full" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "partial_mountain", "prefix": "partial" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_mountain", "prefix": "full" } ], "serial_control_id": "Qwen38Dense_27B_IQ4XS_CUDA2_ROCm2_TPPP_ActFP32_KVFP16_MTPOff" }, "movement_evidence": "not_applicable", "server_args": [ "--mpi-procs", "1", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--backend", "heterogeneous", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--tensor-parallelism-degree", "1", "--tp-scope", "auto", "--define-domain", "parity_stage_0=localhost:-1:cuda:0,localhost:-1:cuda:1;scope=rank_local;backend=nccl;owner=0;weights=0.5,0.5", "--define-domain", "parity_stage_1=localhost:-1:rocm:0,localhost:-1:rocm:1;scope=rank_local;backend=rccl;owner=0;weights=0.5,0.5", "--pp-stage", "0=parity_stage_0:0-31", "--pp-stage", "1=parity_stage_1:32-63", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "256", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "1.5", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1300", "--moe-dynamic-min-improvement-permille", "50", "--moe-dynamic-max-swaps-per-layer", "4", "--moe-dynamic-max-plan-entries-per-wave", "16", "--moe-dynamic-min-window-activations", "64", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "15", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "256", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "128", "--moe-device-rebalance-max-post-wave-load-spread-permille", "100", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ] } }, "cpu_isa": "AVX512", "hardware": { "accelerators": [ "0000:03:00.0 VGA compatible controller [0300]: ASPEED Technology, Inc. ASPEED Graphics Family [1a03:2000] (rev 41)", "0000:3f:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:42:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:8c:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:91:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:97:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:9a:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)" ], "cpu": { "Architecture:": "x86_64", "CPU(s):": "112", "Core(s) per socket:": "28", "Model name:": "Intel(R) Xeon(R) Gold 6238R CPU @ 2.20GHz", "NUMA node(s):": "2", "NUMA node0 CPU(s):": "0-27,56-83", "NUMA node1 CPU(s):": "28-55,84-111", "Socket(s):": "2", "Thread(s) per core:": "2" } }, "model_bytes": 15705861088, "model_files": { "Qwen3.8-27B-IQ4_XS.gguf": 15705861088 }, "prefill_tokens": 512, "workload": { "decode_tokens": 256, "measurement_iterations": 3, "prompt_repetitions": 512, "prompt_unit": " test", "schema": 1, "seed": 42, "temperature": 0, "top_k": 1, "top_p": 1, "warmup_iterations": 1 } }, "prefill_tokens": 512, "samples": { "decode": [ 31.994631354057706, 32.17514397243227, 31.945179490323 ], "prefill": [ 301.367589759008, 301.66045284430544, 302.3965242297808 ] }, "tokens_per_second": { "decode": 31.994631354057706, "prefill": 301.66045284430544 } }, { "case": "Qwen36MoEExpertOverlay/Qwen36MoEExpertOverlayParityTest.ProductionParity/Ornith15MoE_35B_Q4KM_LocalTP_RCCL_2xROCm_ExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "identity": { "case": "Qwen36MoEExpertOverlay/Qwen36MoEExpertOverlayParityTest.ProductionParity/Ornith15MoE_35B_Q4KM_LocalTP_RCCL_2xROCm_ExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "configuration": { "benchmark": { "args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--auto", "--only-backends", "rocm", "--auto-device-counts", "rocm=2", "--only-strategies", "tp", "--moe-routed-expert-owner-order", "ordinal", "--mtp", "--mtp-depth-policy", "dynamic", "--moe-residency-maintenance", "dynamic" ], "context_length": 8192, "policy": "production_defaults", "schema": 1 }, "cross_host_e2e": [], "e2e": { "cell_timeout_seconds": { "AVX2": 900, "AVX512": 900 }, "context_length": 8192, "generation_tokens": 2048, "minimum_prompt_tokens": 4096, "movement_evidence": "required", "planning": { "device_counts": { "rocm": 2 }, "mode": "auto", "mpi_ranks": 1, "strategy": "tp" }, "readiness_timeout_seconds": 60, "request_timeout_seconds": 600, "server_args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--tp-allreduce-precision", "schema", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--auto", "--only-backends", "rocm", "--auto-device-counts", "rocm=2", "--only-strategies", "tp", "--moe-routed-expert-owner-order", "ordinal", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "1", "--moe-residency-maintenance-max-window", "1", "--moe-residency-maintenance-window-growth", "1", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1000", "--moe-dynamic-min-improvement-permille", "0", "--moe-dynamic-max-swaps-per-layer", "20", "--moe-dynamic-max-plan-entries-per-wave", "20", "--moe-dynamic-min-window-activations", "0", "--moe-device-rebalance-maintenance-slack-tokens", "0", "--moe-device-rebalance-min-maintenance-period-tokens", "16", "--moe-device-rebalance-initial-maintenance-period-tokens", "2", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "0", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-max-post-wave-load-spread-permille", "1000", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ], "thinking_modes": "both", "tool_calling": "required" }, "id": "Ornith15MoE_35B_Q4KM_LocalTP_RCCL_2xROCm_ExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "model": "Ornith-1.5-35B-Q4_K_M.gguf", "model_parity_schema": 1, "runtime": { "context_length": 4096, "generation": { "max_tokens": 384, "minimum_completion_tokens": 384, "mtp_policy": "dynamic", "mtp_verify_mode": "speculative-sampling", "prefix_state": "hybrid_recurrent", "readiness_timeout_seconds": 60, "requests": [ { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "fresh_harbor", "prefix": "fresh" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_harbor", "prefix": "full" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "partial_mountain", "prefix": "partial" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_mountain", "prefix": "full" } ], "serial_control_id": "Ornith15MoE_35B_Q4KM_LocalTP_RCCL_2xROCm_ExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPOff" }, "movement_evidence": "required", "server_args": [ "--mpi-procs", "1", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--backend", "rccl", "--tp-allreduce-precision", "schema", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--moe-routed-expert-placement", "single-domain", "--moe-routed-expert-continuation-domain", "qwen36_moe_rocm_local_tp", "--moe-routed-expert-base-model-domain", "qwen36_moe_rocm_local_tp", "--moe-routed-expert-shared-domain", "qwen36_moe_rocm_local_tp", "--moe-continuation-dense-policy", "tensor-parallel", "--moe-routed-expert-owner-order", "ordinal", "--moe-routed-expert-residency", "rebalanced", "--define-domain", "qwen36_moe_rocm_local_tp=localhost:-1:rocm:0,localhost:-1:rocm:1;scope=rank_local;backend=rccl;routed_compute=apportioned;routed_phase=uniform;routed_decode_assignment=static-owner;routed_prefill_assignment=static-owner", "--moe-routed-expert-domain", "qwen36_moe_rocm_local_tp=localhost:-1:rocm:0,localhost:-1:rocm:1;scope=rank_local;backend=rccl;routed_compute=apportioned;routed_phase=uniform;routed_decode_assignment=static-owner;routed_prefill_assignment=static-owner", "--moe-routed-expert-tier", "priority_0@qwen36_moe_rocm_local_tp;priority=0;max-experts-per-layer=0;memory-mb=auto", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "1", "--moe-residency-maintenance-max-window", "1", "--moe-residency-maintenance-window-growth", "1", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1000", "--moe-dynamic-min-improvement-permille", "0", "--moe-dynamic-max-swaps-per-layer", "20", "--moe-dynamic-max-plan-entries-per-wave", "20", "--moe-dynamic-min-window-activations", "0", "--moe-device-rebalance-maintenance-slack-tokens", "0", "--moe-device-rebalance-min-maintenance-period-tokens", "16", "--moe-device-rebalance-initial-maintenance-period-tokens", "2", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "0", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-max-post-wave-load-spread-permille", "1000", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ] } }, "cpu_isa": "AVX512", "hardware": { "accelerators": [ "0000:03:00.0 VGA compatible controller [0300]: ASPEED Technology, Inc. ASPEED Graphics Family [1a03:2000] (rev 41)", "0000:3f:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:42:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:8c:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:91:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:97:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:9a:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)" ], "cpu": { "Architecture:": "x86_64", "CPU(s):": "112", "Core(s) per socket:": "28", "Model name:": "Intel(R) Xeon(R) Gold 6238R CPU @ 2.20GHz", "NUMA node(s):": "2", "NUMA node0 CPU(s):": "0-27,56-83", "NUMA node1 CPU(s):": "28-55,84-111", "Socket(s):": "2", "Thread(s) per core:": "2" } }, "model_bytes": 21713463040, "model_files": { "Ornith-1.5-35B-Q4_K_M.gguf": 21713463040 }, "prefill_tokens": 512, "workload": { "decode_tokens": 256, "measurement_iterations": 3, "prompt_repetitions": 512, "prompt_unit": " test", "schema": 1, "seed": 42, "temperature": 0, "top_k": 1, "top_p": 1, "warmup_iterations": 1 } }, "prefill_tokens": 512, "samples": { "decode": [ 101.99609599742959, 101.96936322324409, 101.81157763906045 ], "prefill": [ 775.0034046792638, 775.5705469201858, 775.307896003185 ] }, "tokens_per_second": { "decode": 101.96936322324409, "prefill": 775.307896003185 } }, { "case": "Qwen36MoE/Qwen36MoESingleDeviceParityTest.ProductionParity/Qwen36MoE_35B_IQ3S_ROCm0_ActFP32_KVFP16_MTPDynamicDepth", "identity": { "case": "Qwen36MoE/Qwen36MoESingleDeviceParityTest.ProductionParity/Qwen36MoE_35B_IQ3S_ROCm0_ActFP32_KVFP16_MTPDynamicDepth", "configuration": { "benchmark": { "args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--auto", "--only-backends", "rocm", "--auto-device-counts", "rocm=1", "--only-strategies", "single", "--mtp", "--mtp-depth-policy", "dynamic" ], "context_length": 8192, "policy": "production_defaults", "schema": 1 }, "cross_host_e2e": [ { "frontend": "plan-apply", "id": "Qwen36MoE_35B_IQ3S_ROCm0_ActFP32_KVFP16_MTPDynamicDepth_RemoteCPU1_plan-apply", "movement_evidence": "required", "owner_order": "ordinal", "planning": { "device_counts": { "cpu": 1, "rocm": 1 }, "mode": "auto", "strategy": "expert-overlay" }, "schema": 1, "server_policy_args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--tp-allreduce-precision", "schema", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--only-backends", "rocm,cpu", "--only-strategies", "expert-overlay", "--auto-hosts", "all", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "1", "--moe-residency-maintenance-max-window", "1", "--moe-residency-maintenance-window-growth", "1", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1000", "--moe-dynamic-min-improvement-permille", "0", "--moe-dynamic-max-swaps-per-layer", "20", "--moe-dynamic-max-plan-entries-per-wave", "20", "--moe-dynamic-min-window-activations", "0", "--moe-device-rebalance-maintenance-slack-tokens", "0", "--moe-device-rebalance-min-maintenance-period-tokens", "16", "--moe-device-rebalance-initial-maintenance-period-tokens", "2", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "0", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-max-post-wave-load-spread-permille", "1000", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10", "--auto-device-counts", "rocm=1,cpu=1" ], "topology": { "continuation_backend": "rocm", "continuation_devices": 1, "continuation_priority": 0, "cpu_ranks_per_host": 1, "execution_ranks": 2, "kind": "cross-host-expert-overlay", "remote_cpu_hosts": 1, "remote_priority": 1 } }, { "frontend": "auto-serve", "id": "Qwen36MoE_35B_IQ3S_ROCm0_ActFP32_KVFP16_MTPDynamicDepth_RemoteCPU1_auto-serve", "movement_evidence": "required", "owner_order": "ordinal", "planning": { "device_counts": { "cpu": 1, "rocm": 1 }, "mode": "auto", "strategy": "expert-overlay" }, "schema": 1, "server_policy_args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--tp-allreduce-precision", "schema", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--only-backends", "rocm,cpu", "--only-strategies", "expert-overlay", "--auto-hosts", "all", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "1", "--moe-residency-maintenance-max-window", "1", "--moe-residency-maintenance-window-growth", "1", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1000", "--moe-dynamic-min-improvement-permille", "0", "--moe-dynamic-max-swaps-per-layer", "20", "--moe-dynamic-max-plan-entries-per-wave", "20", "--moe-dynamic-min-window-activations", "0", "--moe-device-rebalance-maintenance-slack-tokens", "0", "--moe-device-rebalance-min-maintenance-period-tokens", "16", "--moe-device-rebalance-initial-maintenance-period-tokens", "2", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "0", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-max-post-wave-load-spread-permille", "1000", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10", "--auto-device-counts", "rocm=1,cpu=1" ], "topology": { "continuation_backend": "rocm", "continuation_devices": 1, "continuation_priority": 0, "cpu_ranks_per_host": 1, "execution_ranks": 2, "kind": "cross-host-expert-overlay", "remote_cpu_hosts": 1, "remote_priority": 1 } }, { "frontend": "plan-apply", "id": "Qwen36MoE_35B_IQ3S_ROCm0_ActFP32_KVFP16_MTPDynamicDepth_RemoteCPU2_plan-apply", "movement_evidence": "required", "owner_order": "ordinal", "planning": { "device_counts": { "cpu": 2, "rocm": 1 }, "mode": "auto", "strategy": "expert-overlay" }, "schema": 1, "server_policy_args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--tp-allreduce-precision", "schema", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--only-backends", "rocm,cpu", "--only-strategies", "expert-overlay", "--auto-hosts", "all", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "1", "--moe-residency-maintenance-max-window", "1", "--moe-residency-maintenance-window-growth", "1", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1000", "--moe-dynamic-min-improvement-permille", "0", "--moe-dynamic-max-swaps-per-layer", "20", "--moe-dynamic-max-plan-entries-per-wave", "20", "--moe-dynamic-min-window-activations", "0", "--moe-device-rebalance-maintenance-slack-tokens", "0", "--moe-device-rebalance-min-maintenance-period-tokens", "16", "--moe-device-rebalance-initial-maintenance-period-tokens", "2", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "0", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-max-post-wave-load-spread-permille", "1000", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10", "--auto-device-counts", "rocm=1,cpu=2" ], "topology": { "continuation_backend": "rocm", "continuation_devices": 1, "continuation_priority": 0, "cpu_ranks_per_host": 1, "execution_ranks": 3, "kind": "cross-host-expert-overlay", "remote_cpu_hosts": 2, "remote_priority": 1 } }, { "frontend": "auto-serve", "id": "Qwen36MoE_35B_IQ3S_ROCm0_ActFP32_KVFP16_MTPDynamicDepth_RemoteCPU2_auto-serve", "movement_evidence": "required", "owner_order": "ordinal", "planning": { "device_counts": { "cpu": 2, "rocm": 1 }, "mode": "auto", "strategy": "expert-overlay" }, "schema": 1, "server_policy_args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--tp-allreduce-precision", "schema", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--only-backends", "rocm,cpu", "--only-strategies", "expert-overlay", "--auto-hosts", "all", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "1", "--moe-residency-maintenance-max-window", "1", "--moe-residency-maintenance-window-growth", "1", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1000", "--moe-dynamic-min-improvement-permille", "0", "--moe-dynamic-max-swaps-per-layer", "20", "--moe-dynamic-max-plan-entries-per-wave", "20", "--moe-dynamic-min-window-activations", "0", "--moe-device-rebalance-maintenance-slack-tokens", "0", "--moe-device-rebalance-min-maintenance-period-tokens", "16", "--moe-device-rebalance-initial-maintenance-period-tokens", "2", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "0", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "0", "--moe-device-rebalance-max-post-wave-load-spread-permille", "1000", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10", "--auto-device-counts", "rocm=1,cpu=2" ], "topology": { "continuation_backend": "rocm", "continuation_devices": 1, "continuation_priority": 0, "cpu_ranks_per_host": 1, "execution_ranks": 3, "kind": "cross-host-expert-overlay", "remote_cpu_hosts": 2, "remote_priority": 1 } } ], "e2e": { "cell_timeout_seconds": { "AVX2": 900, "AVX512": 900 }, "context_length": 8192, "generation_tokens": 2048, "minimum_prompt_tokens": 4096, "movement_evidence": "not_applicable", "planning": { "device_counts": { "rocm": 1 }, "mode": "auto", "mpi_ranks": 1, "strategy": "single" }, "readiness_timeout_seconds": 60, "request_timeout_seconds": 600, "server_args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--tp-allreduce-precision", "schema", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--auto", "--only-backends", "rocm", "--auto-device-counts", "rocm=1", "--only-strategies", "single", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "256", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "1.5", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1300", "--moe-dynamic-min-improvement-permille", "50", "--moe-dynamic-max-swaps-per-layer", "4", "--moe-dynamic-max-plan-entries-per-wave", "16", "--moe-dynamic-min-window-activations", "64", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "15", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "256", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "128", "--moe-device-rebalance-max-post-wave-load-spread-permille", "100", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ], "thinking_modes": "both", "tool_calling": "required" }, "id": "Qwen36MoE_35B_IQ3S_ROCm0_ActFP32_KVFP16_MTPDynamicDepth", "model": "Qwen3.6-35B-A3B-UD-IQ3_S.gguf", "model_parity_schema": 1, "runtime": { "context_length": 4096, "generation": { "max_tokens": 384, "minimum_completion_tokens": 384, "mtp_policy": "dynamic", "mtp_verify_mode": "speculative-sampling", "prefix_state": "hybrid_recurrent", "readiness_timeout_seconds": 60, "requests": [ { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "fresh_harbor", "prefix": "fresh" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_harbor", "prefix": "full" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "partial_mountain", "prefix": "partial" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_mountain", "prefix": "full" } ], "serial_control_id": "Qwen36MoE_35B_IQ3S_ROCm0_ActFP32_KVFP16_MTPOff" }, "movement_evidence": "not_applicable", "server_args": [ "--mpi-procs", "1", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--backend", "host", "--tp-allreduce-precision", "schema", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--device", "localhost:-1:rocm:0", "--tensor-parallelism-degree", "1", "--tp-scope", "auto", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "256", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "1.5", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1300", "--moe-dynamic-min-improvement-permille", "50", "--moe-dynamic-max-swaps-per-layer", "4", "--moe-dynamic-max-plan-entries-per-wave", "16", "--moe-dynamic-min-window-activations", "64", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "15", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "256", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "128", "--moe-device-rebalance-max-post-wave-load-spread-permille", "100", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ] } }, "cpu_isa": "AVX512", "hardware": { "accelerators": [ "0000:03:00.0 VGA compatible controller [0300]: ASPEED Technology, Inc. ASPEED Graphics Family [1a03:2000] (rev 41)", "0000:3f:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:42:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:8c:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:91:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:97:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:9a:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)" ], "cpu": { "Architecture:": "x86_64", "CPU(s):": "112", "Core(s) per socket:": "28", "Model name:": "Intel(R) Xeon(R) Gold 6238R CPU @ 2.20GHz", "NUMA node(s):": "2", "NUMA node0 CPU(s):": "0-27,56-83", "NUMA node1 CPU(s):": "28-55,84-111", "Socket(s):": "2", "Thread(s) per core:": "2" } }, "model_bytes": 15346432288, "model_files": { "Qwen3.6-35B-A3B-UD-IQ3_S.gguf": 15346432288 }, "prefill_tokens": 512, "workload": { "decode_tokens": 256, "measurement_iterations": 3, "prompt_repetitions": 512, "prompt_unit": " test", "schema": 1, "seed": 42, "temperature": 0, "top_k": 1, "top_p": 1, "warmup_iterations": 1 } }, "prefill_tokens": 512, "samples": { "decode": [ 148.50967120487607, 148.51387996045443, 148.5540181803134 ], "prefill": [ 830.9007604621221, 828.8342405773975, 831.5052074215399 ] }, "tokens_per_second": { "decode": 148.51387996045443, "prefill": 830.9007604621221 } }, { "case": "Qwen36MoE/Qwen36MoESingleDeviceParityTest.ProductionParity/Ornith15MoE_35B_Q4KM_ROCm0_ActFP32_KVFP16_MTPDynamicDepth", "identity": { "case": "Qwen36MoE/Qwen36MoESingleDeviceParityTest.ProductionParity/Ornith15MoE_35B_Q4KM_ROCm0_ActFP32_KVFP16_MTPDynamicDepth", "configuration": { "benchmark": { "args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--auto", "--only-backends", "rocm", "--auto-device-counts", "rocm=1", "--only-strategies", "single", "--mtp", "--mtp-depth-policy", "dynamic" ], "context_length": 8192, "policy": "production_defaults", "schema": 1 }, "cross_host_e2e": [], "e2e": { "cell_timeout_seconds": { "AVX2": 900, "AVX512": 900 }, "context_length": 8192, "generation_tokens": 2048, "minimum_prompt_tokens": 4096, "movement_evidence": "not_applicable", "planning": { "device_counts": { "rocm": 1 }, "mode": "auto", "mpi_ranks": 1, "strategy": "single" }, "readiness_timeout_seconds": 60, "request_timeout_seconds": 600, "server_args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--tp-allreduce-precision", "schema", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--auto", "--only-backends", "rocm", "--auto-device-counts", "rocm=1", "--only-strategies", "single", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "256", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "1.5", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1300", "--moe-dynamic-min-improvement-permille", "50", "--moe-dynamic-max-swaps-per-layer", "4", "--moe-dynamic-max-plan-entries-per-wave", "16", "--moe-dynamic-min-window-activations", "64", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "15", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "256", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "128", "--moe-device-rebalance-max-post-wave-load-spread-permille", "100", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ], "thinking_modes": "both", "tool_calling": "required" }, "id": "Ornith15MoE_35B_Q4KM_ROCm0_ActFP32_KVFP16_MTPDynamicDepth", "model": "Ornith-1.5-35B-Q4_K_M.gguf", "model_parity_schema": 1, "runtime": { "context_length": 4096, "generation": { "max_tokens": 384, "minimum_completion_tokens": 384, "mtp_policy": "dynamic", "mtp_verify_mode": "speculative-sampling", "prefix_state": "hybrid_recurrent", "readiness_timeout_seconds": 60, "requests": [ { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "fresh_harbor", "prefix": "fresh" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_harbor", "prefix": "full" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "partial_mountain", "prefix": "partial" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_mountain", "prefix": "full" } ], "serial_control_id": "Ornith15MoE_35B_Q4KM_ROCm0_ActFP32_KVFP16_MTPOff" }, "movement_evidence": "not_applicable", "server_args": [ "--mpi-procs", "1", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--backend", "host", "--tp-allreduce-precision", "schema", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--device", "localhost:-1:rocm:0", "--tensor-parallelism-degree", "1", "--tp-scope", "auto", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "256", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "1.5", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1300", "--moe-dynamic-min-improvement-permille", "50", "--moe-dynamic-max-swaps-per-layer", "4", "--moe-dynamic-max-plan-entries-per-wave", "16", "--moe-dynamic-min-window-activations", "64", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "15", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "256", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "128", "--moe-device-rebalance-max-post-wave-load-spread-permille", "100", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ] } }, "cpu_isa": "AVX512", "hardware": { "accelerators": [ "0000:03:00.0 VGA compatible controller [0300]: ASPEED Technology, Inc. ASPEED Graphics Family [1a03:2000] (rev 41)", "0000:3f:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:42:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:8c:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:91:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:97:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:9a:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)" ], "cpu": { "Architecture:": "x86_64", "CPU(s):": "112", "Core(s) per socket:": "28", "Model name:": "Intel(R) Xeon(R) Gold 6238R CPU @ 2.20GHz", "NUMA node(s):": "2", "NUMA node0 CPU(s):": "0-27,56-83", "NUMA node1 CPU(s):": "28-55,84-111", "Socket(s):": "2", "Thread(s) per core:": "2" } }, "model_bytes": 21713463040, "model_files": { "Ornith-1.5-35B-Q4_K_M.gguf": 21713463040 }, "prefill_tokens": 512, "workload": { "decode_tokens": 256, "measurement_iterations": 3, "prompt_repetitions": 512, "prompt_unit": " test", "schema": 1, "seed": 42, "temperature": 0, "top_k": 1, "top_p": 1, "warmup_iterations": 1 } }, "prefill_tokens": 512, "samples": { "decode": [ 166.16601941928573, 166.2539517716767, 166.2487332156208 ], "prefill": [ 821.0518686961728, 822.0823126614099, 821.7761947393609 ] }, "tokens_per_second": { "decode": 166.2487332156208, "prefill": 821.7761947393609 } }, { "case": "Qwen38Dense/Qwen38DenseMultiDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_ROCm2_TP_ActFP32_KVFP16_MTPDynamicDepth", "identity": { "case": "Qwen38Dense/Qwen38DenseMultiDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_ROCm2_TP_ActFP32_KVFP16_MTPDynamicDepth", "configuration": { "benchmark": { "args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--auto", "--only-backends", "rocm", "--auto-device-counts", "rocm=2", "--only-strategies", "tp", "--mtp", "--mtp-depth-policy", "dynamic" ], "context_length": 8192, "policy": "production_defaults", "schema": 1 }, "cross_host_e2e": [], "e2e": { "cell_timeout_seconds": { "AVX2": 900, "AVX512": 900 }, "context_length": 8192, "generation_tokens": 2048, "minimum_prompt_tokens": 4096, "movement_evidence": "not_applicable", "planning": { "device_counts": { "rocm": 2 }, "mode": "auto", "mpi_ranks": 1, "strategy": "tp" }, "readiness_timeout_seconds": 60, "request_timeout_seconds": 600, "server_args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--auto", "--only-backends", "rocm", "--auto-device-counts", "rocm=2", "--only-strategies", "tp", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "256", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "1.5", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1300", "--moe-dynamic-min-improvement-permille", "50", "--moe-dynamic-max-swaps-per-layer", "4", "--moe-dynamic-max-plan-entries-per-wave", "16", "--moe-dynamic-min-window-activations", "64", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "15", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "256", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "128", "--moe-device-rebalance-max-post-wave-load-spread-permille", "100", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ], "thinking_modes": "both", "tool_calling": "required" }, "id": "Qwen38Dense_27B_IQ4XS_ROCm2_TP_ActFP32_KVFP16_MTPDynamicDepth", "model": "Qwen3.8-27B-IQ4_XS.gguf", "model_parity_schema": 1, "runtime": { "context_length": 4096, "generation": { "max_tokens": 384, "minimum_completion_tokens": 384, "mtp_policy": "dynamic", "mtp_verify_mode": "speculative-sampling", "prefix_state": "hybrid_recurrent", "readiness_timeout_seconds": 60, "requests": [ { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "fresh_harbor", "prefix": "fresh" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_harbor", "prefix": "full" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "partial_mountain", "prefix": "partial" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_mountain", "prefix": "full" } ], "serial_control_id": "Qwen38Dense_27B_IQ4XS_ROCm2_TP_ActFP32_KVFP16_MTPOff" }, "movement_evidence": "not_applicable", "server_args": [ "--mpi-procs", "1", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--backend", "rccl", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--tp-devices", "localhost:-1:rocm:0,localhost:-1:rocm:1", "--tensor-parallelism-degree", "2", "--tp-scope", "rank_local", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "256", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "1.5", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1300", "--moe-dynamic-min-improvement-permille", "50", "--moe-dynamic-max-swaps-per-layer", "4", "--moe-dynamic-max-plan-entries-per-wave", "16", "--moe-dynamic-min-window-activations", "64", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "15", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "256", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "128", "--moe-device-rebalance-max-post-wave-load-spread-permille", "100", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ] } }, "cpu_isa": "AVX512", "hardware": { "accelerators": [ "0000:03:00.0 VGA compatible controller [0300]: ASPEED Technology, Inc. ASPEED Graphics Family [1a03:2000] (rev 41)", "0000:3f:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:42:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:8c:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:91:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:97:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:9a:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)" ], "cpu": { "Architecture:": "x86_64", "CPU(s):": "112", "Core(s) per socket:": "28", "Model name:": "Intel(R) Xeon(R) Gold 6238R CPU @ 2.20GHz", "NUMA node(s):": "2", "NUMA node0 CPU(s):": "0-27,56-83", "NUMA node1 CPU(s):": "28-55,84-111", "Socket(s):": "2", "Thread(s) per core:": "2" } }, "model_bytes": 15705861088, "model_files": { "Qwen3.8-27B-IQ4_XS.gguf": 15705861088 }, "prefill_tokens": 512, "workload": { "decode_tokens": 256, "measurement_iterations": 3, "prompt_repetitions": 512, "prompt_unit": " test", "schema": 1, "seed": 42, "temperature": 0, "top_k": 1, "top_p": 1, "warmup_iterations": 1 } }, "prefill_tokens": 512, "samples": { "decode": [ 39.13832092204711, 39.08806322210646, 39.13709618663582 ], "prefill": [ 247.11624882009409, 247.06009500897653, 247.27092584783452 ] }, "tokens_per_second": { "decode": 39.13709618663582, "prefill": 247.11624882009409 } }, { "case": "Qwen38Dense/Qwen38DenseMultiDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_ROCm2_PP_ActFP32_KVFP16_MTPDynamicDepth", "identity": { "case": "Qwen38Dense/Qwen38DenseMultiDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_ROCm2_PP_ActFP32_KVFP16_MTPDynamicDepth", "configuration": { "benchmark": { "args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--auto", "--only-backends", "rocm", "--auto-device-counts", "rocm=2", "--only-strategies", "pp", "--mtp", "--mtp-depth-policy", "dynamic" ], "context_length": 8192, "policy": "production_defaults", "schema": 1 }, "cross_host_e2e": [], "e2e": { "cell_timeout_seconds": { "AVX2": 900, "AVX512": 900 }, "context_length": 8192, "generation_tokens": 2048, "minimum_prompt_tokens": 4096, "movement_evidence": "not_applicable", "planning": { "device_counts": { "rocm": 2 }, "mode": "auto", "mpi_ranks": 1, "pipeline_domains": [ { "backend": "rocm", "devices": 1 }, { "backend": "rocm", "devices": 1 } ], "pipeline_layers": 64, "strategy": "pp" }, "readiness_timeout_seconds": 60, "request_timeout_seconds": 600, "server_args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--auto", "--only-backends", "rocm", "--auto-device-counts", "rocm=2", "--only-strategies", "pp", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "256", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "1.5", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1300", "--moe-dynamic-min-improvement-permille", "50", "--moe-dynamic-max-swaps-per-layer", "4", "--moe-dynamic-max-plan-entries-per-wave", "16", "--moe-dynamic-min-window-activations", "64", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "15", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "256", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "128", "--moe-device-rebalance-max-post-wave-load-spread-permille", "100", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ], "thinking_modes": "both", "tool_calling": "required" }, "id": "Qwen38Dense_27B_IQ4XS_ROCm2_PP_ActFP32_KVFP16_MTPDynamicDepth", "model": "Qwen3.8-27B-IQ4_XS.gguf", "model_parity_schema": 1, "runtime": { "context_length": 4096, "generation": { "max_tokens": 384, "minimum_completion_tokens": 384, "mtp_policy": "dynamic", "mtp_verify_mode": "speculative-sampling", "prefix_state": "hybrid_recurrent", "readiness_timeout_seconds": 60, "requests": [ { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "fresh_harbor", "prefix": "fresh" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_harbor", "prefix": "full" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "partial_mountain", "prefix": "partial" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_mountain", "prefix": "full" } ], "serial_control_id": "Qwen38Dense_27B_IQ4XS_ROCm2_PP_ActFP32_KVFP16_MTPOff" }, "movement_evidence": "not_applicable", "server_args": [ "--mpi-procs", "1", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--backend", "rccl", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--tensor-parallelism-degree", "1", "--tp-scope", "auto", "--define-domain", "parity_stage_0=localhost:-1:rocm:0;scope=rank_local;backend=auto;owner=0", "--define-domain", "parity_stage_1=localhost:-1:rocm:1;scope=rank_local;backend=auto;owner=0", "--pp-stage", "0=parity_stage_0:0-31", "--pp-stage", "1=parity_stage_1:32-63", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "256", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "1.5", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1300", "--moe-dynamic-min-improvement-permille", "50", "--moe-dynamic-max-swaps-per-layer", "4", "--moe-dynamic-max-plan-entries-per-wave", "16", "--moe-dynamic-min-window-activations", "64", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "15", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "256", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "128", "--moe-device-rebalance-max-post-wave-load-spread-permille", "100", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ] } }, "cpu_isa": "AVX512", "hardware": { "accelerators": [ "0000:03:00.0 VGA compatible controller [0300]: ASPEED Technology, Inc. ASPEED Graphics Family [1a03:2000] (rev 41)", "0000:3f:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:42:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:8c:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:91:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:97:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:9a:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)" ], "cpu": { "Architecture:": "x86_64", "CPU(s):": "112", "Core(s) per socket:": "28", "Model name:": "Intel(R) Xeon(R) Gold 6238R CPU @ 2.20GHz", "NUMA node(s):": "2", "NUMA node0 CPU(s):": "0-27,56-83", "NUMA node1 CPU(s):": "28-55,84-111", "Socket(s):": "2", "Thread(s) per core:": "2" } }, "model_bytes": 15705861088, "model_files": { "Qwen3.8-27B-IQ4_XS.gguf": 15705861088 }, "prefill_tokens": 512, "workload": { "decode_tokens": 256, "measurement_iterations": 3, "prompt_repetitions": 512, "prompt_unit": " test", "schema": 1, "seed": 42, "temperature": 0, "top_k": 1, "top_p": 1, "warmup_iterations": 1 } }, "prefill_tokens": 512, "samples": { "decode": [ 22.37423908293724, 22.97847757078171, 22.35537398965376 ], "prefill": [ 185.1882273814017, 184.82723917284244, 184.6876131883468 ] }, "tokens_per_second": { "decode": 22.37423908293724, "prefill": 184.82723917284244 } }, { "case": "Qwen38Dense/Qwen38DenseSingleDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_ROCm0_ActFP32_KVFP16_MTPDynamicDepth", "identity": { "case": "Qwen38Dense/Qwen38DenseSingleDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_ROCm0_ActFP32_KVFP16_MTPDynamicDepth", "configuration": { "benchmark": { "args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--auto", "--only-backends", "rocm", "--auto-device-counts", "rocm=1", "--only-strategies", "single", "--mtp", "--mtp-depth-policy", "dynamic" ], "context_length": 32768, "policy": "production_defaults", "schema": 1 }, "cross_host_e2e": [], "e2e": { "cell_timeout_seconds": { "AVX2": 900, "AVX512": 900 }, "context_length": 32768, "generation_tokens": 2048, "minimum_prompt_tokens": 16384, "movement_evidence": "not_applicable", "planning": { "device_counts": { "rocm": 1 }, "mode": "auto", "mpi_ranks": 1, "strategy": "single" }, "readiness_timeout_seconds": 60, "request_timeout_seconds": 600, "server_args": [ "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--auto", "--only-backends", "rocm", "--auto-device-counts", "rocm=1", "--only-strategies", "single", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "256", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "1.5", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1300", "--moe-dynamic-min-improvement-permille", "50", "--moe-dynamic-max-swaps-per-layer", "4", "--moe-dynamic-max-plan-entries-per-wave", "16", "--moe-dynamic-min-window-activations", "64", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "15", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "256", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "128", "--moe-device-rebalance-max-post-wave-load-spread-permille", "100", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ], "thinking_modes": "both", "tool_calling": "required" }, "id": "Qwen38Dense_27B_IQ4XS_ROCm0_ActFP32_KVFP16_MTPDynamicDepth", "model": "Qwen3.8-27B-IQ4_XS.gguf", "model_parity_schema": 1, "runtime": { "context_length": 4096, "generation": { "max_tokens": 384, "minimum_completion_tokens": 384, "mtp_policy": "dynamic", "mtp_verify_mode": "speculative-sampling", "prefix_state": "hybrid_recurrent", "readiness_timeout_seconds": 60, "requests": [ { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "fresh_harbor", "prefix": "fresh" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_harbor", "prefix": "full" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "partial_mountain", "prefix": "partial" }, { "body": { "enable_thinking": false, "max_tokens": 384, "messages": [ { "content": "You are a novelist writing a long, detailed story. Write at least two thousand words. Begin directly with the story and continue the scene without a concluding summary. Use clear sentences, concrete physical details, dialogue, and a consistent chronology. Let the characters notice small changes in their surroundings and explain what they do next. Keep the same names and places throughout. Describe the objects they carry, the sounds they hear, the weather, the route they follow, and the choices they make. Introduce discoveries gradually instead of rushing to an ending. Each paragraph should move the story forward while preserving the facts established in previous paragraphs. Do not discuss these instructions or outline a plan. Do not write a title, a word count, or a list. The reader should be able to follow the journey from one location to the next and understand why each character acts. Keep writing the story in ordinary prose for the requested length.", "role": "system" }, { "content": "Write about a lighthouse keeper discovering an unusual map in an old harbor.", "role": "user" }, { "content": "The keeper unfolded the map on the harbor wall. A narrow line crossed the water and climbed into the mountains, ending at an abandoned observatory. Several days later, a mountain guide named Mira reached its locked gate with the keeper's map in her coat. She heard footsteps inside and called out to whoever was waiting beyond the door.", "role": "assistant" } ], "return_runtime_summary": true, "return_token_ids": true, "seed": 4242, "temperature": 0.7, "top_k": 40, "top_p": 0.9 }, "id": "full_mountain", "prefix": "full" } ], "serial_control_id": "Qwen38Dense_27B_IQ4XS_ROCm0_ActFP32_KVFP16_MTPOff" }, "movement_evidence": "not_applicable", "server_args": [ "--mpi-procs", "1", "--activation-precision", "fp32", "--kv-cache-precision", "fp16", "--backend", "host", "--prefix-cache", "--prefix-cache-storage", "tiered", "--prefix-cache-terminal-state", "auto", "--device", "localhost:-1:rocm:0", "--tensor-parallelism-degree", "1", "--tp-scope", "auto", "--mtp-graph-capacity-draft-tokens", "15", "--mtp", "--mtp-draft-tokens", "15", "--mtp-verify-mode", "speculative-sampling", "--mtp-depth-policy", "dynamic", "--mtp-min-draft-tokens", "1", "--mtp-max-draft-tokens", "15", "--mtp-initial-draft-tokens", "15", "--mtp-depth-window", "1", "--mtp-depth-min-samples", "1", "--mtp-depth-cooldown", "0", "--mtp-depth-promote-windows", "1", "--moe-hot-expert-cache", "off", "--moe-residency-maintenance", "dynamic", "--moe-residency-maintenance-window", "256", "--moe-residency-maintenance-max-window", "4096", "--moe-residency-maintenance-window-growth", "1.5", "--moe-migration-payoff-horizon-tokens", "65536", "--moe-migration-transfer-slots", "5", "--moe-dynamic-imbalance-threshold-permille", "1300", "--moe-dynamic-min-improvement-permille", "50", "--moe-dynamic-max-swaps-per-layer", "4", "--moe-dynamic-max-plan-entries-per-wave", "16", "--moe-dynamic-min-window-activations", "64", "--moe-device-rebalance-min-load-spread-improvement", "0", "--moe-device-rebalance-min-load-spread-improvement-divisor", "15", "--moe-device-rebalance-min-wave-spread-improvement-per-payload-slot", "256", "--moe-device-rebalance-min-foreign-rows-per-critical-path-payload-slot", "0", "--moe-device-rebalance-min-router-spread-improvement-per-payload-slot", "128", "--moe-device-rebalance-max-post-wave-load-spread-permille", "100", "--moe-routed-prefill-assignment-window", "0", "--moe-overlay-prefill-segment-rows", "512", "--moe-routed-prefill-least-loaded-min-routed-rows", "8192", "--moe-routed-prefill-llep-alpha-numerator", "1", "--moe-routed-prefill-llep-alpha-denominator", "1", "--moe-routed-prefill-llep-lambda-numerator", "13", "--moe-routed-prefill-llep-lambda-denominator", "10" ] } }, "cpu_isa": "AVX512", "hardware": { "accelerators": [ "0000:03:00.0 VGA compatible controller [0300]: ASPEED Technology, Inc. ASPEED Graphics Family [1a03:2000] (rev 41)", "0000:3f:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:42:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:8c:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:91:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:97:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:9a:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)" ], "cpu": { "Architecture:": "x86_64", "CPU(s):": "112", "Core(s) per socket:": "28", "Model name:": "Intel(R) Xeon(R) Gold 6238R CPU @ 2.20GHz", "NUMA node(s):": "2", "NUMA node0 CPU(s):": "0-27,56-83", "NUMA node1 CPU(s):": "28-55,84-111", "Socket(s):": "2", "Thread(s) per core:": "2" } }, "model_bytes": 15705861088, "model_files": { "Qwen3.8-27B-IQ4_XS.gguf": 15705861088 }, "prefill_tokens": 512, "workload": { "decode_tokens": 256, "measurement_iterations": 3, "prompt_repetitions": 512, "prompt_unit": " test", "schema": 1, "seed": 42, "temperature": 0, "top_k": 1, "top_p": 1, "warmup_iterations": 1 } }, "prefill_tokens": 512, "samples": { "decode": [ 39.22580948320973, 39.22056699586427, 39.2208238918095 ], "prefill": [ 180.56340115386962, 180.2537655956029, 179.9474201300204 ] }, "tokens_per_second": { "decode": 39.2208238918095, "prefill": 180.2537655956029 } } ], "comparisons": [ { "case": "Qwen36MoEExpertOverlay/Qwen36MoEExpertOverlayParityTest.ProductionParity/Qwen36MoE_35B_IQ3S_NodeTP_2xMPI_CPU_ExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "current": 294.6870673512179, "high_water": null, "passed": true, "phase": "prefill", "status": "new_baseline" }, { "case": "Qwen36MoEExpertOverlay/Qwen36MoEExpertOverlayParityTest.ProductionParity/Qwen36MoE_35B_IQ3S_NodeTP_2xMPI_CPU_ExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "current": 37.00911391591542, "high_water": null, "passed": true, "phase": "decode", "status": "new_baseline" }, { "case": "Qwen36MoEExpertOverlay/Qwen36MoEExpertOverlayParityTest.ProductionParity/Ornith15MoE_35B_Q4KM_NodeTP_2xMPI_CPU_ExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "current": 352.3774183487581, "high_water": null, "passed": true, "phase": "prefill", "status": "new_baseline" }, { "case": "Qwen36MoEExpertOverlay/Qwen36MoEExpertOverlayParityTest.ProductionParity/Ornith15MoE_35B_Q4KM_NodeTP_2xMPI_CPU_ExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "current": 42.009006192223886, "high_water": null, "passed": true, "phase": "decode", "status": "new_baseline" }, { "case": "Qwen35_122B_ExpertOverlay/Qwen35MoENodeExpertOverlayParityTest.ProductionParity/Qwen35_122B_CUDA2_CPU2_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "current": 124.92697548586702, "high_water": null, "passed": true, "phase": "prefill", "status": "new_baseline" }, { "case": "Qwen35_122B_ExpertOverlay/Qwen35MoENodeExpertOverlayParityTest.ProductionParity/Qwen35_122B_CUDA2_CPU2_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "current": 14.0495421473084, "high_water": null, "passed": true, "phase": "decode", "status": "new_baseline" }, { "case": "Qwen35_122B_ExpertOverlay/Qwen35MoENodeExpertOverlayParityTest.ProductionParity/Qwen35_122B_CUDA2_ROCm4_CPU2_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "current": 124.55144778685616, "high_water": null, "passed": true, "phase": "prefill", "status": "new_baseline" }, { "case": "Qwen35_122B_ExpertOverlay/Qwen35MoENodeExpertOverlayParityTest.ProductionParity/Qwen35_122B_CUDA2_ROCm4_CPU2_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "current": 20.119436376706723, "high_water": null, "passed": true, "phase": "decode", "status": "new_baseline" }, { "case": "Qwen35_122B_ExpertOverlay/Qwen35MoENodeExpertOverlayParityTest.ProductionParity/Qwen35_122B_ROCm2_CPU2_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "current": 164.72968452054434, "high_water": null, "passed": true, "phase": "prefill", "status": "new_baseline" }, { "case": "Qwen35_122B_ExpertOverlay/Qwen35MoENodeExpertOverlayParityTest.ProductionParity/Qwen35_122B_ROCm2_CPU2_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "current": 22.61551433948939, "high_water": null, "passed": true, "phase": "decode", "status": "new_baseline" }, { "case": "Qwen35_122B_ExpertOverlay/Qwen35MoENodeExpertOverlayParityTest.ProductionParity/Qwen35_122B_ROCm4_CPU2_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "current": 170.75167157290525, "high_water": null, "passed": true, "phase": "prefill", "status": "new_baseline" }, { "case": "Qwen35_122B_ExpertOverlay/Qwen35MoENodeExpertOverlayParityTest.ProductionParity/Qwen35_122B_ROCm4_CPU2_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "current": 27.478068326425486, "high_water": null, "passed": true, "phase": "decode", "status": "new_baseline" }, { "case": "Qwen36MoEExpertOverlay/Qwen36MoEExpertOverlayParityTest.ProductionParity/Ornith15MoE_35B_Q4KM_LocalTP_NCCL_2xCUDA_ExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "current": 887.3892595832347, "high_water": null, "passed": true, "phase": "prefill", "status": "new_baseline" }, { "case": "Qwen36MoEExpertOverlay/Qwen36MoEExpertOverlayParityTest.ProductionParity/Ornith15MoE_35B_Q4KM_LocalTP_NCCL_2xCUDA_ExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "current": 139.71622483036936, "high_water": null, "passed": true, "phase": "decode", "status": "new_baseline" }, { "case": "Qwen36MoE/Qwen36MoESingleDeviceParityTest.ProductionParity/Qwen36MoE_35B_IQ3S_CUDA0_ActFP32_KVFP16_MTPDynamicDepth", "current": 1737.5849757108697, "high_water": null, "passed": true, "phase": "prefill", "status": "new_baseline" }, { "case": "Qwen36MoE/Qwen36MoESingleDeviceParityTest.ProductionParity/Qwen36MoE_35B_IQ3S_CUDA0_ActFP32_KVFP16_MTPDynamicDepth", "current": 283.0466586455166, "high_water": null, "passed": true, "phase": "decode", "status": "new_baseline" }, { "case": "Qwen38Dense/Qwen38DenseMultiDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_CUDA2_TP_ActFP32_KVFP16_MTPDynamicDepth", "current": 468.1480895375244, "high_water": null, "passed": true, "phase": "prefill", "status": "new_baseline" }, { "case": "Qwen38Dense/Qwen38DenseMultiDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_CUDA2_TP_ActFP32_KVFP16_MTPDynamicDepth", "current": 50.37873541979683, "high_water": null, "passed": true, "phase": "decode", "status": "new_baseline" }, { "case": "Qwen38Dense/Qwen38DenseMultiDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_CUDA2_PP_ActFP32_KVFP16_MTPDynamicDepth", "current": 743.7303214331563, "high_water": null, "passed": true, "phase": "prefill", "status": "new_baseline" }, { "case": "Qwen38Dense/Qwen38DenseMultiDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_CUDA2_PP_ActFP32_KVFP16_MTPDynamicDepth", "current": 40.045097658297664, "high_water": null, "passed": true, "phase": "decode", "status": "new_baseline" }, { "case": "Qwen38Dense/Qwen38DenseSingleDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_CUDA0_ActFP32_KVFP16_MTPDynamicDepth", "current": 718.360566494653, "high_water": null, "passed": true, "phase": "prefill", "status": "new_baseline" }, { "case": "Qwen38Dense/Qwen38DenseSingleDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_CUDA0_ActFP32_KVFP16_MTPDynamicDepth", "current": 62.74931544169534, "high_water": null, "passed": true, "phase": "decode", "status": "new_baseline" }, { "case": "Qwen35_122B_ExpertOverlay/Qwen35MoENodeExpertOverlayParityTest.ProductionParity/Qwen35_122B_CUDA2_ROCm4_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "current": 153.59136216002435, "high_water": null, "passed": true, "phase": "prefill", "status": "new_baseline" }, { "case": "Qwen35_122B_ExpertOverlay/Qwen35MoENodeExpertOverlayParityTest.ProductionParity/Qwen35_122B_CUDA2_ROCm4_2xMPI_NodeExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "current": 25.029439150278442, "high_water": null, "passed": true, "phase": "decode", "status": "new_baseline" }, { "case": "Qwen38Dense/Qwen38DenseMultiDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_CUDA2_ROCm2_TPPP_ActFP32_KVFP16_MTPDynamicDepth", "current": 301.66045284430544, "high_water": null, "passed": true, "phase": "prefill", "status": "new_baseline" }, { "case": "Qwen38Dense/Qwen38DenseMultiDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_CUDA2_ROCm2_TPPP_ActFP32_KVFP16_MTPDynamicDepth", "current": 31.994631354057706, "high_water": null, "passed": true, "phase": "decode", "status": "new_baseline" }, { "case": "Qwen36MoEExpertOverlay/Qwen36MoEExpertOverlayParityTest.ProductionParity/Ornith15MoE_35B_Q4KM_LocalTP_RCCL_2xROCm_ExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "current": 775.307896003185, "high_water": null, "passed": true, "phase": "prefill", "status": "new_baseline" }, { "case": "Qwen36MoEExpertOverlay/Qwen36MoEExpertOverlayParityTest.ProductionParity/Ornith15MoE_35B_Q4KM_LocalTP_RCCL_2xROCm_ExpertOverlay_Dynamic_Ordinal_ActFP32_KVFP16_MTPDynamicDepth", "current": 101.96936322324409, "high_water": null, "passed": true, "phase": "decode", "status": "new_baseline" }, { "case": "Qwen36MoE/Qwen36MoESingleDeviceParityTest.ProductionParity/Qwen36MoE_35B_IQ3S_ROCm0_ActFP32_KVFP16_MTPDynamicDepth", "current": 830.9007604621221, "high_water": null, "passed": true, "phase": "prefill", "status": "new_baseline" }, { "case": "Qwen36MoE/Qwen36MoESingleDeviceParityTest.ProductionParity/Qwen36MoE_35B_IQ3S_ROCm0_ActFP32_KVFP16_MTPDynamicDepth", "current": 148.51387996045443, "high_water": null, "passed": true, "phase": "decode", "status": "new_baseline" }, { "case": "Qwen36MoE/Qwen36MoESingleDeviceParityTest.ProductionParity/Ornith15MoE_35B_Q4KM_ROCm0_ActFP32_KVFP16_MTPDynamicDepth", "current": 821.7761947393609, "high_water": null, "passed": true, "phase": "prefill", "status": "new_baseline" }, { "case": "Qwen36MoE/Qwen36MoESingleDeviceParityTest.ProductionParity/Ornith15MoE_35B_Q4KM_ROCm0_ActFP32_KVFP16_MTPDynamicDepth", "current": 166.2487332156208, "high_water": null, "passed": true, "phase": "decode", "status": "new_baseline" }, { "case": "Qwen38Dense/Qwen38DenseMultiDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_ROCm2_TP_ActFP32_KVFP16_MTPDynamicDepth", "current": 247.11624882009409, "high_water": null, "passed": true, "phase": "prefill", "status": "new_baseline" }, { "case": "Qwen38Dense/Qwen38DenseMultiDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_ROCm2_TP_ActFP32_KVFP16_MTPDynamicDepth", "current": 39.13709618663582, "high_water": null, "passed": true, "phase": "decode", "status": "new_baseline" }, { "case": "Qwen38Dense/Qwen38DenseMultiDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_ROCm2_PP_ActFP32_KVFP16_MTPDynamicDepth", "current": 184.82723917284244, "high_water": null, "passed": true, "phase": "prefill", "status": "new_baseline" }, { "case": "Qwen38Dense/Qwen38DenseMultiDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_ROCm2_PP_ActFP32_KVFP16_MTPDynamicDepth", "current": 22.37423908293724, "high_water": null, "passed": true, "phase": "decode", "status": "new_baseline" }, { "case": "Qwen38Dense/Qwen38DenseSingleDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_ROCm0_ActFP32_KVFP16_MTPDynamicDepth", "current": 180.2537655956029, "high_water": null, "passed": true, "phase": "prefill", "status": "new_baseline" }, { "case": "Qwen38Dense/Qwen38DenseSingleDeviceParityTest.ProductionParity/Qwen38Dense_27B_IQ4XS_ROCm0_ActFP32_KVFP16_MTPDynamicDepth", "current": 39.2208238918095, "high_water": null, "passed": true, "phase": "decode", "status": "new_baseline" } ], "e2e_report_digest": "3c95eedff61447c46ced9e973f95c7fd117de839ddf13eb14e93569c8f95d79e", "hardware": { "accelerators": [ "0000:03:00.0 VGA compatible controller [0300]: ASPEED Technology, Inc. ASPEED Graphics Family [1a03:2000] (rev 41)", "0000:3f:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:42:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA102 [GeForce RTX 3090] [10de:2204] (rev a1)", "0000:8c:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:91:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:97:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)", "0000:9a:00.0 Display controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Vega 20 [Radeon Pro VII/Radeon Instinct MI50 32GB] [1002:66a1] (rev 02)" ], "cpu": { "Architecture:": "x86_64", "CPU(s):": "112", "Core(s) per socket:": "28", "Model name:": "Intel(R) Xeon(R) Gold 6238R CPU @ 2.20GHz", "NUMA node(s):": "2", "NUMA node0 CPU(s):": "0-27,56-83", "NUMA node1 CPU(s):": "28-55,84-111", "Socket(s):": "2", "Thread(s) per core:": "2" } }, "report_digest": "3f1facf36d132ecc4b30cfca46dc4b88753a6d15297da6ea8541148911e7348e", "workload": { "decode_tokens": 256, "measurement_iterations": 3, "prompt_repetitions": 512, "prompt_unit": " test", "schema": 1, "seed": 42, "temperature": 0, "top_k": 1, "top_p": 1, "warmup_iterations": 1 } } }, "workflow_run": "https://github.com/Llaminar/llaminar/actions/runs/36288780200" }