device: NVIDIA GB200
shape: experts=8, tokens/expert=4096, M=32768, K=8192, N=4096
dtype: torch.bfloat16
torch: 2.12.0a0+5aff3928d8.nv26.05
transformer_engine: 2.15.0+42b84005
nvidia-cudnn-frontend: 1.23.0
nvidia-cutlass-dsl: 4.4.1
timing mode: CUDA graph replay

activation: swiglu
shape: experts=8, tokens/expert=4096, M=32768, K=8192, N=4096
fused GEMM N=8192, fused C input N=None, activation output N=4096
fused cuDNN avg: 1.550 ms
unfused TE avg:  1.934 ms
speedup vs TE:   1.25x
fused samples:   [1.357, 1.425, 1.64, 1.667, 1.663]
TE samples:      [2.212, 1.937, 1.814, 1.839, 1.869]

activation: dswiglu
shape: experts=8, tokens/expert=4096, M=32768, K=8192, N=4096
fused GEMM N=4096, fused C input N=8192, activation output N=8192
fused cuDNN avg: 0.870 ms
unfused TE avg:  1.734 ms
speedup vs TE:   1.99x
fused samples:   [0.806, 0.806, 0.793, 0.963, 0.98]
TE samples:      [2.254, 1.639, 1.626, 1.579, 1.571]

activation: srelu
shape: experts=8, tokens/expert=4096, M=32768, K=8192, N=4096
fused GEMM N=4096, fused C input N=None, activation output N=4096
fused cuDNN avg: 0.739 ms
unfused TE avg:  1.041 ms
speedup vs TE:   1.41x
fused samples:   [0.721, 0.718, 0.718, 0.734, 0.803]
TE samples:      [1.1, 1.107, 1.023, 1.008, 0.97]

activation: dsrelu
shape: experts=8, tokens/expert=4096, M=32768, K=8192, N=4096
fused GEMM N=4096, fused C input N=4096, activation output N=4096
fused cuDNN avg: 0.781 ms
unfused TE avg:  1.341 ms
speedup vs TE:   1.72x
fused samples:   [0.731, 0.729, 0.73, 0.848, 0.866]
TE samples:      [1.539, 1.386, 1.27, 1.265, 1.243]

activation: geglu
shape: experts=8, tokens/expert=4096, M=32768, K=8192, N=4096
fused GEMM N=8192, fused C input N=None, activation output N=4096
fused cuDNN avg: 1.544 ms
unfused TE avg:  1.890 ms
speedup vs TE:   1.22x
fused samples:   [1.358, 1.523, 1.6, 1.647, 1.595]
TE samples:      [2.036, 1.827, 1.842, 1.87, 1.874]

activation: dgeglu
shape: experts=8, tokens/expert=4096, M=32768, K=8192, N=4096
fused GEMM N=4096, fused C input N=8192, activation output N=8192
fused cuDNN avg: 0.889 ms
unfused TE avg:  1.820 ms
speedup vs TE:   2.05x
fused samples:   [0.8, 0.795, 0.896, 0.951, 1.003]
TE samples:      [2.369, 1.717, 1.668, 1.702, 1.644]

summary:
activation   cudnn_ms      te_ms   te/cudnn
swiglu          1.550      1.934      1.25x
dswiglu         0.870      1.734      1.99x
srelu           0.739      1.041      1.41x
dsrelu          0.781      1.341      1.72x
geglu           1.544      1.890      1.22x
dgeglu          0.889      1.820      2.05x
