zenlm/zen5-engine
1
0
Fork 0
mirror of https://github.com/zenlm/zen5-engine.git synced 2026-07-26 22:09:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add incremental throughput benchmark

Browse Source
This commit is contained in:
antirez
2026-05-11 13:10:50 +02:00
parent 9deabad750
commit 0c1c0232ad
8 changed files with 24399 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+1
View File
@@ -1,5 +1,6 @@
/ds4
/ds4-server
/ds4-bench
/ds4_native
/ds4_server_test
/ds4_test
Makefile
+18 -4
View File
@@ -30,7 +30,7 @@ endif
.PHONY: all clean test cpu
all: ds4 ds4-server
all: ds4 ds4-server ds4-bench
ifeq ($(UNAME_S),Darwin)
ds4: ds4_cli.o linenoise.o $(CORE_OBJS)
@@ -39,9 +39,13 @@ ds4: ds4_cli.o linenoise.o $(CORE_OBJS)
ds4-server: ds4_server.o rax.o $(CORE_OBJS)
$(CC) $(CFLAGS) -o $@ ds4_server.o rax.o $(CORE_OBJS) $(METAL_LDLIBS)
cpu: ds4_cli_cpu.o ds4_server_cpu.o linenoise.o rax.o $(CPU_CORE_OBJS)
ds4-bench: ds4_bench.o $(CORE_OBJS)
$(CC) $(CFLAGS) -o $@ ds4_bench.o $(CORE_OBJS) $(METAL_LDLIBS)
cpu: ds4_cli_cpu.o ds4_server_cpu.o ds4_bench_cpu.o linenoise.o rax.o $(CPU_CORE_OBJS)
$(CC) $(CFLAGS) -o ds4 ds4_cli_cpu.o linenoise.o $(CPU_CORE_OBJS) $(LDLIBS)
$(CC) $(CFLAGS) -o ds4-server ds4_server_cpu.o rax.o $(CPU_CORE_OBJS) $(LDLIBS)
$(CC) $(CFLAGS) -o ds4-bench ds4_bench_cpu.o $(CPU_CORE_OBJS) $(LDLIBS)
else
ds4: ds4_cli.o linenoise.o $(CORE_OBJS)
$(NVCC) $(NVCCFLAGS) -o $@ $^ $(CUDA_LDLIBS)
@@ -49,9 +53,13 @@ ds4: ds4_cli.o linenoise.o $(CORE_OBJS)
ds4-server: ds4_server.o rax.o $(CORE_OBJS)
$(NVCC) $(NVCCFLAGS) -o $@ $^ $(CUDA_LDLIBS)
cpu: ds4_cli_cpu.o ds4_server_cpu.o linenoise.o rax.o $(CPU_CORE_OBJS)
ds4-bench: ds4_bench.o $(CORE_OBJS)
$(NVCC) $(NVCCFLAGS) -o $@ $^ $(CUDA_LDLIBS)
cpu: ds4_cli_cpu.o ds4_server_cpu.o ds4_bench_cpu.o linenoise.o rax.o $(CPU_CORE_OBJS)
$(CC) $(CFLAGS) -o ds4 ds4_cli_cpu.o linenoise.o $(CPU_CORE_OBJS) $(LDLIBS)
$(CC) $(CFLAGS) -o ds4-server ds4_server_cpu.o rax.o $(CPU_CORE_OBJS) $(LDLIBS)
$(CC) $(CFLAGS) -o ds4-bench ds4_bench_cpu.o $(CPU_CORE_OBJS) $(LDLIBS)
endif
ds4.o: ds4.c ds4.h ds4_gpu.h
@@ -63,6 +71,9 @@ ds4_cli.o: ds4_cli.c ds4.h linenoise.h
ds4_server.o: ds4_server.c ds4.h rax.h
$(CC) $(CFLAGS) -c -o $@ ds4_server.c
ds4_bench.o: ds4_bench.c ds4.h
$(CC) $(CFLAGS) -c -o $@ ds4_bench.c
ds4_test.o: tests/ds4_test.c ds4_server.c ds4.h rax.h
$(CC) $(CFLAGS) -Wno-unused-function -c -o $@ tests/ds4_test.c
@@ -81,6 +92,9 @@ ds4_cli_cpu.o: ds4_cli.c ds4.h linenoise.h
ds4_server_cpu.o: ds4_server.c ds4.h rax.h
$(CC) $(CFLAGS) -DDS4_NO_GPU -c -o $@ ds4_server.c
ds4_bench_cpu.o: ds4_bench.c ds4.h
$(CC) $(CFLAGS) -DDS4_NO_GPU -c -o $@ ds4_bench.c
ds4_metal.o: ds4_metal.m ds4_gpu.h $(METAL_SRCS)
$(CC) $(OBJCFLAGS) -c -o $@ ds4_metal.m
@@ -98,4 +112,4 @@ test: ds4_test
./ds4_test
clean:
rm -f ds4 ds4-server ds4_cpu ds4_native ds4_server_test ds4_test *.o
rm -f ds4 ds4-server ds4-bench ds4_cpu ds4_native ds4_server_test ds4_test *.o
README.md
+30
View File
@@ -113,6 +113,36 @@ Q4 requires the larger-memory machine class, so M3 Max Q4 numbers are `N/A`.
| Mac Studio M3 Ultra, 512 GB | q4 | 12018 tokens | 448.82 t/s | 26.62 t/s |
| DGX Spark GB10, 128 GB | q2 | 7047 tokens | 343.81 t/s | 13.75 t/s |
![M3 Max t/s](bench/m3_max_ts.svg)
## Benchmarking
`ds4-bench` measures instantaneous prefill and generation throughput at context
frontiers instead of reporting one whole-run average. It loads the model once,
walks a fixed token sequence to frontiers such as 2048, 4096, 6144, and uses
incremental prefill so each row measures only the newly-added token interval.
After each frontier it saves the live KV state to memory, generates a fixed
greedy non-EOS probe, restores the memory snapshot, and continues prefill.
```sh
./ds4-bench \
-m ds4flash.gguf \
--prompt-file bench/promessi_sposi.txt \
--ctx-start 2048 \
--ctx-max 65536 \
--step-incr 2048 \
--gen-tokens 128
```
The example file is a cleaned public-domain Project Gutenberg text of
Alessandro Manzoni's *I Promessi Sposi* (ebook #45334), with the Gutenberg
header and footer removed: <https://www.gutenberg.org/ebooks/45334>.
Use `--step-incr N` for different linear spacing, or `--step-mul F` for
exponential sweeps. Output is CSV with one row per frontier: latest prefill
interval tokens/sec, generation tokens/sec at that frontier, and
`kvcache_bytes`.
## CLI
One-shot prompt:
bench/m3_max_ts.svg
+509
View File
@@ -0,0 +1,509 @@
<?xml version="1.0" encoding="utf-8" standalone="no"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
<svg xmlns:xlink="http://www.w3.org/1999/xlink" width="661.68pt" height="374.4pt" viewBox="0 0 661.68 374.4" xmlns="http://www.w3.org/2000/svg" version="1.1">
<metadata>
<rdf:RDF xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:cc="http://creativecommons.org/ns#" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
<cc:Work>
<dc:type rdf:resource="http://purl.org/dc/dcmitype/StillImage"/>
<dc:format>image/svg+xml</dc:format>
<dc:creator>
<cc:Agent>
<dc:title>Matplotlib v3.10.9, https://matplotlib.org/</dc:title>
</cc:Agent>
</dc:creator>
</cc:Work>
</rdf:RDF>
</metadata>
<defs>
<style type="text/css">*{stroke-linejoin: round; stroke-linecap: butt}</style>
</defs>
<g id="figure_1">
<g id="patch_1">
<path d="M 0 374.4
L 661.68 374.4
L 661.68 0
L 0 0
z
" style="fill: #ffffff"/>
</g>
<g id="axes_1">
<g id="patch_2">
<path d="M 49.70149 334.107885
L 617.34101 334.107885
L 617.34101 31.437271
L 49.70149 31.437271
z
" style="fill: #fbfcfe"/>
</g>
<g id="matplotlib.axis_1">
<g id="xtick_1">
<g id="line2d_1">
<path d="M 49.70149 334.107885
L 49.70149 31.437271
" clip-path="url(#pe47501eab8)" style="fill: none; stroke: #d8dee8; stroke-width: 0.85; stroke-linecap: square"/>
</g>
<g id="line2d_2">
<defs>
<path id="md65e8e0f99" d="M 0 0
L 0 3.5
" style="stroke: #46515f; stroke-width: 0.8"/>
</defs>
<g>
<use xlink:href="#md65e8e0f99" x="49.70149" y="334.107885" style="fill: #46515f; stroke: #46515f; stroke-width: 0.8"/>
</g>
</g>
<g id="text_1">
<text style="font-size: 10px; font-family: 'DejaVu Sans'; text-anchor: middle; fill: #46515f" x="49.70149" y="348.706323" transform="rotate(-0 49.70149 348.706323)">0</text>
</g>
</g>
<g id="xtick_2">
<g id="line2d_3">
<path d="M 186.469224 334.107885
L 186.469224 31.437271
" clip-path="url(#pe47501eab8)" style="fill: none; stroke: #d8dee8; stroke-width: 0.85; stroke-linecap: square"/>
</g>
<g id="line2d_4">
<g>
<use xlink:href="#md65e8e0f99" x="186.469224" y="334.107885" style="fill: #46515f; stroke: #46515f; stroke-width: 0.8"/>
</g>
</g>
<g id="text_2">
<text style="font-size: 10px; font-family: 'DejaVu Sans'; text-anchor: middle; fill: #46515f" x="186.469224" y="348.706323" transform="rotate(-0 186.469224 348.706323)">32k</text>
</g>
</g>
<g id="xtick_3">
<g id="line2d_5">
<path d="M 323.236958 334.107885
L 323.236958 31.437271
" clip-path="url(#pe47501eab8)" style="fill: none; stroke: #d8dee8; stroke-width: 0.85; stroke-linecap: square"/>
</g>
<g id="line2d_6">
<g>
<use xlink:href="#md65e8e0f99" x="323.236958" y="334.107885" style="fill: #46515f; stroke: #46515f; stroke-width: 0.8"/>
</g>
</g>
<g id="text_3">
<text style="font-size: 10px; font-family: 'DejaVu Sans'; text-anchor: middle; fill: #46515f" x="323.236958" y="348.706323" transform="rotate(-0 323.236958 348.706323)">64k</text>
</g>
</g>
<g id="xtick_4">
<g id="line2d_7">
<path d="M 460.004691 334.107885
L 460.004691 31.437271
" clip-path="url(#pe47501eab8)" style="fill: none; stroke: #d8dee8; stroke-width: 0.85; stroke-linecap: square"/>
</g>
<g id="line2d_8">
<g>
<use xlink:href="#md65e8e0f99" x="460.004691" y="334.107885" style="fill: #46515f; stroke: #46515f; stroke-width: 0.8"/>
</g>
</g>
<g id="text_4">
<text style="font-size: 10px; font-family: 'DejaVu Sans'; text-anchor: middle; fill: #46515f" x="460.004691" y="348.706323" transform="rotate(-0 460.004691 348.706323)">96k</text>
</g>
</g>
<g id="xtick_5">
<g id="line2d_9">
<path d="M 596.772425 334.107885
L 596.772425 31.437271
" clip-path="url(#pe47501eab8)" style="fill: none; stroke: #d8dee8; stroke-width: 0.85; stroke-linecap: square"/>
</g>
<g id="line2d_10">
<g>
<use xlink:href="#md65e8e0f99" x="596.772425" y="334.107885" style="fill: #46515f; stroke: #46515f; stroke-width: 0.8"/>
</g>
</g>
<g id="text_5">
<text style="font-size: 10px; font-family: 'DejaVu Sans'; text-anchor: middle; fill: #46515f" x="596.772425" y="348.706323" transform="rotate(-0 596.772425 348.706323)">128k</text>
</g>
</g>
<g id="text_6">
<text style="font-size: 12px; font-family: 'DejaVu Sans'; text-anchor: middle; fill: #1f2933" x="333.52125" y="368.904135" transform="rotate(-0 333.52125 368.904135)">ctx size</text>
</g>
</g>
<g id="matplotlib.axis_2">
<g id="ytick_1">
<g id="line2d_11">
<path d="M 49.70149 334.107885
L 617.34101 334.107885
" clip-path="url(#pe47501eab8)" style="fill: none; stroke: #d8dee8; stroke-width: 0.85; stroke-linecap: square"/>
</g>
<g id="line2d_12">
<defs>
<path id="m43c3737daf" d="M 0 0
L -3.5 0
" style="stroke: #2563a8; stroke-width: 0.8"/>
</defs>
<g>
<use xlink:href="#m43c3737daf" x="49.70149" y="334.107885" style="fill: #2563a8; stroke: #2563a8; stroke-width: 0.8"/>
</g>
</g>
<g id="text_7">
<text style="font-size: 10px; font-family: 'DejaVu Sans'; text-anchor: end; fill: #2563a8" x="42.70149" y="337.907104" transform="rotate(-0 42.70149 337.907104)">0</text>
</g>
</g>
<g id="ytick_2">
<g id="line2d_13">
<path d="M 49.70149 273.573762
L 617.34101 273.573762
" clip-path="url(#pe47501eab8)" style="fill: none; stroke: #d8dee8; stroke-width: 0.85; stroke-linecap: square"/>
</g>
<g id="line2d_14">
<g>
<use xlink:href="#m43c3737daf" x="49.70149" y="273.573762" style="fill: #2563a8; stroke: #2563a8; stroke-width: 0.8"/>
</g>
</g>
<g id="text_8">
<text style="font-size: 10px; font-family: 'DejaVu Sans'; text-anchor: end; fill: #2563a8" x="42.70149" y="277.372981" transform="rotate(-0 42.70149 277.372981)">50</text>
</g>
</g>
<g id="ytick_3">
<g id="line2d_15">
<path d="M 49.70149 213.039639
L 617.34101 213.039639
" clip-path="url(#pe47501eab8)" style="fill: none; stroke: #d8dee8; stroke-width: 0.85; stroke-linecap: square"/>
</g>
<g id="line2d_16">
<g>
<use xlink:href="#m43c3737daf" x="49.70149" y="213.039639" style="fill: #2563a8; stroke: #2563a8; stroke-width: 0.8"/>
</g>
</g>
<g id="text_9">
<text style="font-size: 10px; font-family: 'DejaVu Sans'; text-anchor: end; fill: #2563a8" x="42.70149" y="216.838858" transform="rotate(-0 42.70149 216.838858)">100</text>
</g>
</g>
<g id="ytick_4">
<g id="line2d_17">
<path d="M 49.70149 152.505517
L 617.34101 152.505517
" clip-path="url(#pe47501eab8)" style="fill: none; stroke: #d8dee8; stroke-width: 0.85; stroke-linecap: square"/>
</g>
<g id="line2d_18">
<g>
<use xlink:href="#m43c3737daf" x="49.70149" y="152.505517" style="fill: #2563a8; stroke: #2563a8; stroke-width: 0.8"/>
</g>
</g>
<g id="text_10">
<text style="font-size: 10px; font-family: 'DejaVu Sans'; text-anchor: end; fill: #2563a8" x="42.70149" y="156.304736" transform="rotate(-0 42.70149 156.304736)">150</text>
</g>
</g>
<g id="ytick_5">
<g id="line2d_19">
<path d="M 49.70149 91.971394
L 617.34101 91.971394
" clip-path="url(#pe47501eab8)" style="fill: none; stroke: #d8dee8; stroke-width: 0.85; stroke-linecap: square"/>
</g>
<g id="line2d_20">
<g>
<use xlink:href="#m43c3737daf" x="49.70149" y="91.971394" style="fill: #2563a8; stroke: #2563a8; stroke-width: 0.8"/>
</g>
</g>
<g id="text_11">
<text style="font-size: 10px; font-family: 'DejaVu Sans'; text-anchor: end; fill: #2563a8" x="42.70149" y="95.770613" transform="rotate(-0 42.70149 95.770613)">200</text>
</g>
</g>
<g id="ytick_6">
<g id="line2d_21">
<path d="M 49.70149 31.437271
L 617.34101 31.437271
" clip-path="url(#pe47501eab8)" style="fill: none; stroke: #d8dee8; stroke-width: 0.85; stroke-linecap: square"/>
</g>
<g id="line2d_22">
<g>
<use xlink:href="#m43c3737daf" x="49.70149" y="31.437271" style="fill: #2563a8; stroke: #2563a8; stroke-width: 0.8"/>
</g>
</g>
<g id="text_12">
<text style="font-size: 10px; font-family: 'DejaVu Sans'; text-anchor: end; fill: #2563a8" x="42.70149" y="35.23649" transform="rotate(-0 42.70149 35.23649)">250</text>
</g>
</g>
<g id="text_13">
<text style="font-size: 12px; font-family: 'DejaVu Sans'; text-anchor: middle; fill: #2563a8" x="12.118365" y="182.772578" transform="rotate(-90 12.118365 182.772578)">prefill t/s</text>
</g>
</g>
<g id="patch_3">
<path d="M 49.70149 334.107885
L 49.70149 31.437271
" style="fill: none; stroke: #2563a8; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square"/>
</g>
<g id="patch_4">
<path d="M 617.34101 334.107885
L 617.34101 31.437271
" style="fill: none; stroke: #000000; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square"/>
</g>
<g id="patch_5">
<path d="M 49.70149 334.107885
L 617.34101 334.107885
" style="fill: none; stroke: #9aa6b2; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square"/>
</g>
<g id="patch_6">
<path d="M 49.70149 31.437271
L 617.34101 31.437271
" style="fill: none; stroke: #9aa6b2; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square"/>
</g>
<g id="line2d_23">
<path d="M 58.249473 52.712594
L 66.797457 86.637127
L 75.34544 91.258302
L 83.893423 92.49683
L 92.441407 101.8094
L 100.98939 105.307061
L 109.537374 114.63537
L 118.085357 129.76769
L 126.63334 125.134408
L 135.181324 123.539939
L 143.729307 130.877885
L 152.27729 136.339274
L 160.825274 135.208497
L 169.373257 135.173387
L 177.92124 152.569683
L 186.469224 137.329612
L 195.017207 139.49068
L 203.56519 139.620223
L 212.113174 144.431475
L 220.661157 146.415784
L 229.209141 166.930798
L 237.757124 153.820318
L 246.305107 157.920899
L 254.853091 149.63741
L 263.401074 162.390739
L 271.949057 164.631712
L 280.497041 164.546964
L 289.045024 164.114751
L 297.593007 167.049445
L 306.140991 157.796199
L 314.688974 161.881042
L 323.236958 160.895546
L 331.784941 161.466988
L 340.332924 183.598264
L 348.880908 166.941694
L 357.428891 168.792828
L 365.976874 168.194751
L 374.524858 168.649967
L 383.072841 170.847356
L 391.620824 172.548365
L 400.168808 174.215475
L 408.716791 175.029053
L 417.264774 176.093243
L 425.812758 179.598169
L 434.360741 179.362086
L 442.908725 180.027961
L 451.456708 182.484436
L 460.004691 182.507439
L 468.552675 183.727807
L 477.100658 183.869456
L 485.648641 187.534192
L 494.196625 198.693052
L 502.744608 193.452008
L 511.292591 201.016352
L 519.840575 191.571818
L 528.388558 190.577848
L 536.936542 191.780056
L 545.484525 193.234085
L 554.032508 194.186892
L 562.580492 194.748649
L 571.128475 195.935118
L 579.676458 197.033207
L 588.224442 197.179699
L 596.772425 200.193088
L 605.320408 200.78269
" clip-path="url(#pe47501eab8)" style="fill: none; stroke: #2563a8; stroke-width: 2.35; stroke-linecap: round"/>
</g>
<g id="text_14">
<text style="font-weight: 700; font-size: 19px; font-family: 'DejaVu Sans'; text-anchor: middle; fill: #1f2933" x="333.52125" y="17.437271" transform="rotate(-0 333.52125 17.437271)">M3 Max t/s</text>
</g>
<g id="legend_1">
<g id="patch_7">
<path d="M 88.759861 304.894236
L 175.145799 304.894236
L 175.145799 262.934861
L 88.759861 262.934861
z
" style="fill: #ffffff; opacity: 0.96; stroke: #c4ccd6; stroke-linejoin: miter"/>
</g>
<g id="patch_8">
<path d="M 95.759861 278.884861
L 105.759861 278.884861
L 105.759861 269.934861
L 95.759861 269.934861
z
" style="fill: #2563a8; stroke: #2563a8; stroke-linejoin: miter"/>
</g>
<g id="text_15">
<text style="font-size: 10px; font-family: 'DejaVu Sans'; text-anchor: start; fill: #1f2933" x="113.759861" y="277.834861" transform="rotate(-0 113.759861 277.834861)">prefill</text>
</g>
<g id="patch_9">
<path d="M 95.759861 296.864548
L 105.759861 296.864548
L 105.759861 287.914548
L 95.759861 287.914548
z
" style="fill: #d4553f; stroke: #d4553f; stroke-linejoin: miter"/>
</g>
<g id="text_16">
<text style="font-size: 10px; font-family: 'DejaVu Sans'; text-anchor: start; fill: #1f2933" x="113.759861" y="295.814548" transform="rotate(-0 113.759861 295.814548)">generation</text>
</g>
</g>
</g>
<g id="axes_2">
<g id="matplotlib.axis_3">
<g id="ytick_7">
<g id="line2d_24">
<defs>
<path id="m1147778ab9" d="M 0 0
L 3.5 0
" style="stroke: #d4553f; stroke-width: 0.8"/>
</defs>
<g>
<use xlink:href="#m1147778ab9" x="617.34101" y="334.107885" style="fill: #d4553f; stroke: #d4553f; stroke-width: 0.8"/>
</g>
</g>
<g id="text_17">
<text style="font-size: 10px; font-family: 'DejaVu Sans'; text-anchor: start; fill: #d4553f" x="624.34101" y="337.907104" transform="rotate(-0 624.34101 337.907104)">0</text>
</g>
</g>
<g id="ytick_8">
<g id="line2d_25">
<g>
<use xlink:href="#m1147778ab9" x="617.34101" y="273.573762" style="fill: #d4553f; stroke: #d4553f; stroke-width: 0.8"/>
</g>
</g>
<g id="text_18">
<text style="font-size: 10px; font-family: 'DejaVu Sans'; text-anchor: start; fill: #d4553f" x="624.34101" y="277.372981" transform="rotate(-0 624.34101 277.372981)">5</text>
</g>
</g>
<g id="ytick_9">
<g id="line2d_26">
<g>
<use xlink:href="#m1147778ab9" x="617.34101" y="213.03964" style="fill: #d4553f; stroke: #d4553f; stroke-width: 0.8"/>
</g>
</g>
<g id="text_19">
<text style="font-size: 10px; font-family: 'DejaVu Sans'; text-anchor: start; fill: #d4553f" x="624.34101" y="216.838858" transform="rotate(-0 624.34101 216.838858)">10</text>
</g>
</g>
<g id="ytick_10">
<g id="line2d_27">
<g>
<use xlink:href="#m1147778ab9" x="617.34101" y="152.505517" style="fill: #d4553f; stroke: #d4553f; stroke-width: 0.8"/>
</g>
</g>
<g id="text_20">
<text style="font-size: 10px; font-family: 'DejaVu Sans'; text-anchor: start; fill: #d4553f" x="624.34101" y="156.304736" transform="rotate(-0 624.34101 156.304736)">15</text>
</g>
</g>
<g id="ytick_11">
<g id="line2d_28">
<g>
<use xlink:href="#m1147778ab9" x="617.34101" y="91.971394" style="fill: #d4553f; stroke: #d4553f; stroke-width: 0.8"/>
</g>
</g>
<g id="text_21">
<text style="font-size: 10px; font-family: 'DejaVu Sans'; text-anchor: start; fill: #d4553f" x="624.34101" y="95.770613" transform="rotate(-0 624.34101 95.770613)">20</text>
</g>
</g>
<g id="ytick_12">
<g id="line2d_29">
<g>
<use xlink:href="#m1147778ab9" x="617.34101" y="31.437271" style="fill: #d4553f; stroke: #d4553f; stroke-width: 0.8"/>
</g>
</g>
<g id="text_22">
<text style="font-size: 10px; font-family: 'DejaVu Sans'; text-anchor: start; fill: #d4553f" x="624.34101" y="35.23649" transform="rotate(-0 624.34101 35.23649)">25</text>
</g>
</g>
<g id="text_23">
<text style="font-size: 12px; font-family: 'DejaVu Sans'; text-anchor: middle; fill: #d4553f" x="656.184135" y="182.772578" transform="rotate(-90 656.184135 182.772578)">generation t/s</text>
</g>
</g>
<g id="patch_10">
<path d="M 49.70149 334.107885
L 49.70149 31.437271
" style="fill: none; stroke: #000000; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square"/>
</g>
<g id="patch_11">
<path d="M 617.34101 334.107885
L 617.34101 31.437271
" style="fill: none; stroke: #d4553f; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square"/>
</g>
<g id="patch_12">
<path d="M 49.70149 334.107885
L 617.34101 334.107885
" style="fill: none; stroke: #000000; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square"/>
</g>
<g id="line2d_30">
<path d="M 58.249473 103.908723
L 66.797457 134.11525
L 75.34544 114.841186
L 83.893423 123.170681
L 92.441407 124.308722
L 100.98939 131.839167
L 109.537374 137.553588
L 118.085357 142.456852
L 126.63334 140.096022
L 135.181324 137.093529
L 143.729307 146.282609
L 152.27729 147.795962
L 160.825274 147.178514
L 169.373257 151.270621
L 177.92124 150.810561
L 186.469224 146.222075
L 195.017207 144.975072
L 203.56519 146.718455
L 212.113174 155.011629
L 220.661157 161.22243
L 229.209141 153.352994
L 237.757124 146.185754
L 246.305107 155.641184
L 254.853091 155.459582
L 263.401074 160.314419
L 271.949057 162.44522
L 280.497041 160.931867
L 289.045024 162.142549
L 297.593007 155.435368
L 306.140991 152.009137
L 314.688974 157.844626
L 323.236958 150.108365
L 331.784941 168.88605
L 340.332924 158.910027
L 348.880908 152.856615
L 357.428891 153.631451
L 365.976874 154.539463
L 374.524858 155.08427
L 383.072841 156.488662
L 391.620824 157.493528
L 400.168808 162.021481
L 408.716791 157.675131
L 417.264774 159.236911
L 425.812758 159.224804
L 434.360741 159.745398
L 442.908725 161.936733
L 451.456708 164.648662
L 460.004691 161.258751
L 468.552675 163.304804
L 477.100658 167.348484
L 485.648641 165.931985
L 494.196625 165.968306
L 502.744608 176.404388
L 511.292591 176.840234
L 519.840575 167.36059
L 528.388558 170.169374
L 536.936542 167.530086
L 545.484525 169.442964
L 554.032508 167.978039
L 562.580492 169.140294
L 571.128475 168.861837
L 579.676458 168.58338
L 588.224442 168.825516
L 596.772425 170.932104
L 605.320408 170.205694
" clip-path="url(#pe47501eab8)" style="fill: none; stroke: #d4553f; stroke-width: 2.35; stroke-linecap: round"/>
</g>
</g>
</g>
<defs>
<clipPath id="pe47501eab8">
<rect x="49.70149" y="31.437271" width="567.63952" height="302.670614"/>
</clipPath>
</defs>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 18 KiB

bench/promessi_sposi.txt
+23329
View File
File diff suppressed because it is too large Load Diff
ds4.c
+83
View File
@@ -16170,6 +16170,74 @@ int ds4_session_load_payload(ds4_session *s, FILE *fp, uint64_t payload_bytes, c
#endif
}
int ds4_session_save_snapshot(ds4_session *s, ds4_session_snapshot *snap, char *err, size_t errlen) {
if (!s || !snap) {
payload_set_err(err, errlen, "invalid session snapshot save");
return 1;
}
const uint64_t bytes = ds4_session_payload_bytes(s);
if (bytes == 0) {
payload_set_err(err, errlen, "session has no valid checkpoint to snapshot");
return 1;
}
if (bytes > (uint64_t)SIZE_MAX) {
payload_set_err(err, errlen, "session snapshot is too large for this platform");
return 1;
}
if (snap->cap < bytes) {
uint8_t *p = realloc(snap->ptr, (size_t)bytes);
if (!p) {
payload_set_err(err, errlen, "out of memory while allocating session snapshot");
return 1;
}
snap->ptr = p;
snap->cap = bytes;
}
FILE *fp = fmemopen(snap->ptr, (size_t)bytes, "wb");
if (!fp) {
payload_set_err(err, errlen, "failed to open memory stream for session snapshot");
return 1;
}
const int rc = ds4_session_save_payload(s, fp, err, errlen);
if (fclose(fp) != 0 && rc == 0) {
payload_set_err(err, errlen, "failed to finalize memory session snapshot");
return 1;
}
if (rc != 0) return 1;
snap->len = bytes;
return 0;
}
int ds4_session_load_snapshot(ds4_session *s, const ds4_session_snapshot *snap, char *err, size_t errlen) {
if (!s || !snap || !snap->ptr || snap->len == 0) {
payload_set_err(err, errlen, "invalid session snapshot load");
return 1;
}
if (snap->len > (uint64_t)SIZE_MAX) {
payload_set_err(err, errlen, "session snapshot is too large for this platform");
return 1;
}
FILE *fp = fmemopen((void *)snap->ptr, (size_t)snap->len, "rb");
if (!fp) {
payload_set_err(err, errlen, "failed to open memory stream for session snapshot restore");
return 1;
}
const int rc = ds4_session_load_payload(s, fp, snap->len, err, errlen);
if (fclose(fp) != 0 && rc == 0) {
payload_set_err(err, errlen, "failed to close memory session snapshot");
return 1;
}
return rc;
}
void ds4_session_snapshot_free(ds4_session_snapshot *snap) {
if (!snap) return;
free(snap->ptr);
memset(snap, 0, sizeof(*snap));
}
void ds4_engine_dump_tokens(ds4_engine *e, const ds4_tokens *tokens) {
dump_tokens(&e->vocab, tokens);
}
@@ -16877,6 +16945,21 @@ int ds4_session_argmax(ds4_session *s) {
return sample_argmax(s->logits, DS4_N_VOCAB);
}
int ds4_session_argmax_excluding(ds4_session *s, int excluded_id) {
if (!s || !s->logits) return -1;
int best = -1;
float best_logit = DS4_NEG_INF;
for (uint32_t i = 0; i < DS4_N_VOCAB; i++) {
if ((int)i == excluded_id) continue;
const float v = s->logits[i];
if (best < 0 || v > best_logit) {
best = (int)i;
best_logit = v;
}
}
return best;
}
int ds4_session_sample(ds4_session *s, float temperature, int top_k, float top_p, float min_p, uint64_t *rng) {
return sample_top_p_min_p(s->logits, DS4_N_VOCAB, temperature, top_k, top_p, min_p, rng);
}
ds4.h
+10
View File
@@ -82,6 +82,12 @@ typedef struct {
uint32_t comp_cap;
} ds4_context_memory;
typedef struct {
uint8_t *ptr;
uint64_t len;
uint64_t cap;
} ds4_session_snapshot;
int ds4_engine_open(ds4_engine **out, const ds4_engine_options *opt);
void ds4_engine_close(ds4_engine *e);
void ds4_engine_summary(ds4_engine *e);
@@ -152,6 +158,7 @@ ds4_session_rewrite_result ds4_session_rewrite_from_common(
char *err, size_t errlen);
int ds4_session_common_prefix(ds4_session *s, const ds4_tokens *prompt);
int ds4_session_argmax(ds4_session *s);
int ds4_session_argmax_excluding(ds4_session *s, int excluded_id);
int ds4_session_sample(ds4_session *s, float temperature, int top_k, float top_p, float min_p, uint64_t *rng);
int ds4_session_top_logprobs(ds4_session *s, ds4_token_score *out, int k);
int ds4_session_eval(ds4_session *s, int token, char *err, size_t errlen);
@@ -173,5 +180,8 @@ const ds4_tokens *ds4_session_tokens(ds4_session *s);
uint64_t ds4_session_payload_bytes(ds4_session *s);
int ds4_session_save_payload(ds4_session *s, FILE *fp, char *err, size_t errlen);
int ds4_session_load_payload(ds4_session *s, FILE *fp, uint64_t payload_bytes, char *err, size_t errlen);
int ds4_session_save_snapshot(ds4_session *s, ds4_session_snapshot *snap, char *err, size_t errlen);
int ds4_session_load_snapshot(ds4_session *s, const ds4_session_snapshot *snap, char *err, size_t errlen);
void ds4_session_snapshot_free(ds4_session_snapshot *snap);
#endif
ds4_bench.c
+419
View File
@@ -0,0 +1,419 @@
#include "ds4.h"
/* Purpose-built throughput benchmark.
*
* The benchmark walks one fixed token sequence to configurable context
* frontiers, measuring only the newest prefill interval at each frontier. It
* then snapshots the live session in memory, performs a fixed greedy decode
* run without allowing EOS, restores the snapshot, and continues to the next
* frontier. Snapshot save/restore time is intentionally outside both timing
* windows.
*/
#include <errno.h>
#include <limits.h>
#include <math.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
typedef struct {
const char *model_path;
const char *prompt_path;
const char *chat_prompt_path;
const char *system;
const char *csv_path;
ds4_backend backend;
int threads;
int ctx_start;
int ctx_max;
int ctx_alloc;
int step_incr;
int gen_tokens;
double step_mul;
bool warm_weights;
bool quality;
} bench_config;
static double bench_now_sec(void) {
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC, &ts);
return (double)ts.tv_sec + (double)ts.tv_nsec / 1000000000.0;
}
static void usage(FILE *fp) {
fprintf(fp,
"Usage: ds4-bench --prompt-file FILE [options]\n"
"\n"
"Benchmarks instantaneous prefill and generation throughput at context\n"
"frontiers such as 2048, 4096, 6144, ... . Generation is always greedy,\n"
"runs for exactly --gen-tokens tokens, and skips EOS so every row is\n"
"comparable.\n"
"\n"
"Input:\n"
" --prompt-file FILE\n"
" Raw benchmark text. The fixed token sequence is sliced at each frontier.\n"
" --chat-prompt-file FILE\n"
" Render FILE as one no-thinking chat user message, then slice that sequence.\n"
" -sys, --system TEXT\n"
" System prompt used only with --chat-prompt-file.\n"
"\n"
"Model and backend:\n"
" -m, --model FILE GGUF model path. Default: ds4flash.gguf\n"
" --metal | --cuda | --cpu | --backend NAME\n"
" Select backend explicitly. Defaults to Metal on macOS, CUDA elsewhere.\n"
" -t, --threads N CPU helper threads.\n"
" --quality Prefer exact kernels where applicable.\n"
" --warm-weights Touch mapped tensor pages before benchmarking.\n"
"\n"
"Sweep:\n"
" --ctx-start N First measured frontier. Default: 2048\n"
" --ctx-max N Last measured frontier. Default: 32768\n"
" --ctx-alloc N Allocated context. Default: ctx-max + gen-tokens + 1\n"
" --step-mul F Multiplicative step. Default: 1\n"
" --step-incr N Linear step when --step-mul is 1. Default: 2048\n"
" --gen-tokens N Greedy decode tokens per frontier. Default: 128\n"
"\n"
"Output:\n"
" --csv FILE Write CSV there instead of stdout.\n"
" -h, --help Show this help.\n");
}
static int parse_int(const char *s, const char *opt) {
char *end = NULL;
long v = strtol(s, &end, 10);
if (s[0] == '\0' || *end != '\0' || v <= 0 || v > INT_MAX) {
fprintf(stderr, "ds4-bench: invalid value for %s: %s\n", opt, s);
exit(2);
}
return (int)v;
}
static double parse_double_arg(const char *s, const char *opt) {
char *end = NULL;
double v = strtod(s, &end);
if (s[0] == '\0' || *end != '\0' || !isfinite(v)) {
fprintf(stderr, "ds4-bench: invalid value for %s: %s\n", opt, s);
exit(2);
}
return v;
}
static const char *need_arg(int *i, int argc, char **argv, const char *opt) {
if (*i + 1 >= argc) {
fprintf(stderr, "ds4-bench: %s requires an argument\n", opt);
exit(2);
}
return argv[++*i];
}
static ds4_backend parse_backend(const char *s, const char *opt) {
if (!strcmp(s, "metal")) return DS4_BACKEND_METAL;
if (!strcmp(s, "cuda")) return DS4_BACKEND_CUDA;
if (!strcmp(s, "cpu")) return DS4_BACKEND_CPU;
fprintf(stderr, "ds4-bench: invalid value for %s: %s\n", opt, s);
fprintf(stderr, "ds4-bench: valid backends are: metal, cuda, cpu\n");
exit(2);
}
static ds4_backend default_backend(void) {
#ifdef DS4_NO_GPU
return DS4_BACKEND_CPU;
#elif defined(__APPLE__)
return DS4_BACKEND_METAL;
#else
return DS4_BACKEND_CUDA;
#endif
}
static char *read_file(const char *path) {
FILE *fp = fopen(path, "rb");
if (!fp) {
fprintf(stderr, "ds4-bench: failed to open %s: %s\n", path, strerror(errno));
exit(1);
}
if (fseek(fp, 0, SEEK_END) != 0) {
fprintf(stderr, "ds4-bench: failed to seek %s\n", path);
fclose(fp);
exit(1);
}
long n = ftell(fp);
if (n < 0) {
fprintf(stderr, "ds4-bench: failed to tell %s\n", path);
fclose(fp);
exit(1);
}
if (fseek(fp, 0, SEEK_SET) != 0) {
fprintf(stderr, "ds4-bench: failed to rewind %s\n", path);
fclose(fp);
exit(1);
}
char *buf = malloc((size_t)n + 1);
if (!buf) {
fprintf(stderr, "ds4-bench: out of memory reading %s\n", path);
fclose(fp);
exit(1);
}
if (fread(buf, 1, (size_t)n, fp) != (size_t)n) {
fprintf(stderr, "ds4-bench: failed to read %s\n", path);
free(buf);
fclose(fp);
exit(1);
}
fclose(fp);
buf[n] = '\0';
return buf;
}
static bench_config parse_options(int argc, char **argv) {
bench_config c = {
.model_path = "ds4flash.gguf",
.system = "You are a helpful assistant.",
.backend = default_backend(),
.ctx_start = 2048,
.ctx_max = 32768,
.step_incr = 2048,
.gen_tokens = 128,
.step_mul = 1.0,
};
for (int i = 1; i < argc; i++) {
const char *arg = argv[i];
if (!strcmp(arg, "-h") || !strcmp(arg, "--help")) {
usage(stdout);
exit(0);
} else if (!strcmp(arg, "-m") || !strcmp(arg, "--model")) {
c.model_path = need_arg(&i, argc, argv, arg);
} else if (!strcmp(arg, "--prompt-file")) {
c.prompt_path = need_arg(&i, argc, argv, arg);
} else if (!strcmp(arg, "--chat-prompt-file")) {
c.chat_prompt_path = need_arg(&i, argc, argv, arg);
} else if (!strcmp(arg, "-sys") || !strcmp(arg, "--system")) {
c.system = need_arg(&i, argc, argv, arg);
} else if (!strcmp(arg, "--ctx-start")) {
c.ctx_start = parse_int(need_arg(&i, argc, argv, arg), arg);
} else if (!strcmp(arg, "--ctx-max")) {
c.ctx_max = parse_int(need_arg(&i, argc, argv, arg), arg);
} else if (!strcmp(arg, "--ctx-alloc")) {
c.ctx_alloc = parse_int(need_arg(&i, argc, argv, arg), arg);
} else if (!strcmp(arg, "--step-incr")) {
c.step_incr = parse_int(need_arg(&i, argc, argv, arg), arg);
} else if (!strcmp(arg, "--step-mul")) {
c.step_mul = parse_double_arg(need_arg(&i, argc, argv, arg), arg);
} else if (!strcmp(arg, "--gen-tokens") || !strcmp(arg, "--tokens") || !strcmp(arg, "-n")) {
c.gen_tokens = parse_int(need_arg(&i, argc, argv, arg), arg);
} else if (!strcmp(arg, "--csv")) {
c.csv_path = need_arg(&i, argc, argv, arg);
} else if (!strcmp(arg, "-t") || !strcmp(arg, "--threads")) {
c.threads = parse_int(need_arg(&i, argc, argv, arg), arg);
} else if (!strcmp(arg, "--backend")) {
c.backend = parse_backend(need_arg(&i, argc, argv, arg), arg);
} else if (!strcmp(arg, "--metal")) {
c.backend = DS4_BACKEND_METAL;
} else if (!strcmp(arg, "--cuda")) {
c.backend = DS4_BACKEND_CUDA;
} else if (!strcmp(arg, "--cpu")) {
c.backend = DS4_BACKEND_CPU;
} else if (!strcmp(arg, "--quality")) {
c.quality = true;
} else if (!strcmp(arg, "--warm-weights")) {
c.warm_weights = true;
} else {
fprintf(stderr, "ds4-bench: unknown option: %s\n", arg);
usage(stderr);
exit(2);
}
}
if (!!c.prompt_path == !!c.chat_prompt_path) {
fprintf(stderr, "ds4-bench: specify exactly one of --prompt-file or --chat-prompt-file\n");
exit(2);
}
if (c.ctx_start > c.ctx_max) {
fprintf(stderr, "ds4-bench: --ctx-start must be <= --ctx-max\n");
exit(2);
}
if (c.step_mul < 1.0) {
fprintf(stderr, "ds4-bench: --step-mul must be >= 1\n");
exit(2);
}
if (c.step_mul == 1.0 && c.step_incr <= 0) {
fprintf(stderr, "ds4-bench: --step-incr must be positive when --step-mul is 1\n");
exit(2);
}
if (c.ctx_max > INT_MAX - c.gen_tokens - 1) {
fprintf(stderr, "ds4-bench: requested context is too large\n");
exit(2);
}
if (c.ctx_alloc == 0) c.ctx_alloc = c.ctx_max + c.gen_tokens + 1;
if (c.ctx_alloc <= c.ctx_max + c.gen_tokens) {
fprintf(stderr, "ds4-bench: --ctx-alloc must be greater than ctx-max + gen-tokens\n");
exit(2);
}
return c;
}
static int next_frontier(const bench_config *c, int cur) {
if (cur >= c->ctx_max) return c->ctx_max;
int next;
if (c->step_mul == 1.0) {
if (cur > INT_MAX - c->step_incr) next = c->ctx_max;
else next = cur + c->step_incr;
} else {
const double v = ceil((double)cur * c->step_mul);
next = v > (double)INT_MAX ? c->ctx_max : (int)v;
if (next <= cur) next = cur + 1;
}
if (next > c->ctx_max) next = c->ctx_max;
return next;
}
static void log_context_memory(ds4_backend backend, int ctx_size) {
ds4_context_memory m = ds4_context_memory_estimate(backend, ctx_size);
fprintf(stderr,
"ds4-bench: context buffers %.2f MiB (ctx=%d, backend=%s, prefill_chunk=%u, raw_kv_rows=%u, compressed_kv_rows=%u)\n",
(double)m.total_bytes / (1024.0 * 1024.0),
ctx_size,
ds4_backend_name(backend),
m.prefill_cap,
m.raw_cap,
m.comp_cap);
}
int main(int argc, char **argv) {
bench_config cfg = parse_options(argc, argv);
log_context_memory(cfg.backend, cfg.ctx_alloc);
ds4_engine_options opt = {
.model_path = cfg.model_path,
.backend = cfg.backend,
.n_threads = cfg.threads,
.warm_weights = cfg.warm_weights,
.quality = cfg.quality,
};
ds4_engine *engine = NULL;
if (ds4_engine_open(&engine, &opt) != 0) return 1;
char *text = read_file(cfg.prompt_path ? cfg.prompt_path : cfg.chat_prompt_path);
ds4_tokens prompt = {0};
if (cfg.chat_prompt_path) {
ds4_encode_chat_prompt(engine, cfg.system, text, DS4_THINK_NONE, &prompt);
} else {
ds4_tokenize_text(engine, text, &prompt);
}
free(text);
if (prompt.len < cfg.ctx_max) {
fprintf(stderr,
"ds4-bench: prompt has %d tokens, need at least --ctx-max=%d\n",
prompt.len,
cfg.ctx_max);
ds4_tokens_free(&prompt);
ds4_engine_close(engine);
return 1;
}
ds4_session *session = NULL;
if (ds4_session_create(&session, engine, cfg.ctx_alloc) != 0) {
fprintf(stderr, "ds4-bench: failed to create session\n");
ds4_tokens_free(&prompt);
ds4_engine_close(engine);
return 1;
}
FILE *out = stdout;
if (cfg.csv_path) {
out = fopen(cfg.csv_path, "wb");
if (!out) {
fprintf(stderr, "ds4-bench: failed to open %s: %s\n", cfg.csv_path, strerror(errno));
ds4_session_free(session);
ds4_tokens_free(&prompt);
ds4_engine_close(engine);
return 1;
}
}
fprintf(out, "ctx_tokens,prefill_tokens,prefill_tps,gen_tokens,gen_tps,kvcache_bytes\n");
fflush(out);
const int eos = ds4_token_eos(engine);
ds4_session_snapshot snap = {0};
char err[256];
int previous = 0;
int rc = 0;
for (int frontier = cfg.ctx_start; ; frontier = next_frontier(&cfg, frontier)) {
ds4_tokens prefix = {
.v = prompt.v,
.len = frontier,
.cap = frontier,
};
const double prefill_t0 = bench_now_sec();
if (ds4_session_sync(session, &prefix, err, sizeof(err)) != 0) {
fprintf(stderr, "ds4-bench: prefill to %d failed: %s\n", frontier, err);
rc = 1;
break;
}
const double prefill_t1 = bench_now_sec();
const double prefill_sec = prefill_t1 - prefill_t0;
const int prefill_tokens = frontier - previous;
if (ds4_session_save_snapshot(session, &snap, err, sizeof(err)) != 0) {
fprintf(stderr, "ds4-bench: snapshot at %d failed: %s\n", frontier, err);
rc = 1;
break;
}
const double gen_t0 = bench_now_sec();
for (int i = 0; i < cfg.gen_tokens; i++) {
if (ds4_session_pos(session) + 1 >= ds4_session_ctx(session)) {
fprintf(stderr, "ds4-bench: generation would exceed allocated context at frontier %d\n", frontier);
rc = 1;
break;
}
const int token = ds4_session_argmax_excluding(session, eos);
if (token < 0) {
fprintf(stderr, "ds4-bench: failed to choose non-EOS token at frontier %d\n", frontier);
rc = 1;
break;
}
if (ds4_session_eval(session, token, err, sizeof(err)) != 0) {
fprintf(stderr, "ds4-bench: decode at frontier %d failed: %s\n", frontier, err);
rc = 1;
break;
}
}
const double gen_t1 = bench_now_sec();
if (rc != 0) break;
if (ds4_session_load_snapshot(session, &snap, err, sizeof(err)) != 0) {
fprintf(stderr, "ds4-bench: restore at %d failed: %s\n", frontier, err);
rc = 1;
break;
}
const double gen_sec = gen_t1 - gen_t0;
fprintf(out,
"%d,%d,%.2f,%d,%.2f,%llu\n",
frontier,
prefill_tokens,
prefill_sec > 0.0 ? (double)prefill_tokens / prefill_sec : 0.0,
cfg.gen_tokens,
gen_sec > 0.0 ? (double)cfg.gen_tokens / gen_sec : 0.0,
(unsigned long long)snap.len);
fflush(out);
previous = frontier;
if (frontier >= cfg.ctx_max) break;
}
if (out != stdout) fclose(out);
ds4_session_snapshot_free(&snap);
ds4_session_free(session);
ds4_tokens_free(&prompt);
ds4_engine_close(engine);
return rc;
}