mirror of
https://github.com/zenlm/zen5-engine.git
synced 2026-07-27 06:13:34 +00:00
cuda: restore q8 fp16 cache on large-memory devices
Use a smaller default Q8->F16 cache reserve on 112+ GiB CUDA devices so GB10 keeps the fast cached weight path by default. Measured on README.md with the DS4 Flash GGUF: prefill 316.85 t/s, generation 13.83 t/s after the cleanup.
This commit is contained in:
@@ -359,6 +359,10 @@ static uint64_t cuda_q8_f16_cache_reserve_bytes(uint64_t total_bytes) {
|
||||
const uint64_t reserve = cuda_parse_mib_env("DS4_CUDA_Q8_F16_CACHE_RESERVE_MB", &present);
|
||||
if (present) return reserve;
|
||||
|
||||
if (total_bytes >= 112ull * 1024ull * 1024ull * 1024ull) {
|
||||
return 512ull * 1048576ull;
|
||||
}
|
||||
|
||||
/* The expanded Q8->F16 cache is only an acceleration path. Keep enough
|
||||
* device memory free for cuBLAS workspaces, transient graph buffers, and
|
||||
* driver bookkeeping instead of letting optional cached weights consume the
|
||||
|
||||
Reference in New Issue
Block a user