mirror of
https://github.com/zenlm/zen5-engine.git
synced 2026-07-26 22:09:00 +00:00
Add DS4 imatrix and GGUF quantization tools
Add routed-MoE imatrix collection to the DS4 graph path, including dataset generation docs and the tracked calibration prompt corpus. Add a standalone DeepSeek V4 Flash HF-to-GGUF quantizer that reuses GGUF template metadata, emits DS4 Q2/Q4 recipes, supports imatrix-aware routed expert quantization, and internalizes the required Q8_0, Q4_K, Q2_K, and IQ2_XXS quantizers instead of vendoring GGML. Document quantizer provenance, output formats, and the DS4-specific imatrix mapping used by the generated GGUF files.
This commit is contained in:
antirez
@@ -53,6 +53,27 @@ However, we try to keep the project in a usable state, and we are making
|
||||
progresses. If you have issues, make sure to use `--trace` to log the
|
||||
sessions, and open issues including the full trace.
|
||||
|
||||
## More Documentation
|
||||
|
||||
If you are looking for very specific things, we have other
|
||||
sub-README files. Otherwise for normal usage keep reading the
|
||||
next sections.
|
||||
|
||||
- [gguf-tools/README.md](gguf-tools/README.md): offline GGUF generation,
|
||||
imatrix collection, quantization tooling, and quality checks.
|
||||
- [gguf-tools/imatrix/README.md](gguf-tools/imatrix/README.md): how the
|
||||
routed-MoE imatrix is collected and used.
|
||||
- [gguf-tools/imatrix/dataset/README.md](gguf-tools/imatrix/dataset/README.md):
|
||||
how the calibration prompt corpus is generated.
|
||||
- [gguf-tools/quality-testing/README.md](gguf-tools/quality-testing/README.md):
|
||||
how local GGUFs are scored against official DeepSeek V4 Flash continuations.
|
||||
- [dir-steering/README.md](dir-steering/README.md): directional steering data,
|
||||
vector generation, and usage.
|
||||
- [speed-bench/README.md](speed-bench/README.md): benchmark CSV files and graph
|
||||
generation.
|
||||
- [tests/test-vectors/README.md](tests/test-vectors/README.md): official
|
||||
continuation vectors used for regression checks.
|
||||
|
||||
## Model Weights
|
||||
|
||||
This implementation only works with the DeepSeek V4 Flash GGUFs published for
|
||||
@@ -88,6 +109,11 @@ only, without an imatrix. The imatrix variants are preferred.
|
||||
Authentication is optional for public downloads, but `--token TOKEN`,
|
||||
`HF_TOKEN`, or the local Hugging Face token cache are used when present.
|
||||
|
||||
If you want to regenerate GGUF files or collect a new imatrix, see
|
||||
[gguf-tools/README.md](gguf-tools/README.md). Those tools are meant for offline
|
||||
model-building work and can take a long time on the full DeepSeek V4 Flash
|
||||
weights.
|
||||
|
||||
`./download_model.sh mtp` fetches the optional speculative decoding support
|
||||
GGUF. It can be used with q2-imatrix, q4-imatrix, q2, and q4, but must be
|
||||
enabled explicitly with `--mtp`. The current MTP/speculative decoding path is
|
||||
|
||||
@@ -8207,6 +8207,7 @@ typedef struct {
|
||||
ds4_gpu_tensor *batch_routed_mid;
|
||||
ds4_gpu_tensor *batch_routed_down;
|
||||
ds4_gpu_tensor *batch_routed_out;
|
||||
bool batch_routed_mid_is_f16;
|
||||
ds4_gpu_tensor *batch_ffn_out;
|
||||
bool materialize_ffn_out;
|
||||
ds4_gpu_tensor *directional_steering_dirs;
|
||||
@@ -12448,7 +12449,8 @@ static bool metal_graph_encode_layer_ffn_batch(
|
||||
DS4_N_EXPERT_USED,
|
||||
DS4_SWIGLU_CLAMP_EXP,
|
||||
g->batch_ffn_norm,
|
||||
n_tokens) != 0;
|
||||
n_tokens,
|
||||
&g->batch_routed_mid_is_f16) != 0;
|
||||
if (ok) {
|
||||
metal_graph_debug_dump_tensor("ffn_moe_gate_clamped", g->batch_routed_gate,
|
||||
(uint64_t)n_tokens * DS4_N_EXPERT_USED * down_in_dim, il, pos0);
|
||||
@@ -12784,6 +12786,236 @@ static bool metal_graph_eval_mtp_draft(
|
||||
top_id);
|
||||
}
|
||||
|
||||
/* =========================================================================
|
||||
* Imatrix Collection.
|
||||
* =========================================================================
|
||||
*
|
||||
* The 2-bit DS4 quants care most about routed MoE experts. For expert gate
|
||||
* and up matrices the matmul input is the FFN-normalized activation row. For
|
||||
* expert down matrices the matmul input is the routed SwiGLU row after route
|
||||
* weighting. During Metal prefill those tensors are already materialized as
|
||||
* `batch_ffn_norm`, `batch_router_selected`, and `batch_routed_mid`, so the
|
||||
* collector observes the exact release graph without changing inference math.
|
||||
*
|
||||
* The output is llama.cpp's legacy imatrix `.dat` format. Entries are packed
|
||||
* by expert: one tensor entry contains `n_expert * n_columns` floats and the
|
||||
* quantizer slices the vector for each expert.
|
||||
*/
|
||||
typedef struct {
|
||||
float *gate_up_sum2; /* [layer][expert][4096] */
|
||||
float *down_sum2; /* [layer][expert][2048] */
|
||||
uint32_t gate_up_count[DS4_N_LAYER][DS4_N_EXPERT];
|
||||
uint32_t down_count[DS4_N_LAYER][DS4_N_EXPERT];
|
||||
float *ffn_norm_buf;
|
||||
float *routed_mid_buf;
|
||||
uint16_t *routed_mid_f16_buf;
|
||||
int *selected_buf;
|
||||
float *sq_tmp;
|
||||
uint32_t cap_tokens;
|
||||
uint64_t observed_tokens;
|
||||
uint64_t observed_routes;
|
||||
uint32_t chunks;
|
||||
const char *dataset_path;
|
||||
} ds4_imatrix_collector;
|
||||
|
||||
static bool imatrix_collector_init(ds4_imatrix_collector *c, uint32_t cap_tokens, const char *dataset_path) {
|
||||
memset(c, 0, sizeof(*c));
|
||||
c->cap_tokens = cap_tokens ? cap_tokens : 1u;
|
||||
c->dataset_path = dataset_path;
|
||||
const size_t gate_n = (size_t)DS4_N_LAYER * DS4_N_EXPERT * DS4_N_EMBD;
|
||||
const size_t down_n = (size_t)DS4_N_LAYER * DS4_N_EXPERT * DS4_N_FF_EXP;
|
||||
c->gate_up_sum2 = xcalloc(gate_n, sizeof(c->gate_up_sum2[0]));
|
||||
c->down_sum2 = xcalloc(down_n, sizeof(c->down_sum2[0]));
|
||||
c->ffn_norm_buf = xmalloc((size_t)c->cap_tokens * DS4_N_EMBD * sizeof(c->ffn_norm_buf[0]));
|
||||
c->routed_mid_buf = xmalloc((size_t)c->cap_tokens * DS4_N_EXPERT_USED * DS4_N_FF_EXP * sizeof(c->routed_mid_buf[0]));
|
||||
c->routed_mid_f16_buf = xmalloc((size_t)c->cap_tokens * DS4_N_EXPERT_USED * DS4_N_FF_EXP * sizeof(c->routed_mid_f16_buf[0]));
|
||||
c->selected_buf = xmalloc((size_t)c->cap_tokens * DS4_N_EXPERT_USED * sizeof(c->selected_buf[0]));
|
||||
c->sq_tmp = xmalloc((size_t)DS4_N_EMBD * sizeof(c->sq_tmp[0]));
|
||||
return c->gate_up_sum2 && c->down_sum2 && c->ffn_norm_buf &&
|
||||
c->routed_mid_buf && c->routed_mid_f16_buf && c->selected_buf && c->sq_tmp;
|
||||
}
|
||||
|
||||
static void imatrix_collector_free(ds4_imatrix_collector *c) {
|
||||
if (!c) return;
|
||||
free(c->gate_up_sum2);
|
||||
free(c->down_sum2);
|
||||
free(c->ffn_norm_buf);
|
||||
free(c->routed_mid_buf);
|
||||
free(c->routed_mid_f16_buf);
|
||||
free(c->selected_buf);
|
||||
free(c->sq_tmp);
|
||||
memset(c, 0, sizeof(*c));
|
||||
}
|
||||
|
||||
static float *imatrix_gate_up_ptr(ds4_imatrix_collector *c, uint32_t il, uint32_t expert) {
|
||||
return c->gate_up_sum2 + ((size_t)il * DS4_N_EXPERT + expert) * DS4_N_EMBD;
|
||||
}
|
||||
|
||||
static float *imatrix_down_ptr(ds4_imatrix_collector *c, uint32_t il, uint32_t expert) {
|
||||
return c->down_sum2 + ((size_t)il * DS4_N_EXPERT + expert) * DS4_N_FF_EXP;
|
||||
}
|
||||
|
||||
static bool imatrix_collect_layer_batch(
|
||||
ds4_imatrix_collector *c,
|
||||
ds4_gpu_graph *g,
|
||||
uint32_t il,
|
||||
uint32_t n_tokens) {
|
||||
if (!c || n_tokens == 0) return true;
|
||||
if (n_tokens > c->cap_tokens) return false;
|
||||
|
||||
const uint64_t norm_bytes = (uint64_t)n_tokens * DS4_N_EMBD * sizeof(float);
|
||||
const uint64_t mid_elems = (uint64_t)n_tokens * DS4_N_EXPERT_USED * DS4_N_FF_EXP;
|
||||
const uint64_t mid_bytes = mid_elems * (g->batch_routed_mid_is_f16 ? sizeof(uint16_t) : sizeof(float));
|
||||
const uint64_t sel_bytes = (uint64_t)n_tokens * DS4_N_EXPERT_USED * sizeof(int);
|
||||
void *mid_dst = g->batch_routed_mid_is_f16
|
||||
? (void *)c->routed_mid_f16_buf
|
||||
: (void *)c->routed_mid_buf;
|
||||
if (ds4_gpu_tensor_read(g->batch_ffn_norm, 0, c->ffn_norm_buf, norm_bytes) == 0 ||
|
||||
ds4_gpu_tensor_read(g->batch_routed_mid, 0, mid_dst, mid_bytes) == 0 ||
|
||||
ds4_gpu_tensor_read(g->batch_router_selected, 0, c->selected_buf, sel_bytes) == 0)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
for (uint32_t t = 0; t < n_tokens; t++) {
|
||||
const float *x = c->ffn_norm_buf + (size_t)t * DS4_N_EMBD;
|
||||
for (uint32_t i = 0; i < DS4_N_EMBD; i++) c->sq_tmp[i] = x[i] * x[i];
|
||||
|
||||
for (uint32_t slot = 0; slot < DS4_N_EXPERT_USED; slot++) {
|
||||
const int expert = c->selected_buf[(size_t)t * DS4_N_EXPERT_USED + slot];
|
||||
if (expert < 0 || expert >= DS4_N_EXPERT) continue;
|
||||
|
||||
float *gate_up = imatrix_gate_up_ptr(c, il, (uint32_t)expert);
|
||||
for (uint32_t i = 0; i < DS4_N_EMBD; i++) gate_up[i] += c->sq_tmp[i];
|
||||
c->gate_up_count[il][expert]++;
|
||||
|
||||
float *down = imatrix_down_ptr(c, il, (uint32_t)expert);
|
||||
const size_t mid_off = ((size_t)t * DS4_N_EXPERT_USED + slot) * DS4_N_FF_EXP;
|
||||
if (g->batch_routed_mid_is_f16) {
|
||||
const uint16_t *mid = c->routed_mid_f16_buf + mid_off;
|
||||
for (uint32_t i = 0; i < DS4_N_FF_EXP; i++) {
|
||||
const float v = f16_to_f32(mid[i]);
|
||||
down[i] += v * v;
|
||||
}
|
||||
} else {
|
||||
const float *mid = c->routed_mid_buf + mid_off;
|
||||
for (uint32_t i = 0; i < DS4_N_FF_EXP; i++) down[i] += mid[i] * mid[i];
|
||||
}
|
||||
c->down_count[il][expert]++;
|
||||
c->observed_routes++;
|
||||
}
|
||||
}
|
||||
c->observed_tokens += n_tokens;
|
||||
c->chunks++;
|
||||
return true;
|
||||
}
|
||||
|
||||
static void imatrix_write_i32(FILE *fp, int32_t v) {
|
||||
if (fwrite(&v, sizeof(v), 1, fp) != 1) ds4_die("failed to write imatrix");
|
||||
}
|
||||
|
||||
static void imatrix_write_entry(
|
||||
FILE *fp,
|
||||
const char *name,
|
||||
const float *sum2,
|
||||
const uint32_t *counts,
|
||||
uint32_t n_expert,
|
||||
uint32_t n_col) {
|
||||
const int32_t len = (int32_t)strlen(name);
|
||||
const int32_t ncall = 1;
|
||||
const int32_t nval = (int32_t)((uint64_t)n_expert * n_col);
|
||||
imatrix_write_i32(fp, len);
|
||||
if (fwrite(name, 1, (size_t)len, fp) != (size_t)len) ds4_die("failed to write imatrix name");
|
||||
imatrix_write_i32(fp, ncall);
|
||||
imatrix_write_i32(fp, nval);
|
||||
|
||||
float *tmp = xmalloc((size_t)n_col * sizeof(tmp[0]));
|
||||
for (uint32_t e = 0; e < n_expert; e++) {
|
||||
const uint32_t count = counts[e];
|
||||
const float *src = sum2 + (size_t)e * n_col;
|
||||
if (count == 0) {
|
||||
for (uint32_t i = 0; i < n_col; i++) tmp[i] = 1.0f;
|
||||
} else {
|
||||
const float inv = 1.0f / (float)count;
|
||||
for (uint32_t i = 0; i < n_col; i++) tmp[i] = src[i] * inv;
|
||||
}
|
||||
if (fwrite(tmp, sizeof(tmp[0]), n_col, fp) != n_col) ds4_die("failed to write imatrix values");
|
||||
}
|
||||
free(tmp);
|
||||
}
|
||||
|
||||
static bool imatrix_collector_save(
|
||||
const ds4_imatrix_collector *c,
|
||||
const ds4_weights *weights,
|
||||
const char *path) {
|
||||
FILE *fp = fopen(path, "wb");
|
||||
if (!fp) {
|
||||
fprintf(stderr, "ds4: failed to open imatrix output %s: %s\n", path, strerror(errno));
|
||||
return false;
|
||||
}
|
||||
|
||||
const int32_t entries = (int32_t)(DS4_N_LAYER * 3);
|
||||
imatrix_write_i32(fp, entries);
|
||||
for (uint32_t il = 0; il < DS4_N_LAYER; il++) {
|
||||
const ds4_layer_weights *layer = &weights->layer[il];
|
||||
char name[256];
|
||||
snprintf(name, sizeof(name), "%.*s", (int)layer->ffn_gate_exps->name.len, layer->ffn_gate_exps->name.ptr);
|
||||
imatrix_write_entry(fp, name,
|
||||
c->gate_up_sum2 + (size_t)il * DS4_N_EXPERT * DS4_N_EMBD,
|
||||
c->gate_up_count[il],
|
||||
DS4_N_EXPERT,
|
||||
DS4_N_EMBD);
|
||||
snprintf(name, sizeof(name), "%.*s", (int)layer->ffn_up_exps->name.len, layer->ffn_up_exps->name.ptr);
|
||||
imatrix_write_entry(fp, name,
|
||||
c->gate_up_sum2 + (size_t)il * DS4_N_EXPERT * DS4_N_EMBD,
|
||||
c->gate_up_count[il],
|
||||
DS4_N_EXPERT,
|
||||
DS4_N_EMBD);
|
||||
snprintf(name, sizeof(name), "%.*s", (int)layer->ffn_down_exps->name.len, layer->ffn_down_exps->name.ptr);
|
||||
imatrix_write_entry(fp, name,
|
||||
c->down_sum2 + (size_t)il * DS4_N_EXPERT * DS4_N_FF_EXP,
|
||||
c->down_count[il],
|
||||
DS4_N_EXPERT,
|
||||
DS4_N_FF_EXP);
|
||||
}
|
||||
|
||||
const int32_t chunks = (int32_t)c->chunks;
|
||||
imatrix_write_i32(fp, chunks);
|
||||
const char *dataset = c->dataset_path ? c->dataset_path : "";
|
||||
const int32_t dataset_len = (int32_t)strlen(dataset);
|
||||
imatrix_write_i32(fp, dataset_len);
|
||||
if (dataset_len && fwrite(dataset, 1, (size_t)dataset_len, fp) != (size_t)dataset_len) {
|
||||
ds4_die("failed to write imatrix dataset name");
|
||||
}
|
||||
|
||||
if (fclose(fp) != 0) {
|
||||
fprintf(stderr, "ds4: failed to close imatrix output %s: %s\n", path, strerror(errno));
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool metal_graph_reset_prefill_state(ds4_gpu_graph *g) {
|
||||
memset(g->layer_n_index_comp, 0, sizeof(g->layer_n_index_comp));
|
||||
g->mtp_n_raw = 0;
|
||||
for (uint32_t il = 0; il < DS4_N_LAYER; il++) {
|
||||
const uint32_t ratio = ds4_layer_compress_ratio(il);
|
||||
if (ratio == 0) continue;
|
||||
const uint32_t coff = ratio == 4 ? 2u : 1u;
|
||||
const uint64_t attn_width = (uint64_t)coff * DS4_N_HEAD_DIM;
|
||||
const uint64_t attn_rows = (uint64_t)coff * ratio;
|
||||
if (!metal_tensor_fill_f32(g->layer_attn_state_kv[il], 0.0f, attn_width * attn_rows)) return false;
|
||||
if (!metal_tensor_fill_f32(g->layer_attn_state_score[il], DS4_NEG_INF, attn_width * attn_rows)) return false;
|
||||
if (ratio == 4) {
|
||||
const uint64_t index_width = (uint64_t)coff * DS4_N_INDEXER_HEAD_DIM;
|
||||
const uint64_t index_rows = (uint64_t)coff * ratio;
|
||||
if (!metal_tensor_fill_f32(g->layer_index_state_kv[il], 0.0f, index_width * index_rows)) return false;
|
||||
if (!metal_tensor_fill_f32(g->layer_index_state_score[il], DS4_NEG_INF, index_width * index_rows)) return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/* Execute Metal prefill in layer-major order so intermediate activations stay
|
||||
* on the GPU and cache state is built exactly once. */
|
||||
static bool metal_graph_prefill_layer_major(
|
||||
@@ -12793,7 +13025,8 @@ static bool metal_graph_prefill_layer_major(
|
||||
const token_vec *prompt,
|
||||
int n_tokens,
|
||||
float *logits,
|
||||
bool show_progress) {
|
||||
bool show_progress,
|
||||
ds4_imatrix_collector *imatrix) {
|
||||
if (n_tokens <= 0 || n_tokens > prompt->len || (uint32_t)n_tokens > g->prefill_cap) return false;
|
||||
|
||||
bool ok = metal_graph_upload_prompt_tokens(g->prefill_tokens, prompt, 0, (uint32_t)n_tokens);
|
||||
@@ -12808,7 +13041,7 @@ static bool metal_graph_prefill_layer_major(
|
||||
* low overhead, but submit long prompts layer by layer so the display
|
||||
* server gets regular scheduling points.
|
||||
*/
|
||||
const bool split_commands = split_profile || n_tokens > 2048;
|
||||
const bool split_commands = split_profile || n_tokens > 2048 || imatrix != NULL;
|
||||
const bool profile = getenv("DS4_METAL_GRAPH_PREFILL_PROFILE") != NULL || split_profile;
|
||||
const double t0 = profile ? now_sec() : 0.0;
|
||||
double encode_s = 0.0;
|
||||
@@ -12945,6 +13178,7 @@ static bool metal_graph_prefill_layer_major(
|
||||
const double t_ffn_encoded = now_sec();
|
||||
if (ok) ok = ds4_gpu_end_commands() != 0;
|
||||
const double t_ffn_done = now_sec();
|
||||
if (ok && imatrix) ok = imatrix_collect_layer_batch(imatrix, g, il, (uint32_t)n_tokens);
|
||||
|
||||
encode_s += (t_attn_encoded - t_attn0) + (t_ffn_encoded - t_ffn0);
|
||||
execute_s += (t_attn_done - t_attn_encoded) + (t_ffn_done - t_ffn_encoded);
|
||||
@@ -12967,6 +13201,7 @@ static bool metal_graph_prefill_layer_major(
|
||||
const double t_encoded = profile ? now_sec() : 0.0;
|
||||
if (ok) ok = ds4_gpu_end_commands() != 0;
|
||||
const double t_done = profile ? now_sec() : 0.0;
|
||||
if (ok && imatrix) ok = imatrix_collect_layer_batch(imatrix, g, il, (uint32_t)n_tokens);
|
||||
if (profile) {
|
||||
encode_s += t_encoded - t_chunk0;
|
||||
execute_s += t_done - t_encoded;
|
||||
@@ -13052,7 +13287,7 @@ static bool metal_graph_prefill_raw_swa(
|
||||
bool show_progress) {
|
||||
if (n_tokens <= 0 || n_tokens > prompt->len) return false;
|
||||
if ((uint32_t)n_tokens > g->prefill_cap) return false;
|
||||
return metal_graph_prefill_layer_major(g, model, weights, prompt, n_tokens, logits, show_progress);
|
||||
return metal_graph_prefill_layer_major(g, model, weights, prompt, n_tokens, logits, show_progress, NULL);
|
||||
}
|
||||
|
||||
static bool metal_graph_prefill_batch_row_logits(
|
||||
@@ -13098,7 +13333,8 @@ static bool metal_graph_prefill_chunked_range(
|
||||
float *logits,
|
||||
bool show_progress,
|
||||
ds4_session_progress_fn progress,
|
||||
void *progress_ud) {
|
||||
void *progress_ud,
|
||||
ds4_imatrix_collector *imatrix) {
|
||||
if (n_tokens == 0 || g->prefill_cap == 0) return false;
|
||||
if (start > (uint32_t)prompt->len) return false;
|
||||
if (n_tokens > (uint32_t)prompt->len - start) return false;
|
||||
@@ -13163,6 +13399,7 @@ static bool metal_graph_prefill_chunked_range(
|
||||
const double t_encoded = profile ? now_sec() : 0.0;
|
||||
if (ok) ok = ds4_gpu_end_commands() != 0;
|
||||
const double t_done = profile ? now_sec() : 0.0;
|
||||
if (ok && imatrix) ok = imatrix_collect_layer_batch(imatrix, g, il, chunk);
|
||||
if (profile) {
|
||||
encode_s += t_encoded - t_layer0;
|
||||
execute_s += t_done - t_encoded;
|
||||
@@ -13265,7 +13502,8 @@ static bool metal_graph_prefill_chunked(
|
||||
logits,
|
||||
show_progress,
|
||||
progress,
|
||||
progress_ud);
|
||||
progress_ud,
|
||||
NULL);
|
||||
}
|
||||
|
||||
/* Layer-major speculative target verifier for tiny MTP suffixes.
|
||||
@@ -16295,6 +16533,194 @@ int ds4_dump_text_tokenization(const char *model_path, const char *text, FILE *f
|
||||
return 0;
|
||||
}
|
||||
|
||||
#ifndef DS4_NO_GPU
|
||||
static bool imatrix_read_text_file(const char *path, char **out, size_t *len_out) {
|
||||
*out = NULL;
|
||||
*len_out = 0;
|
||||
struct stat st;
|
||||
if (stat(path, &st) != 0) {
|
||||
fprintf(stderr, "ds4: failed to stat imatrix dataset %s: %s\n", path, strerror(errno));
|
||||
return false;
|
||||
}
|
||||
if (st.st_size < 0 || (uint64_t)st.st_size > SIZE_MAX - 1) {
|
||||
fprintf(stderr, "ds4: imatrix dataset is too large: %s\n", path);
|
||||
return false;
|
||||
}
|
||||
FILE *fp = fopen(path, "rb");
|
||||
if (!fp) {
|
||||
fprintf(stderr, "ds4: failed to open imatrix dataset %s: %s\n", path, strerror(errno));
|
||||
return false;
|
||||
}
|
||||
size_t n = (size_t)st.st_size;
|
||||
char *buf = xmalloc(n + 1);
|
||||
if (n != 0 && fread(buf, 1, n, fp) != n) {
|
||||
fprintf(stderr, "ds4: failed to read imatrix dataset %s\n", path);
|
||||
fclose(fp);
|
||||
free(buf);
|
||||
return false;
|
||||
}
|
||||
if (fclose(fp) != 0) {
|
||||
fprintf(stderr, "ds4: failed to close imatrix dataset %s: %s\n", path, strerror(errno));
|
||||
free(buf);
|
||||
return false;
|
||||
}
|
||||
buf[n] = '\0';
|
||||
*out = buf;
|
||||
*len_out = n;
|
||||
return true;
|
||||
}
|
||||
|
||||
static char *imatrix_trim_block(char *p, char *end) {
|
||||
while (p < end && isspace((unsigned char)*p)) p++;
|
||||
while (end > p && isspace((unsigned char)end[-1])) end--;
|
||||
*end = '\0';
|
||||
return p;
|
||||
}
|
||||
#endif
|
||||
|
||||
int ds4_engine_collect_imatrix(ds4_engine *e,
|
||||
const char *dataset_path,
|
||||
const char *output_path,
|
||||
int ctx_size,
|
||||
int max_prompts,
|
||||
int max_tokens) {
|
||||
#ifdef DS4_NO_GPU
|
||||
(void)e;
|
||||
(void)dataset_path;
|
||||
(void)output_path;
|
||||
(void)ctx_size;
|
||||
(void)max_prompts;
|
||||
(void)max_tokens;
|
||||
fprintf(stderr, "ds4: imatrix collection requires a graph backend build\n");
|
||||
return 1;
|
||||
#else
|
||||
if (!e || !dataset_path || !output_path) return 1;
|
||||
if (e->backend != DS4_BACKEND_METAL || !e->metal_ready) {
|
||||
fprintf(stderr, "ds4: imatrix collection currently requires --metal\n");
|
||||
return 1;
|
||||
}
|
||||
if (ctx_size <= 0) ctx_size = 32768;
|
||||
|
||||
char *dataset = NULL;
|
||||
size_t dataset_len = 0;
|
||||
if (!imatrix_read_text_file(dataset_path, &dataset, &dataset_len)) return 1;
|
||||
|
||||
const ds4_model *model = &e->model;
|
||||
const ds4_weights *weights = &e->weights;
|
||||
const uint32_t prefill_cap = metal_graph_prefill_cap_for_prompt(ctx_size);
|
||||
const uint32_t raw_cap = metal_graph_raw_cap_for_context(ctx_size, prefill_cap);
|
||||
|
||||
ds4_gpu_graph g;
|
||||
bool ok = metal_graph_alloc_raw_cap(&g, weights, &weights->layer[0],
|
||||
raw_cap, (uint32_t)ctx_size, prefill_cap, false);
|
||||
if (!ok) {
|
||||
fprintf(stderr, "ds4: failed to allocate imatrix Metal graph runtime\n");
|
||||
free(dataset);
|
||||
return 1;
|
||||
}
|
||||
g.quality = e->quality;
|
||||
|
||||
ds4_imatrix_collector collector;
|
||||
if (!imatrix_collector_init(&collector, prefill_cap, dataset_path)) {
|
||||
fprintf(stderr, "ds4: failed to allocate imatrix collector\n");
|
||||
metal_graph_free(&g);
|
||||
free(dataset);
|
||||
return 1;
|
||||
}
|
||||
|
||||
fprintf(stderr,
|
||||
"ds4: collecting routed-MoE imatrix from %s (ctx=%d, chunk=%u)\n",
|
||||
dataset_path, ctx_size, prefill_cap);
|
||||
|
||||
int prompts_done = 0;
|
||||
int tokens_done = 0;
|
||||
char *cursor = dataset;
|
||||
const char *marker_lit = "===== DS4_IMATRIX_PROMPT";
|
||||
while (*cursor) {
|
||||
char *start = cursor;
|
||||
char *marker = strstr(cursor, marker_lit);
|
||||
if (marker) {
|
||||
char *nl = strchr(marker, '\n');
|
||||
if (!nl) break;
|
||||
start = nl + 1;
|
||||
} else if (prompts_done != 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
char *next = strstr(start, marker_lit);
|
||||
char *end = next ? next : dataset + dataset_len;
|
||||
char saved = *end;
|
||||
char *prompt_text = imatrix_trim_block(start, end);
|
||||
if (prompt_text[0] != '\0') {
|
||||
token_vec prompt = {0};
|
||||
ds4_tokenize_rendered_chat(e, prompt_text, &prompt);
|
||||
if (prompt.len > ctx_size) prompt.len = ctx_size;
|
||||
if (max_tokens > 0 && prompt.len > max_tokens - tokens_done) {
|
||||
prompt.len = max_tokens - tokens_done;
|
||||
}
|
||||
if (prompt.len > 0) {
|
||||
if (!metal_graph_reset_prefill_state(&g)) {
|
||||
fprintf(stderr, "ds4: failed to reset imatrix graph state\n");
|
||||
ok = false;
|
||||
} else if ((uint32_t)prompt.len > prefill_cap) {
|
||||
ok = metal_graph_prefill_chunked_range(&g, model, weights,
|
||||
&prompt, 0,
|
||||
(uint32_t)prompt.len,
|
||||
NULL, false,
|
||||
NULL, NULL,
|
||||
&collector);
|
||||
} else {
|
||||
ok = metal_graph_prefill_layer_major(&g, model, weights,
|
||||
&prompt, prompt.len,
|
||||
NULL, false,
|
||||
&collector);
|
||||
}
|
||||
if (!ok) {
|
||||
fprintf(stderr, "ds4: imatrix prefill failed at prompt %d\n", prompts_done + 1);
|
||||
token_vec_free(&prompt);
|
||||
*end = saved;
|
||||
break;
|
||||
}
|
||||
prompts_done++;
|
||||
tokens_done += prompt.len;
|
||||
if (prompts_done % 10 == 0) {
|
||||
fprintf(stderr,
|
||||
"ds4: imatrix prompts=%d tokens=%d routes=%llu\r",
|
||||
prompts_done,
|
||||
tokens_done,
|
||||
(unsigned long long)collector.observed_routes);
|
||||
fflush(stderr);
|
||||
}
|
||||
}
|
||||
token_vec_free(&prompt);
|
||||
}
|
||||
*end = saved;
|
||||
if (!next) break;
|
||||
cursor = next;
|
||||
if (max_prompts > 0 && prompts_done >= max_prompts) break;
|
||||
if (max_tokens > 0 && tokens_done >= max_tokens) break;
|
||||
}
|
||||
fputc('\n', stderr);
|
||||
|
||||
if (ok) {
|
||||
ok = imatrix_collector_save(&collector, weights, output_path);
|
||||
if (ok) {
|
||||
fprintf(stderr,
|
||||
"ds4: wrote imatrix %s from %d prompts, %d tokens, %llu routed expert observations\n",
|
||||
output_path,
|
||||
prompts_done,
|
||||
tokens_done,
|
||||
(unsigned long long)collector.observed_routes);
|
||||
}
|
||||
}
|
||||
|
||||
imatrix_collector_free(&collector);
|
||||
metal_graph_free(&g);
|
||||
free(dataset);
|
||||
return ok ? 0 : 1;
|
||||
#endif
|
||||
}
|
||||
|
||||
int ds4_engine_generate_argmax(
|
||||
ds4_engine *e,
|
||||
const ds4_tokens *prompt,
|
||||
@@ -16853,7 +17279,8 @@ int ds4_session_sync(ds4_session *s, const ds4_tokens *prompt, char *err, size_t
|
||||
s->logits,
|
||||
false,
|
||||
progress_fn,
|
||||
progress_fn ? &progress : NULL);
|
||||
progress_fn ? &progress : NULL,
|
||||
NULL);
|
||||
if (!ok) {
|
||||
snprintf(err, errlen, "Metal resumed prefill failed while extending checkpoint");
|
||||
s->checkpoint_valid = false;
|
||||
@@ -17037,6 +17464,28 @@ int ds4_session_top_logprobs(ds4_session *s, ds4_token_score *out, int k) {
|
||||
return k;
|
||||
}
|
||||
|
||||
int ds4_session_token_logprob(ds4_session *s, int token, ds4_token_score *out) {
|
||||
if (!s || !out || token < 0 || token >= (int)DS4_N_VOCAB) return 0;
|
||||
|
||||
float max_logit = DS4_NEG_INF;
|
||||
for (uint32_t i = 0; i < DS4_N_VOCAB; i++) {
|
||||
const float v = s->logits[i];
|
||||
if (isfinite(v) && v > max_logit) max_logit = v;
|
||||
}
|
||||
if (!isfinite(max_logit)) return 0;
|
||||
|
||||
double sum = 0.0;
|
||||
for (uint32_t i = 0; i < DS4_N_VOCAB; i++) {
|
||||
const float v = s->logits[i];
|
||||
if (isfinite(v)) sum += exp((double)v - (double)max_logit);
|
||||
}
|
||||
const double logsum = (double)max_logit + log(sum);
|
||||
out->id = token;
|
||||
out->logit = s->logits[token];
|
||||
out->logprob = isfinite(out->logit) ? (float)((double)out->logit - logsum) : DS4_NEG_INF;
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int ds4_session_eval_internal(ds4_session *s, int token, bool probe_mtp,
|
||||
char *err, size_t errlen) {
|
||||
if (!s) return 1;
|
||||
|
||||
@@ -107,6 +107,12 @@ int ds4_engine_generate_argmax(ds4_engine *e, const ds4_tokens *prompt,
|
||||
void *emit_ud,
|
||||
ds4_session_progress_fn progress,
|
||||
void *progress_ud);
|
||||
int ds4_engine_collect_imatrix(ds4_engine *e,
|
||||
const char *dataset_path,
|
||||
const char *output_path,
|
||||
int ctx_size,
|
||||
int max_prompts,
|
||||
int max_tokens);
|
||||
void ds4_engine_dump_tokens(ds4_engine *e, const ds4_tokens *tokens);
|
||||
int ds4_dump_text_tokenization(const char *model_path, const char *text, FILE *fp);
|
||||
int ds4_engine_head_test(ds4_engine *e, const ds4_tokens *prompt);
|
||||
@@ -161,6 +167,7 @@ int ds4_session_argmax(ds4_session *s);
|
||||
int ds4_session_argmax_excluding(ds4_session *s, int excluded_id);
|
||||
int ds4_session_sample(ds4_session *s, float temperature, int top_k, float top_p, float min_p, uint64_t *rng);
|
||||
int ds4_session_top_logprobs(ds4_session *s, ds4_token_score *out, int k);
|
||||
int ds4_session_token_logprob(ds4_session *s, int token, ds4_token_score *out);
|
||||
int ds4_session_eval(ds4_session *s, int token, char *err, size_t errlen);
|
||||
int ds4_session_eval_speculative_argmax(ds4_session *s, int first_token,
|
||||
int max_tokens, int eos_token,
|
||||
|
||||
@@ -34,6 +34,10 @@ typedef struct {
|
||||
bool dump_tokens;
|
||||
const char *dump_logprobs_path;
|
||||
int dump_logprobs_top_k;
|
||||
const char *imatrix_dataset_path;
|
||||
const char *imatrix_output_path;
|
||||
int imatrix_max_prompts;
|
||||
int imatrix_max_tokens;
|
||||
ds4_think_mode think_mode;
|
||||
bool head_test;
|
||||
bool first_token_test;
|
||||
@@ -153,6 +157,14 @@ static void usage(FILE *fp) {
|
||||
" Write greedy continuation top-logprobs as JSON without printing text.\n"
|
||||
" --logprobs-top-k N\n"
|
||||
" Number of local alternatives stored by --dump-logprobs. Default: 20\n"
|
||||
" --imatrix-dataset FILE\n"
|
||||
" Rendered DS4 prompt dataset produced by misc/imatrix_dataset.\n"
|
||||
" --imatrix-out FILE\n"
|
||||
" Collect a routed-MoE activation imatrix and write llama-compatible .dat.\n"
|
||||
" --imatrix-max-prompts N\n"
|
||||
" Stop imatrix collection after N prompts. Default: no prompt limit\n"
|
||||
" --imatrix-max-tokens N\n"
|
||||
" Stop imatrix collection after N prompt tokens. Default: no token limit\n"
|
||||
" --head-test\n"
|
||||
" Run the output HC/logits head after the native slice.\n"
|
||||
" --first-token-test\n"
|
||||
@@ -1256,6 +1268,15 @@ static cli_config parse_options(int argc, char **argv) {
|
||||
c.gen.dump_logprobs_path = need_arg(&i, argc, argv, arg);
|
||||
} else if (!strcmp(arg, "--logprobs-top-k")) {
|
||||
c.gen.dump_logprobs_top_k = parse_int(need_arg(&i, argc, argv, arg), arg);
|
||||
} else if (!strcmp(arg, "--imatrix-dataset")) {
|
||||
c.gen.imatrix_dataset_path = need_arg(&i, argc, argv, arg);
|
||||
} else if (!strcmp(arg, "--imatrix-out")) {
|
||||
c.gen.imatrix_output_path = need_arg(&i, argc, argv, arg);
|
||||
c.engine.backend = DS4_BACKEND_METAL;
|
||||
} else if (!strcmp(arg, "--imatrix-max-prompts")) {
|
||||
c.gen.imatrix_max_prompts = parse_int(need_arg(&i, argc, argv, arg), arg);
|
||||
} else if (!strcmp(arg, "--imatrix-max-tokens")) {
|
||||
c.gen.imatrix_max_tokens = parse_int(need_arg(&i, argc, argv, arg), arg);
|
||||
} else if (!strcmp(arg, "--think")) {
|
||||
c.gen.think_mode = DS4_THINK_HIGH;
|
||||
} else if (!strcmp(arg, "--think-max")) {
|
||||
@@ -1295,6 +1316,14 @@ static cli_config parse_options(int argc, char **argv) {
|
||||
if (c.engine.directional_steering_file && !directional_steering_scale_set) {
|
||||
c.engine.directional_steering_ffn = 1.0f;
|
||||
}
|
||||
if (c.gen.imatrix_output_path && !c.gen.imatrix_dataset_path) {
|
||||
fprintf(stderr, "ds4: --imatrix-out requires --imatrix-dataset\n");
|
||||
exit(2);
|
||||
}
|
||||
if (c.gen.imatrix_dataset_path && !c.gen.imatrix_output_path) {
|
||||
fprintf(stderr, "ds4: --imatrix-dataset requires --imatrix-out\n");
|
||||
exit(2);
|
||||
}
|
||||
|
||||
return c;
|
||||
}
|
||||
@@ -1325,6 +1354,13 @@ int main(int argc, char **argv) {
|
||||
int rc = 0;
|
||||
if (cfg.inspect) {
|
||||
ds4_engine_summary(engine);
|
||||
} else if (cfg.gen.imatrix_output_path) {
|
||||
rc = ds4_engine_collect_imatrix(engine,
|
||||
cfg.gen.imatrix_dataset_path,
|
||||
cfg.gen.imatrix_output_path,
|
||||
cfg.gen.ctx_size,
|
||||
cfg.gen.imatrix_max_prompts,
|
||||
cfg.gen.imatrix_max_tokens);
|
||||
} else if (cfg.gen.prompt == NULL) {
|
||||
rc = run_repl(engine, &cfg);
|
||||
} else {
|
||||
|
||||
@@ -664,7 +664,8 @@ int ds4_gpu_routed_moe_batch_tensor(
|
||||
uint32_t n_expert,
|
||||
float clamp,
|
||||
const ds4_gpu_tensor *x,
|
||||
uint32_t n_tokens);
|
||||
uint32_t n_tokens,
|
||||
bool *mid_is_f16);
|
||||
|
||||
/* =========================================================================
|
||||
* Hyper-Connection Kernels.
|
||||
|
||||
+3
-1
@@ -13056,7 +13056,8 @@ int ds4_gpu_routed_moe_batch_tensor(
|
||||
uint32_t n_expert,
|
||||
float clamp,
|
||||
const ds4_gpu_tensor *x,
|
||||
uint32_t n_tokens) {
|
||||
uint32_t n_tokens,
|
||||
bool *mid_is_f16) {
|
||||
if (!g_initialized && !ds4_gpu_init()) return 0;
|
||||
if (!out || !gate || !up || !mid || !x || !model_map || !selected || !weights ||
|
||||
n_tokens == 0 || n_expert == 0 || n_expert > 6) {
|
||||
@@ -13334,6 +13335,7 @@ int ds4_gpu_routed_moe_batch_tensor(
|
||||
use_mm_id &&
|
||||
use_fused_activation &&
|
||||
request_mid_f16;
|
||||
if (mid_is_f16) *mid_is_f16 = use_mid_f16;
|
||||
if (ok && use_fused_activation) {
|
||||
ok = ds4_gpu_encode_moe_swiglu_weight(cb,
|
||||
gatebuf,
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
deepseek4-quantize
|
||||
quality-testing/score_official
|
||||
quality-testing/data/
|
||||
@@ -0,0 +1,27 @@
|
||||
CC ?= cc
|
||||
UNAME_S := $(shell uname -s)
|
||||
|
||||
ifeq ($(UNAME_S),Darwin)
|
||||
NATIVE_CPU_FLAG ?= -mcpu=native
|
||||
QUALITY_LDLIBS := -lm -pthread -framework Foundation -framework Metal
|
||||
else
|
||||
NATIVE_CPU_FLAG ?= -march=native
|
||||
QUALITY_LDLIBS := -lm -pthread
|
||||
endif
|
||||
|
||||
CFLAGS ?= -O3 -Wall -Wextra -std=c11 $(NATIVE_CPU_FLAG)
|
||||
CPPFLAGS ?= -D_GNU_SOURCE
|
||||
|
||||
.PHONY: all clean quality-score
|
||||
|
||||
all: deepseek4-quantize
|
||||
|
||||
deepseek4-quantize: deepseek4-quantize.c quants.c quants.h
|
||||
$(CC) $(CFLAGS) $(CPPFLAGS) -o $@ deepseek4-quantize.c quants.c -lm -pthread
|
||||
|
||||
quality-score:
|
||||
$(MAKE) -C ..
|
||||
$(CC) $(CFLAGS) -I.. -o quality-testing/score_official quality-testing/score_official.c ../ds4.o ../ds4_metal.o $(QUALITY_LDLIBS)
|
||||
|
||||
clean:
|
||||
rm -f deepseek4-quantize quality-testing/score_official
|
||||
@@ -0,0 +1,134 @@
|
||||
# DS4 GGUF Tools
|
||||
|
||||
This directory contains the offline tools used to build and evaluate DeepSeek
|
||||
V4 Flash GGUF files for `ds4`.
|
||||
|
||||
The important pieces are:
|
||||
|
||||
- `deepseek4-quantize.c`: C HF-safetensors to GGUF quantizer.
|
||||
- `quants.[ch]`: the deliberately small local quantization implementation used
|
||||
by the quantizer. It implements the DS4 output formats we actually ship:
|
||||
`q8_0`, `q4_K`, `q2_K`, and `iq2_xxs`.
|
||||
- `imatrix/`: dataset and instructions for collecting routed-MoE activation
|
||||
importance with `ds4`.
|
||||
- `quality-testing/`: prompts and scripts used to compare local GGUF variants
|
||||
against official DeepSeek V4 Flash continuations.
|
||||
|
||||
## Build
|
||||
|
||||
```sh
|
||||
make -C gguf-tools
|
||||
```
|
||||
|
||||
The quantizer is plain C and does not link GGML. GGUF metadata handling,
|
||||
safetensors loading, FP4/FP8 dequantization, and the quantizers used by our Q2
|
||||
and Q4 recipes live in this directory.
|
||||
|
||||
## Generate An Imatrix
|
||||
|
||||
First regenerate or inspect the calibration dataset:
|
||||
|
||||
```sh
|
||||
python3 gguf-tools/imatrix/dataset/build_ds4_imatrix_dataset.py
|
||||
```
|
||||
|
||||
Then collect activation statistics with the DS4 runtime:
|
||||
|
||||
```sh
|
||||
./ds4 \
|
||||
-m gguf/DeepSeek-V4-Flash-Q4KExperts-F16HC-F16Compressor-F16Indexer-Q8Attn-Q8Shared-Q8Out-chat-v2.gguf \
|
||||
--imatrix-dataset gguf-tools/imatrix/dataset/rendered_prompts.txt \
|
||||
--imatrix-out gguf/DeepSeek-V4-Flash-chat-v2-routed-moe-ds4.dat \
|
||||
--ctx 32768
|
||||
```
|
||||
|
||||
The imatrix file is useful immediately with this DS4 quantizer. Generic GGUF
|
||||
tools need DS4-specific tensor-name mapping and per-expert slicing before they
|
||||
can use it correctly. The accepted imatrix format is the legacy llama.cpp
|
||||
binary `.dat` file emitted by `ds4 --imatrix-out`.
|
||||
|
||||
Generating this `.dat` file locally is possible, but slow: it runs the DS4
|
||||
prefill graph over the full calibration corpus and reads routed-MoE activation
|
||||
statistics back from the GPU. The latest published imatrix-generated GGUF files
|
||||
are available in the antirez Hugging Face repository:
|
||||
|
||||
```text
|
||||
https://huggingface.co/antirez/deepseek-v4-gguf/tree/main
|
||||
```
|
||||
|
||||
## Generate Q2 And Q4 GGUFs
|
||||
|
||||
The template GGUF supplies metadata, tokenizer, tensor order, and logical
|
||||
shapes. Tensor bytes are regenerated from the Hugging Face safetensors. Full
|
||||
generation is intentionally offline and heavy: expect roughly 80-90 GB outputs
|
||||
for the 2-bit template family and roughly 150-170 GB for the 4-bit routed-expert
|
||||
family, plus enough free disk for the temporary output. Use `--dry-run` and
|
||||
`--compare-tensor` before starting a full write, and use `--overwrite` only when
|
||||
you really mean to replace an existing GGUF.
|
||||
|
||||
Q2 routed experts with imatrix:
|
||||
|
||||
```sh
|
||||
gguf-tools/deepseek4-quantize \
|
||||
--hf ../deepseek-v4-quants/hf/DeepSeek-V4-Flash \
|
||||
--template gguf/DeepSeek-V4-Flash-IQ2XXS-w2Q2K-AProjQ8-SExpQ8-OutQ8-chat-v2.gguf \
|
||||
--out gguf/DeepSeek-V4-Flash-IQ2XXS-w2Q2K-AProjQ8-SExpQ8-OutQ8-chat-v2-imatrix.gguf \
|
||||
--imatrix gguf/DeepSeek-V4-Flash-chat-v2-routed-moe-ds4.dat
|
||||
```
|
||||
|
||||
Q4 routed experts with imatrix:
|
||||
|
||||
```sh
|
||||
gguf-tools/deepseek4-quantize \
|
||||
--hf ../deepseek-v4-quants/hf/DeepSeek-V4-Flash \
|
||||
--template gguf/DeepSeek-V4-Flash-Q4KExperts-F16HC-F16Compressor-F16Indexer-Q8Attn-Q8Shared-Q8Out-chat-v2.gguf \
|
||||
--out gguf/DeepSeek-V4-Flash-Q4KExperts-F16HC-F16Compressor-F16Indexer-Q8Attn-Q8Shared-Q8Out-chat-v2-imatrix.gguf \
|
||||
--imatrix gguf/DeepSeek-V4-Flash-chat-v2-routed-moe-ds4.dat
|
||||
```
|
||||
|
||||
You can override tensor families:
|
||||
|
||||
```sh
|
||||
--experts iq2_xxs
|
||||
--routed-w2 q2_k
|
||||
--attention-proj q8_0
|
||||
--shared q8_0
|
||||
--output q8_0
|
||||
```
|
||||
|
||||
Useful checks before writing a full model:
|
||||
|
||||
```sh
|
||||
gguf-tools/deepseek4-quantize \
|
||||
--hf ../deepseek-v4-quants/hf/DeepSeek-V4-Flash \
|
||||
--template MODEL.gguf \
|
||||
--compare-tensor blk.0.attn_q_a.weight
|
||||
```
|
||||
|
||||
`--compare-tensor` regenerates a single tensor and byte-compares it against the
|
||||
template or `--compare-gguf`. `--threads N` controls routed-expert workers.
|
||||
|
||||
## When No Imatrix Is Given
|
||||
|
||||
`iq2_xxs` requires an importance vector. If `--imatrix` is not provided and
|
||||
the target type requires one, `deepseek4-quantize` computes a synthetic fallback
|
||||
from the dequantized weight itself:
|
||||
|
||||
```text
|
||||
importance[column] = sum(row[column]^2) over all rows
|
||||
```
|
||||
|
||||
This is a weight-energy heuristic. It is not as good as measuring real DS4
|
||||
activations, but it gives the quantizer a stable column weighting and was good
|
||||
enough for the first working 2-bit GGUFs.
|
||||
|
||||
## Quality Testing
|
||||
|
||||
See `quality-testing/README.md`. The short version is:
|
||||
|
||||
```sh
|
||||
python3 gguf-tools/quality-testing/collect_official.py
|
||||
make -C gguf-tools quality-score
|
||||
gguf-tools/quality-testing/score_official MODEL.gguf gguf-tools/quality-testing/data/manifest.tsv /tmp/model.tsv 4096
|
||||
python3 gguf-tools/quality-testing/compare_scores.py /tmp/old.tsv /tmp/new.tsv
|
||||
```
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,152 @@
|
||||
# DS4 Imatrix Pipeline
|
||||
|
||||
This directory contains the calibration dataset and instructions used to build
|
||||
activation importance matrices for DeepSeek V4 Flash GGUF quantization.
|
||||
|
||||
The current imatrix target is the routed MoE path. DS4 has 43 layers, 256
|
||||
routed experts per layer, and three routed expert tensors per layer:
|
||||
|
||||
- `blk.N.ffn_gate_exps.weight`
|
||||
- `blk.N.ffn_up_exps.weight`
|
||||
- `blk.N.ffn_down_exps.weight`
|
||||
|
||||
For gate/up tensors, the collector records the squared FFN-normalized input
|
||||
activation. For down tensors, it records the squared routed SwiGLU row after
|
||||
route weighting. The result tells the quantizer which input columns are used
|
||||
more heavily by the actual DS4 inference graph.
|
||||
|
||||
## 1. Build The Calibration Dataset
|
||||
|
||||
The tracked dataset is in `gguf-tools/imatrix/dataset/`. Regenerate it from
|
||||
the repository root with:
|
||||
|
||||
```sh
|
||||
python3 gguf-tools/imatrix/dataset/build_ds4_imatrix_dataset.py
|
||||
```
|
||||
|
||||
The important output is:
|
||||
|
||||
```text
|
||||
gguf-tools/imatrix/dataset/rendered_prompts.txt
|
||||
```
|
||||
|
||||
It contains DS4-rendered chat prompts, separated by visible
|
||||
`DS4_IMATRIX_PROMPT` markers. The prompts include:
|
||||
|
||||
- C/Metal source-review prompts from this repository.
|
||||
- Long-context snippets.
|
||||
- Agent/tool-call prompts using DS4's DSML syntax.
|
||||
- English and Italian prompts.
|
||||
- Both thinking and non-thinking assistant prefixes.
|
||||
|
||||
The current tracked dataset has 1952 rendered prompts and roughly 1.44M tokens
|
||||
by the coarse bytes/4 estimate. Check
|
||||
`gguf-tools/imatrix/dataset/manifest.json` for the exact generated-file
|
||||
summary.
|
||||
|
||||
## 2. Collect The Imatrix
|
||||
|
||||
Use the DS4 runtime itself to collect routed MoE activation statistics:
|
||||
|
||||
```sh
|
||||
./ds4 \
|
||||
-m ../deepseek-v4-quants/gguf/DeepSeek-V4-Flash-Q4KExperts-F16HC-F16Compressor-F16Indexer-Q8Attn-Q8Shared-Q8Out-chat-v2.gguf \
|
||||
--imatrix-dataset gguf-tools/imatrix/dataset/rendered_prompts.txt \
|
||||
--imatrix-out ../deepseek-v4-quants/imatrix/DeepSeek-V4-Flash-chat-v2-routed-moe-ds4-1p5m.dat \
|
||||
--ctx 32768
|
||||
```
|
||||
|
||||
Useful smoke-test limits:
|
||||
|
||||
```sh
|
||||
./ds4 \
|
||||
-m MODEL.gguf \
|
||||
--imatrix-dataset gguf-tools/imatrix/dataset/rendered_prompts.txt \
|
||||
--imatrix-out /tmp/ds4-test.imatrix.dat \
|
||||
--imatrix-max-prompts 1 \
|
||||
--imatrix-max-tokens 4096
|
||||
```
|
||||
|
||||
The collector is Metal-only because it hooks the layer-major Metal prefill graph.
|
||||
It does not change inference math; it reads the already materialized MoE inputs
|
||||
and accumulates `sum(x[column]^2)` per routed expert.
|
||||
|
||||
The output format is llama.cpp's legacy binary `.dat` imatrix format. DS4 packs
|
||||
per-expert vectors into one entry per routed expert tensor:
|
||||
|
||||
```text
|
||||
entry length = n_expert * n_columns
|
||||
```
|
||||
|
||||
The quantizer slices the right expert's segment when quantizing each expert.
|
||||
|
||||
## 3. Generate GGUF Files With The Imatrix
|
||||
|
||||
The local C quantization tool in `gguf-tools/` supports:
|
||||
|
||||
```text
|
||||
--imatrix FILE
|
||||
--imatrix-strict
|
||||
```
|
||||
|
||||
Example Q4 routed-expert regeneration:
|
||||
|
||||
```sh
|
||||
gguf-tools/deepseek4-quantize \
|
||||
--hf ../deepseek-v4-quants/hf/DeepSeek-V4-Flash \
|
||||
--template ../deepseek-v4-quants/gguf/DeepSeek-V4-Flash-Q4KExperts-F16HC-F16Compressor-F16Indexer-Q8Attn-Q8Shared-Q8Out-chat-v2.gguf \
|
||||
--out ../deepseek-v4-quants/gguf/DeepSeek-V4-Flash-Q4KExperts-F16HC-F16Compressor-F16Indexer-Q8Attn-Q8Shared-Q8Out-chat-v2-imatrix.gguf \
|
||||
--imatrix ../deepseek-v4-quants/imatrix/DeepSeek-V4-Flash-chat-v2-routed-moe-ds4-1p5m.dat
|
||||
```
|
||||
|
||||
Example Q2 regeneration:
|
||||
|
||||
```sh
|
||||
gguf-tools/deepseek4-quantize \
|
||||
--hf ../deepseek-v4-quants/hf/DeepSeek-V4-Flash \
|
||||
--template ../deepseek-v4-quants/gguf/DeepSeek-V4-Flash-IQ2XXS-w2Q2K-AProjQ8-SExpQ8-OutQ8-chat-v2.gguf \
|
||||
--out ../deepseek-v4-quants/gguf/DeepSeek-V4-Flash-IQ2XXS-w2Q2K-AProjQ8-SExpQ8-OutQ8-chat-v2-imatrix.gguf \
|
||||
--imatrix ../deepseek-v4-quants/imatrix/DeepSeek-V4-Flash-chat-v2-routed-moe-ds4-1p5m.dat
|
||||
```
|
||||
|
||||
For Q4, the imatrix does not change the runtime tensor type: routed experts
|
||||
remain `Q4_K`. It changes how quantization error is weighted while choosing
|
||||
scales and codes. For Q2, it replaces the previous synthetic weight-energy
|
||||
fallback used for `IQ2_XXS` gate/up experts with real activation statistics.
|
||||
|
||||
## 4. Evaluate
|
||||
|
||||
Useful local tools:
|
||||
|
||||
```text
|
||||
misc/quant_eval.c
|
||||
gguf-tools/quality-testing/
|
||||
```
|
||||
|
||||
`misc/quant_eval.c` compares local GGUF variants by greedy/top-logit behavior.
|
||||
`gguf-tools/quality-testing/` can score local GGUFs against official DeepSeek
|
||||
API continuations by target-token negative log likelihood.
|
||||
|
||||
The Q4 imatrix file uploaded to Hugging Face was tested on 100 official
|
||||
DeepSeek V4 Flash continuations:
|
||||
|
||||
```text
|
||||
old Q4 avg NLL: 0.177357819
|
||||
Q4 imatrix avg NLL: 0.173895148
|
||||
relative NLL change: -1.95%
|
||||
case wins: 54 imatrix / 46 old
|
||||
first-token matches: 83 imatrix / 81 old
|
||||
avg greedy LCP: 12.21 imatrix / 11.94 old
|
||||
```
|
||||
|
||||
## Compatibility
|
||||
|
||||
The `.dat` file is intentionally in llama.cpp's legacy imatrix format, so the
|
||||
data is not conceptually tied to DS4. In practice, it is immediately useful
|
||||
only with a quantizer that understands DS4's tensor names and packed per-expert
|
||||
entries. The current `deepseek4-quantize` tooling does that.
|
||||
|
||||
Other GGUF creation tools can use the same imatrix if they implement the same
|
||||
name mapping and per-expert slicing convention. Without that DS4-specific
|
||||
mapping, a generic imatrix loader will see valid data but will not know how to
|
||||
apply the packed routed-expert vectors correctly.
|
||||
@@ -0,0 +1,33 @@
|
||||
# DS4 Imatrix Calibration Dataset
|
||||
|
||||
This directory contains DS4-rendered chat prompts for collecting activation
|
||||
statistics before building new low-bit GGUF files.
|
||||
|
||||
Run:
|
||||
|
||||
```sh
|
||||
python3 gguf-tools/imatrix/dataset/build_ds4_imatrix_dataset.py
|
||||
```
|
||||
|
||||
Generated files:
|
||||
|
||||
- `prompts.jsonl`: structured records with messages and rendered prompt text.
|
||||
- `rendered_prompts.txt`: all rendered prompts, separated by visible markers.
|
||||
- `rendered_prompts_nothink.txt`: only prompts ending with `</think>`.
|
||||
- `rendered_prompts_think.txt`: only prompts ending with `<think>`.
|
||||
- `manifest.json`: counts, byte totals, and rough token estimate.
|
||||
|
||||
The renderer mirrors the server prompt shape:
|
||||
|
||||
```text
|
||||
<｜begin▁of▁sentence｜>system<｜User｜>...<｜Assistant｜><think>
|
||||
<｜begin▁of▁sentence｜>system<｜User｜>...<｜Assistant｜></think>
|
||||
```
|
||||
|
||||
Some records include DSML tool schemas, sampled DSML tool calls, and tool-result
|
||||
turns so the imatrix sees the same special-token patterns used by agent clients.
|
||||
The corpus also includes Italian prompts, long-context source excerpts, Metal/C
|
||||
code review tasks, and inference-specific debugging tasks.
|
||||
|
||||
For normal imatrix collection, use `rendered_prompts.txt` so calibration covers
|
||||
both thinking and non-thinking modes. Split files are provided for ablations.
|
||||
@@ -0,0 +1,439 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build a local calibration corpus for DeepSeek V4 Flash imatrix collection.
|
||||
|
||||
The quantizer needs activation statistics from prompts that look like the real
|
||||
workload. This script creates deterministic DS4-rendered chat prompts from the
|
||||
repo itself, agent/tool conversations, bilingual tasks, and long-context code
|
||||
reviews. The output is intentionally plain JSONL/text so the imatrix collector
|
||||
can consume it without depending on this script.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import random
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
|
||||
|
||||
BOS = "<｜begin▁of▁sentence｜>"
|
||||
USER = "<｜User｜>"
|
||||
ASSISTANT = "<｜Assistant｜>"
|
||||
EOS = "<｜end▁of▁sentence｜>"
|
||||
|
||||
DEFAULT_SYSTEM = (
|
||||
"You are DeepSeek V4 Flash running locally. Answer accurately, preserve "
|
||||
"technical details, and use tools only when the prompt asks for tool use."
|
||||
)
|
||||
|
||||
TOOLS_PROMPT = """## Tools
|
||||
|
||||
You have access to a set of tools to help answer the user question. You can invoke tools by writing a "<｜DSML｜tool_calls>" block like the following:
|
||||
|
||||
<｜DSML｜tool_calls>
|
||||
<｜DSML｜invoke name="$TOOL_NAME">
|
||||
<｜DSML｜parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</｜DSML｜parameter>
|
||||
...
|
||||
</｜DSML｜invoke>
|
||||
</｜DSML｜tool_calls>
|
||||
|
||||
String parameters should be specified as raw text and set `string="true"`. Preserve characters such as `>`, `&`, and `&&` exactly; never replace normal string characters with XML or HTML entity escapes. Only if a string value itself contains the exact closing parameter tag `</｜DSML｜parameter>`, write that tag as `</｜DSML｜parameter>` inside the value.
|
||||
|
||||
### Available Tool Schemas
|
||||
|
||||
[
|
||||
{"type":"function","function":{"name":"bash","description":"Run a shell command","parameters":{"type":"object","properties":{"command":{"type":"string"},"timeout":{"type":"integer"},"description":{"type":"string"}},"required":["command"]}}},
|
||||
{"type":"function","function":{"name":"read","description":"Read a file","parameters":{"type":"object","properties":{"path":{"type":"string"},"start":{"type":"integer"},"lines":{"type":"integer"}},"required":["path"]}}},
|
||||
{"type":"function","function":{"name":"edit","description":"Apply a small source edit","parameters":{"type":"object","properties":{"path":{"type":"string"},"old":{"type":"string"},"new":{"type":"string"}},"required":["path","old","new"]}}},
|
||||
{"type":"function","function":{"name":"grep","description":"Search source files","parameters":{"type":"object","properties":{"pattern":{"type":"string"},"path":{"type":"string"}},"required":["pattern"]}}}
|
||||
]
|
||||
|
||||
You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls. Use the exact parameter names from the schemas."""
|
||||
|
||||
DSML_LIST_FILES = """
|
||||
<｜DSML｜tool_calls>
|
||||
<｜DSML｜invoke name="bash">
|
||||
<｜DSML｜parameter name="description" string="true">list files in the project root</｜DSML｜parameter>
|
||||
<｜DSML｜parameter name="command" string="true">find . -maxdepth 2 -type f | sort | head -200</｜DSML｜parameter>
|
||||
<｜DSML｜parameter name="timeout" string="false">10</｜DSML｜parameter>
|
||||
</｜DSML｜invoke>
|
||||
</｜DSML｜tool_calls>"""
|
||||
|
||||
DSML_READ_FILE = """
|
||||
<｜DSML｜tool_calls>
|
||||
<｜DSML｜invoke name="read">
|
||||
<｜DSML｜parameter name="path" string="true">ds4.c</｜DSML｜parameter>
|
||||
<｜DSML｜parameter name="start" string="false">14000</｜DSML｜parameter>
|
||||
<｜DSML｜parameter name="lines" string="false">120</｜DSML｜parameter>
|
||||
</｜DSML｜invoke>
|
||||
</｜DSML｜tool_calls>"""
|
||||
|
||||
|
||||
@dataclass
|
||||
class Record:
|
||||
rid: str
|
||||
category: str
|
||||
mode: str
|
||||
source: str
|
||||
messages: list[dict[str, str]]
|
||||
rendered: str
|
||||
|
||||
|
||||
def escape_tool_result(text: str) -> str:
|
||||
"""Match the renderer's intent: tool outputs must not become live DSML."""
|
||||
|
||||
return (
|
||||
text.replace("&", "&")
|
||||
.replace("<", "<")
|
||||
.replace(">", ">")
|
||||
)
|
||||
|
||||
|
||||
def render(messages: list[dict[str, str]], mode: str, tools: bool = False) -> str:
|
||||
"""Render the subset of DS4 chat syntax needed for calibration.
|
||||
|
||||
This mirrors ds4_server.c's prompt shape: BOS, concatenated system text,
|
||||
user/tool/assistant turns, then an assistant prefix using <think> or
|
||||
</think>. Existing assistant answers close with EOS.
|
||||
"""
|
||||
|
||||
system_parts = [m.get("content", "") for m in messages if m.get("role") == "system"]
|
||||
if tools:
|
||||
system_parts.append(TOOLS_PROMPT)
|
||||
system = "\n\n".join(p for p in system_parts if p)
|
||||
think = mode == "think"
|
||||
|
||||
out = [BOS, system]
|
||||
pending_assistant = False
|
||||
pending_tool_result = False
|
||||
last_user_idx = max((i for i, m in enumerate(messages) if m.get("role") == "user"), default=-1)
|
||||
|
||||
for i, msg in enumerate(messages):
|
||||
role = msg.get("role", "")
|
||||
content = msg.get("content", "")
|
||||
if role == "system":
|
||||
continue
|
||||
if role == "user":
|
||||
out.extend([USER, content])
|
||||
pending_assistant = True
|
||||
pending_tool_result = False
|
||||
elif role in ("tool", "function"):
|
||||
if not pending_tool_result:
|
||||
out.append(USER)
|
||||
out.extend(["<tool_result>", escape_tool_result(content), "</tool_result>"])
|
||||
pending_assistant = True
|
||||
pending_tool_result = True
|
||||
elif role == "assistant":
|
||||
if pending_assistant:
|
||||
out.append(ASSISTANT)
|
||||
if think and i > last_user_idx:
|
||||
out.extend(["<think>", msg.get("reasoning", ""), "</think>"])
|
||||
else:
|
||||
out.append("</think>")
|
||||
out.append(content)
|
||||
if msg.get("dsml"):
|
||||
out.append(msg["dsml"])
|
||||
out.append(EOS)
|
||||
pending_assistant = False
|
||||
pending_tool_result = False
|
||||
|
||||
if pending_assistant:
|
||||
out.extend([ASSISTANT, "<think>" if think else "</think>"])
|
||||
|
||||
return "".join(out)
|
||||
|
||||
|
||||
def stable_id(category: str, source: str, mode: str, text: str) -> str:
|
||||
h = hashlib.sha1()
|
||||
h.update(category.encode())
|
||||
h.update(b"\0")
|
||||
h.update(source.encode())
|
||||
h.update(b"\0")
|
||||
h.update(mode.encode())
|
||||
h.update(b"\0")
|
||||
h.update(text[:4096].encode("utf-8", "ignore"))
|
||||
return f"{category}-{h.hexdigest()[:12]}"
|
||||
|
||||
|
||||
def add_record(records: list[Record], category: str, source: str,
|
||||
messages: list[dict[str, str]], *, tools: bool = False,
|
||||
modes: Iterable[str] = ("nothink", "think")) -> None:
|
||||
for mode in modes:
|
||||
rendered = render(messages, mode, tools=tools)
|
||||
rid = stable_id(category, source, mode, rendered)
|
||||
records.append(Record(rid, category, mode, source, messages, rendered))
|
||||
|
||||
|
||||
def read_text(path: Path) -> str:
|
||||
try:
|
||||
return path.read_text(encoding="utf-8", errors="replace")
|
||||
except OSError:
|
||||
return ""
|
||||
|
||||
|
||||
def chunk_text(text: str, size: int, overlap: int) -> list[str]:
|
||||
text = re.sub(r"\n{4,}", "\n\n\n", text).strip()
|
||||
chunks = []
|
||||
i = 0
|
||||
while i < len(text):
|
||||
j = min(len(text), i + size)
|
||||
if j < len(text):
|
||||
nl = text.rfind("\n", i + size // 2, j)
|
||||
if nl > i:
|
||||
j = nl
|
||||
part = text[i:j].strip()
|
||||
if part:
|
||||
chunks.append(part)
|
||||
if j == len(text):
|
||||
break
|
||||
i = max(j - overlap, i + 1)
|
||||
return chunks
|
||||
|
||||
|
||||
def code_prompt(path: str, chunk: str, task: str) -> list[dict[str, str]]:
|
||||
lang = "metal" if path.endswith(".metal") else "c"
|
||||
return [
|
||||
{"role": "system", "content": DEFAULT_SYSTEM},
|
||||
{"role": "user", "content": f"{task}\n\nFile: {path}\n\n```{lang}\n{chunk}\n```"},
|
||||
]
|
||||
|
||||
|
||||
def doc_prompt(path: str, chunk: str, task: str) -> list[dict[str, str]]:
|
||||
return [
|
||||
{"role": "system", "content": DEFAULT_SYSTEM},
|
||||
{"role": "user", "content": f"{task}\n\nDocument: {path}\n\n{chunk}"},
|
||||
]
|
||||
|
||||
|
||||
def make_source_records(root: Path, records: list[Record]) -> None:
|
||||
files = [
|
||||
"ds4.c", "ds4_server.c", "ds4_cli.c", "ds4_metal.m", "ds4.h", "ds4_gpu.h",
|
||||
"README.md", "AGENT.md", "gguf-tools/README.md",
|
||||
"gguf-tools/imatrix/README.md", "gguf-tools/imatrix/dataset/README.md",
|
||||
"gguf-tools/quality-testing/README.md",
|
||||
]
|
||||
files += [str(p.relative_to(root)) for p in sorted((root / "metal").glob("*.metal"))]
|
||||
tasks = [
|
||||
"Review this excerpt for correctness risks, memory lifetime issues, and performance bottlenecks.",
|
||||
"Explain what this code does and identify the inference stage it belongs to.",
|
||||
"Suggest a minimal correctness-preserving optimization for this excerpt.",
|
||||
"Trova bug sottili e spiega quali invarianti dell'inferenza possono rompersi.",
|
||||
]
|
||||
|
||||
for name in files:
|
||||
text = read_text(root / name)
|
||||
if not text:
|
||||
continue
|
||||
is_code = name.endswith((".c", ".h", ".m", ".metal"))
|
||||
size = 2600 if is_code else 3200
|
||||
chunks = chunk_text(text, size=size, overlap=180)
|
||||
for idx, chunk in enumerate(chunks):
|
||||
task = tasks[(idx + len(name)) % len(tasks)]
|
||||
msgs = code_prompt(name, chunk, task) if is_code else doc_prompt(name, chunk, task)
|
||||
add_record(records, "source", f"{name}:{idx}", msgs)
|
||||
|
||||
|
||||
def make_agent_records(records: list[Record]) -> None:
|
||||
scenarios = [
|
||||
(
|
||||
"opencode-list-files",
|
||||
[
|
||||
{"role": "system", "content": "You are a coding agent operating in a local repository. Be terse and use tools for filesystem inspection."},
|
||||
{"role": "user", "content": "list files in the current dir"},
|
||||
{"role": "assistant", "content": "", "dsml": DSML_LIST_FILES},
|
||||
{"role": "tool", "content": ".git\nREADME.md\nds4.c\nds4_server.c\nmetal/\nmisc/\ntests/\n"},
|
||||
{"role": "user", "content": "What files matter for the CLI?"},
|
||||
],
|
||||
True,
|
||||
),
|
||||
(
|
||||
"opencode-read-source",
|
||||
[
|
||||
{"role": "system", "content": "You are a coding agent. Inspect before editing."},
|
||||
{"role": "user", "content": "Find where DS4 renders the chat template and explain how tool calls are preserved."},
|
||||
{"role": "assistant", "content": "", "dsml": DSML_READ_FILE},
|
||||
{"role": "tool", "content": "static char *render_chat_prompt_text(...) { /* abbreviated renderer output */ }\n"},
|
||||
{"role": "user", "content": "Now summarize the invariants that must not drift."},
|
||||
],
|
||||
True,
|
||||
),
|
||||
(
|
||||
"kv-cache-debug",
|
||||
[
|
||||
{"role": "system", "content": DEFAULT_SYSTEM},
|
||||
{"role": "user", "content": "A server request has prompt=16358 cached=11412 suffix=4946. Explain why suffix prefill must be chunked instead of decoded token by token."},
|
||||
{"role": "assistant", "content": "The suffix is long enough that single-token decode would waste the model's batched prefill path. The implementation should start at the cached prefix and process the remaining tokens in 2048-token chunks."},
|
||||
{"role": "user", "content": "Now write a checklist for testing this with context cache hits."},
|
||||
],
|
||||
False,
|
||||
),
|
||||
(
|
||||
"metal-performance-debug",
|
||||
[
|
||||
{"role": "system", "content": DEFAULT_SYSTEM},
|
||||
{"role": "user", "content": "The prefill chunk log starts at 230 tok/s and drops toward 150 tok/s at 16k context. Rank likely causes in DS4 Flash attention."},
|
||||
{"role": "assistant", "content": "The slope usually comes from indexed compressed attention, raw window attention, compressor state, and common MoE/HC cost. Measure per-layer ratio-4 and ratio-128 stages separately."},
|
||||
{"role": "user", "content": "Which measurements would distinguish score-kernel slope from attention-scan slope?"},
|
||||
],
|
||||
False,
|
||||
),
|
||||
]
|
||||
for name, msgs, tools in scenarios:
|
||||
add_record(records, "agent", name, msgs, tools=tools)
|
||||
|
||||
|
||||
def make_general_records(records: list[Record]) -> None:
|
||||
english = [
|
||||
"Explain how a B-tree insertion works, including splits and the root special case.",
|
||||
"Write a concise design for a TCP echo server that handles slow clients without blocking other clients.",
|
||||
"Compare mmap-backed model loading with copying all weights into private buffers on macOS.",
|
||||
"Derive why RMSNorm can be implemented with one sum of squares and one scale pass.",
|
||||
"Explain indexed sparse attention in a mixture-of-experts language model.",
|
||||
"Create a careful migration plan from a C++ graph executor to a pure C model-specific executor.",
|
||||
"Find the bug in a ring buffer where speculative writes may overwrite still-visible rows.",
|
||||
"Explain why quantization imatrices should be collected from realistic prompts instead of random text.",
|
||||
"Summarize the difference between prefill and decode in transformer inference.",
|
||||
"Write a troubleshooting guide for an OpenAI-compatible local chat server that hangs during tool calls.",
|
||||
]
|
||||
italian = [
|
||||
"Spiega come funziona l'inserimento in un B-tree, inclusi split e caso della radice.",
|
||||
"Scrivi un progetto conciso per un server TCP echo che gestisce client lenti senza bloccare gli altri.",
|
||||
"Confronta il caricamento mmap dei pesi con la copia completa in buffer privati su macOS.",
|
||||
"Deriva perche RMSNorm richiede una somma dei quadrati e un passaggio di scala.",
|
||||
"Spiega l'attenzione sparsa indicizzata in un modello linguistico mixture-of-experts.",
|
||||
"Crea un piano di migrazione da un executor a grafo C++ a un executor specifico in C puro.",
|
||||
"Trova il bug in un ring buffer dove scritture speculative possono sovrascrivere righe ancora visibili.",
|
||||
"Spiega perche le imatrix di quantizzazione vanno raccolte da prompt realistici e non testo casuale.",
|
||||
"Riassumi la differenza tra prefill e decode nell'inferenza transformer.",
|
||||
"Scrivi una guida di debug per un server chat locale compatibile OpenAI che si blocca nei tool call.",
|
||||
]
|
||||
for idx, prompt in enumerate(english + italian):
|
||||
add_record(records, "general", f"general:{idx}", [
|
||||
{"role": "system", "content": DEFAULT_SYSTEM},
|
||||
{"role": "user", "content": prompt},
|
||||
])
|
||||
|
||||
|
||||
def make_long_context_records(root: Path, records: list[Record]) -> None:
|
||||
sources = [
|
||||
("README.md", read_text(root / "README.md")),
|
||||
("AGENT.md", read_text(root / "AGENT.md")),
|
||||
("METAL.md", read_text(root / "METAL.md")),
|
||||
("ds4_server.c", read_text(root / "ds4_server.c")),
|
||||
("ds4_metal.m", read_text(root / "ds4_metal.m")),
|
||||
("metal/dsv4_hc.metal", read_text(root / "metal/dsv4_hc.metal")),
|
||||
("metal/moe.metal", read_text(root / "metal/moe.metal")),
|
||||
]
|
||||
blocks = []
|
||||
for name, text in sources:
|
||||
for i, chunk in enumerate(chunk_text(text, size=5200, overlap=120)[:6]):
|
||||
blocks.append(f"### {name} chunk {i}\n{chunk}")
|
||||
random.Random(7).shuffle(blocks)
|
||||
for i in range(0, len(blocks), 4):
|
||||
group = blocks[i:i + 4]
|
||||
if len(group) < 2:
|
||||
continue
|
||||
body = "\n\n".join(group)
|
||||
msgs = [
|
||||
{"role": "system", "content": DEFAULT_SYSTEM},
|
||||
{"role": "user", "content": (
|
||||
"Read the following repository excerpts as one long context. "
|
||||
"Identify the three most important invariants for correctness and "
|
||||
"the two most likely performance bottlenecks.\n\n" + body
|
||||
)},
|
||||
]
|
||||
add_record(records, "long_context", f"long:{i//4}", msgs)
|
||||
|
||||
|
||||
def write_outputs(outdir: Path, records: list[Record]) -> None:
|
||||
outdir.mkdir(parents=True, exist_ok=True)
|
||||
records = sorted(records, key=lambda r: (r.category, r.source, r.mode, r.rid))
|
||||
|
||||
jsonl = outdir / "prompts.jsonl"
|
||||
with jsonl.open("w", encoding="utf-8") as f:
|
||||
for r in records:
|
||||
f.write(json.dumps({
|
||||
"id": r.rid,
|
||||
"category": r.category,
|
||||
"mode": r.mode,
|
||||
"source": r.source,
|
||||
"messages": r.messages,
|
||||
"rendered": r.rendered,
|
||||
}, ensure_ascii=False, separators=(",", ":")) + "\n")
|
||||
|
||||
def write_rendered(path: Path, rows: Iterable[Record]) -> int:
|
||||
count = 0
|
||||
with path.open("w", encoding="utf-8") as f:
|
||||
for count, r in enumerate(rows, start=1):
|
||||
f.write(f"\n\n===== DS4_IMATRIX_PROMPT {r.rid} {r.category} {r.mode} {r.source} =====\n")
|
||||
f.write(r.rendered)
|
||||
return count
|
||||
|
||||
write_rendered(outdir / "rendered_prompts.txt", records)
|
||||
write_rendered(outdir / "rendered_prompts_nothink.txt", [r for r in records if r.mode == "nothink"])
|
||||
write_rendered(outdir / "rendered_prompts_think.txt", [r for r in records if r.mode == "think"])
|
||||
|
||||
categories: dict[str, int] = {}
|
||||
modes: dict[str, int] = {}
|
||||
bytes_by_category: dict[str, int] = {}
|
||||
for r in records:
|
||||
categories[r.category] = categories.get(r.category, 0) + 1
|
||||
modes[r.mode] = modes.get(r.mode, 0) + 1
|
||||
bytes_by_category[r.category] = bytes_by_category.get(r.category, 0) + len(r.rendered.encode("utf-8"))
|
||||
|
||||
rendered_bytes = sum(len(r.rendered.encode("utf-8")) for r in records)
|
||||
manifest = {
|
||||
"version": 1,
|
||||
"purpose": "DeepSeek V4 Flash imatrix calibration prompts",
|
||||
"record_count": len(records),
|
||||
"rendered_utf8_bytes": rendered_bytes,
|
||||
"rough_token_estimate_bytes_div_4": rendered_bytes // 4,
|
||||
"categories": categories,
|
||||
"modes": modes,
|
||||
"bytes_by_category": bytes_by_category,
|
||||
"files": {
|
||||
"jsonl": "prompts.jsonl",
|
||||
"all_rendered": "rendered_prompts.txt",
|
||||
"nothink_rendered": "rendered_prompts_nothink.txt",
|
||||
"think_rendered": "rendered_prompts_think.txt",
|
||||
},
|
||||
}
|
||||
(outdir / "manifest.json").write_text(json.dumps(manifest, indent=2, ensure_ascii=False) + "\n",
|
||||
encoding="utf-8")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--out", default=None, help="Output directory. Defaults to this script's directory.")
|
||||
args = parser.parse_args()
|
||||
|
||||
script_dir = Path(__file__).resolve().parent
|
||||
root = find_repo_root(script_dir)
|
||||
outdir = Path(args.out).resolve() if args.out else script_dir
|
||||
|
||||
records: list[Record] = []
|
||||
make_source_records(root, records)
|
||||
make_agent_records(records)
|
||||
make_general_records(records)
|
||||
make_long_context_records(root, records)
|
||||
write_outputs(outdir, records)
|
||||
|
||||
manifest = json.loads((outdir / "manifest.json").read_text(encoding="utf-8"))
|
||||
print(f"wrote {manifest['record_count']} prompts to {outdir}")
|
||||
print(f"rendered bytes: {manifest['rendered_utf8_bytes']}")
|
||||
print(f"rough tokens: {manifest['rough_token_estimate_bytes_div_4']}")
|
||||
|
||||
|
||||
def find_repo_root(start: Path) -> Path:
|
||||
for path in [start, *start.parents]:
|
||||
if (path / "ds4.c").exists() and (path / "metal").is_dir():
|
||||
return path
|
||||
raise RuntimeError(f"could not find ds4.c repository root from {start}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"version": 1,
|
||||
"purpose": "DeepSeek V4 Flash imatrix calibration prompts",
|
||||
"record_count": 1952,
|
||||
"rendered_utf8_bytes": 5773104,
|
||||
"rough_token_estimate_bytes_div_4": 1443276,
|
||||
"categories": {
|
||||
"agent": 8,
|
||||
"general": 40,
|
||||
"long_context": 18,
|
||||
"source": 1886
|
||||
},
|
||||
"modes": {
|
||||
"nothink": 976,
|
||||
"think": 976
|
||||
},
|
||||
"bytes_by_category": {
|
||||
"agent": 13100,
|
||||
"general": 11902,
|
||||
"long_context": 371581,
|
||||
"source": 5376521
|
||||
},
|
||||
"files": {
|
||||
"jsonl": "prompts.jsonl",
|
||||
"all_rendered": "rendered_prompts.txt",
|
||||
"nothink_rendered": "rendered_prompts_nothink.txt",
|
||||
"think_rendered": "rendered_prompts_think.txt"
|
||||
}
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,71 @@
|
||||
# Official-Continuation Quality Testing
|
||||
|
||||
This directory contains the 100 prompts and scripts used to compare local GGUF
|
||||
variants against official DeepSeek V4 Flash continuations.
|
||||
|
||||
The metric is target-token negative log likelihood: collect a deterministic
|
||||
official continuation, then ask each local GGUF how much probability it assigns
|
||||
to that exact continuation token by token. This avoids judging quality from one
|
||||
sampled answer.
|
||||
|
||||
## 1. Collect Official Continuations
|
||||
|
||||
```sh
|
||||
export DEEPSEEK_API_KEY=...
|
||||
python3 gguf-tools/quality-testing/collect_official.py \
|
||||
--prompts gguf-tools/quality-testing/prompts.jsonl \
|
||||
--out gguf-tools/quality-testing/data \
|
||||
--count 100 \
|
||||
--max-tokens 24
|
||||
```
|
||||
|
||||
The script writes:
|
||||
|
||||
- `data/prompts/case_*.txt`
|
||||
- `data/continuations/case_*.txt`
|
||||
- `data/responses/case_*.json`
|
||||
- `data/manifest.tsv`
|
||||
|
||||
The prompt list is tracked in `prompts.jsonl`; the official responses are not
|
||||
tracked because they are derived from an external API.
|
||||
|
||||
## 2. Build The Local Scorer
|
||||
|
||||
```sh
|
||||
make -C gguf-tools quality-score
|
||||
```
|
||||
|
||||
The scorer links against the DS4 runtime and uses Metal by default.
|
||||
|
||||
## 3. Score GGUF Variants
|
||||
|
||||
```sh
|
||||
gguf-tools/quality-testing/score_official \
|
||||
../deepseek-v4-quants/gguf/OLD.gguf \
|
||||
gguf-tools/quality-testing/data/manifest.tsv \
|
||||
/tmp/old.tsv \
|
||||
4096
|
||||
|
||||
gguf-tools/quality-testing/score_official \
|
||||
../deepseek-v4-quants/gguf/NEW.gguf \
|
||||
gguf-tools/quality-testing/data/manifest.tsv \
|
||||
/tmp/new.tsv \
|
||||
4096
|
||||
```
|
||||
|
||||
## 4. Compare
|
||||
|
||||
```sh
|
||||
python3 gguf-tools/quality-testing/compare_scores.py /tmp/old.tsv /tmp/new.tsv
|
||||
```
|
||||
|
||||
Output fields:
|
||||
|
||||
- `avg_nll`: average negative log likelihood; lower is better.
|
||||
- `delta_new_minus_old`: negative means the new GGUF fits the official
|
||||
continuation better.
|
||||
- `case_wins_new_old_ties`: per-prompt NLL wins.
|
||||
- `first_token_matches`: how often the local greedy first token matches the
|
||||
official first token.
|
||||
- `avg_greedy_lcp`: average greedy longest common prefix against the official
|
||||
continuation.
|
||||
@@ -0,0 +1,233 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Collect official DeepSeek V4 Flash continuations for local quant scoring."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
MODEL = "deepseek-v4-flash"
|
||||
ENDPOINT = "https://api.deepseek.com/chat/completions"
|
||||
|
||||
|
||||
PROMPTS = [
|
||||
"Explain B-tree insertion, including splits and the root special case.",
|
||||
"Write a concise design for a TCP echo server that handles slow clients.",
|
||||
"Compare mmaped model weights with copying all weights into private buffers on macOS.",
|
||||
"Derive why RMSNorm needs a sum of squares and a scaling pass.",
|
||||
"Explain why KV cache checkpointing helps agentic sessions.",
|
||||
"Give pseudocode for a binary heap push operation.",
|
||||
"What are the tradeoffs between top-p sampling and temperature zero decoding?",
|
||||
"Summarize how speculative decoding preserves exact greedy output.",
|
||||
"Explain why a GPU prefill path can be faster than single-token decode.",
|
||||
"Write three invariants for a tokenizer special-token table.",
|
||||
"Spiega come funziona l'inserimento in un B-tree, inclusi split e radice.",
|
||||
"Scrivi un progetto conciso per un server TCP echo con client lenti.",
|
||||
"Confronta mmap dei pesi e copia completa dei pesi su macOS.",
|
||||
"Deriva perche RMSNorm richiede somma dei quadrati e riscalamento.",
|
||||
"Spiega perche la cache KV su disco aiuta nelle sessioni agentiche.",
|
||||
"Scrivi pseudocodice per inserire un elemento in un heap binario.",
|
||||
"Quali sono i pro e contro di top-p rispetto a temperatura zero?",
|
||||
"Riassumi perche la decodifica speculativa puo mantenere l'output esatto.",
|
||||
"Spiega perche il prefill GPU puo essere piu veloce del decode token singolo.",
|
||||
"Scrivi tre invarianti per una tabella di token speciali.",
|
||||
"Given a sorted array, describe how binary search finds the insertion point.",
|
||||
"Write a short C function that clamps an integer to a range.",
|
||||
"Explain the difference between a mutex and an atomic counter.",
|
||||
"What does backpressure mean in a network server?",
|
||||
"Describe a safe format for storing a model checkpoint header.",
|
||||
"Explain how a ring buffer wraps and how to avoid overwriting unread data.",
|
||||
"Write a brief plan for testing long-context prompt chunking.",
|
||||
"What is an importance matrix in low-bit quantization?",
|
||||
"Explain why grouped MoE expert execution can improve prefill.",
|
||||
"Describe how to compare two model quantizations without relying on one answer.",
|
||||
"Data una lista ordinata, descrivi la ricerca binaria del punto di inserimento.",
|
||||
"Scrivi una breve funzione C che limita un intero a un intervallo.",
|
||||
"Spiega la differenza tra mutex e contatore atomico.",
|
||||
"Che cosa significa backpressure in un server di rete?",
|
||||
"Descrivi un formato sicuro per salvare l'header di un checkpoint modello.",
|
||||
"Spiega come funziona un ring buffer e come evitare sovrascritture.",
|
||||
"Scrivi un piano breve per testare il chunking di prompt lunghi.",
|
||||
"Che cos'e una matrice di importanza nella quantizzazione low-bit?",
|
||||
"Spiega perche raggruppare gli esperti MoE puo accelerare il prefill.",
|
||||
"Descrivi come confrontare due quantizzazioni senza fidarsi di una sola risposta.",
|
||||
"A user reports generation slows at long context. List the first five measurements to take.",
|
||||
"Why can small logit differences change a greedy continuation?",
|
||||
"Explain online softmax in attention using simple variables.",
|
||||
"Give a minimal JSON schema for a tool call with name and arguments.",
|
||||
"How would you test that a file-backed mmap is not repeatedly remapped?",
|
||||
"Explain why one large Metal buffer can be worse than multiple overlapping views.",
|
||||
"What is the role of a router in a mixture-of-experts layer?",
|
||||
"Describe the difference between raw KV rows and compressed KV rows.",
|
||||
"Write a checklist for validating that a quantized model still follows instructions.",
|
||||
"Explain why evaluating 50 prompts is more informative than one favorite prompt.",
|
||||
"Un utente segnala decode lento a contesto lungo. Elenca cinque misure iniziali.",
|
||||
"Perche piccole differenze nei logit possono cambiare una continuazione greedy?",
|
||||
"Spiega online softmax nell'attenzione con variabili semplici.",
|
||||
"Dai uno schema JSON minimo per una tool call con nome e argomenti.",
|
||||
"Come testeresti che un mmap su file non venga rimappato ripetutamente?",
|
||||
"Spiega perche un grande buffer Metal puo essere peggiore di viste sovrapposte.",
|
||||
"Qual e il ruolo del router in un layer mixture-of-experts?",
|
||||
"Descrivi la differenza tra righe KV raw e righe KV compresse.",
|
||||
"Scrivi una checklist per validare che un modello quantizzato segua ancora le istruzioni.",
|
||||
"Perche valutare 50 prompt e piu utile di un singolo prompt preferito?",
|
||||
"Write a tiny Python function that returns the median of three numbers.",
|
||||
"Explain why sorted arrays make membership tests faster with binary search.",
|
||||
"What does eventual consistency mean in a distributed database?",
|
||||
"Give a short example of a race condition in C.",
|
||||
"Explain the difference between latency and throughput.",
|
||||
"How does a trie represent a set of strings?",
|
||||
"Describe how to test a command-line parser with edge cases.",
|
||||
"Why can mmap page residency differ from virtual address space size?",
|
||||
"Explain why quantization error can affect rare experts more than common experts.",
|
||||
"Write a short checklist for reviewing a pull request that touches memory lifetimes.",
|
||||
"Scrivi una piccola funzione Python che restituisce la mediana di tre numeri.",
|
||||
"Spiega perche array ordinati rendono piu veloce la ricerca di appartenenza.",
|
||||
"Che cosa significa consistenza eventuale in un database distribuito?",
|
||||
"Fai un breve esempio di race condition in C.",
|
||||
"Spiega la differenza tra latenza e throughput.",
|
||||
"Come rappresenta un trie un insieme di stringhe?",
|
||||
"Descrivi come testare un parser da riga di comando con casi limite.",
|
||||
"Perche la residenza delle pagine mmap puo differire dalla dimensione virtuale?",
|
||||
"Spiega perche l'errore di quantizzazione puo colpire piu gli esperti rari.",
|
||||
"Scrivi una breve checklist per revisionare lifetime di memoria in una PR.",
|
||||
"Complete this sentence: A good benchmark should measure",
|
||||
"Complete this C comment: /* This lock protects",
|
||||
"Complete this Italian sentence: Il vantaggio principale della cache e",
|
||||
"Translate to Italian: The model should answer only after reading the whole prompt.",
|
||||
"Translate to English: La quantizzazione riduce memoria ma puo alterare i logit.",
|
||||
"In one paragraph, explain how a compiler uses an abstract syntax tree.",
|
||||
"In one paragraph, explain why checksums catch accidental corruption.",
|
||||
"Give three examples of useful server metrics.",
|
||||
"Why should a tokenizer treat special tags carefully?",
|
||||
"Explain how a hash table handles collisions.",
|
||||
"Describe the role of calibration data when quantizing a neural network.",
|
||||
"What is a confidence interval, in plain language?",
|
||||
"Write a simple SQL query that counts rows per category.",
|
||||
"Explain why a page cache can make a second file read faster.",
|
||||
"Give a short answer: what is the capital of Japan?",
|
||||
"Give a short answer: what is the derivative of x squared?",
|
||||
"Give a short answer: who wrote The Divine Comedy?",
|
||||
"Rispondi brevemente: qual e la capitale del Giappone?",
|
||||
"Rispondi brevemente: quanto fa 17 per 23?",
|
||||
"Rispondi brevemente: chi ha scritto la Divina Commedia?",
|
||||
]
|
||||
|
||||
|
||||
def request_one(api_key: str, prompt: str, max_tokens: int) -> dict:
|
||||
payload = {
|
||||
"model": MODEL,
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"temperature": 0,
|
||||
"max_tokens": max_tokens,
|
||||
"logprobs": True,
|
||||
"top_logprobs": 5,
|
||||
"thinking": {"type": "disabled"},
|
||||
"stream": False,
|
||||
}
|
||||
req = urllib.request.Request(
|
||||
ENDPOINT,
|
||||
data=json.dumps(payload).encode("utf-8"),
|
||||
headers={
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
method="POST",
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=120) as fp:
|
||||
return json.loads(fp.read().decode("utf-8"))
|
||||
|
||||
|
||||
def fetch_with_retry(api_key: str, prompt: str, max_tokens: int) -> dict:
|
||||
delay = 1.0
|
||||
for attempt in range(6):
|
||||
try:
|
||||
return request_one(api_key, prompt, max_tokens)
|
||||
except urllib.error.HTTPError as e:
|
||||
body = e.read().decode("utf-8", "replace")
|
||||
if e.code < 500 and e.code != 429:
|
||||
raise RuntimeError(f"HTTP {e.code}: {body}") from e
|
||||
last = RuntimeError(f"HTTP {e.code}: {body}")
|
||||
except Exception as e: # noqa: BLE001 - command-line retry wrapper.
|
||||
last = e
|
||||
if attempt == 5:
|
||||
raise last
|
||||
time.sleep(delay)
|
||||
delay *= 1.7
|
||||
raise AssertionError("unreachable")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--out", default="gguf-tools/quality-testing/data")
|
||||
ap.add_argument("--prompts", default="gguf-tools/quality-testing/prompts.jsonl")
|
||||
ap.add_argument("--count", type=int, default=100)
|
||||
ap.add_argument("--max-tokens", type=int, default=24)
|
||||
args = ap.parse_args()
|
||||
|
||||
api_key = os.environ.get("DEEPSEEK_API_KEY")
|
||||
if not api_key:
|
||||
raise SystemExit("DEEPSEEK_API_KEY is not set")
|
||||
prompts = load_prompts(Path(args.prompts))
|
||||
|
||||
out = Path(args.out)
|
||||
(out / "prompts").mkdir(parents=True, exist_ok=True)
|
||||
(out / "continuations").mkdir(parents=True, exist_ok=True)
|
||||
(out / "responses").mkdir(parents=True, exist_ok=True)
|
||||
|
||||
manifest = out / "manifest.tsv"
|
||||
rows = []
|
||||
total = min(args.count, len(prompts))
|
||||
for i, prompt in enumerate(prompts[: args.count]):
|
||||
case_id = f"case_{i:03d}"
|
||||
print(f"official {i + 1}/{total}: {case_id}", file=sys.stderr, flush=True)
|
||||
response = fetch_with_retry(api_key, prompt, args.max_tokens)
|
||||
choice = response["choices"][0]
|
||||
content = choice.get("message", {}).get("content", "")
|
||||
if not content:
|
||||
print(f"warning: empty continuation for {case_id}", file=sys.stderr)
|
||||
|
||||
prompt_path = out / "prompts" / f"{case_id}.txt"
|
||||
cont_path = out / "continuations" / f"{case_id}.txt"
|
||||
resp_path = out / "responses" / f"{case_id}.json"
|
||||
prompt_path.write_text(prompt, encoding="utf-8")
|
||||
cont_path.write_text(content, encoding="utf-8")
|
||||
resp_path.write_text(json.dumps(response, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
rows.append((case_id, prompt_path, cont_path, resp_path))
|
||||
time.sleep(0.05)
|
||||
|
||||
with manifest.open("w", encoding="utf-8") as fp:
|
||||
fp.write("# id\tprompt_file\tcontinuation_file\tresponse_file\n")
|
||||
for row in rows:
|
||||
fp.write("\t".join([row[0], str(row[1]), str(row[2]), str(row[3])]) + "\n")
|
||||
print(f"wrote {manifest}", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
|
||||
def load_prompts(path: Path) -> list[str]:
|
||||
if not path.exists():
|
||||
return PROMPTS
|
||||
prompts = []
|
||||
with path.open(encoding="utf-8") as fp:
|
||||
for line in fp:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
obj = json.loads(line)
|
||||
prompt = obj.get("prompt") if isinstance(obj, dict) else None
|
||||
if not prompt:
|
||||
raise SystemExit(f"bad prompt row in {path}: {line[:120]}")
|
||||
prompts.append(prompt)
|
||||
if not prompts:
|
||||
raise SystemExit(f"no prompts found in {path}")
|
||||
return prompts
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,88 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compare two local model scores on official continuations."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def load(path: Path) -> dict[str, dict[str, float]]:
|
||||
with path.open(newline="", encoding="utf-8") as fp:
|
||||
rows = {}
|
||||
for row in csv.DictReader(fp, delimiter="\t"):
|
||||
rows[row["id"]] = {
|
||||
"target_tokens": int(row["target_tokens"]),
|
||||
"nll": float(row["nll"]),
|
||||
"avg_nll": float(row["avg_nll"]),
|
||||
"first_match": int(row["first_match"]),
|
||||
"greedy_lcp": int(row["greedy_lcp"]),
|
||||
}
|
||||
return rows
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if len(sys.argv) != 3:
|
||||
print(f"usage: {sys.argv[0]} OLD.tsv NEW.tsv", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
old = load(Path(sys.argv[1]))
|
||||
new = load(Path(sys.argv[2]))
|
||||
ids = sorted(set(old) & set(new))
|
||||
if not ids:
|
||||
raise SystemExit("no common cases")
|
||||
|
||||
old_nll = new_nll = 0.0
|
||||
old_first = new_first = 0
|
||||
old_lcp = new_lcp = 0
|
||||
tokens = 0
|
||||
new_case_wins = old_case_wins = ties = 0
|
||||
deltas = []
|
||||
|
||||
for case_id in ids:
|
||||
o = old[case_id]
|
||||
n = new[case_id]
|
||||
if o["target_tokens"] != n["target_tokens"]:
|
||||
raise SystemExit(f"token-count mismatch for {case_id}")
|
||||
t = int(o["target_tokens"])
|
||||
tokens += t
|
||||
old_nll += o["nll"]
|
||||
new_nll += n["nll"]
|
||||
old_first += int(o["first_match"])
|
||||
new_first += int(n["first_match"])
|
||||
old_lcp += int(o["greedy_lcp"])
|
||||
new_lcp += int(n["greedy_lcp"])
|
||||
delta = n["nll"] - o["nll"]
|
||||
deltas.append((delta, case_id, t, o["avg_nll"], n["avg_nll"]))
|
||||
if delta < -1e-9:
|
||||
new_case_wins += 1
|
||||
elif delta > 1e-9:
|
||||
old_case_wins += 1
|
||||
else:
|
||||
ties += 1
|
||||
|
||||
avg_old = old_nll / tokens
|
||||
avg_new = new_nll / tokens
|
||||
print(f"cases\t{len(ids)}")
|
||||
print(f"tokens\t{tokens}")
|
||||
print(f"old_avg_nll\t{avg_old:.9f}")
|
||||
print(f"new_avg_nll\t{avg_new:.9f}")
|
||||
print(f"delta_new_minus_old\t{avg_new - avg_old:.9f}")
|
||||
print(f"relative_nll_change\t{(avg_new / avg_old - 1.0) * 100.0:.3f}%")
|
||||
print(f"case_wins_new_old_ties\t{new_case_wins}\t{old_case_wins}\t{ties}")
|
||||
print(f"first_token_matches_old_new\t{old_first}\t{new_first}")
|
||||
print(f"avg_greedy_lcp_old_new\t{old_lcp / len(ids):.3f}\t{new_lcp / len(ids):.3f}")
|
||||
|
||||
print("\nnew best cases:")
|
||||
for delta, case_id, t, old_avg, new_avg in sorted(deltas)[:8]:
|
||||
print(f"{case_id}\tdelta_nll={delta:.6f}\ttokens={t}\told={old_avg:.6f}\tnew={new_avg:.6f}")
|
||||
|
||||
print("\nold best cases:")
|
||||
for delta, case_id, t, old_avg, new_avg in sorted(deltas, reverse=True)[:8]:
|
||||
print(f"{case_id}\tdelta_nll={delta:.6f}\ttokens={t}\told={old_avg:.6f}\tnew={new_avg:.6f}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,100 @@
|
||||
{"id": "case_000", "prompt": "Explain B-tree insertion, including splits and the root special case."}
|
||||
{"id": "case_001", "prompt": "Write a concise design for a TCP echo server that handles slow clients."}
|
||||
{"id": "case_002", "prompt": "Compare mmaped model weights with copying all weights into private buffers on macOS."}
|
||||
{"id": "case_003", "prompt": "Derive why RMSNorm needs a sum of squares and a scaling pass."}
|
||||
{"id": "case_004", "prompt": "Explain why KV cache checkpointing helps agentic sessions."}
|
||||
{"id": "case_005", "prompt": "Give pseudocode for a binary heap push operation."}
|
||||
{"id": "case_006", "prompt": "What are the tradeoffs between top-p sampling and temperature zero decoding?"}
|
||||
{"id": "case_007", "prompt": "Summarize how speculative decoding preserves exact greedy output."}
|
||||
{"id": "case_008", "prompt": "Explain why a GPU prefill path can be faster than single-token decode."}
|
||||
{"id": "case_009", "prompt": "Write three invariants for a tokenizer special-token table."}
|
||||
{"id": "case_010", "prompt": "Spiega come funziona l'inserimento in un B-tree, inclusi split e radice."}
|
||||
{"id": "case_011", "prompt": "Scrivi un progetto conciso per un server TCP echo con client lenti."}
|
||||
{"id": "case_012", "prompt": "Confronta mmap dei pesi e copia completa dei pesi su macOS."}
|
||||
{"id": "case_013", "prompt": "Deriva perche RMSNorm richiede somma dei quadrati e riscalamento."}
|
||||
{"id": "case_014", "prompt": "Spiega perche la cache KV su disco aiuta nelle sessioni agentiche."}
|
||||
{"id": "case_015", "prompt": "Scrivi pseudocodice per inserire un elemento in un heap binario."}
|
||||
{"id": "case_016", "prompt": "Quali sono i pro e contro di top-p rispetto a temperatura zero?"}
|
||||
{"id": "case_017", "prompt": "Riassumi perche la decodifica speculativa puo mantenere l'output esatto."}
|
||||
{"id": "case_018", "prompt": "Spiega perche il prefill GPU puo essere piu veloce del decode token singolo."}
|
||||
{"id": "case_019", "prompt": "Scrivi tre invarianti per una tabella di token speciali."}
|
||||
{"id": "case_020", "prompt": "Given a sorted array, describe how binary search finds the insertion point."}
|
||||
{"id": "case_021", "prompt": "Write a short C function that clamps an integer to a range."}
|
||||
{"id": "case_022", "prompt": "Explain the difference between a mutex and an atomic counter."}
|
||||
{"id": "case_023", "prompt": "What does backpressure mean in a network server?"}
|
||||
{"id": "case_024", "prompt": "Describe a safe format for storing a model checkpoint header."}
|
||||
{"id": "case_025", "prompt": "Explain how a ring buffer wraps and how to avoid overwriting unread data."}
|
||||
{"id": "case_026", "prompt": "Write a brief plan for testing long-context prompt chunking."}
|
||||
{"id": "case_027", "prompt": "What is an importance matrix in low-bit quantization?"}
|
||||
{"id": "case_028", "prompt": "Explain why grouped MoE expert execution can improve prefill."}
|
||||
{"id": "case_029", "prompt": "Describe how to compare two model quantizations without relying on one answer."}
|
||||
{"id": "case_030", "prompt": "Data una lista ordinata, descrivi la ricerca binaria del punto di inserimento."}
|
||||
{"id": "case_031", "prompt": "Scrivi una breve funzione C che limita un intero a un intervallo."}
|
||||
{"id": "case_032", "prompt": "Spiega la differenza tra mutex e contatore atomico."}
|
||||
{"id": "case_033", "prompt": "Che cosa significa backpressure in un server di rete?"}
|
||||
{"id": "case_034", "prompt": "Descrivi un formato sicuro per salvare l'header di un checkpoint modello."}
|
||||
{"id": "case_035", "prompt": "Spiega come funziona un ring buffer e come evitare sovrascritture."}
|
||||
{"id": "case_036", "prompt": "Scrivi un piano breve per testare il chunking di prompt lunghi."}
|
||||
{"id": "case_037", "prompt": "Che cos'e una matrice di importanza nella quantizzazione low-bit?"}
|
||||
{"id": "case_038", "prompt": "Spiega perche raggruppare gli esperti MoE puo accelerare il prefill."}
|
||||
{"id": "case_039", "prompt": "Descrivi come confrontare due quantizzazioni senza fidarsi di una sola risposta."}
|
||||
{"id": "case_040", "prompt": "A user reports generation slows at long context. List the first five measurements to take."}
|
||||
{"id": "case_041", "prompt": "Why can small logit differences change a greedy continuation?"}
|
||||
{"id": "case_042", "prompt": "Explain online softmax in attention using simple variables."}
|
||||
{"id": "case_043", "prompt": "Give a minimal JSON schema for a tool call with name and arguments."}
|
||||
{"id": "case_044", "prompt": "How would you test that a file-backed mmap is not repeatedly remapped?"}
|
||||
{"id": "case_045", "prompt": "Explain why one large Metal buffer can be worse than multiple overlapping views."}
|
||||
{"id": "case_046", "prompt": "What is the role of a router in a mixture-of-experts layer?"}
|
||||
{"id": "case_047", "prompt": "Describe the difference between raw KV rows and compressed KV rows."}
|
||||
{"id": "case_048", "prompt": "Write a checklist for validating that a quantized model still follows instructions."}
|
||||
{"id": "case_049", "prompt": "Explain why evaluating 50 prompts is more informative than one favorite prompt."}
|
||||
{"id": "case_050", "prompt": "Un utente segnala decode lento a contesto lungo. Elenca cinque misure iniziali."}
|
||||
{"id": "case_051", "prompt": "Perche piccole differenze nei logit possono cambiare una continuazione greedy?"}
|
||||
{"id": "case_052", "prompt": "Spiega online softmax nell'attenzione con variabili semplici."}
|
||||
{"id": "case_053", "prompt": "Dai uno schema JSON minimo per una tool call con nome e argomenti."}
|
||||
{"id": "case_054", "prompt": "Come testeresti che un mmap su file non venga rimappato ripetutamente?"}
|
||||
{"id": "case_055", "prompt": "Spiega perche un grande buffer Metal puo essere peggiore di viste sovrapposte."}
|
||||
{"id": "case_056", "prompt": "Qual e il ruolo del router in un layer mixture-of-experts?"}
|
||||
{"id": "case_057", "prompt": "Descrivi la differenza tra righe KV raw e righe KV compresse."}
|
||||
{"id": "case_058", "prompt": "Scrivi una checklist per validare che un modello quantizzato segua ancora le istruzioni."}
|
||||
{"id": "case_059", "prompt": "Perche valutare 50 prompt e piu utile di un singolo prompt preferito?"}
|
||||
{"id": "case_060", "prompt": "Write a tiny Python function that returns the median of three numbers."}
|
||||
{"id": "case_061", "prompt": "Explain why sorted arrays make membership tests faster with binary search."}
|
||||
{"id": "case_062", "prompt": "What does eventual consistency mean in a distributed database?"}
|
||||
{"id": "case_063", "prompt": "Give a short example of a race condition in C."}
|
||||
{"id": "case_064", "prompt": "Explain the difference between latency and throughput."}
|
||||
{"id": "case_065", "prompt": "How does a trie represent a set of strings?"}
|
||||
{"id": "case_066", "prompt": "Describe how to test a command-line parser with edge cases."}
|
||||
{"id": "case_067", "prompt": "Why can mmap page residency differ from virtual address space size?"}
|
||||
{"id": "case_068", "prompt": "Explain why quantization error can affect rare experts more than common experts."}
|
||||
{"id": "case_069", "prompt": "Write a short checklist for reviewing a pull request that touches memory lifetimes."}
|
||||
{"id": "case_070", "prompt": "Scrivi una piccola funzione Python che restituisce la mediana di tre numeri."}
|
||||
{"id": "case_071", "prompt": "Spiega perche array ordinati rendono piu veloce la ricerca di appartenenza."}
|
||||
{"id": "case_072", "prompt": "Che cosa significa consistenza eventuale in un database distribuito?"}
|
||||
{"id": "case_073", "prompt": "Fai un breve esempio di race condition in C."}
|
||||
{"id": "case_074", "prompt": "Spiega la differenza tra latenza e throughput."}
|
||||
{"id": "case_075", "prompt": "Come rappresenta un trie un insieme di stringhe?"}
|
||||
{"id": "case_076", "prompt": "Descrivi come testare un parser da riga di comando con casi limite."}
|
||||
{"id": "case_077", "prompt": "Perche la residenza delle pagine mmap puo differire dalla dimensione virtuale?"}
|
||||
{"id": "case_078", "prompt": "Spiega perche l'errore di quantizzazione puo colpire piu gli esperti rari."}
|
||||
{"id": "case_079", "prompt": "Scrivi una breve checklist per revisionare lifetime di memoria in una PR."}
|
||||
{"id": "case_080", "prompt": "Complete this sentence: A good benchmark should measure"}
|
||||
{"id": "case_081", "prompt": "Complete this C comment: /* This lock protects"}
|
||||
{"id": "case_082", "prompt": "Complete this Italian sentence: Il vantaggio principale della cache e"}
|
||||
{"id": "case_083", "prompt": "Translate to Italian: The model should answer only after reading the whole prompt."}
|
||||
{"id": "case_084", "prompt": "Translate to English: La quantizzazione riduce memoria ma puo alterare i logit."}
|
||||
{"id": "case_085", "prompt": "In one paragraph, explain how a compiler uses an abstract syntax tree."}
|
||||
{"id": "case_086", "prompt": "In one paragraph, explain why checksums catch accidental corruption."}
|
||||
{"id": "case_087", "prompt": "Give three examples of useful server metrics."}
|
||||
{"id": "case_088", "prompt": "Why should a tokenizer treat special tags carefully?"}
|
||||
{"id": "case_089", "prompt": "Explain how a hash table handles collisions."}
|
||||
{"id": "case_090", "prompt": "Describe the role of calibration data when quantizing a neural network."}
|
||||
{"id": "case_091", "prompt": "What is a confidence interval, in plain language?"}
|
||||
{"id": "case_092", "prompt": "Write a simple SQL query that counts rows per category."}
|
||||
{"id": "case_093", "prompt": "Explain why a page cache can make a second file read faster."}
|
||||
{"id": "case_094", "prompt": "Give a short answer: what is the capital of Japan?"}
|
||||
{"id": "case_095", "prompt": "Give a short answer: what is the derivative of x squared?"}
|
||||
{"id": "case_096", "prompt": "Give a short answer: who wrote The Divine Comedy?"}
|
||||
{"id": "case_097", "prompt": "Rispondi brevemente: qual e la capitale del Giappone?"}
|
||||
{"id": "case_098", "prompt": "Rispondi brevemente: quanto fa 17 per 23?"}
|
||||
{"id": "case_099", "prompt": "Rispondi brevemente: chi ha scritto la Divina Commedia?"}
|
||||
@@ -0,0 +1,166 @@
|
||||
#include "ds4.h"
|
||||
|
||||
#include <errno.h>
|
||||
#include <stdbool.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
static void die(const char *msg) {
|
||||
fprintf(stderr, "%s\n", msg);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
static char *read_file(const char *path) {
|
||||
FILE *fp = fopen(path, "rb");
|
||||
if (!fp) {
|
||||
fprintf(stderr, "open %s: %s\n", path, strerror(errno));
|
||||
exit(1);
|
||||
}
|
||||
if (fseek(fp, 0, SEEK_END) != 0) die("fseek failed");
|
||||
long n = ftell(fp);
|
||||
if (n < 0) die("ftell failed");
|
||||
if (fseek(fp, 0, SEEK_SET) != 0) die("fseek failed");
|
||||
char *buf = malloc((size_t)n + 1);
|
||||
if (!buf) die("out of memory");
|
||||
if (n && fread(buf, 1, (size_t)n, fp) != (size_t)n) die("read failed");
|
||||
buf[n] = '\0';
|
||||
fclose(fp);
|
||||
return buf;
|
||||
}
|
||||
|
||||
static void strip_newline(char *s) {
|
||||
size_t n = strlen(s);
|
||||
while (n && (s[n - 1] == '\n' || s[n - 1] == '\r')) s[--n] = '\0';
|
||||
}
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
if (argc != 4 && argc != 5) {
|
||||
fprintf(stderr, "usage: %s MODEL manifest.tsv OUT.tsv [ctx]\n", argv[0]);
|
||||
return 2;
|
||||
}
|
||||
|
||||
const char *model_path = argv[1];
|
||||
const char *manifest_path = argv[2];
|
||||
const char *out_path = argv[3];
|
||||
int ctx_size = argc == 5 ? atoi(argv[4]) : 4096;
|
||||
if (ctx_size < 1024) ctx_size = 1024;
|
||||
|
||||
ds4_engine_options opt = {
|
||||
.model_path = model_path,
|
||||
.backend = DS4_BACKEND_METAL,
|
||||
.n_threads = 0,
|
||||
.warm_weights = false,
|
||||
.quality = false,
|
||||
};
|
||||
|
||||
ds4_engine *engine = NULL;
|
||||
if (ds4_engine_open(&engine, &opt) != 0) die("failed to open model");
|
||||
|
||||
ds4_session *session = NULL;
|
||||
if (ds4_session_create(&session, engine, ctx_size) != 0) die("failed to create session");
|
||||
|
||||
FILE *mf = fopen(manifest_path, "rb");
|
||||
if (!mf) {
|
||||
fprintf(stderr, "open %s: %s\n", manifest_path, strerror(errno));
|
||||
return 1;
|
||||
}
|
||||
FILE *out = fopen(out_path, "wb");
|
||||
if (!out) {
|
||||
fprintf(stderr, "open %s: %s\n", out_path, strerror(errno));
|
||||
return 1;
|
||||
}
|
||||
fprintf(out, "id\tprompt_tokens\ttarget_tokens\tnll\tavg_nll\tfirst_match\tgreedy_lcp\n");
|
||||
|
||||
char line[8192];
|
||||
int case_n = 0;
|
||||
double total_nll = 0.0;
|
||||
long total_tokens = 0;
|
||||
long total_lcp = 0;
|
||||
long first_matches = 0;
|
||||
char err[256];
|
||||
|
||||
while (fgets(line, sizeof(line), mf)) {
|
||||
strip_newline(line);
|
||||
if (!line[0] || line[0] == '#') continue;
|
||||
|
||||
char *id = strtok(line, "\t");
|
||||
char *prompt_path = strtok(NULL, "\t");
|
||||
char *cont_path = strtok(NULL, "\t");
|
||||
if (!id || !prompt_path || !cont_path) die("bad manifest row");
|
||||
|
||||
char *prompt_text = read_file(prompt_path);
|
||||
char *cont_text = read_file(cont_path);
|
||||
|
||||
ds4_tokens prompt = {0};
|
||||
ds4_tokens target = {0};
|
||||
ds4_encode_chat_prompt(engine, NULL, prompt_text, DS4_THINK_NONE, &prompt);
|
||||
ds4_tokenize_text(engine, cont_text, &target);
|
||||
|
||||
if (prompt.len + target.len + 1 >= ctx_size) {
|
||||
fprintf(stderr, "%s exceeds ctx=%d\n", id, ctx_size);
|
||||
return 1;
|
||||
}
|
||||
if (ds4_session_sync(session, &prompt, err, sizeof(err)) != 0) {
|
||||
fprintf(stderr, "%s sync failed: %s\n", id, err);
|
||||
return 1;
|
||||
}
|
||||
|
||||
double nll = 0.0;
|
||||
int lcp = 0;
|
||||
bool still_matching = true;
|
||||
bool first_match = false;
|
||||
for (int i = 0; i < target.len; i++) {
|
||||
const int greedy = ds4_session_argmax(session);
|
||||
if (i == 0) first_match = (greedy == target.v[i]);
|
||||
if (still_matching && greedy == target.v[i]) lcp++;
|
||||
else still_matching = false;
|
||||
|
||||
ds4_token_score score;
|
||||
if (!ds4_session_token_logprob(session, target.v[i], &score)) {
|
||||
fprintf(stderr, "%s logprob failed at target token %d\n", id, i);
|
||||
return 1;
|
||||
}
|
||||
nll += -(double)score.logprob;
|
||||
|
||||
if (ds4_session_eval(session, target.v[i], err, sizeof(err)) != 0) {
|
||||
fprintf(stderr, "%s eval failed at target token %d: %s\n", id, i, err);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
const double avg = target.len ? nll / (double)target.len : 0.0;
|
||||
fprintf(out, "%s\t%d\t%d\t%.9f\t%.9f\t%d\t%d\n",
|
||||
id, prompt.len, target.len, nll, avg, first_match ? 1 : 0, lcp);
|
||||
fflush(out);
|
||||
|
||||
case_n++;
|
||||
total_nll += nll;
|
||||
total_tokens += target.len;
|
||||
total_lcp += lcp;
|
||||
first_matches += first_match ? 1 : 0;
|
||||
fprintf(stderr,
|
||||
"%s cases=%d prompt=%d target=%d avg_nll=%.6f lcp=%d\n",
|
||||
id, case_n, prompt.len, target.len, avg, lcp);
|
||||
|
||||
ds4_tokens_free(&prompt);
|
||||
ds4_tokens_free(&target);
|
||||
free(prompt_text);
|
||||
free(cont_text);
|
||||
}
|
||||
|
||||
fprintf(stderr,
|
||||
"summary cases=%d tokens=%ld avg_nll=%.9f first_match=%ld avg_lcp=%.3f\n",
|
||||
case_n,
|
||||
total_tokens,
|
||||
total_tokens ? total_nll / (double)total_tokens : 0.0,
|
||||
first_matches,
|
||||
case_n ? (double)total_lcp / (double)case_n : 0.0);
|
||||
|
||||
fclose(out);
|
||||
fclose(mf);
|
||||
ds4_session_free(session);
|
||||
ds4_engine_close(engine);
|
||||
return 0;
|
||||
}
|
||||
+1109
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,75 @@
|
||||
#ifndef DS4_QUANTS_H
|
||||
#define DS4_QUANTS_H
|
||||
|
||||
/*
|
||||
* Narrow quantization API used by the DS4 GGUF writer.
|
||||
*
|
||||
* The enum values intentionally match GGUF/GGML type IDs so template metadata
|
||||
* can be copied without translation. Only the formats used by the DS4 Flash
|
||||
* quantization recipes are implemented as output targets.
|
||||
*/
|
||||
|
||||
#include <stdbool.h>
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#define DS4Q_MAX_DIMS 4
|
||||
|
||||
typedef enum {
|
||||
DS4Q_TYPE_F32 = 0,
|
||||
DS4Q_TYPE_F16 = 1,
|
||||
DS4Q_TYPE_Q4_0 = 2,
|
||||
DS4Q_TYPE_Q4_1 = 3,
|
||||
DS4Q_TYPE_Q5_0 = 6,
|
||||
DS4Q_TYPE_Q5_1 = 7,
|
||||
DS4Q_TYPE_Q8_0 = 8,
|
||||
DS4Q_TYPE_Q8_1 = 9,
|
||||
DS4Q_TYPE_Q2_K = 10,
|
||||
DS4Q_TYPE_Q3_K = 11,
|
||||
DS4Q_TYPE_Q4_K = 12,
|
||||
DS4Q_TYPE_Q5_K = 13,
|
||||
DS4Q_TYPE_Q6_K = 14,
|
||||
DS4Q_TYPE_Q8_K = 15,
|
||||
DS4Q_TYPE_IQ2_XXS = 16,
|
||||
DS4Q_TYPE_IQ2_XS = 17,
|
||||
DS4Q_TYPE_IQ3_XXS = 18,
|
||||
DS4Q_TYPE_IQ1_S = 19,
|
||||
DS4Q_TYPE_IQ4_NL = 20,
|
||||
DS4Q_TYPE_IQ3_S = 21,
|
||||
DS4Q_TYPE_IQ2_S = 22,
|
||||
DS4Q_TYPE_IQ4_XS = 23,
|
||||
DS4Q_TYPE_I8 = 24,
|
||||
DS4Q_TYPE_I16 = 25,
|
||||
DS4Q_TYPE_I32 = 26,
|
||||
DS4Q_TYPE_I64 = 27,
|
||||
DS4Q_TYPE_F64 = 28,
|
||||
DS4Q_TYPE_IQ1_M = 29,
|
||||
DS4Q_TYPE_BF16 = 30,
|
||||
DS4Q_TYPE_TQ1_0 = 34,
|
||||
DS4Q_TYPE_TQ2_0 = 35,
|
||||
DS4Q_TYPE_MXFP4 = 39,
|
||||
DS4Q_TYPE_NVFP4 = 40,
|
||||
DS4Q_TYPE_Q1_0 = 41,
|
||||
DS4Q_TYPE_COUNT = 42,
|
||||
} ds4q_type;
|
||||
|
||||
static inline size_t ds4q_pad(size_t x, size_t n) {
|
||||
return ((x + n - 1) / n) * n;
|
||||
}
|
||||
|
||||
const char *ds4q_type_name(ds4q_type type);
|
||||
bool ds4q_can_quantize(ds4q_type type);
|
||||
int64_t ds4q_block_size(ds4q_type type);
|
||||
size_t ds4q_row_size(ds4q_type type, int64_t ne);
|
||||
bool ds4q_requires_imatrix(ds4q_type type);
|
||||
void ds4q_quantize_init(ds4q_type type);
|
||||
size_t ds4q_quantize_chunk(ds4q_type type, const float *src, void *dst,
|
||||
int64_t start, int64_t nrows, int64_t ncols,
|
||||
const float *imatrix);
|
||||
|
||||
float ds4q_f16_to_f32(uint16_t bits);
|
||||
float ds4q_bf16_to_f32(uint16_t bits);
|
||||
void ds4q_f32_to_f16_row(const float *src, uint16_t *dst, int64_t n);
|
||||
void ds4q_f32_to_bf16_row(const float *src, uint16_t *dst, int64_t n);
|
||||
|
||||
#endif
|
||||
Reference in New Issue
Block a user