mirror of
https://github.com/zenlm/zen5-engine.git
synced 2026-07-26 22:09:00 +00:00
Add long-context story recall regression
Replace the default long-context continuation check with a deterministic prose-story retrieval test. The fixture embeds spelled-out person-number assignments in a long rendered prompt, and ds4_test now validates the generated Name=number list instead of brittle sampled prose.
This commit is contained in:
antirez
+4
-2
@@ -43,8 +43,10 @@ What they cover:
|
||||
- `--logprob-vectors`: compares local token bytes and top-logprob slices against
|
||||
official DeepSeek V4 Flash continuation vectors. This catches tokenizer,
|
||||
template, attention, and logits regressions.
|
||||
- `--long-context`: runs a long-context continuation regression from
|
||||
`tests/long_context_security_prompt.txt`.
|
||||
- `--long-context`: runs a long-context story fact-recall regression from
|
||||
`tests/long_context_story_prompt.txt`. The model must retrieve spelled-out
|
||||
person-number assignments from a long prose prompt and return `Name=number`
|
||||
lines that the test parses.
|
||||
- `--tool-call-quality`: exercises actual model behavior for DSML tool-call
|
||||
emission in both fast and exact paths.
|
||||
- `--metal-kernels`: isolated Metal kernel numeric checks.
|
||||
|
||||
+86
-18
@@ -178,15 +178,84 @@ static char *test_read_file(const char *path) {
|
||||
return s;
|
||||
}
|
||||
|
||||
static int test_count_substr(const char *s, const char *needle) {
|
||||
int count = 0;
|
||||
size_t n = strlen(needle);
|
||||
const char *p = s;
|
||||
while ((p = strstr(p, needle)) != NULL) {
|
||||
count++;
|
||||
p += n;
|
||||
typedef struct {
|
||||
const char *name;
|
||||
int number;
|
||||
} test_long_fact;
|
||||
|
||||
static const test_long_fact test_long_facts[] = {
|
||||
{"Bob", 34},
|
||||
{"Alice", 52},
|
||||
{"Clara", 71},
|
||||
{"Diego", 93},
|
||||
{"Elena", 16},
|
||||
{"Felix", 88},
|
||||
{"Greta", 47},
|
||||
{"Hugo", 29},
|
||||
{"Iris", 64},
|
||||
{"Jonas", 12},
|
||||
{"Kira", 81},
|
||||
{"Leo", 39},
|
||||
{"Marta", 76},
|
||||
{"Nadia", 23},
|
||||
{"Owen", 58},
|
||||
{"Priya", 97},
|
||||
};
|
||||
|
||||
static bool test_is_name_boundary(char c) {
|
||||
unsigned char uc = (unsigned char)c;
|
||||
return c == '\0' || !(isalnum(uc) || c == '_');
|
||||
}
|
||||
|
||||
static bool test_parse_assignment_value(const char *p, int *value) {
|
||||
while (*p == ' ' || *p == '\t') p++;
|
||||
if (*p != '=') return false;
|
||||
p++;
|
||||
while (*p == ' ' || *p == '\t') p++;
|
||||
if (!isdigit((unsigned char)*p)) return false;
|
||||
|
||||
int v = 0;
|
||||
while (isdigit((unsigned char)*p)) {
|
||||
v = v * 10 + (*p - '0');
|
||||
p++;
|
||||
}
|
||||
return count;
|
||||
*value = v;
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool test_output_has_fact(const char *text, const test_long_fact *fact) {
|
||||
const size_t name_len = strlen(fact->name);
|
||||
const char *p = text;
|
||||
bool saw_wrong_assignment = false;
|
||||
int wrong_value = -1;
|
||||
|
||||
while ((p = strstr(p, fact->name)) != NULL) {
|
||||
const bool before_ok = p == text || test_is_name_boundary(p[-1]);
|
||||
const bool after_ok = test_is_name_boundary(p[name_len]) ||
|
||||
p[name_len] == ' ' ||
|
||||
p[name_len] == '\t' ||
|
||||
p[name_len] == '=';
|
||||
if (before_ok && after_ok) {
|
||||
int value = 0;
|
||||
if (test_parse_assignment_value(p + name_len, &value)) {
|
||||
if (value == fact->number) return true;
|
||||
saw_wrong_assignment = true;
|
||||
wrong_value = value;
|
||||
}
|
||||
}
|
||||
p += name_len;
|
||||
}
|
||||
|
||||
if (saw_wrong_assignment) {
|
||||
fprintf(stderr,
|
||||
"ds4-test: long-context wrong assignment for %s: got %d expected %d\n",
|
||||
fact->name, wrong_value, fact->number);
|
||||
} else {
|
||||
fprintf(stderr,
|
||||
"ds4-test: long-context missing assignment for %s=%d\n",
|
||||
fact->name, fact->number);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static int test_hex_digit(char c) {
|
||||
@@ -227,10 +296,10 @@ static void test_long_prefill_progress(void *ud, const char *event, int current,
|
||||
}
|
||||
}
|
||||
|
||||
static void test_long_security_continuation(void) {
|
||||
static void test_long_story_fact_recall(void) {
|
||||
const char *prompt_path = getenv("DS4_TEST_LONG_PROMPT");
|
||||
if (!prompt_path || !prompt_path[0]) {
|
||||
prompt_path = "tests/long_context_security_prompt.txt";
|
||||
prompt_path = "tests/long_context_story_prompt.txt";
|
||||
}
|
||||
char *prompt_text = test_read_file(prompt_path);
|
||||
TEST_ASSERT(prompt_text != NULL);
|
||||
@@ -263,8 +332,8 @@ static void test_long_security_continuation(void) {
|
||||
uint64_t rng = 12345;
|
||||
int generated = 0;
|
||||
bool decode_ok = true;
|
||||
for (; generated < 700; generated++) {
|
||||
int token = ds4_session_sample(session, 0.8f, 40, 0.95f, 0.05f, &rng);
|
||||
for (; generated < 350; generated++) {
|
||||
int token = ds4_session_sample(session, 0.0f, 0, 1.0f, 0.0f, &rng);
|
||||
if (token == ds4_token_eos(engine)) break;
|
||||
|
||||
size_t piece_len = 0;
|
||||
@@ -281,10 +350,9 @@ static void test_long_security_continuation(void) {
|
||||
const char *text = out.ptr ? out.ptr : "";
|
||||
TEST_ASSERT(decode_ok);
|
||||
TEST_ASSERT(generated > 0);
|
||||
TEST_ASSERT(strstr(text, "</think>") != NULL);
|
||||
TEST_ASSERT(test_count_substr(text, "</think>") == 1);
|
||||
TEST_ASSERT(test_count_substr(text, "The most critical security issue") == 1);
|
||||
TEST_ASSERT(strstr(text, "arbitrary file") != NULL);
|
||||
for (size_t i = 0; i < sizeof(test_long_facts) / sizeof(test_long_facts[0]); i++) {
|
||||
TEST_ASSERT(test_output_has_fact(text, &test_long_facts[i]));
|
||||
}
|
||||
|
||||
buf_free(&out);
|
||||
ds4_session_free(session);
|
||||
@@ -579,7 +647,7 @@ typedef struct {
|
||||
|
||||
static const ds4_test_entry test_entries[] = {
|
||||
#ifndef DS4_NO_GPU
|
||||
{"--long-context", "long-context", "long Metal continuation regression", test_long_security_continuation},
|
||||
{"--long-context", "long-context", "long-context story fact-recall regression", test_long_story_fact_recall},
|
||||
{"--tool-call-quality", "tool-call-quality", "model emits valid DSML tool calls", test_tool_call_quality},
|
||||
{"--logprob-vectors", "logprob-vectors", "official API top-logprob vector comparison", test_official_logprob_vectors},
|
||||
{"--metal-kernels", "metal-kernels", "isolated Metal kernel numeric regressions", test_metal_f16_matvec_fast_nr0_4},
|
||||
@@ -601,7 +669,7 @@ static void test_print_help(const char *prog) {
|
||||
puts(" Show this help.");
|
||||
puts("\nEnvironment:");
|
||||
puts(" DS4_TEST_MODEL=FILE Model path. Default: ds4flash.gguf");
|
||||
puts(" DS4_TEST_LONG_PROMPT=FILE Rendered long-context regression prompt.");
|
||||
puts(" DS4_TEST_LONG_PROMPT=FILE Rendered long-context story fact prompt.");
|
||||
puts(" DS4_TEST_VECTOR_FILE=FILE Simple official-vector fixture.");
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,196 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate the long-context fact-recall prompt used by ds4_test.
|
||||
|
||||
The fixture is intentionally prose instead of a synthetic table. The model has
|
||||
to retrieve person -> number assignments scattered through a long story, convert
|
||||
the spelled-out numbers to digits, and emit a parseable list.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import random
|
||||
from pathlib import Path
|
||||
|
||||
BOS = "<｜begin▁of▁sentence｜>"
|
||||
USER = "<｜User｜>"
|
||||
ASSISTANT = "<｜Assistant｜>"
|
||||
|
||||
FACTS = [
|
||||
("Bob", "thirty-four", 34),
|
||||
("Alice", "fifty-two", 52),
|
||||
("Clara", "seventy-one", 71),
|
||||
("Diego", "ninety-three", 93),
|
||||
("Elena", "sixteen", 16),
|
||||
("Felix", "eighty-eight", 88),
|
||||
("Greta", "forty-seven", 47),
|
||||
("Hugo", "twenty-nine", 29),
|
||||
("Iris", "sixty-four", 64),
|
||||
("Jonas", "twelve", 12),
|
||||
("Kira", "eighty-one", 81),
|
||||
("Leo", "thirty-nine", 39),
|
||||
("Marta", "seventy-six", 76),
|
||||
("Nadia", "twenty-three", 23),
|
||||
("Owen", "fifty-eight", 58),
|
||||
("Priya", "ninety-seven", 97),
|
||||
]
|
||||
|
||||
OPENING = """\
|
||||
You are reading a long story from the harbor town of Bellwether. The story is
|
||||
ordinary on purpose: people speak, walk, remember, repair things, argue about
|
||||
weather, and sometimes receive a private assignment number written out in
|
||||
words. Your job at the end is to recover the assignment numbers.
|
||||
|
||||
Important rule while reading: only assignments stated as "was assigned the
|
||||
number ..." count. Other ages, prices, dates, distances, room numbers, rumors,
|
||||
or guesses do not count. The assignment numbers in the story are written in
|
||||
words, not numerals.
|
||||
|
||||
"""
|
||||
|
||||
SCENE_TEMPLATES = [
|
||||
"""\
|
||||
At first light the harbor smelled of rope, rain, and cedar smoke. {lead} crossed
|
||||
the quay with a folded map tucked under one arm, stopping whenever gulls made a
|
||||
mess of the chalk marks near the fish stalls. {friend} had promised to fix the
|
||||
south gate before supper, but the hinges complained so loudly that everyone
|
||||
pretended not to hear them. In the bakery window, loaves cooled beneath linen
|
||||
while a child counted shells in a wooden bowl. No one was in a hurry, because
|
||||
Bellwether moved by tide and habit, not by the bells on the council tower.
|
||||
|
||||
The archivist Mara wrote notes in brown ink, never black, because black ink made
|
||||
old ledgers look like court summonses. She watched {lead} and {friend} pass the
|
||||
fountain, then added a line about the morning fog. Her notes often wandered into
|
||||
small details: the color of a scarf, the chipped rim of a cup, the way a door
|
||||
kept opening after it had been firmly shut.
|
||||
""",
|
||||
"""\
|
||||
By noon the market had filled with baskets of pears, lamp oil, brass hooks, and
|
||||
paper flowers. {lead} bargained for twine while {friend} listened to a sailor
|
||||
describe a storm that seemed to grow taller every time he retold it. The town
|
||||
clock had stopped again, but nobody agreed on when, so every shopkeeper chose a
|
||||
different hour and defended it with confidence.
|
||||
|
||||
Mara sat outside the apothecary and copied the day's ordinary business into the
|
||||
festival ledger. She liked ordinary business best. Extraordinary business came
|
||||
with signatures, seals, and people who leaned over her shoulder. Ordinary
|
||||
business arrived quietly, sat down, and became history before anyone noticed.
|
||||
""",
|
||||
"""\
|
||||
In the afternoon, a rehearsal for the midsummer play blocked the west road.
|
||||
{lead} carried a crate of lantern glass through the crowd while {friend} read
|
||||
lines from a damp script. Someone had painted the moon too blue on the backdrop,
|
||||
and three people argued about whether a theatrical moon was allowed to be wrong.
|
||||
The argument lasted longer than the scene.
|
||||
|
||||
The ledger lay open on a bench. Mara kept it weighted with two smooth stones
|
||||
from the beach. She recorded who borrowed the theater ladder, who returned the
|
||||
wrong kettle, and who claimed the missing red umbrella. The handwriting was calm
|
||||
even when the town was not.
|
||||
""",
|
||||
"""\
|
||||
Evening brought a quiet wind and the sound of shutters being latched one after
|
||||
another. {lead} helped carry chairs into the assembly hall, where the floor had
|
||||
been scrubbed until it smelled faintly of salt. {friend} found a lost button
|
||||
near the door and pinned it to the notice board with a note that said, simply,
|
||||
"lonely."
|
||||
|
||||
Mara walked the perimeter of the hall with the ledger pressed to her chest. She
|
||||
had learned that important facts hid best inside unimportant days. A missing
|
||||
button, a changed route, a corrected name, a number assigned without ceremony:
|
||||
these were the things that later made sense of everything else.
|
||||
""",
|
||||
"""\
|
||||
Rain arrived after midnight and softened every sound in Bellwether. {lead}
|
||||
stood beneath the awning of the rope-maker's shop, waiting for {friend}, who had
|
||||
gone back for a forgotten satchel. The street lamps shone in puddles like coins
|
||||
that nobody could spend. From the hill, the lighthouse blinked with patient
|
||||
regularity.
|
||||
|
||||
Mara remained awake in the archive room. She sharpened a pencil, rejected it,
|
||||
and returned to brown ink. The festival ledger had grown heavy with the week:
|
||||
weather notes, repairs, errands, apologies, and a few facts she underlined only
|
||||
once so they would not look too important.
|
||||
""",
|
||||
]
|
||||
|
||||
BRIDGE_SENTENCES = [
|
||||
"The town talked around the matter without naming it directly.",
|
||||
"A kettle whistled somewhere nearby and broke the silence at exactly the right moment.",
|
||||
"Mara did not decorate the sentence; she wanted it to be easy to find later.",
|
||||
"The phrase was spoken once, then folded into the rest of the day's business.",
|
||||
"No one treated the entry like a puzzle, which is why it survived unchanged.",
|
||||
"The ledger page smelled of dust, salt, and the faint sweetness of drying glue.",
|
||||
]
|
||||
|
||||
|
||||
def assignment_sentence(name: str, word: str) -> str:
|
||||
return (
|
||||
f"During that same scene, {name} was assigned the number {word}. "
|
||||
f"Mara wrote the assignment in words, closed the ledger for a moment, "
|
||||
f"and then returned to the smaller gossip of the harbor."
|
||||
)
|
||||
|
||||
|
||||
def make_story() -> str:
|
||||
rng = random.Random(20260513)
|
||||
names = [name for name, _, _ in FACTS]
|
||||
fact_by_scene = {7 + i * 11: fact for i, fact in enumerate(FACTS)}
|
||||
scenes: list[str] = []
|
||||
|
||||
for scene_index in range(190):
|
||||
lead = rng.choice(names)
|
||||
friend = rng.choice([n for n in names if n != lead])
|
||||
template = SCENE_TEMPLATES[scene_index % len(SCENE_TEMPLATES)]
|
||||
scene = template.format(lead=lead, friend=friend)
|
||||
|
||||
if scene_index in fact_by_scene:
|
||||
name, word, _ = fact_by_scene[scene_index]
|
||||
scene += "\n" + rng.choice(BRIDGE_SENTENCES) + " " + assignment_sentence(name, word) + "\n"
|
||||
elif scene_index % 9 == 3:
|
||||
scene += (
|
||||
"\nMara heard someone mention an old rumor about a numbered key, "
|
||||
"but she crossed it out because it was not an assignment and did "
|
||||
"not belong in the final list.\n"
|
||||
)
|
||||
elif scene_index % 11 == 6:
|
||||
scene += (
|
||||
"\nA shop sign advertised a discount in careful words, but prices "
|
||||
"and discounts were not assignment numbers, so Mara ignored them.\n"
|
||||
)
|
||||
|
||||
scenes.append(scene)
|
||||
|
||||
final_names = ", ".join(name for name, _, _ in FACTS)
|
||||
expected_hint = "Bob=34"
|
||||
question = f"""\
|
||||
|
||||
Final task:
|
||||
|
||||
Compile the assignment ledger from the story. Convert the spelled-out numbers to
|
||||
ordinary decimal numerals. Write only lines in the form Name=number. The first
|
||||
example line is {expected_hint}; include that line and all remaining people.
|
||||
|
||||
People to list: {final_names}.
|
||||
|
||||
No bullets, no prose, no explanation.
|
||||
"""
|
||||
return OPENING + "\n".join(scenes) + question
|
||||
|
||||
|
||||
def main() -> None:
|
||||
root = Path(__file__).resolve().parent
|
||||
story = make_story()
|
||||
rendered = (
|
||||
BOS
|
||||
+ "You are a careful assistant. Read the story, remember the assignments, "
|
||||
+ "and answer the final task exactly."
|
||||
+ USER
|
||||
+ story
|
||||
+ ASSISTANT
|
||||
+ "</think>"
|
||||
)
|
||||
(root / "long_context_story_prompt.txt").write_text(rendered, encoding="utf-8")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user