mtfyun/ds4
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

DeepSeek v4.1 Flash support for Metal

Browse Source
This commit is contained in:
Salvatore Sanfilippo authored and antirez committed 2026-09-12 11:47:54 +02:00
1 parent 6289c51627
commit bd66c40207
521 files changed
+1158482 -791
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.gitignore
+10
View File
@@ -26,8 +26,10 @@
/tests/test_layer_pack
/tests/test_metal_session_batch
/tests/test_metal_tp_spec
/tests/test_metal_tp_cancel
/tests/test_metal_moe_prefill
/tests/test_metal_ssd_experts
/tests/test_metal_command_memory
/tests/test_metal_dense_mpp
/tests/test_mxfp4_cuda
/tests/test_mxfp4_dot
@@ -40,6 +42,8 @@
/tests/test_session_state
/tests/test_session_state_gpu
/tests/test_tp_commands
/tests/test_tp_rdma
/tests/test_metal_tp_bulk
/tests/test_glm53_kda
/tests/test_glm53_vision_engine
/tests/test_glm_attention
@@ -47,6 +51,12 @@
/tests/test_linux_memory
/tests/test_rocm_memory
/tests/test_deepseek4_vision_image
/tests/test_deepseek41_metal
/tests/test_deepseek41_gguf
/tests/test_deepseek41_graph
/tests/test_deepseek41_prefill
/tests/test_deepseek41_cli
/tests/test_engram
*.o
*.dSYM/
__pycache__/
LICENSE
+1
View File
@@ -2,6 +2,7 @@ MIT License
Copyright (c) 2026 The ds4.c authors
Copyright (c) 2023-2026 The ggml authors
Copyright (c) 2023 DeepSeek
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
Makefile
+93 -6
View File
@@ -25,7 +25,7 @@ DS4_DSPARK_SUPPORT ?= gguf/DeepSeek-V4-Flash-DSpark-support-0731.gguf
ifeq ($(UNAME_S),Darwin)
METAL_LDLIBS := $(LDLIBS) -framework Foundation -framework Metal
CORE_OBJS = ds4.o ds4_image.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_metal.o ds4_layer_pack.o
CORE_OBJS = ds4.o ds4_image.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_metal.o ds4_layer_pack.o ds4_engram.o
CPU_CORE_OBJS = ds4_cpu.o ds4_image.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o
else
CFLAGS += -D_GNU_SOURCE -fno-finite-math-only
@@ -119,6 +119,9 @@ tests/test_metal_tp_spec.o: tests/test_metal_tp_spec.c ds4.h ds4_tp.h
tests/test_metal_tp_spec: tests/test_metal_tp_spec.o $(CORE_OBJS)
$(CC) $(CFLAGS) -o $@ $^ $(METAL_LDLIBS)
tests/test_metal_tp_cancel: tests/test_metal_tp_cancel.c ds4.h ds4_tp.h $(CORE_OBJS)
$(CC) $(CFLAGS) -I. -o $@ $< $(CORE_OBJS) $(METAL_LDLIBS)
test-metal-session-batch: tests/test_metal_session_batch
DS4_TEST_MODEL="$(DS4_TEST_MODEL)" ./tests/test_metal_session_batch
@@ -168,6 +171,50 @@ tests/test_metal_ssd_experts: tests/test_metal_ssd_experts.o $(CORE_OBJS)
.PHONY: test-metal-ssd-experts
test-metal-ssd-experts: tests/test_metal_ssd_experts
./tests/test_metal_ssd_experts
./tests/test_metal_ssd_experts --q4
./tests/test_metal_ssd_experts --mxfp4
tests/test_metal_command_memory: tests/test_metal_command_memory.c ds4_gpu.h $(CORE_OBJS)
$(CC) $(CFLAGS) -I. -o $@ $< $(CORE_OBJS) $(METAL_LDLIBS)
.PHONY: test-metal-command-memory
test-metal-command-memory: tests/test_metal_command_memory
MTL_DEBUG_LAYER=1 ./tests/test_metal_command_memory
MTL_DEBUG_LAYER=1 ./tests/test_metal_command_memory row
MTL_DEBUG_LAYER=1 ./tests/test_metal_command_memory session
MTL_DEBUG_LAYER=1 ./tests/test_metal_command_memory batch
MTL_DEBUG_LAYER=1 ./tests/test_metal_command_memory big
tests/test_deepseek41_metal.o: tests/test_deepseek41_metal.c ds4_gpu.h
$(CC) $(filter-out -ffast-math,$(CFLAGS)) -I. -c -o $@ $<
tests/test_deepseek41_metal: tests/test_deepseek41_metal.o $(CORE_OBJS)
$(CC) $(filter-out -ffast-math,$(CFLAGS)) -o $@ $^ $(METAL_LDLIBS)
.PHONY: test-deepseek41-metal
test-deepseek41-metal: tests/test_deepseek41_metal
./tests/test_deepseek41_metal
tests/test_deepseek41_graph.o: tests/test_deepseek41_graph.c ds4.c ds4_gpu.h ds4_engram.h
$(CC) $(filter-out -ffast-math,$(CFLAGS)) -Wno-unused-function -I. -c -o $@ $<
tests/test_deepseek41_graph: tests/test_deepseek41_graph.o $(filter-out ds4.o,$(CORE_OBJS))
$(CC) $(filter-out -ffast-math,$(CFLAGS)) -o $@ $^ $(METAL_LDLIBS)
tests/test_deepseek41_prefill.o: tests/test_deepseek41_prefill.c ds4.c ds4_gpu.h ds4_engram.h
$(CC) $(filter-out -ffast-math,$(CFLAGS)) -Wno-unused-function -I. -c -o $@ $<
tests/test_deepseek41_prefill: tests/test_deepseek41_prefill.o $(filter-out ds4.o,$(CORE_OBJS))
$(CC) $(filter-out -ffast-math,$(CFLAGS)) -o $@ $^ $(METAL_LDLIBS)
tests/test_metal_tp_bulk: tests/test_metal_tp_bulk.c ds4_gpu.h ds4_tp.h $(CORE_OBJS)
$(CC) $(filter-out -ffast-math,$(CFLAGS)) -I. -o $@ $< $(CORE_OBJS) $(METAL_LDLIBS)
tests/test_deepseek41_cli.o: tests/test_deepseek41_cli.c ds4_cli.c ds4.h
$(CC) $(CFLAGS) -Wno-unused-function -I. -c -o $@ $<
tests/test_deepseek41_cli: tests/test_deepseek41_cli.o ds4_help.o ds4_prompt_prefix.o linenoise.o ds4_gpu_args.o $(CORE_OBJS)
$(CC) $(CFLAGS) -o $@ $^ $(METAL_LDLIBS)
tests/test_metal_dense_mpp.o: tests/test_metal_dense_mpp.c ds4_gpu.h
$(CC) $(CFLAGS) -fno-fast-math -I. -c -o $@ $<
@@ -308,7 +355,7 @@ test-mxfp4-cuda: tests/test_mxfp4_cuda
./tests/test_mxfp4_cuda
endif
ds4.o: ds4.c ds4.h ds4_ssd.h ds4_distributed.h ds4_gpu.h ds4_linux_memory.h
ds4.o: ds4.c ds4.h ds4_ssd.h ds4_distributed.h ds4_gpu.h ds4_linux_memory.h ds4_engram.h
$(CC) $(CFLAGS) -c -o $@ ds4.c
ds4_image.o: ds4_image.c ds4_image.h third_party/iris/jpeg.h third_party/iris/png.h
@@ -317,6 +364,9 @@ ds4_image.o: ds4_image.c ds4_image.h third_party/iris/jpeg.h third_party/iris/pn
ds4_ssd.o: ds4_ssd.c ds4_ssd.h
$(CC) $(CFLAGS) -c -o $@ ds4_ssd.c
ds4_engram.o: ds4_engram.c ds4_engram.h
$(CC) $(filter-out -ffast-math,$(CFLAGS)) -c -o $@ ds4_engram.c
ds4_cli.o: ds4_cli.c ds4.h ds4_ssd.h ds4_distributed.h ds4_help.h ds4_prompt_prefix.h linenoise.h
$(CC) $(CFLAGS) -c -o $@ ds4_cli.c
@@ -409,7 +459,7 @@ else
endif
tests/test_glm53_vision_prompt.o: tests/test_glm53_vision_prompt.c ds4.h
$(CC) $(CFLAGS) -I. -c -o $@ tests/test_glm53_vision_prompt.c
$(CC) $(CFLAGS) -fno-fast-math -I. -c -o $@ tests/test_glm53_vision_prompt.c
tests/test_glm53_vision_prompt: tests/test_glm53_vision_prompt.o $(CORE_OBJS)
ifeq ($(UNAME_S),Darwin)
@@ -490,6 +540,23 @@ tests/test_ssd_cache: tests/test_ssd_cache.c ds4_ssd.c ds4_ssd.h
test-ssd-cache: tests/test_ssd_cache
./tests/test_ssd_cache
tests/test_engram: tests/test_engram.c ds4_engram.c ds4_engram.h
$(CC) $(filter-out -ffast-math,$(CFLAGS)) -I. -o $@ tests/test_engram.c ds4_engram.c -lm
.PHONY: test-engram
test-engram: tests/test_engram
./tests/test_engram
tests/test_deepseek41_gguf.o: tests/test_deepseek41_gguf.c ds4.c ds4.h ds4_engram.h
$(CC) $(filter-out -ffast-math,$(CFLAGS)) -Wno-unused-function -DDS4_NO_GPU -I. -c -o $@ $<
tests/test_deepseek41_gguf: tests/test_deepseek41_gguf.o ds4_engram.c $(filter-out ds4_cpu.o,$(CPU_CORE_OBJS))
$(CC) $(filter-out -ffast-math,$(CFLAGS)) -I. -o $@ $^ $(LDLIBS)
.PHONY: test-deepseek41-gguf
test-deepseek41-gguf: tests/test_deepseek41_gguf
./tests/test_deepseek41_gguf
ds4_cuda.o: ds4_cuda.cu ds4_gpu.h ds4_gpu_mgpu.h ds4_glm53_vision_gpu.cuh ds4_deepseek4_vision_gpu.cuh ds4_image.h ds4_iq2_tables_cuda.inc cuda/mmq/ds4_mmq.h
$(NVCC) $(NVCCFLAGS) -c -o $@ ds4_cuda.cu
@@ -611,10 +678,17 @@ tests/test_tp_commands.o: tests/test_tp_commands.c ds4_tp.c ds4_tp.h ds4.h
tests/test_tp_commands: tests/test_tp_commands.o $(filter-out ds4_tp.o,$(CPU_CORE_OBJS))
$(CC) $(CFLAGS) -o $@ $^ $(LDLIBS)
tests/test_tp_rdma.o: tests/test_tp_rdma.c ds4_tp.c ds4_tp.h ds4.h
$(CC) $(CFLAGS) -I. -c -o $@ $<
tests/test_tp_rdma: tests/test_tp_rdma.o $(filter-out ds4_tp.o,$(CPU_CORE_OBJS))
$(CC) $(CFLAGS) -o $@ $^ $(LDLIBS)
.PHONY: test-session-state
test-session-state: tests/test_session_state tests/test_tp_commands
test-session-state: tests/test_session_state tests/test_tp_commands tests/test_tp_rdma
./tests/test_session_state
./tests/test_tp_commands
./tests/test_tp_rdma
ifneq ($(UNAME_S),Darwin)
tests/test_gpu_xdev.o: tests/test_gpu_xdev.c ds4_gpu.h ds4_gpu_mgpu.h
@@ -700,7 +774,7 @@ test-frontends: ds4_test ds4_agent_test
./ds4_test --server
./ds4_agent_test
test: ds4_test ds4_agent_test ds4-eval q4k-dot-test mxfp4-dot-test test-session-state test-linux-memory \
test: ds4_test ds4_agent_test ds4-eval q4k-dot-test mxfp4-dot-test test-session-state test-linux-memory test-engram \
tests/test_layer_pack tests/test_engine_mgpu_placement tests/test_gpu_args \
tests/test_deepseek4_vision_image tests/test_prompt_prefix $(SAMPLING_TEST) ds4 ds4-server ds4-bench ds4-agent
./ds4-eval --validate-cases
@@ -748,11 +822,16 @@ mxfp4-dot-test: tests/test_mxfp4_dot.c
$(CC) -O2 -Wall -Wextra -std=c99 -o tests/test_mxfp4_dot tests/test_mxfp4_dot.c -lm
./tests/test_mxfp4_dot
.PHONY: test-download-model
test-download-model:
python3 tests/test_model_download.py
.PHONY: test-quality-api
# Only the scorer's JSON parser is needed; discard the unused engine entry point.
test-quality-api: tests/test_quality_api.c gguf-tools/quality-testing/score_official.c
$(CC) $(QUALITY_CFLAGS) -I. -ffunction-sections -fdata-sections -o tests/test_quality_api tests/test_quality_api.c -Wl,$(if $(filter Darwin,$(UNAME_S)),-dead_strip,--gc-sections) -lm
./tests/test_quality_api
python3 tests/test_collect_official.py
ds4.o ds4_cpu.o ds4_agent.o ds4_agent_cpu.o ds4_server.o ds4_server_cpu.o \
ds4_test.o ds4_agent_test.o \
@@ -761,12 +840,20 @@ tests/test_session_state_gpu.o: ds4_tool_text.h
clean:
rm -f tests/test_metal_ssd_experts
rm -f tests/test_metal_command_memory
rm -f tests/test_deepseek41_metal
rm -f tests/test_deepseek41_gguf
rm -f tests/test_deepseek41_graph tests/test_deepseek41_cli
rm -f tests/test_deepseek41_prefill
rm -f tests/test_metal_tp_bulk
rm -f tests/test_cuda_q8_scratch
rm -f tests/test_cuda_dspark_moe
rm -f tests/test_quality_api
rm -f tests/test_linux_memory tests/test_rocm_memory
rm -f tests/test_glm_attention tests/test_glm_attention_rocm
rm -f tests/test_ssd_cache
rm -f tests/test_ssd_cache tests/test_engram
rm -f tests/test_session_state tests/test_session_state_gpu tests/test_tp_commands
rm -f tests/test_tp_rdma
rm -f tests/test_metal_tp_spec
rm -f tests/test_metal_tp_cancel
rm -f ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_cpu ds4_native ds4_server_test ds4_test ds4_agent_test gguf-tools/quality-testing/score_official gguf-tools/quality-testing/score_official.o speed-bench/metal_decode_schedule_bench speed-bench/metal_prefill_variant_bench speed-bench/*.o tests/test_q4k_dot tests/test_mxfp4_dot tests/test_mxfp4_metal tests/test_mxfp4_rocm tests/test_mxfp4_cuda tests/test_metal_session_batch tests/test_metal_moe_prefill tests/test_metal_dense_mpp tests/test_glm53_kda tests/test_glm53_kda_rocm tests/test_glm53_vision_engine tests/test_glm53_vision_prompt tests/test_deepseek4_vision_image tests/test_prompt_prefix tests/test_gpu_xdev tests/test_gpu_model_cache tests/test_gpu_lookup_cache_strict tests/test_engine_mgpu_refusal tests/test_engine_mgpu_runtime tests/test_engine_correctness tests/test_sampling tests/test_cuda_session_batch tests/test_cuda_mixed_batch tests/*.o *.o tests/cuda_long_context_smoke tests/cuda_long_context_smoke.o
QA_BEFORE_RELEASES.md
+419 -1
View File
@@ -84,6 +84,13 @@ in this system.
replay changed:
`./ds4_test --server`.
- Run `./ds4-eval --self-test-extractors`.
- On Metal, run `make test-metal-command-memory` after command-buffer or copy
changes. With API validation enabled, repeated compute/copy/flush batches
must preserve their output and keep the CPU heap bounded after warmup.
The target also checks row, session, speculative and large-prefill TP gates
with ordered loopback exchanges; physical RDMA remains a separate test.
Long model runs must not accumulate temporary driver objects per token;
fixed KV and expert-cache budgets alone do not rule out this growth.
### Critical Input And Server Regression Pass
@@ -1755,7 +1762,418 @@ its production server without explicit permission for the current QA pass. If
permission is granted, the existing hard floor remains 110 aggregate t/s for
the 16-row decode oracle.
## 17. Release Sign-off
## 17. DeepSeek V4.1 Flash (Metal)
V4.1 is a different architecture and checkpoint, not a replacement filename for
V4 Flash. Use the matching vectors in
`gguf-tools/quality-testing/deepseek-v4.1-flash-20260910/manifest.tsv`.
The dataset README explains the temperature-one API logprobs: compare
teacher-forced probabilities and API top-token agreement, not sampled-prefix
length alone. Never refresh old Flash vectors through an API alias that now
serves V4.1.
Also score the 100 short general prompts in
`gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-general/manifest.tsv`
when comparing quantizations. Keep this broader probability check separate from
the long sparse-boundary tests; neither substitutes for the other.
- Audit the completed GGUF against its pinned source with
`gguf-tools/deepseek41_validate_gguf.py --payload`, including native Engram rows.
Repeat for calibrated weights using the actual imatrix. Record expert coverage;
synthetic fixture statistics must never become a release calibration file.
For Q4, pass `--quant q4` to both converter and validator. Convert from the
original safetensors with the retained imatrix, not from the Q2 GGUF. A valid
tensor inventory does not replace real-model quality tests for that recipe.
- Run `make test-engram test-deepseek41-gguf test-quality-api test-frontends`,
the V4.1 manifest tests and the official RoPE/quantization primitive checks.
Run `make test-download-model` to check verified downloads, damaged files,
partial transfers and the default model link. Check the published artifacts
using `ds41f-q2`, `ds41f-q4` and `ds41f-vision` on a runtime host. Q4 comes
in two transport parts: check interrupted assembly, insufficient disk space,
corrupt parts and the final complete-file checksum before opening the model.
- With real weights, run `tests/test_deepseek41_graph MODEL --session-fixture`
and `tests/test_deepseek41_graph MODEL --partitions` under Metal API validation.
Check restored continuation logits, compression carry, sliding-window wrap,
cancellation and malformed snapshots. Zero-weight fixtures are not quality QA.
On a sufficiently large resident Metal host, run
`tests/test_deepseek41_prefill --resident MODEL speed-bench/promessi_sposi.txt`.
This grows two sessions beyond 130K tokens while mixing scalar, layer-major
and deferred prefills; it checks progress, complete state and restored decode.
Omit `--resident` for the bounded SSD variant through 113K. Never run the
resident variant on a 128 GiB Mac.
Run `tests/test_deepseek41_graph MODEL --long-sessions PROMPT_FILE` with the
dataset's `prompts/case_016.txt` for incremental prefill and exact restored
continuation checks through 16385 tokens. This uses two sessions and a shared
64 GiB expert-cache budget; monitor memory and do not overlap another inference job.
- After prefill scheduling changes, run
`tests/test_deepseek41_graph MODEL --prefill-parity speed-bench/promessi_sposi.txt`
under Metal API validation. This compares complete logits and all saved cache
spans with token-major execution, including large continued chunks, snapshot
restoration and cancellation. An in-flight partial layer stack must never
be accepted as a valid snapshot.
Run `make test-metal-ssd-experts test-metal-moe-prefill` too: seeded cache
replacement must release physical buffers, not just reset entry counters.
The SSD test covers IQ2 eight-expert and Q4/MXFP4 six-expert routes, cold/hot
eviction, batched outputs and repeated physical-cache release. Its
`--table-admission` mode must reject persistent full-expert tables during
streaming, even with diagnostic table overrides. A resident one-row batch
must not silently populate the SSD cache. For new large Q4 artifacts, start
with a small cache and one continuation before the full scorer. Monitor host
wired memory and swap independently of the engine's planned allocation;
retained Metal residency sets can otherwise escape its accounting.
`tests/test_metal_moe_prefill --ssd-address` checks the IQ2/Q2 address kernel
across row counts that previously overflowed an unused narrow thread index.
Compare short/long, initial/continued prefill with the same cache budget on
the same host. Include first-token latency and steady decoding after prefill;
a faster prefill that leaves a cold expert cache can lose overall.
- For the long-prompt resident-encoder schedule, run
`tests/test_deepseek41_graph MODEL --encoder-parity speed-bench/promessi_sposi.txt`
and `--encoder-long-parity` with the same prompt file. These need an automatic
expert cache large enough to hold the encoder; use one inference process per
128 GiB host. Compare complete logits and every cache span at 16,385 tokens,
then append and decode, including snapshot restoration. `--encoder-cancel`
separately checks interruption after pinning the full encoder and rebuilding
the session; the short test also cancels a partially loaded encoder.
Confirm the cache budget is restored and no encoder pages remain pinned.
Memory must replace the expert cache, not sit beside it. A smaller budget or
failed residency request must retain the two-layer path, with Engram on disk.
Measure both schedules at 8k and 16k, including loading, first decode latency
and at least 512 generated tokens. Use
`DS4_METAL_DISABLE_V41_ENCODER_RESIDENCY=1` for the two-layer control.
- Score both single-host SSD streaming and two-host TP over real RDMA. Cover
127/128/129, 511/512/513, 1023/1024/1025 and 16383/16384/16385 prompt tokens,
then continued prefills and a real CLI/agent/server coding task with tool calls.
Compare bootstrap and calibrated weights on the same held-out cases.
Also use the `20260911-long` and `20260911-extended` V4.1 manifests for
8/16/32K and 64/96K prefixes. Compare whole-prompt paired probability scores;
sampled API output is not a full-logit oracle. Investigate persistent TP
differences instead of assuming that a different reduction order explains them.
- Exercise native resident and TP batching with unrelated sessions. Run
`tests/test_metal_session_batch` with `DS4_TEST_BATCH_ISOLATION=1`,
`DS4_TEST_SESSION_COUNT=2`, `4` and `8`, and `DS4_TEST_DECODE_STEPS=32`.
Supply `DS4_TEST_MODEL` and an admitted `DS4_TEST_CONTEXT_SIZE`; repeat two
sessions at 64K using `DS4_TEST_PROMPT_FILE`. This oracle changes companion
prompts and row order, checks complete target logits exactly, tests invalid
batches without advancing state, and resumes after a mixed prefill/decode call.
Isolation does not establish equivalence to serial inference: separately run
`score_official --session-batch N` on the short and long manifests. Measure
aggregate throughput without Metal validation, including two-session cases.
Require the `native_ds41=1` trace for admitted native shapes, including mixed
batches with nonzero `prefill_rows`; a passing ordered fallback is not evidence
for the native path. Image-bearing sessions currently use that fallback.
Real concurrent server requests must also cover cancellation and prefix reuse;
a serial prefill followed by batched decode is not a native mixed-layer path.
Run `tests/test_metal_moe_prefill --v41-decode` for rows 1-9, including both
TP ownership partitions. Its `--v41-q4-decode` mode checks the Q4 shape at
rows 1-8 against independent one-row calls, both ownership partitions and
untouched output tails. Run both: IQ2 success does not cover Q4 dispatch.
Run `tests/test_deepseek41_graph --batch-admission`
and `tests/test_deepseek41_graph MODEL --batch-head`; the latter compares
every output logit with the scalar head and selected rows with double math.
Use `DS4_TEST_MIXED_SHAPES=1` with `tests/test_metal_session_batch` for 1-7
appended tokens sharing a batch with independent decoders. Repeat at 73,728
context on a large resident host, not the 128 GiB RDMA pair: this fixture
allocates ten sessions. Check causal state and subsequent decode at 64K.
Keep HTTP identical-output comparisons at a fixed prefill chunk geometry;
the default idle and concurrent schedules can round differently. Also run
the default schedule for independent coding correctness and prefix reuse.
- Run `tests/test_metal_tp_cancel` over the physical RDMA pair. Interrupt at
small and large prefill boundaries, reject a partial snapshot, then rebuild
and decode on the same connection. Both peers must exit normally. Repeat
shared transport changes with older Flash and GLM, not just V4.1.
Run `make test-session-state` and delayed physical `tests/test_metal_tp_bulk`
exchanges for decode, verify and bulk shapes. Both receive windows must be
posted before either peer sends. Keep matching protocol versions on both
hosts; a longer timeout is not a fix for a missing receive window.
- For vision, audit the separate encoder GGUF against the pinned source and run
`tests/test_deepseek41_graph --vision-routing`. Compare
`--vision-encoder VISION_GGUF IMAGE OUTPUT.f32` with independently evaluated
official BF16 encoder outputs for OCR, layout, diagram and screenshot images.
Build `tests/test_glm53_vision_prompt` (the shared vision-prompt oracle) and
run its `--logit-oracle` check for non-finite values. Test
`tests/test_glm53_vision_prompt MODEL VISION_GGUF IMAGE` in resident,
SSD and physical TP modes, including SSD `--quality`. Check image replay,
changed embeddings and restored logits, not only a plausible description.
Run the server vision-cache suite with tool-result images, changed/reordered
images, multiple sessions and thinking replay. An unchanged image must retain
its prefix; a changed old image must invalidate the affected cache. Exercise
native agent image tools too. Keep the older Flash encoder regression gates.
Run `python3 tests/test_agent_vision.py --binary ./ds4-agent --model MODEL
--vision VISION_GGUF --ssd-streaming --output /tmp/agent-vision-qa`, then
repeat with `--ctx 73728 --archive-words 50000` and a fresh output directory.
It must inspect two images separately, edit code from their contents, pass
an independent output check and retain the prefix at every tool continuation.
Omit `--ssd-streaming` only on a host admitted for full residency. Allow
headroom above the 50K input for image tokens, tools and system reminders;
a task compacted into a short context does not pass the long-context gate.
- Verify that Engram tables remain unmapped and unpinned in every mode, including
weight warming and TP. On 128 GiB Macs, never try full main-model residency;
use SSD streaming or one half of a two-Mac TP setup. Increase context gradually
and monitor memory/swap. Larger-Mac residency requires separate physical QA.
With both released Q2 and Q4 GGUFs, run `tests/test_deepseek41_graph MODEL --memory-plan`.
It rejects Q4 residency on 128/256 GiB hosts and a Q4 shard on a 128 GiB rank,
while admitting larger hosts and fitting the SSD cache to its remaining budget.
The Q2 variant checks simulated 128/256/512 GiB admission and TP shard sizing.
Neither variant allocates weights or context buffers; this is not physical-host QA.
Repeat startup with `--warm-weights`: TP must warm only its owned shard,
SSD streaming must skip the full-file pass, and inspection must not warm weights.
- Exercise `--think-level 0`, `1`, `25`, `100`, `/think 25`, `/think`, `--think`
and `--think-max`; reject malformed/out-of-range values. Check changing effort
in a live conversation invalidates the old prefix without losing its messages.
- Run `tests/test_agent_compaction.py` first at 4K, then at least 64K with
the appropriate SSD or resident configuration. Check real read/edit/test
tasks, automatic and mid-generation compaction, interruption, text session
save/restore, cached-prefix reuse, generation budgets and recovery after an
oversized message. Compile and independently test the generated code.
The small-context run must exercise mid-generation compaction. The long run
must retain at least 75% of its context for the coding turn, not compact
before it; allow room for the system reminder inserted after 50K tokens.
Repeat with `--interrupt-only` and a fresh output directory to interrupt
generation and verify a real tool call in the same conversation afterward.
For a served V4.1 model, run `tests/test_server_vision_agent.py` with
`--model deepseek-v4.1-flash` against Chat, Responses and Anthropic. Require
independently checked image-driven code edits and cached-prefix reuse,
not just successful HTTP responses.
- Unsupported backends, pipeline and speculative modes must fail explicitly until
their V4.1 implementation is validated; an older graph/drafter is not a fallback.
Historical September 10 focused V4.1 pass, implementation at `e4a7c0e1`, on the two
128 GiB M5 Max hosts only. Source revision
`df42c109f1defefcbfcedbe7d905718a12266e40`; routed gate/up IQ2_XXS, down Q2_K,
attention/shared/head Q8_0, native disk-only FP8 Engram tables. Both GGUFs
are 340.598 GiB, of which 188.833 GiB is Engram data. The calibrated artifact
is `DeepSeek-V4.1-Flash-IQ2_XXS-Q2_K-imatrix.gguf`, SHA-256
`1ce6a8f8806205c13330d7ca287bd198331dc5ca35ccc5d8a9a92a188a6f6f42`.
The 8,192-token calibration observed 14,922 of 15,360 experts (97.15%);
unobserved experts retain the bootstrap fallback. Both source/payload audits
passed, including native Engram rows and source-reencoded expert samples.
The general 100-case fixture contains 2,994 teacher-forced API output tokens.
All three runs used Metal API validation and 4,096 allocated context; SSD
quality runs used a 64 GiB expert-cache budget.
| Weights and execution | Average NLL | API top-token agreement | Target logprob MAE |
| --- | ---: | ---: | ---: |
| Bootstrap, single-Mac SSD | 0.395514736 | 2692/2994 (89.91%) | 0.248143951 |
| Calibrated, single-Mac SSD | 0.364576009 | 2697/2994 (90.08%) | 0.227531809 |
| Calibrated, two-Mac RDMA TP | 0.366961107 | 2699/2994 (90.15%) | 0.228669737 |
Calibration lowers NLL by 7.82% on this set; top-token agreement is effectively
unchanged. Whole-prompt paired resampling gives a 95% interval of
`[-0.05563, -0.00745]` for the calibrated-minus-bootstrap NLL difference.
This supports the calibrated artifact, not universal superiority or bit-exact
API parity. The initial five-case subset had worse calibrated NLL, which is
why the broader comparison was required. TP uses a different reduction order;
real-weight partition tests and the independent API scores both pass.
Bootstrap SSD and TP passed all 17 short/boundary cases, including every
retrieval through 16,385 tokens. Calibrated SSD and TP passed cases 000-013
plus 016; the calibrated 16,383/16,384 cases were not separately repeated.
Real-weight incremental prefill and snapshot restoration preserved exact
next-token logits at all twelve tested boundaries through 16,385. Native
agent coding passed with `--think-level 25`; real HTTP read/edit/test loops
passed Chat, Responses replay/live-tail and Anthropic, including reversed
parallel tool results. A bootstrap SSD coding task began with 18,379 tokens
and reused its live prefix on every follow-up. Independent tests passed
without edits to the test files.
Production-mode calibrated benchmarks used `speed-bench/promessi_sposi.txt`,
4,096 allocated context, 512-token prefill intervals and 128 greedy output
tokens per frontier, without speculation or API validation. These are single
sweeps, not three-run release medians. SSD used automatic cache sizing.
| Context frontier | SSD prefill / decode | RDMA TP prefill / decode |
| --- | ---: | ---: |
| 512 | 14.18 / 16.42 t/s | 25.87 / 25.85 t/s |
| 1024 | 17.21 / 16.61 t/s | 26.65 / 25.72 t/s |
| 1536 | 17.30 / 16.70 t/s | 26.54 / 25.63 t/s |
| 2048 | 17.16 / 16.35 t/s | 26.59 / 25.58 t/s |
These baseline runs used token-by-token prefill, before the layer-major SSD
schedule was added. Automatic SSD allocation planned about 92.5 GiB total, including
a 75.97 GiB expert cache. US swap grew to about 641 MiB during its agent run,
then remained bounded and decreased during benchmarking. TP held about
80.7 GiB per rank. No OOM, watchdog reset or reboot occurred.
Warning-free Metal builds, frontend sanitizer checks, shared kernel/cache tests
and focused older Flash/GLM continuation regressions passed. This is not full
release sign-off: no M3, larger-Mac residency, CUDA, ROCm or pipeline execution
was tested. Physical V4.1 TP cancellation/recovery was not specifically run;
single-host cancellation and shared transport protocol tests do not replace it.
At that revision, V4.1 vision, DSpark and native session batching were unsupported.
September 12 completion pass, Metal only; numerical qualifications are recorded below:
- Q2 full residency was exercised on a 512 GiB M3 Ultra. Its short official
scores match the single-host SSD aggregates above. The main model occupies
about 151.8 GiB; the 188.8 GiB Engram tables remain disk-only.
Mixed-size prefill, cache/history, progress, snapshot and next-decode checks
pass through 130,062 tokens on that Ultra, with no swap used. The bounded
64 GiB-cache SSD counterpart passes through 113,678 tokens on M5 Max.
- Resident and SSD vision prompt tests pass, as does physical two-Mac RDMA
vision. Independent BF16 encoder comparisons across four fixtures have cosine
similarity 0.999677 through 0.999907. Six resident server image questions and
the image-prefix cache suite pass. These are focused tests, not a vision benchmark.
- Physical TP cancellation/reuse passes on V4.1, older Flash and GLM. Rebuilt
continuation logits are exact, with the same connection retained.
- Native SSD agent compaction/save/recovery checks pass at 4K and 64K on M5
Max. The long coding turn retained 54,884 tokens; all 80 generated C functions
compiled and passed an independent value check. Save/restore resumed tools
with the cached prefix, and oversized-input rejection remained recoverable.
- Native vision-agent SSD checks pass at short and long context on M5 Max.
The long run began with 53,945 tokens and reached 55,738, using a 73,728-token
window. Both image-based code edits passed the independent oracle, and all
six tool continuations reused the prefix without compaction. Planned memory
was 102.32 GiB; the host reported about 1 MiB of swap afterward.
- Native decode isolation passes at 2/4/8 rows on the Ultra and 2/8 on RDMA,
including two sessions at 64K on both. Eight-row, 64-step full-logit controls
differ by at most 1.15e-5; intermediate layer-state comparisons are exact.
Small expert kernels pass the consumed-output oracle at rows 1-9, and the
batch output head passes scalar and independent double-reference checks.
- Resident native batches score the extended six-case set at NLL 0.558586174
(two rows) and 0.558586151 (eight), versus scalar 0.558586149. API top-token
agreement stays 327/384. TP native two-row scoring matches scalar aggregates:
short NLL 0.361471746 versus 0.361471738, 2699/2994 top tokens; extended
0.560655391 versus 0.560655370, 328/384. Complete manifests, finite values
and count consistency were checked before comparing scores.
- Balanced timings without validation: Ultra two-row decode 34.1 versus
19.7 aggregate t/s serial; eight-row 44.2 versus 19.9. Physical TP reaches
36.1/40.7/45.7 aggregate t/s for 3/4/8 rows, versus about 30.8 serial.
Two-row TP batching measured 29.0 versus 30.8 and is therefore not the
default. These are balanced focused runs, not release-wide medians.
- Native mixed batches (1-7 prefix rows, eight total rows) pass complete-logit,
invalid-input and subsequent-decode checks on the Ultra and physical TP.
The Ultra long variant crosses 16K, 32K and 64K boundaries; maximum logit
differences are 7.63e-6 on the Ultra and 5.73e-6 on TP. At 64K, seven prefix
rows plus one decoder took 0.249 seconds versus 0.507 for serialized work.
That timing was collected under validation, not as a release benchmark.
- Resident small prefills now batch from eight tokens. A 128-token append
measured 0.74 versus 6.24 seconds on the Ultra. Official 100-case NLL is
0.365680596 versus 0.364576009, with a paired 95% interval spanning zero
(-0.00513 to 0.00766); API top-token agreement is 2705 versus 2697/2994.
Continued-128 long scoring similarly shows no established regression;
the two 64K/96K control cases improve target NLL. Complete-state checks
were repeated through 130,062 tokens after this scheduling change.
- Real Pi image/code tasks pass Chat, Responses and Anthropic, with prefix
reuse after both images. Eight-slot JSON/SSE identical short-prompt tests
pass after cancelled requests. Mixed-length identical-pair tests pass with
a fixed 2048-token prefill quantum; the default variable-geometry run is
not bit-identical and is not reported as an exact-output pass.
- Native-agent generation interruption and immediate same-session tool
recovery pass on M5 SSD streaming. Physical cancellation/reuse was repeated
with protocol 12 on V4.1, older Flash and GLM. Older Flash Vision Experimental
with its matching DSpark passes 512 TP target tokens with zero logit gap;
GLM Flash Q4 retains its ten-case NLL 0.319056979 baseline.
- Earlier accepted RDMA timing reached 1111.30 t/s for 32K prefill,
1083.48 t/s for continued prefill and 30.13 t/s for short-context decode.
These supersede the token-by-token baseline table, but are not new release
medians. Keep cold-start loading and cache handoff in subsequent comparisons.
- The first Q4 SSD scorer rebooted M5 US with a 92-second watchdog timeout:
wired memory reached about 123.4 GiB despite only 76.69 GiB planned. The
shared PRO/Q4 indirect expert-table path retained whole-layer residency
outside the bounded cache. Full-table dispatch is now resident-only; SSD
uses the selected-expert cache. The admission test fails on the old code
and passes with the fix. Six-expert Q4/MXFP4 eviction/batch/release checks and
the existing IQ2 suite pass under validation on M5 Max and M3 Ultra.
A guarded 32 GiB-cache Q4 continuation passes with 31/32 API top tokens,
about 30 GiB wired and no swap. The subsequent 64 GiB-cache Q4 scorer
passes all 100 short prompts and nine 8K/16K/32K prompts: NLL 0.246523256
and 0.335651410, API top-token agreement
2896/2994 and 542/576. Short target-logprob MAE is 0.070084928 versus Q2's
0.227531809; the paired short NLL difference interval is [-0.14129, -0.09452].
Full result tables pass manifest, finite-value and count checks. Wired memory
stayed bounded and swap remained zero. The reboot is not erased by the
passing rerun.
- Calibrated Q2, Q4 and the vision encoder are published in
`antirez/deepseek-v4.1-flash-gguf`. Q4 contains 294.15 GiB of main weights
and 188.83 GiB of disk-only Engram data. Its two transport parts reconstruct
the audited 518,596,067,328-byte GGUF without changing any tensor bytes.
The actual Ultra download passed both part checks and the full SHA-256
`a5e2e2c3ada4b2e98d9f9e4b50f6d9c2a12c2c96f5da165c07e13aff9264984e`.
The downloader's eight-case offline suite passed on M5 Max and M3 Ultra.
- Q4 resident short scoring on the 512 GiB Ultra gives NLL 0.252409161,
2896/2994 API top tokens and target-logprob MAE 0.070639113. Compared with
M5 SSD above, the paired short NLL difference interval is
[0.000496, 0.011493]: a small measurable difference, not bit-exact parity.
The nine 8K/16K/32K cases give NLL 0.328760094 and 538/576 top tokens;
their NLL difference interval [-0.02705, 0.01503] spans zero. Disabling
layer-major prefill on the same Ultra reproduces the M5 SSD short-score
file byte for byte. This isolates the short difference to batched versus
scalar prefill, not residency or the host. The small target-NLL loss remains
measured; unchanged top-token totals are not proof of numerical equivalence.
- The complete six-case Q4 64K/96K set on the Ultra scores NLL 0.366465846,
360/384 API top tokens and target-logprob MAE 0.095093210. The resident Q2
control scores 0.558586149, 327/384 and 0.358345897. Whole-prompt paired
intervals favor Q4 for both NLL [-0.29963, -0.10130] and top-token agreement
[0.046875, 0.132813]. All six cases and 384 targets passed the finite-value
and count checks. This is a measured quantization comparison on the stored
fixture, not a claim of exact hosted-model parity.
- Q4 resident state checks pass through 130,062 tokens: cache spans, Engram
history, full logits, progress, snapshot restore and subsequent decode.
Native eight-row decoding initially failed the 0.0002 full-logit bound with
a 0.141911 difference. Reusing the scalar expert reductions through eight
Q4 rows fixes it: the primitive is exact for all 27 solo/ownership shapes,
while the old dispatch fails. Full-model eight-row controls pass 64 steps
with a maximum difference of 1.53e-5. The tolerance was not increased.
- Q4 native eight-row official scoring gives NLL 0.252409123 versus serial
0.252409161, with the same 2896/2994 API top-token matches. All seven mixed
prefill/decode shapes and image replay/change/restoration checks pass.
The final resident rerun stayed below 309.1 GiB wired, with zero swap.
- The shared MoE fixture now uses an explicitly unfused reference. Fused
kernels need not write optional gate/up scratch; the test checks consumed
intermediates and final outputs, retaining finite checks and exact bulk
comparisons. The previous fixture stopped on untouched scratch on M3,
before it could exercise the actual fused-output comparison.
- A 512-token warm-SSD batching threshold was not retained. Balanced timings
reduced the append from about 29.2 to 16.8 seconds, but the nine long API
cases scored NLL 0.578360615 versus 0.560790592 for the existing schedule;
the paired difference interval [0.00456, 0.03568] does not support neutrality.
The six extended cases show no established NLL difference, but do not negate
this regression on the separate nine-case set.
Keep the existing 1024-token threshold when at least half the experts fit.
- The six-case extended API set shows a TP-versus-M5-SSD NLL difference
(0.560655 versus 0.540163). The M3 resident score is 0.558586. On the two
dominant 64K/96K cases, M3 SSD and residency produce identical score files;
M5 SSD with Metal 4 disabled also matches those files exactly. This isolates
a hardware/kernel arithmetic difference from residency itself, but does not
prove every TP difference harmless or establish bit-exact cross-host parity.
- A matched physical TP scalar-prefill control covers the two dominant 64K/96K
prompts, with the same source, resident shards and 106,496-token allocation.
Disabling layer-major prefill on both ranks gives NLL 0.705645662 and 101/128
API top tokens; production batched prefill gives 0.689299268 and 105/128.
The NLL changes have opposite signs across the two prompts, with a paired
interval [-0.07303, 0.04034]. This small control does not establish general
equivalence, but does not implicate batched prefill as the source of the
earlier cross-path quality gap. Both ranks exited normally after about
94 minutes, using about 84 GiB wired each with no swap growth.
- Warning-free focused Metal builds and Engram, GGUF, frontend, download,
SSD-cache and expert-kernel regressions pass. Final forced builds and the
Q4/IQ2 tiny-batch, shared MoE, three-quant SSD cache, table-admission, dispatch
and finite-vision tests pass on the Ultra and both M5 Max hosts. CUDA, ROCm
and pipeline support are outside this pass. No M3 Max inference was run on
the editing machine.
Q4 full-resident timing on the 512 GiB M3 Ultra, September 12: default paths,
Engram disk-only, 36,864-token allocation, no Metal validation or speculation.
These are medians of three complete context sweeps after discarding a warmup;
each frontier generates 128 tokens. All retained runs produced identical,
coherent prose. Loading is excluded. Only the first row is an initial prefill;
the others append to the preceding frontier.
| Context frontier | Added tokens | Prefill t/s | Generation t/s |
| ---: | ---: | ---: | ---: |
| 4,096 | 4,096 | 341.75 | 18.69 |
| 8,192 | 4,096 | 339.10 | 18.67 |
| 16,384 | 8,192 | 641.49 | 18.32 |
| 32,768 | 16,384 | 715.96 | 18.11 |
```sh
./ds4-bench -m gguf/DeepSeek-V4.1-Flash-Q4.gguf \
--prompt-file speed-bench/promessi_sposi.txt --ctx-start 4096 --ctx-max 32768 \
--step-mul 2 --ctx-alloc 36864 --gen-tokens 128 --show-output --csv RESULT.csv
```
## 18. Release Sign-off
Do not sign off until:
README.md
+15 -4
View File
@@ -7,6 +7,7 @@ language models on consumer hardware (that is, hardware that people
can actually own). To reach this goal, we are building
a small native inference engine optimized first for
**DeepSeek V4 Flash** (including the experimental vision model),
**DeepSeek V4.1 Flash** (Metal only),
and additionally **GLM 5.2 and 5.3**, **GLM 5.3 Flash** and
**DeepSeek V4 PRO**. The code is self-contained and
deliberately narrow, not a general GGUF runner: you need to use the
@@ -163,10 +164,16 @@ the [client setup guide](docs/CLIENTS.md).
requirements. DeepSeek Vision Experimental uses a different checkpoint from
Flash 0731; GLM 5.3 Flash adds vision to the same text model.
DeepSeek V4.1 Flash text and vision run on Metal. Q2 runs with SSD streaming
on one 128 GB Mac, or resident across two using RDMA. Q4 needs SSD streaming
or a 512 GB Mac. Engram tables remain on disk in every mode, so use a fast
local SSD. See the [model guide](docs/MODELS.md#deepseek-v41-flash) for downloads
and setup.
With the matching encoder passed as `--vision FILE`, use `/read image.png`
in the CLI or `view_image` in the native agent.
Speculative decoding is opt-in. GLM uses `--mtp`; Flash DSpark needs a matching
Speculative decoding is opt-in. GLM uses `--mtp`; V4 Flash DSpark needs a matching
support GGUF. It can improve generation, but not every workload benefits.
Read [speculative decoding](docs/SPECULATIVE_DECODING.md) for setup and the
difference between default opportunistic sampling and `--mtp-exact-sampling`.
@@ -175,13 +182,17 @@ difference between default opportunistic sampling and `--mtp-exact-sampling`.
Thinking is enabled by default. Use `--nothink` or `/nothink` for direct
answers, and `--think` or `/think` to enable it again.
For V4.1, `ds4` and `ds4-agent` also accept
`--think-level 25` or `/think 25`: 1 to 100 sets the reasoning effort, and
0 disables thinking. `--think` selects 75, `--think-max` selects 100.
Changing the level in a conversation rebuilds its cached prefix.
The normal sampling defaults are temperature 1, top-p 1, and min-p 0.05;
`--temp 0` selects greedy output.
For DeepSeek, `--power N` trades throughput for lower sustained GPU load.
The default is 100. GLM currently requires `--power 100`.
For DeepSeek V4, `--power N` trades throughput for lower sustained GPU load.
The default is 100. V4.1 and GLM currently require `--power 100`.
DeepSeek Flash and GLM 5.3 Flash also support directional steering. Load a
DeepSeek V4 Flash and GLM 5.3 Flash also support directional steering. Load a
vector with `--dir-steering-file FILE`; `/steer F` adjusts its scale for
subsequent tokens in a local CLI or agent session, without rebuilding the
existing KV cache. See [steering documentation](dir-steering/README.md).
docs/DISTRIBUTED.md
+10 -7
View File
@@ -17,13 +17,14 @@ artifacts must agree. Update all TP peers together when changing versions.
## Tensor parallelism between two Macs
This is a 50/50 split with exactly one worker. Do not pass `--layers`.
Routed experts are sharded; dense weights remain replicated. Both GPUs work
on the same token and exchange partial results. This can reduce generation
Routed experts are sharded; attention partitioning depends on the model. Both
GPUs work on the same token and exchange partial results. This can reduce generation
latency, but the gain depends on the model, link, and comparison setup.
Two 128 GB Macs are useful for Flash Q4/MXFP4 or GLM 5.3 Flash Q4.
GLM 5.2 IQ2_XXS is another tested capacity setup. A larger quant may need
larger machines even though its tensor layout is supported.
Two 128 GB Macs are useful for V4 Flash Q4/MXFP4 or GLM 5.3 Flash Q4.
GLM 5.2 IQ2_XXS is another tested capacity setup. V4.1 Flash Q2 also runs
on two 128 GB Macs, with disk-only Engram tables.
A larger quant may need larger machines even though its tensor layout is supported.
### Link setup
@@ -86,9 +87,11 @@ Do not treat repeated handshake or RDMA timeouts as successful QA merely
because a retry works.
The coordinator can be `ds4`, `ds4-agent`, `ds4-server`, or `ds4-bench`;
workers run `ds4`. Pass the same `--vision FILE` to both for image input.
workers run `ds4`. For models with vision support, pass the same `--vision FILE`
to both for image input.
For GLM MTP, enable `--mtp` on both. For DeepSeek DSpark, both need the
matching support model and DSpark options.
matching support model and DSpark options. V4.1 supports vision but not
speculative decoding.
TP disk-cache restore currently rebuilds the exact saved token prefix on both
ranks rather than restoring the coordinator alone. Expect prefill on restore.
docs/METAL.md
+5 -2
View File
@@ -32,6 +32,9 @@ Leave other GPU and memory-heavy applications idle when comparing performance.
| 256 GB | Flash Q4/MXFP4 or GLM 5.3 Flash Q4 |
| 512 GB | Larger models, including PRO Q2 |
Here, Flash means DeepSeek V4 Flash. V4.1 Flash has different memory
requirements; see its [model guide](MODELS.md#deepseek-v41-flash).
These are starting points, not guarantees that every context or session count
will fit. GLM 5.3 Flash Q2 is about 90 GiB before runtime allocations.
Stop other memory-heavy workloads before loading it resident.
@@ -64,5 +67,5 @@ For more than two machines, use [pipeline parallelism](DISTRIBUTED.md#pipeline-p
- [Batched serving](SERVER.md#multiple-sessions)
- [Benchmarking](PERFORMANCE.md)
For DeepSeek, `--power 70` trades throughput for lower sustained GPU load.
GLM currently requires `--power 100`.
For DeepSeek V4 Flash and PRO, `--power 70` trades throughput for lower
sustained GPU load. V4.1 and GLM currently require `--power 100`.
docs/MODELS.md
+71
View File
@@ -43,6 +43,76 @@ path, including pipeline execution for models too large for one host.
To build weights rather than download them, see [GGUF tools](../gguf-tools/README.md).
## DeepSeek V4.1 Flash
V4.1 Flash text and vision inference work on Metal. It needs its own GGUF,
tokenizer and inference graph; V4 Flash weights and DSpark
support files are not interchangeable with it.
| Target | File size | Main weights |
| --- | ---: | ---: |
| `ds41f-q2` | 341 GiB | 152 GiB |
| `ds41f-q4` | 483 GiB | 294 GiB |
Both use imatrix-calibrated routed experts and include 189 GiB of Engram
tables. Engram rows are read directly from the file as needed in every mode,
never loaded as a resident table. Keep the GGUF on a fast local SSD.
On one 128 GB Mac, use SSD streaming. Leave the expert cache budget automatic:
```sh
./download_model.sh ds41f-q2
./ds4 -m gguf/DeepSeek-V4.1-Flash-Q2.gguf \
--ssd-streaming --ctx 32768
```
Use `ds4-agent` or `ds4-server` with the same model and memory options.
`--think-level 25` sets reasoning effort explicitly; the range is 1 to 100,
with 0 disabling thinking. `/think 25` changes it in the CLI or native agent.
`--think` selects 75 and `--think-max` selects 100.
For two 128 GB Macs, follow the [TP/RDMA setup](DISTRIBUTED.md), passing this
GGUF with `-m` on both ranks and omitting `--ssd-streaming`. Each rank holds
about 81 GiB of main weights, plus context and runtime buffers. Both machines
need the complete GGUF on disk. A 256 GB or larger Mac can instead hold all
main weights; full residency has been tested on an M3 Ultra with 512 GB.
Q4 needs SSD streaming on smaller Macs, or a 512 GB Mac for full residency.
It does not fit resident TP across two 128 GB Macs. Download it with
`./download_model.sh ds41f-q4` and select `gguf/DeepSeek-V4.1-Flash-Q4.gguf`.
The download comes in two parts; the script joins and verifies them automatically.
Allow another 37 GiB of free disk space while joining. Rerun the command to resume an
interrupted download or join.
Large SSD prefills process layers in wide batches, overlapping computation
with the next layer's reads. Short appends keep using the expert cache.
Resident and TP inference also batch continued prefills automatically.
For concurrent serving, see [session batching](SERVER.md#multiple-sessions).
Each slot needs its own context memory; start with `--ctx 4096` before
increasing both context and slot count. DSpark, pipeline execution and
non-Metal backends are not implemented for V4.1.
Scalar, batched and tensor-parallel execution are not numerically identical.
Q4 batched prefill shows a small probability-score loss on the short official
continuation test, with unchanged overall top-token agreement. See the
[QA results](../QA_BEFORE_RELEASES.md#17-deepseek-v41-flash-metal) for details
and remaining differences.
For images, download the matching encoder and add it to the same command:
```sh
./download_model.sh ds41f-vision
./ds4-agent -m gguf/DeepSeek-V4.1-Flash-Q2.gguf \
--ssd-streaming --vision gguf/DeepSeek-V4.1-Flash-Vision.gguf
```
Vision works with SSD streaming, full residency and two-Mac TP. Pass the encoder
on both TP ranks. Use `/read image.png` in `ds4`, `view_image` in `ds4-agent`,
or the [server image API](SERVER.md#images). V4 Flash vision encoders do not
work with V4.1. See [conversion](../gguf-tools/README.md#convert-deepseek-v41-flash)
to build the GGUFs from safetensors.
## GLM 5.3 Flash
| Target | Approximate file size | Use |
@@ -100,6 +170,7 @@ Directional steering is supported for GLM 5.3, not GLM 5.2.
PNG and JPEG input works in the CLI, native agent, and HTTP server on Metal,
single-GPU CUDA, and ROCm. The encoder must match the model.
V4.1 Flash is currently Metal-only; its setup is [above](#deepseek-v41-flash).
### DeepSeek Flash Vision Experimental
docs/SERVER.md
+5 -1
View File
@@ -63,6 +63,9 @@ four times. Idle slots can be cached before reuse; active requests are not evict
| Backend/model | Decode execution |
| --- | --- |
| Metal, resident Flash | Native shared-expert/QKV batching where supported |
| Metal, resident V4.1 Flash | Native decoding for 2-8 sessions |
| Metal RDMA TP, V4.1 Flash | Native decoding for 3-8 sessions; ordered fallback for two |
| Metal SSD streaming, V4.1 Flash | Ordered fallback |
| Metal, GLM 5.2 | Ordered fallback |
| Metal, GLM 5.3 | Native batching through 2051 visible tokens; ordered fallback afterward |
| CUDA, supported multi-GPU Flash TP layout | Native grouped decode and mixed prefill/decode |
@@ -70,7 +73,8 @@ four times. Idle slots can be cached before reuse; active requests are not evict
Fallback executes the rows separately. It provides concurrency and scheduling
fairness, not the aggregate speedup of native batching. Native grouping may
change floating-point reduction order slightly.
change floating-point reduction order slightly. V4.1 sessions containing images
use the ordered fallback.
Long prefills yield to active decoders in bounded intervals, normally 128
tokens. `--mixed-prefill-quantum N` changes that interval for testing.
download_model.sh
+168 -2
View File
@@ -6,6 +6,10 @@ GLM_ANTIREZ_REPO="antirez/GLM-5.2-GGUF"
GLM53_REPO="antirez/glm-5.3-flash-gguf"
GLM53_FULL_REPO="antirez/glm-5.3-gguf"
REPO="antirez/deepseek-v4-gguf"
DS41_REPO="antirez/deepseek-v4.1-flash-gguf"
DS41_Q2_FILE="DeepSeek-V4.1-Flash-Q2.gguf"
DS41_Q4_FILE="DeepSeek-V4.1-Flash-Q4.gguf"
DS41_VISION_FILE="DeepSeek-V4.1-Flash-Vision.gguf"
DS4F_Q2_FILE="DeepSeek-V4-Flash-IQ2XXS-w2Q2K-AProjQ8-SExpQ8-OutQ8-chat-v2-imatrix-0731.gguf"
DS4F_Q4_FILE="DeepSeek-V4-Flash-Q4KExperts-F16HC-F16Compressor-F16Indexer-Q8Attn-Q8Shared-Q8Out-chat-v2-imatrix-0731.gguf"
DS4F_MXFP4_FILE="DeepSeek-V4-Flash-MXFP4Experts-F16HC-F16Compressor-F16Indexer-Q8Attn-Q8Shared-Q8Out-chat-v2-mxfp4-0731.gguf"
@@ -54,6 +58,9 @@ Usage:
./download_model.sh ds4f-vision-mxfp4 [--token TOKEN]
./download_model.sh ds4f-vision-encoder [--token TOKEN]
./download_model.sh ds4f-vision-dspark [--token TOKEN]
./download_model.sh ds41f-q2 [--token TOKEN]
./download_model.sh ds41f-q4 [--token TOKEN]
./download_model.sh ds41f-vision [--token TOKEN]
./download_model.sh pro-q2-imatrix [--token TOKEN]
./download_model.sh pro-q4-layers00-30 [--token TOKEN]
./download_model.sh pro-q4-layers31-output [--token TOKEN]
@@ -113,6 +120,21 @@ Targets:
Matching DSpark speculative decoding support for Vision Experimental,
about 5.6 GiB. It is not compatible with the 0731 language checkpoint.
ds41f-q2
DeepSeek V4.1 Flash calibrated Q2, about 341 GiB on disk. Main weights
occupy 152 GiB; Engram tables stay on disk. Metal only: use SSD streaming
on one 128 GB Mac, tensor parallelism on two, or a larger resident Mac.
ds41f-vision
Matching V4.1 Flash vision encoder, about 0.9 GiB. Add --vision FILE
to the language-model command. Does not update ./ds4flash.gguf.
ds41f-q4
DeepSeek V4.1 Flash calibrated Q4, about 483 GiB on disk. Main weights
occupy 294 GiB; Engram tables stay on disk. Metal only: use SSD streaming
on smaller Macs or full residency on a 512 GB Mac. Downloads two parts
and joins them automatically; allow another 37 GiB of free disk space.
pro-q2-imatrix
DeepSeek V4 PRO 0813 q2 imatrix quant, as a single GGUF file. About
430 GB on disk; intended for 512 GB RAM machines.
@@ -185,7 +207,7 @@ Then the default commands work:
After downloading DSpark support, enable it explicitly:
./ds4 --dspark --mtp-model <download directory>/$DS4F_DSPARK_FILE
PRO and GLM files are downloaded with the official Hugging Face downloader
PRO, V4.1 and GLM files use the official Hugging Face downloader
because they are too large, sharded, or nested for the curl path used by the
smaller DeepSeek Flash GGUF files.
EOF
@@ -234,6 +256,22 @@ case "$MODEL" in
FORCE_HF_DOWNLOAD=1
LINK_MODEL=0
;;
ds41f-q2)
REPO=$DS41_REPO
MODEL_FILE=$DS41_Q2_FILE
FORCE_HF_DOWNLOAD=1
;;
ds41f-q4)
REPO=$DS41_REPO
MODEL_FILE=$DS41_Q4_FILE
FORCE_HF_DOWNLOAD=1
;;
ds41f-vision)
REPO=$DS41_REPO
MODEL_FILE=$DS41_VISION_FILE
FORCE_HF_DOWNLOAD=1
LINK_MODEL=0
;;
pro-q2-imatrix) MODEL_FILE=$PRO_Q2_IMATRIX_FILE ;;
pro-q4-layers00-30) MODEL_FILE=$PRO_Q4_LAYERS00_30_FILE; LINK_MODEL=0 ;;
pro-q4-layers31-output) MODEL_FILE=$PRO_Q4_LAYERS31_OUTPUT_FILE; LINK_MODEL=0 ;;
@@ -363,6 +401,51 @@ local_download_name() {
fi
}
artifact_identity() {
case "$1" in
"$DS41_Q2_FILE")
expected_bytes=365713686528
expected_sha=1ce6a8f8806205c13330d7ca287bd198331dc5ca35ccc5d8a9a92a188a6f6f42
;;
"$DS41_Q4_FILE")
expected_bytes=518596067328
expected_sha=a5e2e2c3ada4b2e98d9f9e4b50f6d9c2a12c2c96f5da165c07e13aff9264984e
;;
"$DS41_Q4_FILE.part1")
expected_bytes=480000000000
expected_sha=6442b1f9224079662c02003c0ef9ef6be6e2aff509510f681dab9e6cc41df246
;;
"$DS41_Q4_FILE.part2")
expected_bytes=38596067328
expected_sha=7c3e10646c918eeaffbc39305a75ec96117450262c61454ff194cef00d7617f0
;;
"$DS41_VISION_FILE")
expected_bytes=970555552
expected_sha=cc283f032b3e8b8d78aeb5fccaa14e97b859b0c53aae3cd6bffa690ddf0e9e15
;;
*) return 1 ;;
esac
}
verify_download() {
artifact_identity "$1" || return 0
if [ "$(wc -c < "$2")" -ne "$expected_bytes" ]; then
echo "Incorrect file size: $2. Move the incomplete file aside and retry." >&2
exit 1
fi
echo "Verifying SHA-256: $2"
if command -v sha256sum >/dev/null 2>&1; then
actual_sha=$(sha256sum < "$2")
else
actual_sha=$(shasum -a 256 < "$2")
fi
actual_sha=${actual_sha%% *}
if [ "$actual_sha" != "$expected_sha" ]; then
echo "Checksum mismatch: $2. The file was not accepted." >&2
exit 1
fi
}
download_one_hf() {
file=$1
local_file=$(local_download_name "$file")
@@ -373,6 +456,7 @@ download_one_hf() {
mkdir -p "$(dirname "$out")"
if [ -s "$out" ]; then
verify_download "$file" "$out"
echo "Already downloaded: $out"
return
fi
@@ -412,6 +496,7 @@ download_one_hf() {
echo "Hugging Face download finished but expected file is missing: $out" >&2
exit 1
fi
verify_download "$file" "$out"
}
download_one() {
@@ -453,7 +538,88 @@ download_one() {
mv "$part" "$out"
}
if [ -n "$MODEL_FILES" ]; then
download_ds41_q4() {
q4_out="$OUT_DIR/$DS41_Q4_FILE"
if [ -e "$q4_out" ]; then
verify_download "$DS41_Q4_FILE" "$q4_out"
echo "Already downloaded: $q4_out"
return
fi
if ! command -v python3 >/dev/null 2>&1; then
echo "Joining the Q4 download requires Python 3." >&2
exit 1
fi
if [ ! -e "$q4_out.assembling" ]; then
download_one_hf "$DS41_Q4_FILE.part1"
fi
download_one_hf "$DS41_Q4_FILE.part2"
artifact_identity "$DS41_Q4_FILE"
q4_bytes=$expected_bytes
q4_sha=$expected_sha
artifact_identity "$DS41_Q4_FILE.part1"
python3 - "$q4_out" "$expected_bytes" "$q4_bytes" "$q4_sha" <<'PY'
import fcntl
import hashlib
import os
from pathlib import Path
import shutil
import sys
out = Path(sys.argv[1])
boundary, total = map(int, sys.argv[2:4])
expected = sys.argv[4]
first, second, pending = (Path(str(out) + suffix)
for suffix in (".part1", ".part2", ".assembling"))
def verify(path):
if path.stat().st_size != total:
sys.exit("Incorrect file size: " + str(path))
print("Verifying SHA-256: " + str(path), flush=True)
digest = hashlib.sha256()
with path.open("rb") as fp:
for block in iter(lambda: fp.read(16 << 20), b""):
digest.update(block)
if digest.hexdigest() != expected:
sys.exit("Checksum mismatch: " + str(path) +
". Move this file aside before retrying; it was not accepted.")
# Keep this lock file: unlinking it could allow two different locks for the
# same download. The first part becomes the output without a second full copy.
with Path(str(pending) + ".lock").open("a") as lock:
fcntl.flock(lock, fcntl.LOCK_EX)
if out.exists():
verify(out)
sys.exit(0)
if not pending.exists():
if first.stat().st_size != boundary:
sys.exit("Incorrect file size: " + str(first))
first.rename(pending)
size = pending.stat().st_size
if not boundary <= size <= total:
sys.exit("Invalid partial assembly: " + str(pending) +
". Move it aside before retrying.")
if second.stat().st_size != total - boundary:
sys.exit("Incorrect file size: " + str(second))
# An interrupted append restarts only the smaller tail, never the prefix.
with pending.open("r+b") as dst:
dst.truncate(boundary)
dst.seek(boundary)
if shutil.disk_usage(out.parent).free < total - boundary + (1 << 30):
sys.exit("Not enough disk space to join Q4; keep the parts and retry.")
print("Joining Q4 download parts", flush=True)
with second.open("rb") as src:
shutil.copyfileobj(src, dst, 16 << 20)
dst.flush()
os.fsync(dst.fileno())
verify(pending)
pending.rename(out)
second.unlink()
PY
}
if [ "$MODEL" = "ds41f-q4" ]; then
download_ds41_q4
elif [ -n "$MODEL_FILES" ]; then
for file in $MODEL_FILES; do
download_one "$file"
done
ds4.c
+3006 -102
View File
File diff suppressed because it is too large. Load diff
ds4.h
+12 -3
View File
@@ -27,6 +27,8 @@ typedef enum {
DS4_THINK_HIGH,
DS4_THINK_MAX,
} ds4_think_mode;
/* Explicit numeric effort lives outside the stable named-mode values. */
#define DS4_THINK_LEVEL_BASE 1000
typedef enum {
DS4_LOG_DEFAULT,
@@ -305,6 +307,8 @@ bool ds4_engine_is_glm_dsa(ds4_engine *e);
bool ds4_engine_is_glm53(ds4_engine *e);
const char *ds4_backend_name(ds4_backend backend);
bool ds4_think_mode_enabled(ds4_think_mode mode);
int ds4_think_mode_level(ds4_think_mode mode);
bool ds4_think_mode_parse_level(const char *text, ds4_think_mode *out);
const char *ds4_think_mode_name(ds4_think_mode mode);
const char *ds4_think_max_prefix(void);
const char *ds4_glm_reasoning_effort_text(ds4_think_mode mode);
@@ -344,9 +348,12 @@ int ds4_dump_chat_tokenization(const char *model_path,
const char *system,
const char *prompt,
ds4_think_mode think_mode,
int ctx_size,
FILE *fp);
int ds4_engine_head_test(ds4_engine *e, const ds4_tokens *prompt);
bool ds4_engine_is_glm_dsa(ds4_engine *e);
bool ds4_engine_is_deepseek41(ds4_engine *e);
const char *ds4_deepseek41_reasoning_effort_text(ds4_think_mode mode);
int ds4_engine_first_token_test(ds4_engine *e, const ds4_tokens *prompt);
int ds4_engine_metal_graph_test(ds4_engine *e, const ds4_tokens *prompt);
int ds4_engine_metal_graph_full_test(ds4_engine *e, const ds4_tokens *prompt);
@@ -367,6 +374,7 @@ void ds4_encode_chat_prompt(
ds4_think_mode think_mode,
ds4_tokens *out);
void ds4_chat_append_max_effort_prefix(ds4_engine *e, ds4_tokens *tokens);
void ds4_chat_append_think_prefix(ds4_engine *e, ds4_tokens *tokens, ds4_think_mode mode);
void ds4_chat_append_message(ds4_engine *e, ds4_tokens *tokens, const char *role, const char *content);
void ds4_chat_append_assistant_prefix(ds4_engine *e, ds4_tokens *tokens, ds4_think_mode think_mode);
@@ -400,9 +408,10 @@ void ds4_session_set_progress(ds4_session *s, ds4_session_progress_fn fn, void *
/* UI-only progress. It may report fine-grained progress inside a prefill chunk;
* callers must not treat it as a durable KV checkpoint boundary. */
void ds4_session_set_display_progress(ds4_session *s, ds4_session_progress_fn fn, void *ud);
/* Optional cooperative cancellation. ds4_session_sync() checks it only at
* safe boundaries where the live checkpoint is either unchanged or represents a
* valid token prefix, and returns DS4_SESSION_SYNC_INTERRUPTED when it stops. */
/* Cooperative cancellation for ds4_session_sync(), which drains pending work
* before returning DS4_SESSION_SYNC_INTERRUPTED. A complete prefix may remain
* usable, but a partial layer-major pass is invalidated and the next sync
* rebuilds it. Do not assume an interrupted session can be saved or decoded. */
void ds4_session_set_cancel(ds4_session *s, ds4_session_cancel_fn fn, void *ud);
void ds4_session_report_progress(ds4_session *s, const char *event, int current, int total);
/* Distributed coordinator sessions return 1 when the full layer route is
ds4_agent.c
+243 -68
View File
@@ -20,6 +20,7 @@
#include <pthread.h>
#include <regex.h>
#include <signal.h>
#include <spawn.h>
#include <stdarg.h>
#include <stdbool.h>
#include <stdint.h>
@@ -38,6 +39,8 @@
#include <sys/xattr.h>
#endif
extern char **environ;
/* This is intentionally not in linenoise.h, but it is part of the existing
* multiplexed editor implementation. The agent uses it only to restore text
* after Enter is pressed while the model is still busy. */
@@ -159,6 +162,8 @@ typedef struct {
bool compact_requested;
bool power_requested;
int requested_power;
bool think_requested;
ds4_think_mode requested_think;
int progress_base;
bool progress_direct;
double progress_started_at;
@@ -289,6 +294,7 @@ typedef struct {
typedef enum {
AGENT_TOOL_SYNTAX_DSML,
AGENT_TOOL_SYNTAX_GLM,
AGENT_TOOL_SYNTAX_DSML41,
} agent_tool_syntax;
typedef enum {
@@ -404,7 +410,24 @@ static int agent_compact_reserve_tokens(agent_worker *w);
static agent_tool_syntax agent_tool_syntax_for_engine(ds4_engine *engine) {
return ds4_engine_is_glm_dsa(engine) ? AGENT_TOOL_SYNTAX_GLM
: AGENT_TOOL_SYNTAX_DSML;
: ds4_engine_is_deepseek41(engine) ? AGENT_TOOL_SYNTAX_DSML41
: AGENT_TOOL_SYNTAX_DSML;
}
static const char *agent_dsml_tag_name(agent_tool_syntax syntax, const char *name) {
if (syntax != AGENT_TOOL_SYNTAX_DSML41) return name;
if (!strcmp(name, "tool_calls")) return " calls";
if (!strcmp(name, "invoke")) return " invoke";
return " parameter";
}
static const char *agent_tool_start(agent_tool_syntax syntax) {
if (syntax == AGENT_TOOL_SYNTAX_GLM) return "<tool_call>";
return syntax == AGENT_TOOL_SYNTAX_DSML41 ? "<｜DSML｜ calls>" : "<｜DSML｜tool_calls>";
}
static const char *agent_dsml_param_close(agent_tool_syntax syntax) {
return syntax == AGENT_TOOL_SYNTAX_DSML41 ? "</｜DSML｜ parameter>" : "</｜DSML｜parameter>";
}
static bool agent_tool_syntax_assistant_turn_uses_eos(agent_tool_syntax syntax) {
@@ -605,6 +628,7 @@ static bool agent_slash_command_known(const char *cmd) {
!strcmp(cmd, "/exit") ||
!strcmp(cmd, "/new") ||
agent_slash_command_with_args(cmd, "/power") ||
agent_slash_command_with_args(cmd, "/think") ||
agent_slash_command_with_args(cmd, "/steer") ||
agent_slash_command_with_args(cmd, "/hints") ||
agent_slash_command_with_args(cmd, "/switch") ||
@@ -831,6 +855,11 @@ static agent_config parse_options(int argc, char **argv) {
c.gen.min_p_set = true;
} else if (!strcmp(arg, "--seed")) {
c.gen.seed = parse_u64(need_arg(&i, argc, argv, arg), arg);
} else if (!strcmp(arg, "--think-level")) {
if (!ds4_think_mode_parse_level(need_arg(&i, argc, argv, arg), &c.gen.think_mode)) {
fprintf(stderr, "ds4-agent: --think-level requires an integer from 0 to 100\n");
exit(2);
}
} else if (!strcmp(arg, "--think")) {
c.gen.think_mode = DS4_THINK_HIGH;
} else if (!strcmp(arg, "--think-max")) {
@@ -1340,10 +1369,18 @@ static char *agent_build_glm_tools_prompt(bool edit_upto, bool vision) {
return out;
}
static char *agent_dsml41_tools_prompt(const char *source);
static char *agent_build_tools_prompt(ds4_engine *engine, bool edit_upto) {
if (agent_tool_syntax_for_engine(engine) == AGENT_TOOL_SYNTAX_GLM)
return agent_build_glm_tools_prompt(edit_upto, ds4_engine_has_vision(engine));
return agent_build_dsml_tools_prompt(edit_upto, ds4_engine_has_vision(engine));
char *prompt = agent_build_dsml_tools_prompt(edit_upto, ds4_engine_has_vision(engine));
if (ds4_engine_is_deepseek41(engine)) {
char *next = agent_dsml41_tools_prompt(prompt);
free(prompt);
prompt = next;
}
return prompt;
}
static const char agent_dsml_syntax_reminder[] =
@@ -1359,6 +1396,14 @@ static const char agent_glm_syntax_reminder[] =
"<tool_call>$TOOL_NAME<arg_key>$PARAMETER_NAME</arg_key>"
"<arg_value>$PARAMETER_VALUE</arg_value></tool_call>\n";
static const char agent_dsml41_syntax_reminder[] =
"DSML syntax reminder:\n"
"<｜DSML｜ calls>\n"
"<｜DSML｜ invoke name=\"$TOOL_NAME\">\n"
"<｜DSML｜ parameter name=\"$PARAMETER_NAME\" string=\"true|false\">$PARAMETER_VALUE</｜DSML｜ parameter>\n"
"</｜DSML｜ invoke>\n"
"</｜DSML｜ calls>\n";
#define AGENT_SYSTEM_PROMPT_REMINDER_TOKENS 50000
static const char agent_hints_prompt[] =
@@ -1439,8 +1484,11 @@ static void agent_append_system_prompt(ds4_engine *engine, ds4_tokens *tokens,
char *tools_prompt = agent_build_tools_prompt(engine, edit_upto);
if (agent_tool_syntax_for_engine(engine) == AGENT_TOOL_SYNTAX_GLM)
ds4_chat_append_message(engine, tokens, "system", tools_prompt);
else
else {
if (ds4_engine_is_deepseek41(engine))
ds4_chat_append_message(engine, tokens, "system", "");
ds4_tokenize_rendered_chat(engine, tools_prompt, tokens);
}
free(tools_prompt);
if (!extra || !extra[0]) return;
@@ -1829,11 +1877,11 @@ static bool agent_dsml_close_tag_at(const char *s, const char *name, size_t *tag
* handled by agent_dsml_close_tag_at(); this helper exists for online behavior:
* terminal rendering must hide partial close tags without waiting for the whole
* parameter to finish. */
static bool agent_dsml_parameter_close_tail(const char *tail, size_t len,
static bool agent_dsml_parameter_close_tail(agent_tool_syntax syntax, const char *tail, size_t len,
bool *complete) {
static const char prefix[] = "</｜DSML｜parameter";
const char *prefix = agent_dsml_param_close(syntax);
static const char dsml_bar[] = "｜";
const size_t prefix_len = sizeof(prefix) - 1;
const size_t prefix_len = strlen(prefix) - 1;
const size_t bar_len = sizeof(dsml_bar) - 1;
*complete = false;
if (len <= prefix_len) return memcmp(prefix, tail, len) == 0;
@@ -1874,7 +1922,7 @@ static bool agent_tool_value_close_tail(agent_tool_syntax syntax,
bool *complete) {
if (syntax == AGENT_TOOL_SYNTAX_GLM)
return agent_glm_arg_value_close_tail(tail, len, complete);
return agent_dsml_parameter_close_tail(tail, len, complete);
return agent_dsml_parameter_close_tail(syntax, tail, len, complete);
}
static void agent_dsml_update_param_close_prefix(agent_dsml_parser *p) {
@@ -2072,16 +2120,19 @@ static void agent_dsml_parse(agent_dsml_parser *p) {
return;
}
const char *calls = agent_dsml_tag_name(p->syntax, "tool_calls");
const char *invoke = agent_dsml_tag_name(p->syntax, "invoke");
const char *parameter = agent_dsml_tag_name(p->syntax, "parameter");
while (p->state == AGENT_DSML_STRUCTURAL || p->state == AGENT_DSML_PARAM_VALUE) {
if (p->state == AGENT_DSML_PARAM_VALUE) {
size_t end_tag_len = 0;
char *end = agent_dsml_find_close_tag(p->raw + p->param_value_start,
"parameter", &end_tag_len);
parameter, &end_tag_len);
if (!end) return;
agent_tool_call_add_arg(&p->current, p->param_name ? p->param_name : "",
p->raw + p->param_value_start,
(size_t)(end - (p->raw + p->param_value_start)),
p->param_is_string, "</｜DSML｜parameter>");
p->param_is_string, agent_dsml_param_close(p->syntax));
p->param_close_prefix = false;
free(p->param_name);
p->param_name = NULL;
@@ -2097,13 +2148,13 @@ static void agent_dsml_parse(agent_dsml_parser *p) {
if (p->parse_pos >= p->raw_len) return;
size_t close_len = 0;
if (agent_dsml_close_tag_at(p->raw + p->parse_pos, "tool_calls", &close_len)) {
if (agent_dsml_close_tag_at(p->raw + p->parse_pos, calls, &close_len)) {
agent_tool_calls_push(&p->calls, &p->current);
p->parse_pos += close_len;
p->state = AGENT_DSML_DONE;
return;
}
if (agent_dsml_close_tag_at(p->raw + p->parse_pos, "invoke", &close_len)) {
if (agent_dsml_close_tag_at(p->raw + p->parse_pos, invoke, &close_len)) {
agent_tool_calls_push(&p->calls, &p->current);
p->parse_pos += close_len;
continue;
@@ -2114,7 +2165,7 @@ static void agent_dsml_parse(agent_dsml_parser *p) {
size_t tag_len = (size_t)(tag_end - (p->raw + p->parse_pos)) + 1;
char *tag = xstrndup(p->raw + p->parse_pos, tag_len);
if (agent_dsml_open_tag_is(tag, "invoke")) {
if (agent_dsml_open_tag_is(tag, invoke)) {
agent_tool_call_free(&p->current);
p->current.name = agent_parse_attr(tag, "name");
if (!p->current.name) {
@@ -2123,7 +2174,7 @@ static void agent_dsml_parse(agent_dsml_parser *p) {
return;
}
p->parse_pos += tag_len;
} else if (agent_dsml_open_tag_is(tag, "parameter")) {
} else if (agent_dsml_open_tag_is(tag, parameter)) {
free(p->param_name);
p->param_name = agent_parse_attr(tag, "name");
char *is_string = agent_parse_attr(tag, "string");
@@ -2150,8 +2201,7 @@ static void agent_dsml_parse(agent_dsml_parser *p) {
}
static void agent_dsml_start(agent_dsml_parser *p) {
const char *start = p->syntax == AGENT_TOOL_SYNTAX_GLM ?
"<tool_call>" : "<｜DSML｜tool_calls>";
const char *start = agent_tool_start(p->syntax);
p->state = AGENT_DSML_STRUCTURAL;
p->search_len = 0;
agent_dsml_raw_append(p, start, strlen(start));
@@ -2159,8 +2209,7 @@ static void agent_dsml_start(agent_dsml_parser *p) {
}
static void agent_dsml_feed(agent_dsml_parser *p, const char *s, size_t n) {
const char *start = p->syntax == AGENT_TOOL_SYNTAX_GLM ?
"<tool_call>" : "<｜DSML｜tool_calls>";
const char *start = agent_tool_start(p->syntax);
const size_t start_len = strlen(start);
if (p->state == AGENT_DSML_DONE || p->state == AGENT_DSML_ERROR) return;
@@ -4005,10 +4054,11 @@ static bool agent_stream_dsml_start_match(agent_tool_syntax syntax,
return false;
}
static const char canonical[] = "<｜DSML｜tool_calls>";
static const char missing_bar[] = "<DSML｜tool_calls>";
static const char invoke[] = "<｜DSML｜invoke";
static const char invoke_missing_bar[] = "<DSML｜invoke";
const bool v41 = syntax == AGENT_TOOL_SYNTAX_DSML41;
const char *canonical = agent_tool_start(syntax);
const char *missing_bar = v41 ? "<DSML｜ calls>" : "<DSML｜tool_calls>";
const char *invoke = v41 ? "<｜DSML｜ invoke" : "<｜DSML｜invoke";
const char *invoke_missing_bar = v41 ? "<DSML｜ invoke" : "<DSML｜invoke";
struct {
const char *text;
bool implicit_invoke;
@@ -4085,9 +4135,9 @@ static void agent_stream_note_plain_dsml_byte(agent_stream_renderer *sr,
* the DSML detector. The detector must hold short prefixes because the model
* can split "<｜DSML｜tool_calls>" across arbitrary tokens. */
static void agent_stream_normal_byte(agent_stream_renderer *sr, char c) {
static const char canonical_invoke[] = "<｜DSML｜invoke";
const char *start = sr->syntax == AGENT_TOOL_SYNTAX_GLM ?
"<tool_call>" : "<｜DSML｜tool_calls>";
const char *canonical_invoke = sr->syntax == AGENT_TOOL_SYNTAX_DSML41 ?
"<｜DSML｜ invoke" : "<｜DSML｜invoke";
const char *start = agent_tool_start(sr->syntax);
if (sr->parser->state == AGENT_DSML_ERROR) return;
agent_stream_note_thinking_dsml_byte(sr, c);
@@ -4097,7 +4147,7 @@ static void agent_stream_normal_byte(agent_stream_renderer *sr, char c) {
* printing them makes tool calls appear after odd empty lines. We only
* suppress whitespace in this very narrow post-thinking window; once the
* first non-space byte arrives, normal rendering resumes. */
if (sr->post_think_gap &&
if (sr->post_think_gap && !sr->dsml_start_len &&
(c == ' ' || c == '\t' || c == '\r' || c == '\n'))
{
return;
@@ -4119,8 +4169,8 @@ static void agent_stream_normal_byte(agent_stream_renderer *sr, char c) {
* implicit tool_calls block; the model often knows it wants a
* tool but forgets the outer wrapper. */
agent_stream_start_dsml(sr, sr->in_think);
if (sr->syntax == AGENT_TOOL_SYNTAX_DSML && implicit_invoke) {
for (size_t i = 0; i < sizeof(canonical_invoke) - 1; i++)
if (sr->syntax != AGENT_TOOL_SYNTAX_GLM && implicit_invoke) {
for (size_t i = 0; i < strlen(canonical_invoke); i++)
agent_stream_feed_dsml_byte(sr, canonical_invoke[i]);
}
}
@@ -4372,6 +4422,30 @@ static char *agent_buf_take(agent_buf *b) {
return p;
}
/* Adapt only our trusted examples, including their escaped closing tags.
* Never translate sampled text or user/tool payloads between model formats. */
static char *agent_dsml41_tools_prompt(const char *source) {
const char marker[] = "｜DSML｜";
const char *names[] = {"tool_calls", "invoke", "parameter"};
agent_buf out = {0};
const char *p = source, *next;
while ((next = strstr(p, marker)) != NULL) {
next += sizeof(marker) - 1;
agent_buf_append(&out, p, (size_t)(next - p));
p = next;
for (size_t i = 0; i < sizeof(names) / sizeof(*names); i++) {
const size_t n = strlen(names[i]);
if (!strncmp(p, names[i], n) && (p[n] == '>' || p[n] == ' ')) {
agent_buf_puts(&out, agent_dsml_tag_name(AGENT_TOOL_SYNTAX_DSML41, names[i]));
p += n;
break;
}
}
}
agent_buf_puts(&out, p);
return agent_buf_take(&out);
}
static bool agent_tokens_equal(const ds4_tokens *a, const ds4_tokens *b) {
if (!a || !b || a->len != b->len) return false;
for (int i = 0; i < a->len; i++) {
@@ -4820,13 +4894,7 @@ static bool agent_kv_save_path(agent_worker *w, const char *path,
static void agent_worker_build_system_tokens(agent_worker *w, ds4_tokens *out) {
ds4_chat_begin(w->engine, out);
ds4_think_mode think_mode = effective_think_mode(w->cfg);
if (agent_tool_syntax_for_engine(w->engine) == AGENT_TOOL_SYNTAX_GLM) {
const char *effort = ds4_glm_reasoning_effort_text(think_mode);
if (effort) ds4_chat_append_message(w->engine, out, "system", effort);
} else if (w->cfg->gen.think_mode == DS4_THINK_MAX &&
think_mode == DS4_THINK_MAX) {
ds4_chat_append_max_effort_prefix(w->engine, out);
}
ds4_chat_append_think_prefix(w->engine, out, think_mode);
agent_append_system_prompt(w->engine, out, w->cfg->gen.system,
w->cfg->edit_upto);
ds4_prompt_prefix_append(w->engine, out, &w->cfg->gen.prefix);
@@ -8531,6 +8599,36 @@ static void *agent_bash_monitor(void *arg) {
return NULL;
}
/* Do not fork the inference process: copying its wired, private model mappings
* can exhaust memory before the child gets to exec, even with copy-on-write. */
static int agent_bash_spawn(pid_t *pid, int tmpfd, const int pipefd[2],
const char *cmd) {
posix_spawnattr_t attr;
posix_spawn_file_actions_t actions;
int rc = posix_spawnattr_init(&attr);
if (rc) return rc;
rc = posix_spawn_file_actions_init(&actions);
if (rc) {
posix_spawnattr_destroy(&attr);
return rc;
}
rc = posix_spawnattr_setpgroup(&attr, 0);
if (!rc) rc = posix_spawnattr_setflags(&attr, POSIX_SPAWN_SETPGROUP);
if (!rc) rc = posix_spawn_file_actions_addclose(&actions, tmpfd);
if (!rc) rc = posix_spawn_file_actions_addclose(&actions, pipefd[0]);
if (!rc) rc = posix_spawn_file_actions_adddup2(&actions, pipefd[1], STDOUT_FILENO);
if (!rc) rc = posix_spawn_file_actions_adddup2(&actions, pipefd[1], STDERR_FILENO);
if (!rc) rc = posix_spawn_file_actions_addclose(&actions, pipefd[1]);
/* A tool must not inherit or change the live terminal's input mode. */
if (!rc) rc = posix_spawn_file_actions_addopen(&actions, STDIN_FILENO,
"/dev/null", O_RDONLY, 0);
char *argv[] = {"sh", "-c", (char *)(cmd ? cmd : ""), NULL};
if (!rc) rc = posix_spawn(pid, "/bin/sh", &actions, &attr, argv, environ);
posix_spawn_file_actions_destroy(&actions);
posix_spawnattr_destroy(&attr);
return rc;
}
/* Spawn a shell command into its own process group so bash_stop/timeout can
* kill grandchildren created by the shell, not just the /bin/sh wrapper. */
static agent_bash_job *agent_bash_start(agent_worker *w, const char *cmd,
@@ -8549,43 +8647,28 @@ static agent_bash_job *agent_bash_start(agent_worker *w, const char *cmd,
unlink(tmp_path);
return NULL;
}
fcntl(tmpfd, F_SETFD, FD_CLOEXEC);
fcntl(pipefd[0], F_SETFD, FD_CLOEXEC);
fcntl(pipefd[1], F_SETFD, FD_CLOEXEC);
pid_t pid = fork();
if (pid < 0) {
snprintf(err, err_len, "failed to fork: %s", strerror(errno));
int rc = 0;
/* Keep the source for both dup2 actions out of the standard fd range,
* including when the agent itself was started with closed descriptors. */
if (pipefd[1] <= STDERR_FILENO) {
int fd = fcntl(pipefd[1], F_DUPFD_CLOEXEC, STDERR_FILENO + 1);
if (fd < 0) rc = errno;
else { close(pipefd[1]); pipefd[1] = fd; }
}
if (!rc && (fcntl(tmpfd, F_SETFD, FD_CLOEXEC) < 0 ||
fcntl(pipefd[0], F_SETFD, FD_CLOEXEC) < 0 ||
fcntl(pipefd[1], F_SETFD, FD_CLOEXEC) < 0)) rc = errno;
pid_t pid;
if (!rc) rc = agent_bash_spawn(&pid, tmpfd, pipefd, cmd);
if (rc) {
snprintf(err, err_len, "failed to spawn shell: %s", strerror(rc));
close(pipefd[0]);
close(pipefd[1]);
close(tmpfd);
unlink(tmp_path);
return NULL;
}
if (pid == 0) {
setpgid(0, 0);
close(tmpfd);
/* The bash tool is not interactive. Give the shell /dev/null as
* stdin so it does not inherit the live linenoise terminal and reset
* it from raw mode to cooked mode behind the agent's back. */
int null_fd = open("/dev/null", O_RDONLY);
if (null_fd >= 0) {
if (dup2(null_fd, STDIN_FILENO) < 0)
close(STDIN_FILENO);
if (null_fd != STDIN_FILENO)
close(null_fd);
} else {
close(STDIN_FILENO);
}
close(pipefd[0]);
dup2(pipefd[1], STDOUT_FILENO);
dup2(pipefd[1], STDERR_FILENO);
close(pipefd[1]);
execl("/bin/sh", "sh", "-c", cmd ? cmd : "", (char *)NULL);
_exit(127);
}
close(pipefd[1]);
setpgid(pid, pid);
int old_flags;
set_nonblock(pipefd[0], true, &old_flags);
@@ -10314,7 +10397,8 @@ static int worker_run_turn(agent_worker *w, const char *user_text) {
agent_tool_observation_puts(&observation, "\n");
agent_tool_observation_puts(
&observation, tool_syntax == AGENT_TOOL_SYNTAX_GLM ?
agent_glm_syntax_reminder : agent_dsml_syntax_reminder);
agent_glm_syntax_reminder : tool_syntax == AGENT_TOOL_SYNTAX_DSML41 ?
agent_dsml41_syntax_reminder : agent_dsml_syntax_reminder);
} else {
agent_tool_observation_free(&observation);
observation = agent_execute_tool_observation(w, &dsml.calls);
@@ -10564,6 +10648,65 @@ static void worker_request_power(agent_worker *w, int power) {
pthread_mutex_unlock(&w->mu);
}
static void agent_numeric_think_prefix(ds4_engine *engine, ds4_think_mode mode,
ds4_tokens *prefix) {
ds4_chat_begin(engine, prefix);
ds4_chat_append_think_prefix(engine, prefix, mode);
ds4_chat_append_message(engine, prefix, "system", "");
}
static void worker_apply_requested_think(agent_worker *w) {
pthread_mutex_lock(&w->mu);
const ds4_think_mode mode = w->requested_think;
w->think_requested = false;
pthread_mutex_unlock(&w->mu);
if (!ds4_engine_is_deepseek41(w->engine) || w->cfg->gen.raw_prompt) {
agent_publishf(w, "\n/think requires a V4.1 chat session\n");
return;
}
/* Restored sessions may have a different effort from the current CLI
* setting. Match their actual token prefix, not the current preference. */
int old_len = 0;
for (int level = 0; level <= 100; level++) {
ds4_tokens candidate = {0};
agent_numeric_think_prefix(w->engine,
(ds4_think_mode)(DS4_THINK_LEVEL_BASE + level), &candidate);
if (candidate.len > old_len && ds4_tokens_starts_with(&w->transcript, &candidate))
old_len = candidate.len;
ds4_tokens_free(&candidate);
}
ds4_tokens next = {0};
agent_numeric_think_prefix(w->engine, mode, &next);
const int delta = next.len - old_len;
if (!old_len || (int64_t)w->transcript.len + delta >= agent_worker_effective_ctx_size(w)) {
agent_publishf(w, "\ncannot change thinking prefix: incompatible session or no context room\n");
ds4_tokens_free(&next);
return;
}
const bool changed = delta != 0 || memcmp(next.v, w->transcript.v,
(size_t)old_len * sizeof(int)) != 0;
if (changed) {
for (int i = old_len; i < w->transcript.len; i++)
ds4_tokens_push(&next, w->transcript.v[i]);
pthread_mutex_lock(&w->mu);
ds4_tokens_free(&w->transcript);
w->transcript = next;
memset(&next, 0, sizeof(next));
for (size_t i = 0; i < w->image_count; i++)
w->images[i].token_start = (uint32_t)((int64_t)w->images[i].token_start + delta);
pthread_mutex_unlock(&w->mu);
ds4_session_invalidate(w->session);
}
ds4_tokens_free(&next);
pthread_mutex_lock(&w->mu);
w->cfg->gen.think_mode = mode;
w->session_dirty |= changed;
w->status.ctx_used = w->transcript.len;
agent_wake_locked(w);
pthread_mutex_unlock(&w->mu);
agent_publishf(w, "\nThinking mode: %s.\n", ds4_think_mode_name(mode));
}
static bool worker_take_save_requested(agent_worker *w) {
pthread_mutex_lock(&w->mu);
bool requested = w->save_requested;
@@ -10675,7 +10818,7 @@ static void *worker_main(void *arg) {
while (true) {
pthread_mutex_lock(&w->mu);
while (!w->stop && !w->cmd_text && !w->save_requested &&
!w->compact_requested && !w->power_requested)
!w->compact_requested && !w->power_requested && !w->think_requested)
pthread_cond_wait(&w->cond, &w->mu);
if (w->stop) {
pthread_mutex_unlock(&w->mu);
@@ -10686,6 +10829,16 @@ static void *worker_main(void *arg) {
worker_apply_pending_power(w);
continue;
}
if (w->think_requested) {
w->status.state = AGENT_WORKER_PREFILL;
pthread_mutex_unlock(&w->mu);
worker_apply_requested_think(w);
pthread_mutex_lock(&w->mu);
w->status.state = w->cmd_text ? AGENT_WORKER_PREFILL : AGENT_WORKER_IDLE;
agent_wake_locked(w);
pthread_mutex_unlock(&w->mu);
continue;
}
if (!w->cmd_text && w->save_requested) {
pthread_mutex_unlock(&w->mu);
worker_run_deferred_save(w);
@@ -10754,7 +10907,8 @@ static void drain_wake_fd(int fd) {
* the UI can keep the typed text editable instead of silently queueing it. */
static bool worker_submit(agent_worker *w, const char *text) {
pthread_mutex_lock(&w->mu);
bool ok = w->initialized && w->status.state == AGENT_WORKER_IDLE && !w->cmd_text;
bool ok = w->initialized && w->status.state == AGENT_WORKER_IDLE &&
!w->cmd_text && !w->think_requested;
if (ok) {
w->cmd_text = xstrdup(text);
/* A submitted turn is no longer idle, even if the worker thread has
@@ -10838,7 +10992,7 @@ static void worker_get_status(agent_worker *w, agent_status *status) {
static bool worker_is_idle(agent_worker *w) {
pthread_mutex_lock(&w->mu);
bool idle = w->initialized &&
bool idle = w->initialized && !w->think_requested &&
(w->status.state == AGENT_WORKER_IDLE ||
w->status.state == AGENT_WORKER_ERROR);
pthread_mutex_unlock(&w->mu);
@@ -12805,6 +12959,22 @@ static int run_agent(ds4_engine *engine, agent_config *cfg) {
worker_request_power(&worker, power);
}
}
} else if (agent_slash_command_with_args(cmd, "/think")) {
char *arg = cmd + strlen("/think");
while (*arg == ' ' || *arg == '\t') arg++;
ds4_think_mode mode = DS4_THINK_HIGH;
if (!ds4_engine_is_deepseek41(worker.engine) ||
(arg[0] && !ds4_think_mode_parse_level(arg, &mode))) {
printf("usage: /think [0..100] (V4.1 only)\n");
} else {
pthread_mutex_lock(&worker.mu);
worker.requested_think = mode;
worker.think_requested = true;
pthread_cond_signal(&worker.cond);
agent_wake_locked(&worker);
pthread_mutex_unlock(&worker.mu);
if (busy) printf("thinking change scheduled after this turn\n");
}
} else if (agent_slash_command_with_args(cmd, "/hints")) {
char *arg = cmd + strlen("/hints");
while (*arg == ' ' || *arg == '\t') arg++;
@@ -13056,6 +13226,11 @@ int main(int argc, char **argv) {
} else if (ds4_engine_open(&engine, &cfg.engine) != 0) {
return 1;
}
if (ds4_think_mode_level(cfg.gen.think_mode) >= 0 && !ds4_engine_is_deepseek41(engine)) {
fprintf(stderr, "ds4-agent: --think-level requires a DeepSeek V4.1 model\n");
ds4_engine_close(engine);
return 2;
}
ds4_tp *tp_leader = NULL;
if (cfg.engine.tp.role == DS4_TP_LEADER) {
char tp_err[256] = "";
ds4_cli.c
+54 -40
View File
@@ -505,12 +505,7 @@ static void build_chat_prompt(ds4_engine *engine,
ds4_tokens *out) {
const ds4_think_mode think_mode = cli_effective_think_mode(gen);
ds4_chat_begin(engine, out);
if (ds4_engine_is_glm_dsa(engine)) {
const char *effort = ds4_glm_reasoning_effort_text(think_mode);
if (effort) ds4_chat_append_message(engine, out, "system", effort);
} else if (think_mode == DS4_THINK_MAX) {
ds4_chat_append_max_effort_prefix(engine, out);
}
ds4_chat_append_think_prefix(engine, out, think_mode);
if (gen->system && gen->system[0])
ds4_chat_append_message(engine, out, "system", gen->system);
ds4_prompt_prefix_append(engine, out, &gen->prefix);
@@ -1291,8 +1286,8 @@ static char *trim_inplace(char *s) {
static void print_repl_help(void) {
puts("Commands:");
puts(" /help Show this help.");
puts(" /think Use normal thinking mode.");
puts(" /think-max Use Think Max only when context is at least 393216 tokens.");
puts(" /think [N] Use normal thinking, or V4.1 effort 0..100 (0 disables thinking).");
puts(" /think-max Use maximum thinking (V4.1: 100; V4: requires ctx >= 393216).");
puts(" /nothink Disable thinking mode.");
puts(" /ctx N Set context size for following prompts.");
puts(" /power N Set GPU duty cycle percentage, 1..100.");
@@ -1337,6 +1332,7 @@ typedef struct {
int ctx_size;
int think_prefix_pos;
int think_prefix_tokens;
bool initial_system_text;
} repl_chat;
static void repl_chat_free(repl_chat *chat);
@@ -1389,34 +1385,22 @@ static void tokens_remove(ds4_tokens *dst, int pos, int n) {
dst->len -= n;
}
static const char *repl_glm_reasoning_effort_text(ds4_think_mode mode) {
switch (mode) {
case DS4_THINK_HIGH: return "Reasoning Effort: High";
case DS4_THINK_MAX: return "Reasoning Effort: Max";
case DS4_THINK_NONE: return NULL;
}
return NULL;
}
static void repl_chat_build_think_prefix(ds4_engine *engine,
ds4_think_mode mode,
ds4_tokens *prefix) {
if (ds4_engine_is_glm_dsa(engine)) {
const char *effort = repl_glm_reasoning_effort_text(mode);
if (effort) ds4_chat_append_message(engine, prefix, "system", effort);
} else if (mode == DS4_THINK_MAX) {
ds4_chat_append_max_effort_prefix(engine, prefix);
}
ds4_chat_append_think_prefix(engine, prefix, mode);
}
/* Insert/replace the model-family thinking prefix inside the existing
* transcript. It lives immediately after the BOS sequence and before any
* user/system text, matching the GGUF chat templates. */
static void repl_chat_apply_think_prefix(ds4_engine *engine,
static bool repl_chat_apply_think_prefix(ds4_engine *engine,
repl_chat *chat,
ds4_think_mode mode) {
ds4_tokens prefix = {0};
repl_chat_build_think_prefix(engine, mode, &prefix);
if (ds4_engine_is_deepseek41(engine) && chat->initial_system_text && !prefix.len)
ds4_chat_append_message(engine, &prefix, "system", "");
bool same = chat->think_prefix_tokens == prefix.len;
if (same && prefix.len > 0) {
@@ -1426,11 +1410,16 @@ static void repl_chat_apply_think_prefix(ds4_engine *engine,
}
if (!same) {
const int old_prefix_tokens = chat->think_prefix_tokens;
const int shift = prefix.len - old_prefix_tokens;
if (chat->ctx_size > 0 && (int64_t)chat->transcript.len + shift >= chat->ctx_size) {
fprintf(stderr, "ds4: no context room to change the thinking prefix\n");
ds4_tokens_free(&prefix);
return false;
}
tokens_remove(&chat->transcript, chat->think_prefix_pos,
chat->think_prefix_tokens);
tokens_insert(&chat->transcript, chat->think_prefix_pos, &prefix);
chat->think_prefix_tokens = prefix.len;
const int shift = prefix.len - old_prefix_tokens;
for (size_t i = 0; i < chat->image_count; i++) {
chat->images[i].token_start =
(uint32_t)((int64_t)chat->images[i].token_start + shift);
@@ -1438,6 +1427,7 @@ static void repl_chat_apply_think_prefix(ds4_engine *engine,
if (chat->session) ds4_session_invalidate(chat->session);
}
ds4_tokens_free(&prefix);
return true;
}
static int repl_chat_create_session(ds4_engine *engine, repl_chat *chat, int ctx_size) {
@@ -1454,6 +1444,7 @@ static int repl_chat_create_session(ds4_engine *engine, repl_chat *chat, int ctx
static int repl_chat_init(ds4_engine *engine, repl_chat *chat, const cli_config *cfg) {
memset(chat, 0, sizeof(*chat));
chat->initial_system_text = cfg->gen.system && cfg->gen.system[0];
ds4_chat_begin(engine, &chat->transcript);
chat->think_prefix_pos = chat->transcript.len;
repl_chat_apply_think_prefix(engine, chat, cli_effective_think_mode(&cfg->gen));
@@ -1530,7 +1521,7 @@ static int run_chat_turn(ds4_engine *engine, cli_config *cfg, repl_chat *chat,
ds4_think_mode think_mode = ds4_think_mode_for_context(cfg->gen.think_mode,
chat->ctx_size);
repl_chat_apply_think_prefix(engine, chat, think_mode);
if (!repl_chat_apply_think_prefix(engine, chat, think_mode)) return 1;
const int rollback_len = chat->transcript.len;
const size_t rollback_images = chat->image_count;
if (image) {
@@ -1760,22 +1751,31 @@ static int run_repl(ds4_engine *engine, cli_config *cfg) {
if (!strcmp(cmd, "/help")) {
print_repl_help();
} else if (!strcmp(cmd, "/think")) {
cfg->gen.think_mode = DS4_THINK_HIGH;
repl_chat_apply_think_prefix(engine, &chat, DS4_THINK_HIGH);
puts("Thinking mode: high.");
} else if (!strncmp(cmd, "/think", 6) &&
(cmd[6] == '\0' || isspace((unsigned char)cmd[6]))) {
const char *arg = trim_inplace(cmd + 6);
ds4_think_mode mode = DS4_THINK_HIGH;
if (arg[0] && (!ds4_engine_is_deepseek41(engine) ||
!ds4_think_mode_parse_level(arg, &mode))) {
fprintf(stderr, "ds4: /think N requires V4.1 and a level from 0 to 100\n");
} else if (repl_chat_apply_think_prefix(engine, &chat, mode)) {
cfg->gen.think_mode = mode;
printf("Thinking mode: %s.\n", ds4_think_mode_name(mode));
}
} else if (!strcmp(cmd, "/think-max")) {
cfg->gen.think_mode = DS4_THINK_MAX;
bool active = ds4_think_mode_for_context(cfg->gen.think_mode,
bool active = ds4_think_mode_for_context(DS4_THINK_MAX,
chat.ctx_size) == DS4_THINK_MAX;
repl_chat_apply_think_prefix(engine, &chat,
active ? DS4_THINK_MAX : DS4_THINK_HIGH);
cli_warn_think_max_downgraded(&cfg->gen, "/think-max");
printf("Thinking mode: %s.\n", active ? "max" : "high (ctx below 393216)");
if (repl_chat_apply_think_prefix(engine, &chat,
active ? DS4_THINK_MAX : DS4_THINK_HIGH)) {
cfg->gen.think_mode = DS4_THINK_MAX;
cli_warn_think_max_downgraded(&cfg->gen, "/think-max");
printf("Thinking mode: %s.\n", active ? "max" : "high (ctx below 393216)");
}
} else if (!strcmp(cmd, "/nothink")) {
cfg->gen.think_mode = DS4_THINK_NONE;
repl_chat_apply_think_prefix(engine, &chat, DS4_THINK_NONE);
puts("Thinking mode: none.");
if (repl_chat_apply_think_prefix(engine, &chat, DS4_THINK_NONE)) {
cfg->gen.think_mode = DS4_THINK_NONE;
puts("Thinking mode: none.");
}
} else if (!strncmp(cmd, "/power", 6) && (cmd[6] == '\0' || isspace((unsigned char)cmd[6]))) {
char *arg = trim_inplace(cmd + 6);
if (!arg[0]) {
@@ -2162,6 +2162,11 @@ static cli_config parse_options(int argc, char **argv) {
} else if (!strcmp(arg, "--imatrix-min-expert-samples")) {
c.gen.imatrix_min_expert_samples =
parse_int(need_arg(&i, argc, argv, arg), arg);
} else if (!strcmp(arg, "--think-level")) {
if (!ds4_think_mode_parse_level(need_arg(&i, argc, argv, arg), &c.gen.think_mode)) {
fprintf(stderr, "ds4: --think-level requires an integer from 0 to 100\n");
exit(2);
}
} else if (!strcmp(arg, "--think")) {
c.gen.think_mode = DS4_THINK_HIGH;
} else if (!strcmp(arg, "--think-max")) {
@@ -2283,7 +2288,8 @@ int main(int argc, char **argv) {
rc = ds4_dump_chat_tokenization(cfg.engine.model_path,
cfg.gen.system,
cfg.gen.prompt,
cli_effective_think_mode(&cfg.gen),
cfg.gen.think_mode,
cfg.gen.ctx_size,
stdout);
}
ds4_dist_options_free(cfg.dist);
@@ -2338,6 +2344,14 @@ int main(int argc, char **argv) {
free(cfg.prompt_owned);
return 1;
}
if (ds4_think_mode_level(cfg.gen.think_mode) >= 0 && !ds4_engine_is_deepseek41(engine)) {
fprintf(stderr, "ds4: --think-level requires a DeepSeek V4.1 model\n");
ds4_engine_close(engine);
ds4_dist_options_free(cfg.dist);
ds4_prompt_prefix_free(&cfg.gen.prefix);
free(cfg.prompt_owned);
return 2;
}
cli_apply_model_sampling_defaults(engine, &cfg.gen);
if (cfg.engine.tp.role == DS4_TP_WORKER) {
int rc = ds4_tp_worker_run(engine, &cfg.engine.tp);
ds4_engram.c
+277
View File
@@ -0,0 +1,277 @@
#define _DARWIN_C_SOURCE
#define _POSIX_C_SOURCE 200809L
#include "ds4_engram.h"
#include <errno.h>
#include <fcntl.h>
#include <limits.h>
#include <math.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <unistd.h>
#ifdef __APPLE__
#include <dispatch/dispatch.h>
#endif
bool ds4_engram_layout_valid(const ds4_engram_layout *l) {
if (!l || !l->token_map || !l->vocab_size ||
!l->compressed_vocab_size || l->compressed_vocab_size > INT32_MAX ||
l->pad_id >= l->compressed_vocab_size) return false;
for (uint32_t i = 0; i < l->vocab_size; i++)
if (l->token_map[i] >= l->compressed_vocab_size) return false;
for (int layer = 0; layer < DS4_ENGRAM_LAYERS; layer++) {
for (int i = 0; i < DS4_ENGRAM_NGRAM; i++) {
uint64_t m = l->multipliers[layer][i];
if (!(m & 1) || m > (uint64_t)INT64_MAX / l->compressed_vocab_size)
return false;
}
uint64_t total = 0;
for (int i = 0; i < DS4_ENGRAM_COLS; i++) {
if (l->primes[layer][i] < 2) return false;
total += l->primes[layer][i];
}
if (total != l->rows[layer]) return false;
}
return true;
}
void ds4_engram_history_reset(ds4_engram_history *h) {
for (int i = 0; i < DS4_ENGRAM_NGRAM - 1; i++) h->tail[i] = DS4_ENGRAM_DEAD;
}
bool ds4_engram_hash(const ds4_engram_layout *l, ds4_engram_history *h,
const int *tokens, const uint8_t *mask, size_t count,
uint32_t *rows) {
if (!l || !h || !l->token_map || (count && (!tokens || !rows)) ||
count > SIZE_MAX / (DS4_ENGRAM_LAYERS * DS4_ENGRAM_COLS * sizeof(*rows)))
return false;
for (int i = 0; i < DS4_ENGRAM_NGRAM - 1; i++) {
if (h->tail[i] < DS4_ENGRAM_DEAD ||
(h->tail[i] >= 0 && (uint32_t)h->tail[i] >= l->compressed_vocab_size))
return false;
}
for (size_t i = 0; i < count; i++) {
if (tokens[i] < 0 || (uint32_t)tokens[i] >= l->vocab_size) return false;
}
for (size_t i = 0; i < count; i++) {
int32_t current = mask && !mask[i] ? DS4_ENGRAM_DEAD :
(int32_t)l->token_map[tokens[i]];
uint32_t ids[DS4_ENGRAM_NGRAM];
bool blocked = false;
for (int j = 0; j < DS4_ENGRAM_NGRAM; j++) {
int32_t id = j ? h->tail[j - 1] : current;
blocked |= id == DS4_ENGRAM_DEAD;
ids[j] = blocked ? l->pad_id : (uint32_t)id;
}
for (int layer = 0; layer < DS4_ENGRAM_LAYERS; layer++) {
uint64_t hash = (uint64_t)ids[0] * l->multipliers[layer][0];
uint32_t offset = 0;
for (int j = 1; j < DS4_ENGRAM_NGRAM; j++) {
hash ^= (uint64_t)ids[j] * l->multipliers[layer][j];
for (int head = 0; head < DS4_ENGRAM_HEADS; head++) {
int col = (j - 1) * DS4_ENGRAM_HEADS + head;
uint32_t prime = l->primes[layer][col];
*rows++ = (uint32_t)(hash % prime) + offset;
offset += prime;
}
}
}
for (int j = DS4_ENGRAM_NGRAM - 2; j > 0; j--) h->tail[j] = h->tail[j - 1];
h->tail[0] = current;
}
return true;
}
bool ds4_engram_table_open(ds4_engram_table *t, const char *path,
uint64_t offset, uint32_t rows) {
if (!t) return false;
*t = (ds4_engram_table){.fd = -1};
uint64_t bytes = (uint64_t)rows * DS4_ENGRAM_ROW_BYTES;
if (!path || !rows || offset > INT64_MAX || bytes > INT64_MAX - offset) {
errno = EINVAL;
return false;
}
int fd = open(path, O_RDONLY | O_CLOEXEC);
if (fd < 0) return false;
struct stat st;
if (fstat(fd, &st) != 0) goto fail;
if (!S_ISREG(st.st_mode) || st.st_size < 0 || offset + bytes > (uint64_t)st.st_size) {
errno = EINVAL;
goto fail;
}
#ifdef __APPLE__
if (fcntl(fd, F_NOCACHE, 1) != 0 || fcntl(fd, F_RDAHEAD, 0) != 0) goto fail;
#endif
*t = (ds4_engram_table){.fd = fd, .offset = offset, .rows = rows};
return true;
fail: {
int saved = errno;
close(fd);
errno = saved;
return false;
}
}
void ds4_engram_table_close(ds4_engram_table *t) {
if (!t) return;
if (t->fd >= 0) close(t->fd);
*t = (ds4_engram_table){.fd = -1};
}
static bool read_row(int fd, uint64_t offset, uint8_t row[DS4_ENGRAM_ROW_BYTES]) {
size_t done = 0;
while (done < DS4_ENGRAM_ROW_BYTES) {
ssize_t n = pread(fd, row + done, DS4_ENGRAM_ROW_BYTES - done,
(off_t)(offset + done));
if (n < 0 && errno == EINTR) continue;
if (n <= 0) {
if (n == 0) errno = EIO;
return false;
}
done += (size_t)n;
}
return true;
}
static float e4m3(uint8_t byte) {
int exponent = (byte >> 3) & 15, mantissa = byte & 7;
float value = exponent ? ldexpf((float)(8 + mantissa), exponent - 10) :
ldexpf((float)mantissa, -9);
return byte & 128 ? -value : value;
}
bool ds4_engram_read(const ds4_engram_table *t, const uint32_t *rows,
size_t count, float *out) {
if (!t || t->fd < 0 || (count && (!rows || !out)) ||
count > SIZE_MAX / (DS4_ENGRAM_DIM * sizeof(*out))) {
errno = EINVAL;
return false;
}
for (size_t i = 0; i < count; i++) {
if (rows[i] >= t->rows) {
errno = EINVAL;
return false;
}
}
uint8_t raw[DS4_ENGRAM_ROW_BYTES];
for (size_t i = 0; i < count; i++) {
if (!read_row(t->fd, t->offset + (uint64_t)rows[i] * sizeof(raw), raw)) return false;
for (int j = 0; j < DS4_ENGRAM_DIM; j++) {
uint8_t code = raw[j], scale = raw[DS4_ENGRAM_DIM + j / 32];
if ((code & 127) == 127 || scale == 255) {
errno = EDOM;
return false;
}
float value = ldexpf(e4m3(code), (int)scale - 127);
uint32_t bits;
memcpy(&bits, &value, sizeof(bits));
bits = (bits + 0x7fffu + ((bits >> 16) & 1u)) & 0xffff0000u;
memcpy(&value, &bits, sizeof(value));
if (!isfinite(value)) {
errno = EDOM;
return false;
}
out[i * DS4_ENGRAM_DIM + j] = value;
}
}
return true;
}
typedef struct {
uint32_t row, output;
} engram_request;
static int request_order(const void *a, const void *b) {
const engram_request *x = a, *y = b;
return (x->row > y->row) - (x->row < y->row);
}
enum { ENGRAM_READERS = 16 };
typedef struct {
const ds4_engram_table *table;
const engram_request *request;
float *out;
size_t count, readers;
int error[ENGRAM_READERS];
} engram_batch;
static void read_batch_part(void *context, size_t part) {
engram_batch *batch = context;
const engram_request *request = batch->request;
const size_t begin = batch->count * part / batch->readers;
const size_t end = batch->count * (part + 1) / batch->readers;
const float *previous = NULL;
for (size_t i = begin; i < end; i++) {
float *dst = batch->out + (size_t)request[i].output * DS4_ENGRAM_DIM;
if (i > begin && request[i].row == request[i - 1].row) {
memcpy(dst, previous, DS4_ENGRAM_DIM * sizeof(*dst));
} else {
if (!ds4_engram_read(batch->table, &request[i].row, 1, dst)) {
batch->error[part] = errno ? errno : EIO;
return;
}
previous = dst;
}
}
}
bool ds4_engram_read_batch(const ds4_engram_table *t, const uint32_t *rows,
size_t tokens, size_t stride, float *out) {
if (!t || t->fd < 0 || (tokens && (!rows || !out || stride < DS4_ENGRAM_COLS)) ||
tokens > SIZE_MAX / (DS4_ENGRAM_COLS * DS4_ENGRAM_DIM * sizeof(*out)) ||
(tokens && tokens - 1 > (SIZE_MAX / sizeof(*rows) - DS4_ENGRAM_COLS) / stride)) {
errno = EINVAL;
return false;
}
for (size_t i = 0; i < tokens; i++) {
for (size_t j = 0; j < DS4_ENGRAM_COLS; j++) {
if (rows[i * stride + j] >= t->rows) {
errno = EINVAL;
return false;
}
}
}
if (!tokens) return true;
enum { BATCH_TOKENS = 2048 };
const size_t cap = tokens < BATCH_TOKENS ? tokens : BATCH_TOKENS;
engram_request *request = malloc(cap * DS4_ENGRAM_COLS * sizeof(*request));
if (!request) return false;
bool ok = true;
for (size_t start = 0; ok && start < tokens; start += cap) {
const size_t n = tokens - start < cap ? tokens - start : cap;
const size_t count = n * DS4_ENGRAM_COLS;
for (size_t i = 0; i < count; i++) {
request[i] = (engram_request){
rows[(start + i / DS4_ENGRAM_COLS) * stride + i % DS4_ENGRAM_COLS],
(uint32_t)i
};
}
qsort(request, count, sizeof(*request), request_order);
engram_batch batch = {.table = t, .request = request, .count = count,
.out = out + start * DS4_ENGRAM_COLS * DS4_ENGRAM_DIM, .readers = 1};
#ifdef __APPLE__
/* Fixed concurrency hides random-read latency without caching the table.
* Each worker owns disjoint output rows; all finish before GPU use. */
if (count >= 256) {
batch.readers = ENGRAM_READERS;
dispatch_apply_f(batch.readers,
dispatch_get_global_queue(QOS_CLASS_USER_INITIATED, 0), &batch, read_batch_part);
} else
#endif
read_batch_part(&batch, 0);
for (size_t i = 0; i < batch.readers; i++) {
if (batch.error[i]) {
errno = batch.error[i];
ok = false;
break;
}
}
}
int saved = errno;
free(request);
errno = saved;
return ok;
}
ds4_engram.h
+63
View File
@@ -0,0 +1,63 @@
#ifndef DS4_ENGRAM_H
#define DS4_ENGRAM_H
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
enum {
DS4_ENGRAM_LAYERS = 2,
DS4_ENGRAM_NGRAM = 4,
DS4_ENGRAM_HEADS = 8,
DS4_ENGRAM_COLS = 24,
DS4_ENGRAM_DIM = 256,
DS4_ENGRAM_ROW_BYTES = 264,
DS4_ENGRAM_DEAD = -1
};
typedef struct {
const uint32_t *token_map;
uint32_t vocab_size, compressed_vocab_size, pad_id;
uint32_t rows[DS4_ENGRAM_LAYERS];
uint64_t multipliers[DS4_ENGRAM_LAYERS][DS4_ENGRAM_NGRAM];
uint32_t primes[DS4_ENGRAM_LAYERS][DS4_ENGRAM_COLS];
} ds4_engram_layout;
/* Newest first; copying this value also snapshots the complete hash state. */
typedef struct {
int32_t tail[DS4_ENGRAM_NGRAM - 1];
} ds4_engram_history;
bool ds4_engram_layout_valid(const ds4_engram_layout *layout);
void ds4_engram_history_reset(ds4_engram_history *history);
/* Validate the layout once at model load. Mask zero breaks all n-grams that
* cross that position. Output is [token][layer][column], including masked rows;
* the graph must suppress Engram at masked positions, as in the reference. */
bool ds4_engram_hash(const ds4_engram_layout *layout,
ds4_engram_history *history,
const int *tokens, const uint8_t *mask, size_t count,
uint32_t *rows);
typedef struct {
int fd;
uint64_t offset;
uint32_t rows;
} ds4_engram_table;
/* A separate uncached file descriptor, never an mmap or Metal model view.
* Each GGUF I8 row is 256 E4M3 bytes followed by 8 original E8M0 scales. */
bool ds4_engram_table_open(ds4_engram_table *table, const char *path,
uint64_t offset, uint32_t rows);
void ds4_engram_table_close(ds4_engram_table *table);
/* Output uses F32 storage for the reference's BF16-rounded values. No whole
* table allocation; caller owns count * DIM floats. Failure invalidates output. */
bool ds4_engram_read(const ds4_engram_table *table, const uint32_t *rows,
size_t count, float *out);
/* Read COLS rows per token, restoring token order after deduplicated disk reads.
* Input stride is in row IDs; output is packed [token][COLS][DIM]. Temporary
* storage is bounded to 384 KiB, independent of the table and prefix size.
* On macOS, large batches use bounded concurrent pread readers. */
bool ds4_engram_read_batch(const ds4_engram_table *table, const uint32_t *rows,
size_t tokens, size_t stride, float *out);
#endif
ds4_gpu.h
+101 -2
View File
@@ -79,6 +79,103 @@ int ds4_gpu_flush_encoder(void);
int ds4_gpu_flush_commands(void);
int ds4_gpu_commands_active(void);
#ifdef __APPLE__
/* V4.1 activation/cache formats. Buffers are float-addressable but the
* rounded values follow the released BF16/FP8/FP4 inference graph. */
typedef enum {
DS4_V41_BF16 = 0,
DS4_V41_FP8_E8M0 = 1,
DS4_V41_FP4_E8M0 = 2,
DS4_V41_FP4_E4M3 = 3,
} ds4_v41_activation_format;
int ds4_gpu_dsv41_quantize(ds4_gpu_tensor *x, uint32_t width, uint32_t rows,
ds4_v41_activation_format format);
/* Full-head prefill, with BF16 rounding between the two Q8 projections. */
int ds4_gpu_dsv41_attention_output_batch(
ds4_gpu_tensor *out, ds4_gpu_tensor *low,
const void *model_map, uint64_t model_size,
uint64_t out_a_offset, uint64_t out_b_offset,
const ds4_gpu_tensor *heads, uint32_t n_tokens);
/* Packed local 32-head input and BF16 low projection; output is an unrounded
* rank partial. The graph sums ranks before rounding the attention block. */
int ds4_gpu_dsv41_attention_output_tp_batch(
ds4_gpu_tensor *out, ds4_gpu_tensor *low,
const void *model_map, uint64_t model_size,
uint64_t out_a_offset, uint64_t out_b_offset,
const ds4_gpu_tensor *heads, uint32_t n_tokens, uint32_t tp_rank);
/* Adjacent-pair, unit-magnitude RoPE with the released V4.1 frequencies. */
int ds4_gpu_dsv41_rope(ds4_gpu_tensor *x, uint32_t width, uint32_t heads,
uint32_t rows, uint32_t start, bool compressed, bool inverse);
/* Compressed pairs advance two absolute token positions per stored row. */
int ds4_gpu_dsv41_rope_stride(ds4_gpu_tensor *x, uint32_t width, uint32_t heads,
uint32_t rows, uint32_t start, uint32_t stride,
bool compressed, bool inverse);
int ds4_gpu_dsv41_engram_add(ds4_gpu_tensor *residual,
const ds4_gpu_tensor *kv,
const ds4_gpu_tensor *q_weight,
const ds4_gpu_tensor *k_weight,
const ds4_gpu_tensor *mask,
uint32_t width, uint32_t rows, float eps);
/* Pool complete pairs and retain the last unpaired projection in previous_*.
* start is the absolute token position, including earlier chunks. */
int ds4_gpu_dsv41_pool2(ds4_gpu_tensor *out,
const ds4_gpu_tensor *kv, const ds4_gpu_tensor *scores,
ds4_gpu_tensor *previous_kv, ds4_gpu_tensor *previous_scores,
uint32_t width, uint32_t rows, uint32_t start);
/* Candidate blocks contain eight compressed positions. Produce causal block
* maxima, pinning the newest block; filter consumes a 0/-inf block mask. */
int ds4_gpu_dsv41_candidate_blocks(ds4_gpu_tensor *blocks,
const ds4_gpu_tensor *scores,
uint32_t width, uint32_t rows,
uint32_t start, uint32_t ratio);
int ds4_gpu_dsv41_candidate_filter(ds4_gpu_tensor *scores,
const ds4_gpu_tensor *block_mask,
uint32_t width, uint32_t rows,
uint32_t start, uint32_t ratio);
/* Causal index scores over ratio-1/2 compressed keys, without an extra cast
* of the already quantized FP4 queries/keys. Scores have source_rows stride. */
int ds4_gpu_dsv41_indexer_scores_batch(ds4_gpu_tensor *scores,
const ds4_gpu_tensor *q,
const ds4_gpu_tensor *weights,
const ds4_gpu_tensor *keys,
uint32_t source_rows, uint32_t rows,
uint32_t start, uint32_t ratio);
int ds4_gpu_dsv41_tensor_ops_available(void);
/* Reuse exact BF16 views of FP4 queries/keys across score tiles. Invalid
* casts retain the F32 arithmetic for the affected tile. */
uint64_t ds4_gpu_dsv41_indexer_packed_bytes(uint32_t source_rows, uint32_t rows);
int ds4_gpu_dsv41_indexer_pack(ds4_gpu_tensor *packed,
const ds4_gpu_tensor *q, const ds4_gpu_tensor *keys,
uint32_t source_rows, uint32_t rows);
int ds4_gpu_dsv41_indexer_scores_packed(ds4_gpu_tensor *scores,
const ds4_gpu_tensor *q,
const ds4_gpu_tensor *weights,
const ds4_gpu_tensor *keys,
const ds4_gpu_tensor *packed,
uint32_t source_rows, uint32_t rows,
uint32_t start, uint32_t ratio,
uint32_t packed_rows, uint32_t offset);
/* Exact row-sort ordering with independent causal widths; at least 1024
* visible keys per row. Output stride is 512 indices. */
int ds4_gpu_dsv41_indexer_topk_batch(ds4_gpu_tensor *selected,
const ds4_gpu_tensor *scores,
uint32_t width, uint32_t rows,
uint32_t start, uint32_t ratio);
enum { DS4_V41_CARRY_BF16, DS4_V41_CARRY_MASK, DS4_V41_CARRY_F32 };
/* Lossless storage for already-BF16 activations or 0/-inf candidate masks.
* Packed rows are padded to whole uint32_t words. Plain rows remain F32. */
int ds4_gpu_dsv41_carry_copy(ds4_gpu_tensor *packed, uint32_t row_offset,
ds4_gpu_tensor *plain, uint32_t width, uint32_t rows,
uint32_t format, bool pack);
/* Batched F16 projections with the same arithmetic as individual matvecs. */
int ds4_gpu_dsv41_projection_rows(ds4_gpu_tensor *out,
const void *model_map, uint64_t model_size,
uint64_t weight_offset, uint32_t width,
uint32_t outputs, uint32_t rows,
const ds4_gpu_tensor *in);
/* Gather 512-wide F32 KV rows; IDs must come from top-k over source_rows. */
int ds4_gpu_dsv41_gather_kv(ds4_gpu_tensor *out, const ds4_gpu_tensor *source,
const ds4_gpu_tensor *ids, uint32_t source_rows,
uint32_t selected_rows);
int ds4_gpu_parallel_ffn_finish(void);
void ds4_gpu_parallel_ffn_abort(void);
int ds4_gpu_parallel_ffn_start(
@@ -427,8 +524,9 @@ int ds4_gpu_matmul_q8_0_kslice_tensor(
uint64_t out_dim,
const ds4_gpu_tensor *x,
uint64_t x_elem_off);
/* CUDA multi-row variant. Each input row contains only the owned contiguous
* K slice, while each output row spans the full projection width. */
/* CUDA input rows contain only the owned contiguous K slice. Metal input
* rows span full_in_dim and the kernel reads the slice at k_off within them.
* Both backends produce full-width output rows. */
int ds4_gpu_matmul_q8_0_kslice_rows_tensor(
ds4_gpu_tensor *out,
const void *model_map,
@@ -3180,6 +3278,7 @@ typedef struct {
uint64_t mtp_visual_router_bias[DS4_DEEPSEEK4_MTP_LAYERS];
uint64_t hash_router_bias[3];
ds4_deepseek4_vision_layer_weights layer[DS4_DEEPSEEK4_VISION_LAYERS];
uint32_t projection_dim;
} ds4_deepseek4_vision_weights;
#endif
ds4_help.c
+11 -8
View File
@@ -196,7 +196,7 @@ static void print_model_runtime(FILE *fp, const help_colors *c,
opt(fp, c, "--dspark-confidence F", "DSpark confidence pruning threshold 0..1.");
}
opt(fp, c, "--quality", "Prefer exact kernels where faster approximate paths exist.");
opt(fp, c, "--warm-weights", "Touch mapped tensor pages at startup to reduce first-use stalls.");
opt(fp, c, "--warm-weights", "Touch resident weights at startup to reduce first-use stalls.");
if (tool == DS4_HELP_DS4 || tool == DS4_HELP_BENCH) {
opt(fp, c, "--expert-profile FILE", "Metal-only: write routed expert locality/cache simulation JSON.");
}
@@ -204,7 +204,7 @@ static void print_model_runtime(FILE *fp, const help_colors *c,
fputc('\n', fp);
}
static void print_sampling(FILE *fp, const help_colors *c, bool full) {
static void print_sampling(FILE *fp, const help_colors *c, bool full, ds4_help_tool tool) {
title(fp, c, "Prompt And Sampling");
opt(fp, c, "-n, --tokens N", "Maximum generated tokens.");
opt(fp, c, "--temp F", "Sampling temperature. 0 is greedy/deterministic.");
@@ -212,8 +212,10 @@ static void print_sampling(FILE *fp, const help_colors *c, bool full) {
opt(fp, c, "--min-p F", "Keep tokens scoring at least F times the top token.");
opt(fp, c, "--seed N", "Sampling seed for reproducible non-greedy runs.");
para(fp, c, "GLM CLI and agent runs default to temperature 1.0, top-p 0.95, and min-p 0 unless those options are set explicitly.");
opt(fp, c, "--think", "Use normal thinking mode.");
opt(fp, c, "--think-max", "Use Think Max when context is large enough.");
opt(fp, c, "--think", "Use normal thinking mode (V4.1: effort 75).");
opt(fp, c, "--think-max", "Use maximum thinking (V4.1: 100; V4: requires ctx >= 393216).");
if (tool == DS4_HELP_DS4 || tool == DS4_HELP_AGENT)
opt(fp, c, "--think-level N", "V4.1 thinking effort, 1..100; 0 disables thinking.");
opt(fp, c, "--nothink", "Disable thinking and ask for direct replies.");
if (full) {
opt(fp, c, "-sys, --system TEXT", "System prompt. Empty string disables the default where supported.");
@@ -300,7 +302,7 @@ static void print_cli_diagnostics(FILE *fp, const help_colors *c) {
static void print_cli_commands(FILE *fp, const help_colors *c) {
title_red(fp, c, "Interactive Commands");
opt(fp, c, "/help", "Show interactive commands.");
opt(fp, c, "/think, /think-max, /nothink", "Switch thinking mode.");
opt(fp, c, "/think [N], /think-max, /nothink", "Switch thinking mode; V4.1 accepts effort 0..100.");
opt(fp, c, "/ctx N", "Restart the interactive session with a new context size.");
opt(fp, c, "/power N", "Set GPU duty cycle percentage, 1..100.");
opt(fp, c, "/read FILE", "Submit a text file, PNG, or JPEG as the next user message.");
@@ -333,6 +335,7 @@ static void print_agent_sessions(FILE *fp, const help_colors *c) {
opt(fp, c, "/strip ID", "Remove KV payload; the text history can be rebuilt later.");
opt(fp, c, "/history [N]", "Show N recent user turns from the current session.");
opt(fp, c, "/hints on|off", "Enable or disable brief programming hints. New and resumed sessions start off.");
opt(fp, c, "/think [N]", "V4.1: set effort 0..100 (default 75) after this turn; rebuilds the prefix.");
opt(fp, c, "/power N", "Set GPU duty cycle percentage, 1..100.");
opt(fp, c, "/new", "Start a fresh session from the system prompt.");
opt(fp, c, "/quit, /exit", "Exit.");
@@ -528,7 +531,7 @@ static void print_examples(FILE *fp, const help_colors *c, ds4_help_tool tool, c
static void print_topic(FILE *fp, const help_colors *c, ds4_help_tool tool, const char *topic) {
if (streq(topic, "all")) {
print_model_runtime(fp, c, tool, true);
if (tool_has_topic(tool, "sampling")) print_sampling(fp, c, true);
if (tool_has_topic(tool, "sampling")) print_sampling(fp, c, true, tool);
if (tool_has_topic(tool, "steering")) print_steering(fp, c);
print_distributed(fp, c);
if (tool == DS4_HELP_DS4) {
@@ -550,7 +553,7 @@ static void print_topic(FILE *fp, const help_colors *c, ds4_help_tool tool, cons
}
if (streq(topic, "runtime")) print_model_runtime(fp, c, tool, true);
else if (streq(topic, "sampling")) print_sampling(fp, c, true);
else if (streq(topic, "sampling")) print_sampling(fp, c, true, tool);
else if (streq(topic, "steering")) print_steering(fp, c);
else if (streq(topic, "distributed")) print_distributed(fp, c);
else if (tool == DS4_HELP_DS4 && streq(topic, "diagnostics")) print_cli_diagnostics(fp, c);
@@ -575,7 +578,7 @@ static void print_default(FILE *fp, const help_colors *c, ds4_help_tool tool) {
if (tool == DS4_HELP_DS4) {
print_cli_specific(fp, c, true);
print_sampling(fp, c, false);
print_sampling(fp, c, false, tool);
} else if (tool == DS4_HELP_SERVER) {
print_server_api(fp, c);
print_kv_cache(fp, c);
ds4_image.c
+75 -22
View File
@@ -574,12 +574,15 @@ static uint32_t ds4_deepseek4_grid_tokens(
uint32_t height,
uint32_t width,
uint32_t *grid_height,
uint32_t *grid_width) {
uint32_t *grid_width,
bool v41) {
const uint32_t h = (height / 14u + 2u) / 3u;
const uint32_t w = (width / 14u + 2u) / 3u;
uint64_t count = (uint64_t)h * (w + 1u) + 2u;
if (h & 1u) count += w + 1u;
if ((((h + 1u) / 2u) * (w + 1u)) & 1u) count += 2u;
if (!v41) {
if (h & 1u) count += w + 1u;
if ((((h + 1u) / 2u) * (w + 1u)) & 1u) count += 2u;
}
if (grid_height) *grid_height = h;
if (grid_width) *grid_width = w;
return count <= UINT32_MAX ? (uint32_t)count : UINT32_MAX;
@@ -590,7 +593,8 @@ static int ds4_deepseek4_solve_resize(
uint32_t width,
uint32_t budget,
uint32_t *best_height,
uint32_t *best_width) {
uint32_t *best_width,
bool v41) {
const double ratio = (double)height / width;
const double max_w_float = sqrt(((double)budget - 2.0) / ratio + 0.25) - 0.5;
const double max_h_float = max_w_float * ratio;
@@ -598,19 +602,20 @@ static int ds4_deepseek4_solve_resize(
if (max_w_float < 1.0) {
max_w = 1;
max_h = (budget - 2u) / (max_w + 1u);
max_h &= ~1u;
if (!v41) max_h &= ~1u;
if (max_h == 0) return 0;
*best_width = max_w * 42u;
*best_height = max_h * 42u;
} else if (max_h_float < 2.0) {
max_h = 2;
} else if (max_h_float < (v41 ? 1.0 : 2.0)) {
max_h = v41 ? 1u : 2u;
max_w = (budget - 2u) / max_h - 1u;
if (max_w <= 1u) return 0;
*best_width = max_w * 42u;
*best_height = max_h * 42u;
} else {
max_w = (uint32_t)floor(max_w_float);
max_h = (uint32_t)floor(max_h_float) & ~1u;
max_h = (uint32_t)floor(max_h_float);
if (!v41) max_h &= ~1u;
if (max_w == 0 || max_h == 0) return 0;
const double scale = fmin((double)max_w * 42.0 / width,
(double)max_h * 42.0 / height);
@@ -626,28 +631,31 @@ static int ds4_deepseek4_safe_resize(
uint32_t *best_height,
uint32_t *best_width,
uint32_t *llm_height,
uint32_t *llm_width) {
uint32_t budget = 384u - 3u;
uint32_t *llm_width,
bool v41) {
const uint32_t max_tokens = v41 ? 1024u : 381u;
uint32_t budget = max_tokens;
uint32_t tokens = ds4_deepseek4_grid_tokens(
*best_height, *best_width, llm_height, llm_width);
while (tokens > 381u) {
*best_height, *best_width, llm_height, llm_width, v41);
while (tokens > max_tokens) {
if (budget <= 4u ||
!ds4_deepseek4_solve_resize(height, width, budget,
best_height, best_width)) {
best_height, best_width, v41)) {
return 0;
}
tokens = ds4_deepseek4_grid_tokens(
*best_height, *best_width, llm_height, llm_width);
*best_height, *best_width, llm_height, llm_width, v41);
budget--;
}
return 1;
}
int ds4_image_preprocess_deepseek4(
static int ds4_image_preprocess_deepseek(
ds4_deepseek4_image_patches *out,
const ds4_image *image,
char *error,
size_t error_cap) {
size_t error_cap,
bool v41) {
if (!out) return 0;
memset(out, 0, sizeof(*out));
if (!image || !image->rgb || image->width == 0 || image->height == 0) {
@@ -658,13 +666,14 @@ int ds4_image_preprocess_deepseek4(
uint32_t planned_width = image->width;
uint32_t planned_height = image->height;
if ((uint64_t)planned_width > (uint64_t)planned_height * 8u) {
if (!v41 && (uint64_t)planned_width > (uint64_t)planned_height * 8u) {
planned_width = planned_height * 8u;
}
const uint64_t planned_pixels =
(uint64_t)planned_width * planned_height;
if (planned_pixels < 147456u) {
const double scale = sqrt(147456.0 / planned_pixels);
const uint32_t min_pixels = v41 ? 295936u : 147456u;
if (planned_pixels < min_pixels) {
const double scale = sqrt((double)min_pixels / planned_pixels);
planned_width = (uint32_t)(planned_width * scale);
planned_height = (uint32_t)(planned_height * scale);
if (planned_width == 0) planned_width = 1;
@@ -676,9 +685,9 @@ int ds4_image_preprocess_deepseek4(
uint32_t llm_height = 0, llm_width = 0;
if (!ds4_deepseek4_safe_resize(planned_height, planned_width,
&best_height, &best_width,
&llm_height, &llm_width)) {
&llm_height, &llm_width, v41)) {
ds4_image_error(error, error_cap,
"unable to fit image within the 384-token vision budget");
"unable to fit image within the vision token budget");
return 0;
}
if (best_width > DS4_IMAGE_MAX_DIMENSION ||
@@ -701,7 +710,7 @@ int ds4_image_preprocess_deepseek4(
uint32_t content_height = best_height;
uint32_t offset_x = 0, offset_y = 0;
const bool force_resize =
(uint64_t)image->width >= (uint64_t)image->height * 8u;
!v41 && (uint64_t)image->width >= (uint64_t)image->height * 8u;
if (!force_resize) {
const double scale = fmin((double)best_width / image->width,
(double)best_height / image->height);
@@ -768,6 +777,50 @@ int ds4_image_preprocess_deepseek4(
return 1;
}
int ds4_image_preprocess_deepseek4(
ds4_deepseek4_image_patches *out, const ds4_image *image,
char *error, size_t error_cap) {
return ds4_image_preprocess_deepseek(out, image, error, error_cap, false);
}
int ds4_image_preprocess_deepseek41(
ds4_deepseek4_image_patches *out, const ds4_image *image,
char *error, size_t error_cap) {
return ds4_image_preprocess_deepseek(out, image, error, error_cap, true);
}
int ds4_deepseek41_image_layout_build(
ds4_deepseek4_image_layout *out, uint32_t height, uint32_t width,
char *error, size_t error_cap) {
if (!out) return 0;
memset(out, 0, sizeof(*out));
const uint64_t count = (uint64_t)height * ((uint64_t)width + 1u) + 2u;
if (!height || !width || count > 1024u) {
ds4_image_error(error, error_cap, "invalid V4.1 image token grid");
return 0;
}
uint8_t *types = malloc((size_t)count);
uint32_t *perm = malloc((size_t)height * width * sizeof(*perm));
if (!types || !perm) {
free(types); free(perm);
ds4_image_error(error, error_cap, "unable to allocate V4.1 image layout");
return 0;
}
uint32_t token = 0, image = 0;
types[token++] = DS4_DEEPSEEK4_IMAGE_START;
for (uint32_t row = 0; row < height; row++) {
for (uint32_t col = 0; col < width; col++) {
types[token++] = DS4_DEEPSEEK4_IMAGE;
perm[image] = image;
image++;
}
types[token++] = DS4_DEEPSEEK4_IMAGE_NEWLINE;
}
types[token++] = DS4_DEEPSEEK4_IMAGE_END;
*out = (ds4_deepseek4_image_layout){token, image, types, perm};
return 1;
}
int ds4_deepseek4_image_layout_build(
ds4_deepseek4_image_layout *out,
uint32_t grid_height,
ds4_image.h
+8
View File
@@ -90,6 +90,14 @@ int ds4_image_preprocess_deepseek4(
char *error,
size_t error_cap);
int ds4_image_preprocess_deepseek41(
ds4_deepseek4_image_patches *out, const ds4_image *image,
char *error, size_t error_cap);
int ds4_deepseek41_image_layout_build(
ds4_deepseek4_image_layout *out, uint32_t height, uint32_t width,
char *error, size_t error_cap);
int ds4_deepseek4_image_layout_build(
ds4_deepseek4_image_layout *out,
uint32_t grid_height,
ds4_metal.m
+901 -271
View File
File diff suppressed because it is too large. Load diff
ds4_server.c
+543 -49
View File
@@ -671,8 +671,16 @@ typedef enum {
typedef enum {
SERVER_MODEL_SYNTAX_DEEPSEEK,
SERVER_MODEL_SYNTAX_GLM,
SERVER_MODEL_SYNTAX_DEEPSEEK41,
} server_model_syntax;
#define DS41_TOOL_CALLS_START "<｜DSML｜ calls>"
#define DS41_TOOL_CALLS_END "</｜DSML｜ calls>"
#define DS41_INVOKE_START "<｜DSML｜ invoke"
#define DS41_INVOKE_END "</｜DSML｜ invoke>"
#define DS41_PARAM_START "<｜DSML｜ parameter"
#define DS41_PARAM_END "</｜DSML｜ parameter>"
static void random_tool_id(char *dst, size_t dstlen, api_style api) {
static uint64_t fallback_ctr;
unsigned char bytes[16];
@@ -740,6 +748,11 @@ typedef struct {
int cap;
} tool_schema_orders;
typedef struct {
size_t begin, end;
char *id, *content;
} tool_result_span;
typedef struct {
char *role;
char *content;
@@ -750,6 +763,8 @@ typedef struct {
int tool_call_ids_len;
int tool_call_ids_cap;
tool_calls calls;
tool_result_span *tool_results;
int tool_results_len;
} chat_msg;
typedef struct {
@@ -776,6 +791,7 @@ static void id_list_push_unique(stop_list *ids, const char *id);
static void id_list_free(stop_list *ids);
static bool responses_live_has_call_id(server *s, const char *id);
static bool anthropic_live_has_call_id(server *s, const char *id);
static stop_list live_tool_call_order(server *s, api_style api, const stop_list *ids);
typedef struct {
req_kind kind;
@@ -885,6 +901,11 @@ static void chat_msg_free(chat_msg *m) {
for (int i = 0; i < m->tool_call_ids_len; i++) free(m->tool_call_ids[i]);
free(m->tool_call_ids);
tool_calls_free(&m->calls);
for (int i = 0; i < m->tool_results_len; i++) {
free(m->tool_results[i].id);
free(m->tool_results[i].content);
}
free(m->tool_results);
memset(m, 0, sizeof(*m));
}
@@ -1135,10 +1156,12 @@ static bool model_alias_enables_thinking(const char *model) {
static server_model_syntax server_model_syntax_for_engine(ds4_engine *engine) {
return ds4_engine_is_glm_dsa(engine) ?
SERVER_MODEL_SYNTAX_GLM : SERVER_MODEL_SYNTAX_DEEPSEEK;
SERVER_MODEL_SYNTAX_GLM : ds4_engine_is_deepseek41(engine) ?
SERVER_MODEL_SYNTAX_DEEPSEEK41 : SERVER_MODEL_SYNTAX_DEEPSEEK;
}
static const char *server_model_id_from_engine(ds4_engine *engine) {
if (ds4_engine_is_deepseek41(engine)) return "deepseek-v4.1-flash";
if (ds4_engine_is_glm53(engine)) return "glm-5.3-flash";
if (ds4_engine_is_glm_dsa(engine)) return "glm-5.2";
return ds4_engine_model_id(engine) == 1 ?
@@ -1148,6 +1171,7 @@ static const char *server_model_id_from_engine(ds4_engine *engine) {
static bool server_model_alias_known(const char *id) {
return id &&
(!strcmp(id, "deepseek-v4-flash") ||
!strcmp(id, "deepseek-v4.1-flash") ||
!strcmp(id, "deepseek-v4-pro") ||
!strcmp(id, "glm-5.2") ||
!strcmp(id, "glm-5.2-chat") ||
@@ -2201,9 +2225,18 @@ static bool parse_anthropic_content_block(const char **p, bool allow_tools, chat
chat_msg_add_tool_call_id(msg, id);
buf b = {0};
buf_puts(&b, msg->content ? msg->content : "");
const size_t begin = b.len;
buf_puts(&b, "<tool_result>");
append_tool_result_text(&b, nested.content);
buf_puts(&b, "</tool_result>");
/* Preserve parsed boundaries, not just markup which ordinary user
* text can also contain. V4.1 orders and separates these blocks. */
msg->tool_results = xrealloc(msg->tool_results,
(size_t)(msg->tool_results_len + 1) * sizeof(msg->tool_results[0]));
msg->tool_results[msg->tool_results_len++] = (tool_result_span){
.begin = begin, .end = b.len, .id = id ? xstrdup(id) : NULL,
.content = nested.content};
nested.content = NULL;
free(msg->content);
msg->content = buf_take(&b);
if (nested.images.len) {
@@ -2325,8 +2358,12 @@ static bool parse_anthropic_messages(const char **p, chat_msgs *msgs) {
goto fail;
}
} else if (!strcmp(key, "content")) {
free(msg.content);
msg.content = NULL;
/* Duplicate content members would leave stale block offsets
* and tool IDs. Refuse an ambiguous protocol message. */
if (msg.content || msg.calls.len || msg.tool_results_len) {
free(key);
goto fail;
}
if (!parse_anthropic_content(p, &msg, true)) {
free(key);
goto fail;
@@ -2451,9 +2488,21 @@ bad:
return false;
}
static void append_tools_prompt_text(buf *b, const char *tool_schemas) {
static void append_tools_prompt_text(buf *b, const char *tool_schemas, bool v41) {
if (!tool_schemas || !tool_schemas[0]) return;
buf_puts(b,
if (v41) buf_puts(b,
"## Tools\n\n"
"You can invoke tools using this format:\n\n"
DS41_TOOL_CALLS_START "\n"
DS41_INVOKE_START " name=\"$TOOL_NAME\">\n"
DS41_PARAM_START " name=\"$PARAMETER_NAME\" string=\"true|false\">$PARAMETER_VALUE" DS41_PARAM_END "\n"
DS41_INVOKE_END "\n" DS41_TOOL_CALLS_END "\n\n"
"String values use string=\"true\"; all other values use JSON and string=\"false\". "
"Inside string values only, escape a literal </｜DSML｜ parameter> as &lt;/｜DSML｜ parameter>. "
"To write that escaped spelling literally, use &amp;lt;/｜DSML｜ parameter>. Other HTML entities are unchanged.\n\n"
"Finish reasoning with </think> before tool calls or a final response.\n\n"
"### Available Tool Schemas\n\n");
else buf_puts(b,
"## Tools\n\n"
"You have access to a set of tools to help answer the user question. "
"You can invoke tools by writing a \"<｜DSML｜tool_calls>\" block like the following:\n\n"
@@ -2646,8 +2695,8 @@ static void append_dsml_attr_escaped(buf *b, const char *s) {
}
}
static void append_dsml_parameter_text(buf *b, const char *s) {
const char *end = "</｜DSML｜parameter>";
static void append_dsml_parameter_text(buf *b, const char *s, bool v41) {
const char *end = v41 ? DS41_PARAM_END : "</｜DSML｜parameter>";
for (s = s ? s : ""; *s;) {
if (ds4_tool_text_needs_escape(s, end)) {
buf_puts(b, *s == '<' ? "&lt;" : "&amp;");
@@ -2736,8 +2785,8 @@ static void append_tool_result_text(buf *b, const char *s) {
}
}
static void append_dsml_json_literal(buf *b, const char *s) {
const char *end = "</｜DSML｜parameter>";
static void append_dsml_json_literal(buf *b, const char *s, bool v41) {
const char *end = v41 ? DS41_PARAM_END : "</｜DSML｜parameter>";
const size_t endlen = strlen(end);
for (s = s ? s : ""; *s;) {
if (!strncmp(s, end, endlen)) {
@@ -2749,15 +2798,15 @@ static void append_dsml_json_literal(buf *b, const char *s) {
}
}
static void append_dsml_arg(buf *b, const json_arg *arg) {
buf_puts(b, "<｜DSML｜parameter name=\"");
static void append_dsml_arg(buf *b, const json_arg *arg, bool v41) {
buf_puts(b, v41 ? DS41_PARAM_START " name=\"" : "<｜DSML｜parameter name=\"");
append_dsml_attr_escaped(b, arg->key);
buf_puts(b, "\" string=\"");
buf_puts(b, arg->is_string ? "true" : "false");
buf_puts(b, "\">");
if (arg->is_string) append_dsml_parameter_text(b, arg->value);
else append_dsml_json_literal(b, arg->value);
buf_puts(b, "</｜DSML｜parameter>\n");
if (arg->is_string) append_dsml_parameter_text(b, arg->value, v41);
else append_dsml_json_literal(b, arg->value, v41);
buf_puts(b, v41 ? DS41_PARAM_END "\n" : "</｜DSML｜parameter>\n");
}
static void append_glm_arg(buf *b, const json_arg *arg) {
@@ -2768,20 +2817,20 @@ static void append_glm_arg(buf *b, const json_arg *arg) {
buf_puts(b, "</arg_value>");
}
static bool append_dsml_arguments_from_json(buf *b, const char *json, const tool_schema_order *order) {
static bool append_dsml_arguments_from_json(buf *b, const char *json, const tool_schema_order *order, bool v41) {
json_args args = {0};
if (!json_args_parse(json, &args)) return false;
if (order) {
for (int i = 0; i < order->len; i++) {
int idx = json_args_find_unused(&args, order->prop[i]);
if (idx < 0) continue;
append_dsml_arg(b, &args.v[idx]);
append_dsml_arg(b, &args.v[idx], v41);
args.v[idx].used = true;
}
}
for (int i = 0; i < args.len; i++) {
if (args.v[i].used) continue;
append_dsml_arg(b, &args.v[i]);
append_dsml_arg(b, &args.v[i], v41);
}
json_args_free(&args);
return true;
@@ -2830,26 +2879,27 @@ static void append_json_object_or_empty(buf *b, const char *json) {
json_args_free(&args);
}
static void append_dsml_tool_calls_text(buf *b, const tool_calls *calls) {
static void append_dsml_tool_calls_text(buf *b, const tool_calls *calls, bool v41) {
if (!calls || calls->len == 0) return;
if (calls->raw_tool_text && calls->raw_tool_text[0]) {
buf_puts(b, calls->raw_tool_text);
return;
}
buf_puts(b, "\n\n<｜DSML｜tool_calls>\n");
buf_puts(b, v41 ? "\n\n" DS41_TOOL_CALLS_START "\n" : "\n\n<｜DSML｜tool_calls>\n");
for (int i = 0; i < calls->len; i++) {
const tool_call *tc = &calls->v[i];
buf_puts(b, "<｜DSML｜invoke name=\"");
buf_puts(b, v41 ? DS41_INVOKE_START " name=\"" : "<｜DSML｜invoke name=\"");
append_dsml_attr_escaped(b, tc->name);
buf_puts(b, "\">\n");
if (!append_dsml_arguments_from_json(b, tc->arguments, NULL)) {
buf_puts(b, "<｜DSML｜parameter name=\"arguments\" string=\"true\">");
append_dsml_parameter_text(b, tc->arguments);
buf_puts(b, "</｜DSML｜parameter>\n");
if (!append_dsml_arguments_from_json(b, tc->arguments, NULL, v41)) {
buf_puts(b, v41 ? DS41_PARAM_START " name=\"arguments\" string=\"true\">" :
"<｜DSML｜parameter name=\"arguments\" string=\"true\">");
append_dsml_parameter_text(b, tc->arguments, v41);
buf_puts(b, v41 ? DS41_PARAM_END "\n" : "</｜DSML｜parameter>\n");
}
buf_puts(b, "</｜DSML｜invoke>\n");
buf_puts(b, v41 ? DS41_INVOKE_END "\n" : "</｜DSML｜invoke>\n");
}
buf_puts(b, "</｜DSML｜tool_calls>");
buf_puts(b, v41 ? DS41_TOOL_CALLS_END : "</｜DSML｜tool_calls>");
}
static void append_glm_tool_calls_text(buf *b, const tool_calls *calls,
@@ -2883,7 +2933,7 @@ static void append_tool_calls_text_for_syntax(buf *b,
if (syntax == SERVER_MODEL_SYNTAX_GLM) {
append_glm_tool_calls_text(b, calls, tool_orders);
} else {
append_dsml_tool_calls_text(b, calls);
append_dsml_tool_calls_text(b, calls, syntax == SERVER_MODEL_SYNTAX_DEEPSEEK41);
}
}
@@ -2921,7 +2971,7 @@ static char *render_deepseek_chat_prompt_text(const chat_msgs *msgs, const char
* --kv-cache-boundary-trim-tokens chops a dynamic tail from the client
* message instead of the much larger tool-schema region. */
if (tool_schemas && tool_schemas[0]) {
append_tools_prompt_text(&system, tool_schemas);
append_tools_prompt_text(&system, tool_schemas, false);
}
for (int i = 0; i < msgs->len; i++) {
const chat_msg *m = &msgs->v[i];
@@ -2973,7 +3023,7 @@ static char *render_deepseek_chat_prompt_text(const chat_msgs *msgs, const char
}
}
buf_puts(&out, m->content ? m->content : "");
append_dsml_tool_calls_text(&out, &m->calls);
append_dsml_tool_calls_text(&out, &m->calls, false);
buf_puts(&out, "<｜end▁of▁sentence｜>");
pending_assistant = false;
pending_tool_result = false;
@@ -3088,6 +3138,181 @@ static char *render_glm_chat_prompt_text(const chat_msgs *msgs,
return buf_take(&out);
}
typedef struct {
const chat_msg *msg;
size_t order, position;
} ds41_tool_result_order;
/* Most views borrow the original message. Only split Anthropic blocks own
* strings, kept in fragments so the source history and image list stay intact. */
static chat_msgs ds41_message_views(const chat_msgs *msgs, chat_msgs *fragments,
int *start) {
chat_msgs views = {0};
const int first = *start;
for (int i = 0; msgs && i < msgs->len; i++) {
const chat_msg *m = &msgs->v[i];
if (i == first) *start = views.len;
if (strcmp(m->role, "user") || !m->tool_results_len) {
chat_msgs_push(&views, *m);
continue;
}
size_t begin = 0, length = strlen(m->content);
for (int j = 0; j <= m->tool_results_len; j++) {
const tool_result_span *span = j < m->tool_results_len ? &m->tool_results[j] : NULL;
const size_t end = span ? span->begin : length;
if (end > begin) {
chat_msg text = {.role = xstrdup("user"),
.content = xstrndup(m->content + begin, end - begin)};
chat_msgs_push(fragments, text);
chat_msgs_push(&views, text);
}
if (span) {
chat_msg result = {.role = xstrdup("tool"),
.content = xstrdup(span->content ? span->content : ""),
.tool_call_id = span->id ? xstrdup(span->id) : NULL};
chat_msgs_push(fragments, result);
chat_msgs_push(&views, result);
begin = span->end;
}
}
}
if (!msgs || first >= msgs->len) *start = views.len;
return views;
}
static int ds41_compare_tool_results(const void *a, const void *b) {
const ds41_tool_result_order *x = a, *y = b;
if (x->order != y->order) return x->order < y->order ? -1 : 1;
return (x->position > y->position) - (x->position < y->position);
}
/* Parallel tool replies can arrive out of order. V4.1 renders them in
* call order within each merged user turn, leaving ordinary text in place. */
static const chat_msg **ds41_order_messages(const chat_msgs *msgs, const stop_list *initial_order) {
if (!msgs || !msgs->len) return NULL;
const chat_msg **ordered = xmalloc((size_t)msgs->len * sizeof(*ordered));
ds41_tool_result_order *results = xmalloc((size_t)msgs->len * sizeof(*results));
rax *order = raxNew();
for (int i = 0; initial_order && i < initial_order->len; i++) {
const char *id = initial_order->v[i];
raxInsert(order, (unsigned char *)id, strlen(id), (void *)(uintptr_t)(i + 1), NULL);
}
for (int i = 0; i < msgs->len;) {
const chat_msg *m = &msgs->v[i];
ordered[i] = m;
if (!strcmp(m->role, "assistant") && m->calls.len) {
raxFree(order);
order = raxNew();
for (int j = 0; j < m->calls.len; j++) {
const char *id = m->calls.v[j].id;
if (id && id[0]) raxInsert(order, (unsigned char *)id, strlen(id),
(void *)(uintptr_t)(j + 1), NULL);
}
}
if (!role_is_user_like(m->role)) { i++; continue; }
const int begin = i;
size_t count = 0;
while (i < msgs->len && role_is_user_like(msgs->v[i].role)) {
m = ordered[i] = &msgs->v[i];
if (strcmp(m->role, "user")) {
const char *id = m->tool_call_id;
void *rank = id ? raxFind(order, (unsigned char *)id, strlen(id)) : raxNotFound;
results[count++] = (ds41_tool_result_order){.msg = m,
.order = rank == raxNotFound ? 0 : (uintptr_t)rank - 1,
.position = (size_t)i};
}
i++;
}
qsort(results, count, sizeof(*results), ds41_compare_tool_results);
size_t next = 0;
for (int j = begin; j < i; j++)
if (strcmp(msgs->v[j].role, "user")) ordered[j] = results[next++].msg;
}
raxFree(order);
free(results);
return ordered;
}
/* The same renderer owns full requests and live tool-result tails, so their
* separators and role transitions cannot disagree at the cache boundary. */
static char *render_deepseek41_chat_ordered(const chat_msgs *msgs, int start,
const char *tool_schemas,
ds4_think_mode mode, bool live_tail,
const stop_list *initial_order) {
buf out = {0};
chat_msgs fragments = {0};
chat_msgs views = ds41_message_views(msgs, &fragments, &start);
msgs = &views;
const chat_msg **ordered = ds41_order_messages(msgs, initial_order);
const bool think = ds4_think_mode_enabled(mode);
const bool keep_reasoning = live_tail || chat_history_uses_tool_context(msgs, tool_schemas);
const char *effort = ds4_deepseek41_reasoning_effort_text(mode);
bool initial_system = false, pending = false, user_open = false;
int last_user = -1;
for (int i = 0; msgs && i < msgs->len; i++) {
const chat_msg *m = &msgs->v[i];
if (role_is_user_like(m->role) || (i > 0 && role_is_system(m->role))) last_user = i;
}
if (live_tail) {
buf_puts(&out, "<｜end▁of▁sentence｜>");
} else {
buf_puts(&out, "<｜begin▁of▁sentence｜>");
if (effort || (tool_schemas && tool_schemas[0])) {
buf_puts(&out, "<｜System｜>");
initial_system = true;
if (effort) buf_puts(&out, effort);
if (tool_schemas && tool_schemas[0])
append_tools_prompt_text(&out, tool_schemas, true);
}
}
for (int i = start; msgs && i < msgs->len; i++) {
const chat_msg *m = ordered[i];
if (role_is_system(m->role)) {
if (i == 0 && initial_system) {
if (tool_schemas && tool_schemas[0]) buf_puts(&out, "\n\n");
} else buf_puts(&out, "<｜System｜>");
buf_puts(&out, m->content ? m->content : "");
pending = i > 0 || live_tail;
user_open = false;
} else if (role_is_user_like(m->role)) {
buf_puts(&out, user_open ? "\n\n" : "<｜User｜>");
if (strcmp(m->role, "user")) {
buf_puts(&out, "<tool_result>");
append_tool_result_text(&out, m->content);
buf_puts(&out, "</tool_result>");
} else buf_puts(&out, m->content ? m->content : "");
pending = user_open = true;
} else if (!strcmp(m->role, "assistant")) {
if (pending) {
buf_puts(&out, "<｜Assistant｜>");
if (think && (keep_reasoning || i > last_user)) {
buf_puts(&out, "<think>");
buf_puts(&out, m->reasoning ? m->reasoning : "");
}
buf_puts(&out, "</think>");
}
buf_puts(&out, m->content ? m->content : "");
append_dsml_tool_calls_text(&out, &m->calls, true);
buf_puts(&out, "<｜end▁of▁sentence｜>");
pending = user_open = false;
}
}
if (pending) {
buf_puts(&out, "<｜Assistant｜>");
buf_puts(&out, think ? "<think>" : "</think>");
}
free(ordered);
free(views.v);
chat_msgs_free(&fragments);
return buf_take(&out);
}
static char *render_deepseek41_chat(const chat_msgs *msgs, int start,
const char *tool_schemas,
ds4_think_mode mode, bool live_tail) {
return render_deepseek41_chat_ordered(msgs, start, tool_schemas, mode, live_tail, NULL);
}
static char *render_chat_prompt_text_for_syntax(server_model_syntax syntax,
const chat_msgs *msgs,
const char *tool_schemas,
@@ -3097,6 +3322,8 @@ static char *render_chat_prompt_text_for_syntax(server_model_syntax syntax,
return render_glm_chat_prompt_text(msgs, tool_schemas,
tool_orders, think_mode);
}
if (syntax == SERVER_MODEL_SYNTAX_DEEPSEEK41)
return render_deepseek41_chat(msgs, 0, tool_schemas, think_mode, false);
return render_deepseek_chat_prompt_text(msgs, tool_schemas,
tool_orders, think_mode);
}
@@ -3245,7 +3472,7 @@ static char *render_deepseek_live_tool_tail(const chat_msgs *msgs, int start,
}
}
buf_puts(&out, m->content ? m->content : "");
append_dsml_tool_calls_text(&out, &m->calls);
append_dsml_tool_calls_text(&out, &m->calls, false);
buf_puts(&out, "<｜end▁of▁sentence｜>");
pending_assistant = false;
pending_tool_result = false;
@@ -3306,6 +3533,8 @@ static char *render_live_tool_tail_for_syntax(server_model_syntax syntax,
if (syntax == SERVER_MODEL_SYNTAX_GLM) {
return render_glm_live_tool_tail(msgs, start, tool_orders, think_mode);
}
if (syntax == SERVER_MODEL_SYNTAX_DEEPSEEK41)
return render_deepseek41_chat(msgs, start, NULL, think_mode, true);
return render_deepseek_live_tool_tail(msgs, start, think_mode);
}
@@ -3316,6 +3545,20 @@ static DS4_SERVER_MAYBE_UNUSED char *render_live_tool_tail(
msgs, start, NULL, think_mode);
}
static char *render_request_live_tool_tail(server *s, const request *r,
const chat_msgs *msgs, int start,
const stop_list *ids) {
if (r->model_syntax != SERVER_MODEL_SYNTAX_DEEPSEEK41)
return render_live_tool_tail_for_syntax(r->model_syntax, msgs, start,
&r->tool_orders, r->think_mode);
/* Output-only requests omit the assistant message which declares order.
* Copy the remembered order under its mutex, never borrow mutable IDs. */
stop_list order = live_tool_call_order(s, r->api, ids);
char *text = render_deepseek41_chat_ordered(msgs, start, NULL, r->think_mode, true, &order);
id_list_free(&order);
return text;
}
static void chat_msg_collect_tool_call_ids(const chat_msg *m, stop_list *ids) {
if (!m || !ids) return;
id_list_push_unique(ids, m->tool_call_id);
@@ -3406,7 +3649,7 @@ static bool responses_validate_tool_outputs(server *s, const chat_msgs *msgs,
* server state is still exactly at the remembered token frontier before using
* it. If another request already replaced the session, normal token/text/disk
* prefix matching handles the request instead. */
static void responses_prepare_live_continuation(request *r,
static void responses_prepare_live_continuation(server *s, request *r,
const chat_msgs *msgs) {
if (!r || r->api != API_RESPONSES || !msgs || msgs->len == 0) return;
@@ -3435,8 +3678,7 @@ static void responses_prepare_live_continuation(request *r,
free(r->responses_live_suffix_text);
r->responses_live_suffix_text =
render_live_tool_tail_for_syntax(r->model_syntax, msgs, tail_start,
&r->tool_orders, r->think_mode);
render_request_live_tool_tail(s, r, msgs, tail_start, &r->responses_live_call_ids);
}
static bool anthropic_msg_is_tool_result_tail(const chat_msg *m) {
@@ -3508,7 +3750,7 @@ static bool anthropic_validate_tool_results(server *s, const chat_msgs *msgs,
* model sampled. If the incoming tool_result IDs match the live sampled
* frontier, generate_job() can skip replay matching entirely and append just
* EOS + tool_result + next assistant prefix to the real KV. */
static void anthropic_prepare_live_continuation(request *r,
static void anthropic_prepare_live_continuation(server *s, request *r,
const chat_msgs *msgs) {
if (!r || r->api != API_ANTHROPIC || !msgs || msgs->len == 0) return;
@@ -3530,8 +3772,7 @@ static void anthropic_prepare_live_continuation(request *r,
free(r->anthropic_live_suffix_text);
r->anthropic_live_suffix_text =
render_live_tool_tail_for_syntax(r->model_syntax, msgs, tail_start,
&r->tool_orders, r->think_mode);
render_request_live_tool_tail(s, r, msgs, tail_start, &r->anthropic_live_call_ids);
}
/* The API parsers are intentionally selective JSON parsers: they keep only
@@ -3909,6 +4150,12 @@ static bool parse_anthropic_request(ds4_engine *e, server *s, const char *body,
msg.content = system;
system = NULL;
chat_msgs_push(&msgs, msg);
if (r->model_syntax == SERVER_MODEL_SYNTAX_DEEPSEEK41) {
/* This is the API's initial system prompt, not a later system
* turn, which V4.1 preserves at its original history position. */
memmove(msgs.v + 1, msgs.v, (size_t)(msgs.len - 1) * sizeof(msgs.v[0]));
msgs.v[0] = msg;
}
}
r->has_tools = tool_schemas && tool_schemas[0] && !tool_choice_none;
if (!got_thinking && model_alias_disables_thinking(r->model)) thinking_enabled = false;
@@ -3927,7 +4174,7 @@ static bool parse_anthropic_request(ds4_engine *e, server *s, const char *body,
}
kv_cache_restore_tool_memory_for_messages(s, &msgs);
tool_memory_attach_to_messages(s, &msgs, &r->tool_replay);
anthropic_prepare_live_continuation(r, &msgs);
anthropic_prepare_live_continuation(s, r, &msgs);
const char *active_tool_schemas = r->has_tools ? tool_schemas : NULL;
r->prompt_preserves_reasoning =
chat_history_uses_tool_context(&msgs, active_tool_schemas);
@@ -4960,7 +5207,7 @@ static bool parse_responses_request(ds4_engine *e, server *s, const char *body,
tool_memory_attach_to_messages(s, &msgs, &r->tool_replay);
r->prompt_preserves_reasoning =
chat_history_uses_tool_context(&msgs, active_tool_schemas);
responses_prepare_live_continuation(r, &msgs);
responses_prepare_live_continuation(s, r, &msgs);
r->prompt_text = render_chat_prompt_text_for_syntax(
r->model_syntax, &msgs, active_tool_schemas,
&r->tool_orders, r->think_mode);
@@ -5272,6 +5519,7 @@ static void json_escape_fragment_n(buf *b, const char *s, size_t n) {
static const char *find_any_tool_start(const char *s) {
const char *best = NULL;
const char *candidates[] = {
strstr(s, DS41_TOOL_CALLS_START),
strstr(s, DS4_TOOL_CALLS_START),
strstr(s, DS4_TOOL_CALLS_START_SHORT),
strstr(s, "<tool_calls>"),
@@ -5286,6 +5534,7 @@ static const char *find_any_tool_start(const char *s) {
static const char *find_any_tool_end(const char *s) {
const char *best = NULL;
const char *candidates[] = {
strstr(s, DS41_TOOL_CALLS_END),
strstr(s, DS4_TOOL_CALLS_END),
strstr(s, DS4_TOOL_CALLS_END_SHORT),
strstr(s, "</tool_calls>"),
@@ -5323,6 +5572,7 @@ static const char *find_last_substr(const char *s, const char *needle) {
static const char *find_tool_structural_text(const char *s, const char *needle,
bool last) {
static const char *wrappers[][2] = {
{DS41_PARAM_START, DS41_PARAM_END},
{DS4_PARAM_START, DS4_PARAM_END},
{DS4_PARAM_START_SHORT, DS4_PARAM_END_SHORT},
{"<parameter", "</parameter>"},
@@ -5593,6 +5843,12 @@ static bool parse_deepseek_generated_message_ex(const char *text,
start = strstr(tool_search, "<tool_calls>");
style = start ? 1 : style;
}
const char *v41_start = strstr(tool_search, "\n\n" DS41_TOOL_CALLS_START);
if (!v41_start) v41_start = strstr(tool_search, DS41_TOOL_CALLS_START);
if (v41_start && (!start || v41_start < start)) {
start = v41_start;
style = 3;
}
if (!start) {
split_reasoning_content(text, strlen(text), content_out, reasoning_out);
return true;
@@ -5620,6 +5876,13 @@ static bool parse_deepseek_generated_message_ex(const char *text,
invoke_end = DS4_INVOKE_END_SHORT;
param_start = DS4_PARAM_START_SHORT;
param_end = DS4_PARAM_END_SHORT;
} else if (style == 3) {
tool_calls_start = DS41_TOOL_CALLS_START;
tool_calls_end = DS41_TOOL_CALLS_END;
invoke_start = DS41_INVOKE_START;
invoke_end = DS41_INVOKE_END;
param_start = DS41_PARAM_START;
param_end = DS41_PARAM_END;
}
const char *p = strstr(start, tool_calls_start);
@@ -5936,15 +6199,21 @@ static bool try_repair_dsml(const char *s, size_t len, buf *out) {
/* Detect style from first <tool_calls> tag */
const char *ts, *te, *is, *ie, *ps, *pe;
if (strstr(scan_start, DS4_TOOL_CALLS_START)) {
const char *first = find_any_tool_start(scan_start);
if (!first) return false;
if (!strncmp(first, DS41_TOOL_CALLS_START, strlen(DS41_TOOL_CALLS_START))) {
ts = DS41_TOOL_CALLS_START; te = DS41_TOOL_CALLS_END;
is = DS41_INVOKE_START; ie = DS41_INVOKE_END;
ps = DS41_PARAM_START; pe = DS41_PARAM_END;
} else if (!strncmp(first, DS4_TOOL_CALLS_START, strlen(DS4_TOOL_CALLS_START))) {
ts = DS4_TOOL_CALLS_START; te = DS4_TOOL_CALLS_END;
is = DS4_INVOKE_START; ie = DS4_INVOKE_END;
ps = DS4_PARAM_START; pe = DS4_PARAM_END;
} else if (strstr(scan_start, DS4_TOOL_CALLS_START_SHORT)) {
} else if (!strncmp(first, DS4_TOOL_CALLS_START_SHORT, strlen(DS4_TOOL_CALLS_START_SHORT))) {
ts = DS4_TOOL_CALLS_START_SHORT; te = DS4_TOOL_CALLS_END_SHORT;
is = DS4_INVOKE_START_SHORT; ie = DS4_INVOKE_END_SHORT;
ps = DS4_PARAM_START_SHORT; pe = DS4_PARAM_END_SHORT;
} else if (strstr(scan_start, "<tool_calls>")) {
} else if (!strncmp(first, "<tool_calls>", strlen("<tool_calls>"))) {
ts = "<tool_calls>"; te = "</tool_calls>";
is = "<invoke"; ie = "</invoke>";
ps = "<parameter"; pe = "</parameter>";
@@ -6557,6 +6826,11 @@ typedef struct {
} dsml_syntax;
static const dsml_syntax dsml_syntaxes[] = {
{
DS41_TOOL_CALLS_START, DS41_TOOL_CALLS_END,
DS41_INVOKE_START, DS41_INVOKE_END,
DS41_PARAM_START, DS41_PARAM_END,
},
{
DS4_TOOL_CALLS_START, DS4_TOOL_CALLS_END,
DS4_INVOKE_START, DS4_INVOKE_END,
@@ -7009,7 +7283,14 @@ static bool openai_tool_stream_init(openai_tool_stream *ts, const char *raw,
ts->active = true;
ts->state = DSML_TOOL_BETWEEN_INVOKES;
ts->parse_pos = pos;
if (raw_full_lit(raw, raw_len, pos, DS4_TOOL_CALLS_START)) {
if (raw_full_lit(raw, raw_len, pos, DS41_TOOL_CALLS_START)) {
ts->parse_pos += strlen(DS41_TOOL_CALLS_START);
ts->tool_calls_end = DS41_TOOL_CALLS_END;
ts->invoke_start = DS41_INVOKE_START;
ts->invoke_end = DS41_INVOKE_END;
ts->param_start = DS41_PARAM_START;
ts->param_end = DS41_PARAM_END;
} else if (raw_full_lit(raw, raw_len, pos, DS4_TOOL_CALLS_START)) {
ts->parse_pos += strlen(DS4_TOOL_CALLS_START);
ts->tool_calls_end = DS4_TOOL_CALLS_END;
ts->invoke_start = DS4_INVOKE_START;
@@ -9358,7 +9639,7 @@ static bool server_encode_image(server *s, const server_image_input *input,
input->encoded_len, out, err, errlen))
return false;
server_image_cache_put(&s->image_cache, input, out,
4096, /* Both supported vision encoders emit 4096-wide rows. */
(uint32_t)ds4_engine_embd_dim(s->engine),
SERVER_IMAGE_CACHE_BYTES);
server_log(DS4_LOG_KVCACHE, "ds4-server: vision embedding encoded; cache=%zu bytes",
s->image_cache.bytes);
@@ -9734,6 +10015,26 @@ static bool anthropic_live_has_call_id(server *s, const char *id) {
return found;
}
static stop_list live_tool_call_order(server *s, api_style api, const stop_list *ids) {
stop_list order = {0};
if (!s || !ids || !ids->len || (api != API_RESPONSES && api != API_ANTHROPIC)) return order;
pthread_mutex_lock(&s->tool_mu);
for (int i = 0; i < s->slot_count; i++) {
const live_tool_state *live = api == API_RESPONSES ?
&s->slots[i].responses_live : &s->slots[i].anthropic_live;
if (!live->valid || live->call_ids.len != ids->len) continue;
bool match = true;
for (int j = 0; j < ids->len && match; j++)
match = id_list_contains(&live->call_ids, ids->v[j]);
if (!match) continue;
for (int j = 0; j < live->call_ids.len; j++)
id_list_push_unique(&order, live->call_ids.v[j]);
break;
}
pthread_mutex_unlock(&s->tool_mu);
return order;
}
static bool responses_live_matches_request(server *s, server_slot *slot,
const stop_list *ids,
int live_tokens) {
@@ -10042,6 +10343,8 @@ static const char *find_next_dsml_tool_block(const char *p, const char **end_out
const char *start;
const char *end;
} forms[] = {
{"\n\n" DS41_TOOL_CALLS_START, DS41_TOOL_CALLS_END},
{DS41_TOOL_CALLS_START, DS41_TOOL_CALLS_END},
{"\n\n" DS4_TOOL_CALLS_START, DS4_TOOL_CALLS_END},
{DS4_TOOL_CALLS_START, DS4_TOOL_CALLS_END},
{"\n\n" DS4_TOOL_CALLS_START_SHORT, DS4_TOOL_CALLS_END_SHORT},
@@ -13914,7 +14217,9 @@ static bool send_model(server *s, int fd, const char *id) {
static bool send_models(server *s, int fd) {
buf b = {0};
buf_puts(&b, "{\"object\":\"list\",\"data\":[");
if (ds4_engine_is_glm_dsa(s->engine)) {
if (ds4_engine_is_deepseek41(s->engine)) {
append_model_json(&b, s, server_model_id_from_engine(s->engine));
} else if (ds4_engine_is_glm_dsa(s->engine)) {
append_model_json(&b, s, "glm-5.2");
buf_putc(&b, ',');
append_model_json(&b, s, "glm-5.2-chat");
@@ -16641,7 +16946,7 @@ static void test_render_glm_groups_tool_results(void) {
static void test_dsml_tool_args_preserve_call_order(void) {
tool_calls calls = make_swapped_bash_call();
buf b = {0};
append_dsml_tool_calls_text(&b, &calls);
append_dsml_tool_calls_text(&b, &calls, false);
const char *command = strstr(b.ptr, "name=\"command\"");
const char *description = strstr(b.ptr, "name=\"description\"");
const char *timeout = strstr(b.ptr, "name=\"timeout\"");
@@ -17534,7 +17839,7 @@ static void test_anthropic_live_tail_renders_tool_results_only(void) {
system.content = xstrdup("You are terse.");
chat_msgs_push(&msgs, system);
anthropic_prepare_live_continuation(&r, &msgs);
anthropic_prepare_live_continuation(NULL, &r, &msgs);
TEST_ASSERT(r.anthropic_live_call_ids.len == 1);
TEST_ASSERT(!strcmp(r.anthropic_live_call_ids.v[0], "toolu_live"));
TEST_ASSERT(r.anthropic_live_suffix_text != NULL);
@@ -17714,7 +18019,7 @@ static void test_responses_live_tail_renders_tool_outputs_only(void) {
tool.content = xstrdup("/tmp");
chat_msgs_push(&msgs, tool);
responses_prepare_live_continuation(&r, &msgs);
responses_prepare_live_continuation(NULL, &r, &msgs);
TEST_ASSERT(r.responses_live_call_ids.len == 1);
TEST_ASSERT(!strcmp(r.responses_live_call_ids.v[0], "call_live"));
TEST_ASSERT(r.responses_live_suffix_text != NULL);
@@ -18204,7 +18509,7 @@ static void test_dsml_prompt_escapes_tool_supplied_text(void) {
tool_calls_push(&calls, tc);
buf b = {0};
append_dsml_tool_calls_text(&b, &calls);
append_dsml_tool_calls_text(&b, &calls, false);
TEST_ASSERT(strstr(b.ptr, "echo 2>&1 && echo </｜DSML｜tool_calls>") != NULL);
TEST_ASSERT(strstr(b.ptr, "2&gt;&amp;1") == NULL);
TEST_ASSERT(strstr(b.ptr, "&amp;&amp;") == NULL);
@@ -18217,7 +18522,7 @@ static void test_dsml_prompt_escapes_tool_supplied_text(void) {
tc.arguments = xstrdup("{\"command\":\"echo </｜DSML｜parameter>\",\"count\":1}");
tool_calls_push(&calls, tc);
append_dsml_tool_calls_text(&b, &calls);
append_dsml_tool_calls_text(&b, &calls, false);
TEST_ASSERT(strstr(b.ptr, "echo &lt;/｜DSML｜parameter>") != NULL);
TEST_ASSERT(strstr(b.ptr, "echo </｜DSML｜parameter>") == NULL);
buf_free(&b);
@@ -20164,9 +20469,198 @@ static void test_server_image_embedding_cache(void) {
src.token_count = 1;
server_image_cache_put(&cache, &input, &src, 2, sizeof(data));
TEST_ASSERT(cache.bytes == 0);
/* V4.1 image rows are wider than the older 4096-dimensional encoders. */
float wide[2 * 5120];
for (size_t i = 0; i < sizeof(wide) / sizeof(*wide); i++) wide[i] = (float)i;
src.data = wide;
src.token_count = 2;
server_image_cache_put(&cache, &input, &src, 5120, SERVER_IMAGE_CACHE_BYTES);
TEST_ASSERT(cache.bytes == sizeof(wide) + input.encoded_len);
const bool found = server_image_cache_get(&cache, &input, &out);
TEST_ASSERT(found);
if (found) {
TEST_ASSERT(out.token_count == 2 && !memcmp(out.data, wide, sizeof(wide)));
ds4_vision_embedding_free(&out);
}
server_image_cache_clear(&cache);
}
static void test_deepseek41_server_stream(void) {
const char *raw = "Plan.</think>\n\n" DS41_TOOL_CALLS_START "\n"
DS41_INVOKE_START " name=\"write\">\n"
DS41_PARAM_START " name=\"content\" string=\"true\">"
"literal </think> &lt;/｜DSML｜ parameter> caf\xc3\xa8" DS41_PARAM_END "\n"
DS41_INVOKE_END "\n" DS41_TOOL_CALLS_END;
for (size_t split = 0; split <= strlen(raw); split++) {
int sv[2] = {-1, -1};
TEST_ASSERT(socketpair(AF_UNIX, SOCK_STREAM, 0, sv) == 0);
if (sv[0] < 0 || sv[1] < 0) return;
request r;
request_init(&r, REQ_CHAT, 128);
r.api = API_OPENAI;
r.model_syntax = SERVER_MODEL_SYNTAX_DEEPSEEK41;
r.stream = r.has_tools = true;
r.think_mode = DS4_THINK_HIGH;
openai_stream st;
openai_stream_start(&r, &st);
char *partial = xstrndup(raw, split);
TEST_ASSERT(openai_sse_stream_update(sv[0], NULL, &r, "chatcmpl_v41", &st,
partial, split, false));
free(partial);
TEST_ASSERT(openai_sse_stream_update(sv[0], NULL, &r, "chatcmpl_v41", &st,
raw, strlen(raw), false));
shutdown(sv[0], SHUT_WR);
char *out = read_socket_text(sv[1]);
buf arguments = {0};
const char *p = out;
while ((p = strstr(p, "\"arguments\":")) != NULL) {
p += strlen("\"arguments\":");
char *delta = NULL;
TEST_ASSERT(json_string(&p, &delta));
if (!delta) break;
buf_puts(&arguments, delta);
free(delta);
}
json_args args = {0};
TEST_ASSERT(arguments.ptr && json_args_parse(arguments.ptr, &args));
int index = json_args_find_unused(&args, "content");
TEST_ASSERT(index >= 0 && !strcmp(args.v[index].value,
"literal </think> </｜DSML｜ parameter> caf\xc3\xa8"));
TEST_ASSERT(strstr(out, "\"name\":\"write\""));
TEST_ASSERT(!strstr(out, DS41_TOOL_CALLS_START));
bool complete = index >= 0 && !strcmp(args.v[index].value,
"literal </think> </｜DSML｜ parameter> caf\xc3\xa8");
if (!complete) fprintf(stderr, "V4.1 SSE split=%zu: %s\n", split, out);
json_args_free(&args); buf_free(&arguments); free(out);
openai_stream_free(&st); request_free(&r);
close(sv[0]); close(sv[1]);
if (!complete) break;
}
}
static void test_deepseek41_server_tools(void) {
tool_calls original = {0}, parsed = {0};
tool_call call = {.name = xstrdup("write"),
.arguments = xstrdup("{\"path\":\"a.c\",\"content\":\"literal </think> </｜DSML｜ parameter> &lt;/｜DSML｜ parameter>\"}")};
tool_calls_push(&original, call);
buf raw = {0};
buf_puts(&raw, "Plan.</think>");
append_dsml_tool_calls_text(&raw, &original, true);
char *content = NULL, *reasoning = NULL;
TEST_ASSERT(parse_generated_message_ex_for_syntax(SERVER_MODEL_SYNTAX_DEEPSEEK41,
raw.ptr, true, &content, &reasoning, &parsed));
TEST_ASSERT(parsed.len == 1 && content && !content[0]);
TEST_ASSERT(reasoning && !strcmp(reasoning, "Plan."));
if (parsed.len == 1) {
json_args args = {0};
TEST_ASSERT(json_args_parse(parsed.v[0].arguments, &args));
int index = json_args_find_unused(&args, "content");
TEST_ASSERT(index >= 0 && !strcmp(args.v[index].value,
"literal </think> </｜DSML｜ parameter> &lt;/｜DSML｜ parameter>"));
json_args_free(&args);
}
free(content); free(reasoning);
tool_calls_free(&parsed); tool_calls_free(&original);
buf_free(&raw);
const char *json = "[{\"role\":\"user\",\"content\":\"Hi\"},"
"{\"role\":\"assistant\",\"content\":\"Ready\"},"
"{\"role\":\"tool\",\"content\":\"first\"},"
"{\"role\":\"tool\",\"content\":\"second\"},"
"{\"role\":\"system\",\"content\":\"Continue\"}]";
chat_msgs msgs = {0};
TEST_ASSERT(parse_messages(&json, &msgs));
char *full = render_deepseek41_chat(&msgs, 0, NULL, DS4_THINK_HIGH, false);
char *tail = render_deepseek41_chat(&msgs, 2, NULL, DS4_THINK_HIGH, true);
const char *boundary = strstr(full, "<｜end▁of▁sentence｜>");
TEST_ASSERT(boundary && !strcmp(boundary, tail));
TEST_ASSERT(strstr(tail, "</tool_result>\n\n<tool_result>"));
TEST_ASSERT(strstr(tail, "<｜System｜>Continue<｜Assistant｜><think>"));
free(full); free(tail); chat_msgs_free(&msgs);
}
static void test_deepseek41_anthropic_results(void) {
const char *json = "[{\"role\":\"user\",\"content\":\"Inspect\"},"
"{\"role\":\"assistant\",\"content\":["
"{\"type\":\"tool_use\",\"id\":\"a\",\"name\":\"read\",\"input\":{}},"
"{\"type\":\"tool_use\",\"id\":\"b\",\"name\":\"read\",\"input\":{}}]},"
"{\"content\":["
"{\"type\":\"tool_result\",\"tool_use_id\":\"b\",\"content\":\"SECOND </tool_result> &lt;\"},"
"{\"type\":\"text\",\"text\":\"literal <tool_result>text</tool_result>\"},"
"{\"type\":\"tool_result\",\"tool_use_id\":\"a\",\"content\":\"\"}],\"role\":\"user\"},"
"{\"role\":\"assistant\",\"content\":\"Done\"},"
"{\"role\":\"user\",\"content\":\"Continue\"}]";
chat_msgs msgs = {0};
TEST_ASSERT(parse_anthropic_messages(&json, &msgs));
TEST_ASSERT(msgs.len == 5 && msgs.v[2].tool_results_len == 2);
if (msgs.len != 5) { chat_msgs_free(&msgs); return; }
char *source = xstrdup(msgs.v[2].content);
char *old = render_deepseek_chat_prompt_text(&msgs, NULL, NULL, DS4_THINK_HIGH);
char *full = render_deepseek41_chat(&msgs, 0, NULL, DS4_THINK_HIGH, false);
char *tail = render_deepseek41_chat(&msgs, 2, NULL, DS4_THINK_HIGH, true);
const char *boundary = strstr(full, "<｜end▁of▁sentence｜>");
TEST_ASSERT(boundary && !strcmp(boundary, tail));
TEST_ASSERT(strstr(full, "<tool_result></tool_result>\n\n"
"literal <tool_result>text</tool_result>\n\n"
"<tool_result>SECOND &lt;/tool_result> &lt;</tool_result>"));
free(tail);
tail = render_deepseek41_chat(&msgs, 4, NULL, DS4_THINK_HIGH, true);
TEST_ASSERT(!strcmp(tail, "<｜end▁of▁sentence｜><｜User｜>Continue<｜Assistant｜><think>"));
TEST_ASSERT(!strcmp(source, msgs.v[2].content));
char *unchanged = render_deepseek_chat_prompt_text(&msgs, NULL, NULL, DS4_THINK_HIGH);
TEST_ASSERT(!strcmp(old, unchanged));
free(source); free(old); free(full); free(tail); free(unchanged);
chat_msgs_free(&msgs);
json = "[{\"role\":\"user\",\"content\":[{\"type\":\"tool_result\","
"\"tool_use_id\":\"x\",\"content\":\"old\"}],\"content\":\"new\"}]";
TEST_ASSERT(!parse_anthropic_messages(&json, &msgs));
chat_msgs_free(&msgs);
}
static void test_deepseek41_live_result_order(void) {
server s = {0};
server_slot slot;
test_server_bind_slot(&s, &slot);
pthread_mutex_init(&s.tool_mu, NULL);
for (int anthropic = 0; anthropic < 2; anthropic++) {
request r;
request_init(&r, REQ_CHAT, 128);
r.model_syntax = SERVER_MODEL_SYNTAX_DEEPSEEK41;
r.api = anthropic ? API_ANTHROPIC : API_RESPONSES;
r.think_mode = DS4_THINK_HIGH;
live_tool_state *live = anthropic ? &slot.anthropic_live : &slot.responses_live;
live->valid = true;
id_list_push_unique(&live->call_ids, "a");
id_list_push_unique(&live->call_ids, "b");
const char *json = anthropic ?
"[{\"role\":\"user\",\"content\":["
"{\"type\":\"tool_result\",\"tool_use_id\":\"b\",\"content\":\"SECOND\"},"
"{\"type\":\"tool_result\",\"tool_use_id\":\"a\",\"content\":\"FIRST\"}]}]" :
"[{\"role\":\"tool\",\"tool_call_id\":\"b\",\"content\":\"SECOND\"},"
"{\"role\":\"tool\",\"tool_call_id\":\"a\",\"content\":\"FIRST\"}]";
chat_msgs msgs = {0};
TEST_ASSERT(anthropic ? parse_anthropic_messages(&json, &msgs) : parse_messages(&json, &msgs));
if (anthropic) anthropic_prepare_live_continuation(&s, &r, &msgs);
else responses_prepare_live_continuation(&s, &r, &msgs);
const char *tail = anthropic ? r.anthropic_live_suffix_text : r.responses_live_suffix_text;
TEST_ASSERT(tail && !strcmp(tail,
"<｜end▁of▁sentence｜><｜User｜><tool_result>FIRST</tool_result>\n\n"
"<tool_result>SECOND</tool_result><｜Assistant｜><think>"));
live_tool_state_clear_locked(live);
/* Rendered requests own their bytes even if the slot is replaced. */
TEST_ASSERT(tail && strstr(tail, "FIRST</tool_result>\n\n<tool_result>SECOND"));
chat_msgs_free(&msgs);
request_free(&r);
}
pthread_mutex_destroy(&s.tool_mu);
}
static void ds4_server_unit_tests_run(void) {
test_deepseek41_server_stream();
test_deepseek41_server_tools();
test_deepseek41_anthropic_results();
test_deepseek41_live_result_order();
test_visible_image_key();
test_anthropic_tool_image_output();
test_responses_tool_image_output();
ds4_tp.c
+123 -41
View File
@@ -41,7 +41,7 @@
#define DS4_TP_MAGIC UINT32_C(0x44533454) /* "DS4T" */
#define DS4_TP_BATCH_MAGIC UINT32_C(0x44533442) /* "DS4B" */
#define DS4_TP_PROTOCOL_VERSION 10u
#define DS4_TP_PROTOCOL_VERSION 12u
#define DS4_TP_DEFAULT_TIMEOUT_SEC 300
/* Once both ranks enter a Metal gate, a live exchange normally completes in
@@ -131,7 +131,8 @@ typedef struct {
* (a model-fixed number of gates per token). After any initial bulk prefill,
* decode keeps a receive window posted by sequence number: recv for seq s
* lands in the slab in-slot (s-1) % slots and its completion is the arrival
* signal. */
* signal. The provider also reports CQEs for unsignaled sends: request and
* account for every send, rather than treating one CQE as a completed chain. */
#define DS4_TP_RDMA_MAX_MSG 16384
#define DS4_TP_RDMA_RECV_WINDOW 16
#define DS4_TP_RDMA_BULK_SLOTS 64
@@ -150,7 +151,7 @@ typedef struct {
int gid_index;
uint32_t max_inline;
ds4_tp_rdma_info peer;
uint32_t send_outstanding; /* signaled sends not yet reaped */
uint32_t send_outstanding; /* individual sends not yet reaped */
uint64_t recv_done; /* highest gate seq whose recv completed */
uint64_t last_gate_seq; /* last real decode receive consumed */
bool recv_window_active; /* decode recvs are queued ahead */
@@ -203,6 +204,7 @@ struct ds4_tp {
uint64_t timeout_sec;
uint64_t gate_timeout_ms;
atomic_bool failed;
uint64_t sync_checkpoint_seq;
#ifdef DS4_TP_HAVE_VERBS
ds4_tp_rdma rdma;
#endif
@@ -1128,6 +1130,22 @@ static int tp_rdma_drain_cq(ds4_tp *tp) {
return 1;
}
static int tp_rdma_ensure_work_requests(ds4_tp_rdma *r) {
if (r->win_sge && r->win_rwr && r->win_swr) return 1;
free(r->win_sge); free(r->win_rwr); free(r->win_swr);
const uint32_t recv = r->recv_depth ? r->recv_depth : 64u;
const uint32_t send = r->send_depth ? r->send_depth : 256u;
const uint32_t cap = recv > send ? recv : send;
r->win_sge = calloc(2u * cap, sizeof(*r->win_sge));
r->win_rwr = calloc(cap, sizeof(*r->win_rwr));
r->win_swr = calloc(cap, sizeof(*r->win_swr));
if (!r->win_sge || !r->win_rwr || !r->win_swr) {
fprintf(stderr, "ds4-tp: work-request allocation failed\n");
return 0;
}
return 1;
}
/* Verify-block window helpers. */
static int tp_rdma_block_post_layer(ds4_tp *tp, uint32_t layer) {
ds4_tp_rdma *r = &tp->rdma;
@@ -1135,7 +1153,7 @@ static int tp_rdma_block_post_layer(ds4_tp *tp, uint32_t layer) {
const uint32_t chunks = (uint32_t)((tp->vec_bytes + DS4_TP_RDMA_MAX_MSG - 1u) /
DS4_TP_RDMA_MAX_MSG);
const uint32_t n = rows * chunks;
if (n == 0 || n > r->recv_depth) return 0;
if (n == 0 || n > r->recv_depth || !tp_rdma_ensure_work_requests(r)) return 0;
struct ibv_sge *sge = r->win_sge;
struct ibv_recv_wr *wr = r->win_rwr;
memset(wr, 0, (size_t)n * sizeof(*wr));
@@ -1206,7 +1224,7 @@ static int tp_rdma_block_gate_exchange(ds4_tp *tp, uint32_t layer, uint32_t rows
wr[wi].sg_list = &sge[wi];
wr[wi].num_sge = 1;
wr[wi].opcode = IBV_WR_SEND;
wr[wi].send_flags = wi + 1u == n ? IBV_SEND_SIGNALED : 0;
wr[wi].send_flags = IBV_SEND_SIGNALED;
if (wi != 0) wr[wi - 1u].next = &wr[wi];
wi++;
off += len;
@@ -1218,7 +1236,7 @@ static int tp_rdma_block_gate_exchange(ds4_tp *tp, uint32_t layer, uint32_t rows
layer, strerror(errno));
ok = 0;
} else {
r->send_outstanding++;
r->send_outstanding += n;
}
}
}
@@ -1290,6 +1308,8 @@ static double g_rdma_stat_wait_us[2];
static uint64_t g_rdma_stat_count[2];
static int g_rdma_stat_enabled = -1;
static int tp_rdma_window_barrier(ds4_tp *tp, uint8_t tag);
static int tp_rdma_gate_exchange(ds4_tp *tp, uint32_t layer, uint32_t gate, uint64_t seq) {
ds4_tp_rdma *r = &tp->rdma;
const uint32_t slot = layer * DS4_TP_GATES_PER_LAYER + gate;
@@ -1312,6 +1332,9 @@ static int tp_rdma_gate_exchange(ds4_tp *tp, uint32_t layer, uint32_t gate, uint
if (!r->recv_window_active) {
for (uint64_t s = seq; ok && s < seq + DS4_TP_RDMA_RECV_WINDOW; s++)
ok = tp_rdma_post_gate_recv(tp, s);
/* UC does not retry a send that arrives before the peer posts its
* receive. This is needed once per window, not at every decode gate. */
if (ok) ok = tp_rdma_window_barrier(tp, 0xD1u);
if (ok) r->recv_window_active = true;
}
struct ibv_sge send_sge[2];
@@ -1328,17 +1351,17 @@ static int tp_rdma_gate_exchange(ds4_tp *tp, uint32_t layer, uint32_t gate, uint
send_wr[send_count].sg_list = &send_sge[send_count];
send_wr[send_count].num_sge = 1;
send_wr[send_count].opcode = IBV_WR_SEND;
send_wr[send_count].send_flags = IBV_SEND_SIGNALED;
if (send_count != 0) send_wr[send_count - 1u].next = &send_wr[send_count];
off += len;
}
if (ok) {
send_wr[send_count - 1u].send_flags = IBV_SEND_SIGNALED;
struct ibv_send_wr *bad = NULL;
ok = ibv_post_send(r->qp, send_wr, &bad) == 0;
if (!ok) {
fprintf(stderr, "ds4-tp: rdma post_send: %s\n", strerror(errno));
} else {
r->send_outstanding++;
r->send_outstanding += send_count;
}
}
if (g_rdma_stat_enabled) st1 = tp_now_sec();
@@ -1415,7 +1438,7 @@ static int tp_rdma_drain_decode_window(ds4_tp *tp) {
wr[wi].sg_list = &sge[wi];
wr[wi].num_sge = 1;
wr[wi].opcode = IBV_WR_SEND;
wr[wi].send_flags = wi + 1u == nwr ? IBV_SEND_SIGNALED : 0;
wr[wi].send_flags = IBV_SEND_SIGNALED;
if (wi > 0) wr[wi - 1u].next = &wr[wi];
wi++;
off += len;
@@ -1432,11 +1455,11 @@ static int tp_rdma_drain_decode_window(ds4_tp *tp) {
}
uint32_t recv_done = 0;
int send_done = 0;
uint32_t send_done = 0;
const double deadline =
tp_now_sec() + (double)tp->gate_timeout_ms / 1000.0;
uint32_t peer_poll = 0;
while (recv_done < nwr || !send_done) {
while (recv_done < nwr || send_done < nwr || r->send_outstanding) {
struct ibv_wc wc[DS4_TP_RDMA_RECV_WINDOW * 2u + 1u];
int n = ibv_poll_cq(r->cq,
(int)(DS4_TP_RDMA_RECV_WINDOW * 2u + 1u), wc);
@@ -1454,7 +1477,7 @@ static int tp_rdma_drain_decode_window(ds4_tp *tp) {
if (wc[i].opcode & IBV_WC_RECV) {
recv_done++;
} else if (wc[i].wr_id & DS4_TP_RDMA_BULK_WR_TAG) {
send_done = 1;
send_done++;
} else if (r->send_outstanding > 0) {
r->send_outstanding--;
}
@@ -1502,7 +1525,10 @@ static int tp_rdma_big_gate_exchange(ds4_tp *tp,
void *in,
uint64_t bytes) {
ds4_tp_rdma *r = &tp->rdma;
if (!tp_rdma_big_gate_capable(tp) || r->recv_window_active) return 0;
if (!tp_rdma_big_gate_capable(tp) || r->recv_window_active) {
fprintf(stderr, "ds4-tp: big gate unavailable or decode window still active\n");
return 0;
}
const uintptr_t slab_lo = (uintptr_t)tp->slab;
const uintptr_t slab_hi = slab_lo + tp->slab_bytes;
const uintptr_t out_lo = (uintptr_t)out;
@@ -1524,13 +1550,7 @@ static int tp_rdma_big_gate_exchange(ds4_tp *tp,
if (depth == 0u) return 0;
out_mr = in_mr = r->mr;
}
if (!r->win_sge) {
const uint32_t cap = r->recv_depth > r->send_depth ? r->recv_depth : r->send_depth;
r->win_sge = calloc(2u * cap, sizeof(*r->win_sge));
r->win_rwr = calloc(cap, sizeof(*r->win_rwr));
r->win_swr = calloc(cap, sizeof(*r->win_swr));
if (!r->win_sge || !r->win_rwr || !r->win_swr) return 0;
}
if (!tp_rdma_ensure_work_requests(r)) return 0;
const uint32_t send_depth = r->send_depth ? r->send_depth : 256u;
uint64_t off = 0;
uint8_t tag = 1;
@@ -1581,17 +1601,19 @@ static int tp_rdma_big_gate_exchange(ds4_tp *tp,
return 0;
}
tag++;
/* Sends in sub-batches bounded by the send depth; the last of each
* sub-batch is signaled. */
uint32_t sent = 0, send_done = 0, recv_done = 0, signaled = 0;
/* Apple also completes unsignaled sends. Request and count every
* message: counting those CQEs as completed batches releases staging
* memory while later sends still read it. */
uint32_t sent = 0, send_done = 0, recv_done = 0;
uint64_t recv_seen[4] = {0}, send_seen[4] = {0};
const double deadline = tp_now_sec() + (double)tp->gate_timeout_ms / 1000.0 + 2.0;
uint32_t peer_poll = 0;
while (recv_done < chunks || send_done < signaled || sent < chunks) {
if (sent < chunks && (signaled - send_done) < 4u) {
while (recv_done < chunks || send_done < chunks) {
if (sent < chunks && sent - send_done < send_depth) {
uint32_t n = chunks - sent;
if (n > 64u) n = 64u;
if (n > send_depth / 4u && send_depth / 4u > 0u) n = send_depth / 4u;
if (n == 0u) n = 1u;
if (n > send_depth - (sent - send_done))
n = send_depth - (sent - send_done);
for (uint32_t i = 0; i < n; i++) {
struct ibv_send_wr *w = &r->win_swr[i];
memset(w, 0, sizeof(*w));
@@ -1599,7 +1621,7 @@ static int tp_rdma_big_gate_exchange(ds4_tp *tp,
w->sg_list = &r->win_sge[depth + sent + i];
w->num_sge = 1;
w->opcode = IBV_WR_SEND;
w->send_flags = i + 1u == n ? IBV_SEND_SIGNALED : 0;
w->send_flags = IBV_SEND_SIGNALED;
w->next = i + 1u < n ? &r->win_swr[i + 1u] : NULL;
}
struct ibv_send_wr *bad_send = NULL;
@@ -1608,11 +1630,14 @@ static int tp_rdma_big_gate_exchange(ds4_tp *tp,
return 0;
}
sent += n;
signaled++;
}
struct ibv_wc wc[64];
int nwc = ibv_poll_cq(r->cq, 64, wc);
if (nwc < 0) return 0;
if (nwc < 0) {
fprintf(stderr, "ds4-tp: big gate poll_cq failed at %llu/%llu bytes: %s\n",
(unsigned long long)off, (unsigned long long)bytes, strerror(errno));
return 0;
}
for (int i = 0; i < nwc; i++) {
if (wc[i].status != IBV_WC_SUCCESS) {
fprintf(stderr, "ds4-tp: big gate completion error: %s\n",
@@ -1627,11 +1652,20 @@ static int tp_rdma_big_gate_exchange(ds4_tp *tp,
}
continue;
}
if (wc[i].opcode & IBV_WC_RECV) {
const uint64_t idx = (wc[i].wr_id & ~DS4_TP_RDMA_BULK_WR_TAG) - 1u;
const uint64_t idx = (wc[i].wr_id & ~DS4_TP_RDMA_BULK_WR_TAG) - 1u;
const bool received = (wc[i].opcode & IBV_WC_RECV) != 0;
uint64_t *seen = received ? recv_seen : send_seen;
if (idx >= (received ? chunks : sent) ||
(seen[idx / 64u] & (UINT64_C(1) << (idx % 64u)))) {
fprintf(stderr, "ds4-tp: duplicate or invalid big gate completion %llu\n",
(unsigned long long)wc[i].wr_id);
return 0;
}
seen[idx / 64u] |= UINT64_C(1) << (idx % 64u);
if (received) {
const uint32_t want = idx + 1u < chunks || win_bytes % DS4_TP_RDMA_MAX_MSG == 0u
? (uint32_t)DS4_TP_RDMA_MAX_MSG : (uint32_t)(win_bytes % DS4_TP_RDMA_MAX_MSG);
if (idx >= chunks || wc[i].byte_len != want) {
if (wc[i].byte_len != want) {
fprintf(stderr, "ds4-tp: big gate chunk %llu received %u bytes, expected %u\n",
(unsigned long long)idx, wc[i].byte_len, want);
return 0;
@@ -1647,7 +1681,7 @@ static int tp_rdma_big_gate_exchange(ds4_tp *tp,
}
if (nwc == 0 && tp_now_sec() > deadline) {
fprintf(stderr, "ds4-tp: timeout in big gate window (%u/%u recvs, %u/%u sends, %u sent)\n",
recv_done, chunks, send_done, signaled, sent);
recv_done, chunks, send_done, chunks, sent);
return 0;
}
}
@@ -2090,10 +2124,19 @@ int ds4_tp_big_gate_exchange(ds4_tp *tp, uint32_t layer, uint64_t seq,
if (dbg < 0) dbg = getenv("DS4_TP_BIG_GATE_DEBUG") != NULL;
const double t_start = dbg ? tp_now_sec() : 0.0;
#endif
/* A cold prefill kernel can make one rank arrive much later than the
* other. Match the bulk RDMA window's bounded grace, without relaxing
* the decode timeout or any subsequent payload exchange. */
if (!tp_socket_set_gate_timeout(tp->data_fd, tp->gate_timeout_ms + 2000u)) return 0;
ds4_tp_gate_header h = { DS4_TP_BATCH_MAGIC, (uint16_t)layer, 0xB16u, seq };
if (!tp_write_full(tp->data_fd, &h, sizeof(h))) return 0;
ds4_tp_gate_header ph;
if (!tp_read_full(tp->data_fd, &ph, sizeof(ph))) return 0;
const bool header_ok = tp_write_full(tp->data_fd, &h, sizeof(h)) &&
tp_read_full(tp->data_fd, &ph, sizeof(ph));
if (!header_ok)
fprintf(stderr, "ds4-tp: big gate header exchange failed or peer closed (layer %u seq %llu): %s\n",
layer, (unsigned long long)seq, strerror(errno));
const bool timeout_restored = tp_socket_set_gate_timeout(tp->data_fd, tp->gate_timeout_ms);
if (!header_ok || !timeout_restored) return 0;
#ifdef DS4_TP_HAVE_VERBS
const double t_hs = dbg ? tp_now_sec() : 0.0;
#endif
@@ -2372,8 +2415,8 @@ int ds4_tp_send_command_ack(ds4_tp *tp, uint64_t session_id, int status) {
&ack, sizeof(ack));
}
int ds4_tp_wait_command_ack(ds4_tp *tp, uint64_t session_id,
const char *operation, char *err, size_t errlen) {
int ds4_tp_wait_command_status(ds4_tp *tp, uint64_t session_id, int *status,
const char *operation, char *err, size_t errlen) {
uint32_t type = 0, bytes = 0;
ds4_tp_command_ack ack;
if (!tp_read_frame_header(tp->control_fd, &type, &bytes) ||
@@ -2384,13 +2427,52 @@ int ds4_tp_wait_command_ack(ds4_tp *tp, uint64_t session_id,
operation ? operation : "command");
return 0;
}
if (ack.session_id != session_id || ack.status != 0) {
if (ack.session_id != session_id || ack.reserved != 0) {
ds4_tp_mark_failed(tp);
tp_set_err(err, errlen,
"tp: worker %s failed (session %llu, status %d)",
operation ? operation : "command",
(unsigned long long)ack.session_id, (int)ack.status);
return 0;
}
*status = ack.status;
return 1;
}
int ds4_tp_wait_command_ack(ds4_tp *tp, uint64_t session_id,
const char *operation, char *err, size_t errlen) {
int status;
if (!ds4_tp_wait_command_status(tp, session_id, &status, operation, err, errlen))
return 0;
if (status != 0) {
tp_set_err(err, errlen, "tp: worker %s failed (session %llu, status %d)",
operation ? operation : "command", (unsigned long long)session_id, status);
return 0;
}
return 1;
}
int ds4_tp_sync_checkpoint(ds4_tp *tp, uint32_t point, int current, int total,
bool requested, bool *cancelled) {
if (!tp || !cancelled) return 0;
struct {
uint64_t seq;
uint32_t point;
int32_t current, total;
uint32_t cancelled;
} local = {++tp->sync_checkpoint_seq, point, current, total, requested}, peer;
uint32_t type, bytes;
if (ds4_tp_failed(tp) ||
!tp_send_frame(tp->control_fd, DS4_TP_FRAME_SYNC_CHECKPOINT, &local, sizeof(local)) ||
!tp_read_frame_header(tp->control_fd, &type, &bytes) ||
type != DS4_TP_FRAME_SYNC_CHECKPOINT || bytes != sizeof(peer) ||
!tp_read_full(tp->control_fd, &peer, sizeof(peer)) ||
peer.seq != local.seq || peer.point != point ||
peer.current != current || peer.total != total || peer.cancelled > 1u) {
ds4_tp_mark_failed(tp);
return 0;
}
*cancelled = requested || peer.cancelled;
return 1;
}
@@ -2872,10 +2954,10 @@ int ds4_tp_worker_run(ds4_engine *engine, const ds4_tp_options *opt) {
ds4_session_sync(session, &prompt, err, sizeof(err));
if (!ds4_tp_send_command_ack(tp, command.session_id, sync_rc)) {
rc = 1;
} else if (sync_rc != 0) {
} else if (sync_rc != 0 && sync_rc != DS4_SESSION_SYNC_INTERRUPTED) {
ds4_log(stderr, DS4_LOG_ERROR, "tp worker sync: %s", err);
rc = 1;
} else if (ds4_engine_tp_vocab_split(engine) &&
} else if (sync_rc == 0 && ds4_engine_tp_vocab_split(engine) &&
!tp_worker_send_logits(tp, session, logits, vocab)) {
rc = 1;
}
ds4_tp.h
+7
View File
@@ -177,6 +177,12 @@ int ds4_tp_send_mixed_batch(ds4_tp *tp, uint64_t prefill_session_id,
int ds4_tp_send_command_ack(ds4_tp *tp, uint64_t session_id, int status);
int ds4_tp_wait_command_ack(ds4_tp *tp, uint64_t session_id,
const char *operation, char *err, size_t errlen);
int ds4_tp_wait_command_status(ds4_tp *tp, uint64_t session_id, int *status,
const char *operation, char *err, size_t errlen);
/* Both ranks call at matching prefill boundaries. Cancellation is agreed
* here, never sampled independently while either rank is inside a GPU gate. */
int ds4_tp_sync_checkpoint(ds4_tp *tp, uint32_t point, int current, int total,
bool requested, bool *cancelled);
int ds4_tp_send_stop(ds4_tp *tp);
/* Worker: blocks for the next mirrored command. Frame types below; for
@@ -205,6 +211,7 @@ typedef enum {
DS4_TP_FRAME_RDMA_WARM = 19,
DS4_TP_FRAME_RDMA_POSTED = 20,
DS4_TP_FRAME_GLM_MTP = 21,
DS4_TP_FRAME_SYNC_CHECKPOINT = 22,
} ds4_tp_frame_type;
typedef struct {
gguf-tools/Makefile
+3 -3
View File
@@ -12,9 +12,9 @@ NATIVE_CPU_FLAG ?= -mcpu=native
QUANTS_SHARED := libds4quants.dylib
SHARED_FLAGS := -dynamiclib
QUALITY_LDLIBS := -lm -pthread -framework Foundation -framework Metal
QUALITY_TARGETS := ds4.o ds4_metal.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o ds4_image.o rax.o ds4_gpu_args.o
QUALITY_OBJS := ../ds4.o ../ds4_metal.o ../ds4_distributed.o ../ds4_tp.o ../ds4_ssd.o ../ds4_layer_pack.o ../ds4_image.o ../rax.o ../ds4_gpu_args.o
QUALITY_LINK := $(CC) $(CFLAGS) -I.. -o
QUALITY_TARGETS := ds4.o ds4_metal.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o ds4_engram.o ds4_image.o rax.o ds4_gpu_args.o
QUALITY_OBJS := ../ds4.o ../ds4_metal.o ../ds4_distributed.o ../ds4_tp.o ../ds4_ssd.o ../ds4_layer_pack.o ../ds4_engram.o ../ds4_image.o ../rax.o ../ds4_gpu_args.o
QUALITY_LINK = $(CC) $(CFLAGS) -I.. -o
LLAMA_CPP_LDLIBS ?= -L$(LLAMA_CPP_BIN) -Wl,-rpath,$(abspath $(LLAMA_CPP_BIN)) \
-lllama -lggml -lggml-base -lggml-cpu -lggml-blas -lggml-metal \
-framework Foundation -framework Metal -framework Accelerate
gguf-tools/README.md
+41
View File
@@ -119,6 +119,47 @@ gguf-tools/deepseek4-quantize \
`--compare-tensor` regenerates a single tensor and byte-compares it against the
template or `--compare-gguf`. `--threads N` controls routed-expert workers.
## Convert DeepSeek V4.1 Flash
V4.1 uses its own converter; a V4 template is not compatible. Install NumPy,
tokenizers and SymPy, then build the quantizer library:
```sh
make -C gguf-tools libds4quants.dylib
python3 gguf-tools/deepseek41_quantize.py \
--hf models/DeepSeek-V4.1-Flash \
--source-revision df42c109f1defefcbfcedbe7d905718a12266e40 \
--out gguf/DeepSeek-V4.1-Flash-IQ2_XXS-Q2_K-bootstrap.gguf --dry-run
```
Omit `--dry-run` to write the file. Add `--resume` after an interrupted conversion.
Use `libds4quants.so` on Linux. Gate/up experts use IQ2_XXS and down experts use
Q2_K; attention, shared experts and the output head use Q8_0. Engram rows retain
their original FP8 values and scales, packed together at the end of the GGUF for
disk lookups. Vision and DSpark weights are not included.
The first conversion uses weight-energy importance for IQ2_XXS. After runtime
calibration, add `--imatrix FILE` and choose a new output filename to regenerate
from the original safetensors. Do not requantize the first GGUF.
For Q4, add `--quant q4` and use `DeepSeek-V4.1-Flash-Q4.gguf` as the output.
This changes only the routed experts to Q4_K; the other tensor formats and
disk-only Engram layout stay the same. The same imatrix works for both recipes.
Check the finished artifact against the pinned source before running it:
```sh
python3 gguf-tools/deepseek41_validate_gguf.py \
--hf models/DeepSeek-V4.1-Flash \
--source-revision df42c109f1defefcbfcedbe7d905718a12266e40 \
--gguf gguf/DeepSeek-V4.1-Flash-IQ2_XXS-Q2_K-bootstrap.gguf --payload
```
For a calibrated file, pass the same `--imatrix FILE` used during conversion.
Pass `--quant q4` to the audit as well when checking a Q4 file.
The audit checks the complete layout, all non-expert tensors, sampled experts
and native Engram rows. It does not replace [inference quality tests](quality-testing/deepseek-v4.1-flash-20260910/README.md).
## Convert A DSpark Support Checkpoint
The DSpark Flash checkpoint is published as Hugging Face safetensors and stores
gguf-tools/deepseek41_metadata.py
+154
View File
@@ -0,0 +1,154 @@
#!/usr/bin/env python3
"""DeepSeek V4.1 tokenizer and Engram metadata, derived from the released config.
The normalization, prime allocation and seeded hash multipliers follow DeepSeek's
MIT-licensed inference/engram.py. The runtime consumes the resulting arrays, not
a second implementation of Unicode normalization or NumPy's random generator.
"""
import json
import os
import struct
from glm53_quantize import (
GGUF_ARRAY, GGUF_STRING, GGUF_UINT32, GGUF_UINT64, kv_bool, kv_string,
kv_f32, kv_u32, kv_u32_array, pack_string,
)
# Apple Silicon VM pages are 16 KiB. The Engram extent must be removable
# without also unmapping bytes of the preceding resident tensor.
GGUF_ALIGNMENT = 16384
def array_record(key, kind, values):
header = pack_string(key) + struct.pack("<IIQ", GGUF_ARRAY, kind, len(values))
if kind == GGUF_STRING:
return header + b"".join(pack_string(value) for value in values)
fmt = {GGUF_UINT32: "I", GGUF_UINT64: "Q", 5: "i", 6: "f"}[kind]
return header + struct.pack(f"<{len(values)}{fmt}", *values)
def compressed_token_map(tokenizer):
from tokenizers import Regex, normalizers
sentinel = "\ue000"
normalizer = normalizers.Sequence([
normalizers.NFKC(), normalizers.NFD(), normalizers.StripAccents(),
normalizers.Lowercase(), normalizers.Replace(Regex(r"[ \t\r\n]+"), " "),
normalizers.Replace(Regex(r"^ $"), sentinel), normalizers.Strip(),
normalizers.Replace(sentinel, " "),
])
unique, result = {}, []
for token_id in range(tokenizer.get_vocab_size(with_added_tokens=True)):
text = tokenizer.decode([token_id], skip_special_tokens=False)
if "\ufffd" in text:
key = tokenizer.id_to_token(token_id)
else:
key = normalizer.normalize_str(text) or text
result.append(unique.setdefault(key, len(unique)))
return result, len(unique)
def engram_layout(config, tokenizer):
import numpy as np
from sympy import nextprime
token_map, vocab_size = compressed_token_map(tokenizer)
if vocab_size != config["engram_compressed_vocab_size"]:
raise ValueError(f"Engram vocabulary mismatch: {vocab_size}")
if len(token_map) != config["vocab_size"]:
raise ValueError("Engram requires every released tokenizer ID")
layers = config["engram_layer_ids"]
if len(layers) != 2 or config["engram_max_ngram_size"] != 4 or \
config["engram_n_heads"] != 8 or config["engram_head_dim"] != 256:
raise ValueError("unsupported Engram shape")
seen, primes, multipliers = set(), [], []
bound = (np.iinfo(np.int64).max // vocab_size) // 2
for layer in layers:
row = []
for _ in range(config["engram_max_ngram_size"] - 1):
current = config["engram_vocab_size"] - 1
for _ in range(config["engram_n_heads"]):
current = int(nextprime(current))
while current in seen:
current = int(nextprime(current))
seen.add(current)
row.append(current)
primes.append(row)
rng = np.random.default_rng(10007 * layer)
multipliers.append((rng.integers(0, bound, size=4, dtype=np.int64) * 2 + 1).tolist())
if [sum(row) for row in primes] != config["engram_num_embeddings"]:
raise ValueError("Engram table size differs from hash bucket layout")
return dict(token_map=token_map, compressed_vocab_size=vocab_size,
pad_id=token_map[config["engram_pad_token_id"]],
rows=config["engram_num_embeddings"], layers=layers,
primes=primes, multipliers=multipliers)
def metadata(hf_dir, revision):
from tokenizers import Tokenizer
with open(os.path.join(hf_dir, "config.json"), "rb") as fp:
config = json.load(fp)
if config["model_type"] != "deepseek_v41":
raise ValueError("not a DeepSeek V4.1 source checkpoint")
text = config["text_config"]
tokenizer = Tokenizer.from_file(os.path.join(hf_dir, "tokenizer.json"))
layout = engram_layout(text, tokenizer)
with open(os.path.join(hf_dir, "tokenizer.json"), "rb") as fp:
tok = json.load(fp)
if tok["model"]["type"] != "BPE":
raise ValueError("unsupported tokenizer")
# This tokenizer lists special tokens in both the BPE vocabulary and
# added_tokens. Its parsed ID table resolves that intentional overlap.
tokens = [tokenizer.id_to_token(i) for i in range(text["vocab_size"])]
if any(token is None for token in tokens):
raise ValueError("tokenizer has missing IDs")
special = {item["id"] for item in tok["added_tokens"] if item["special"]}
merges = [" ".join(pair) if isinstance(pair, list) else pair for pair in tok["model"]["merges"]]
records = [
kv_string("general.architecture", "deepseek41"),
kv_string("general.name", "DeepSeek V4.1 Flash"),
kv_string("general.source.url", "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash"),
kv_string("general.source.revision", revision),
kv_u32("general.alignment", GGUF_ALIGNMENT),
kv_string("deepseek41.config", json.dumps(config, sort_keys=True, separators=(",", ":"))),
kv_string("deepseek41.engram.encoding", "e4m3_e8m0_32_row264"),
kv_u32_array("deepseek41.engram.layer_ids", layout["layers"]),
kv_u32_array("deepseek41.engram.rows", layout["rows"]),
kv_u32("deepseek41.engram.compressed_vocab_size", layout["compressed_vocab_size"]),
kv_u32("deepseek41.engram.pad_id", layout["pad_id"]),
kv_u32_array("deepseek41.engram.token_map", layout["token_map"]),
kv_u32_array("deepseek41.engram.primes", sum(layout["primes"], [])),
array_record("deepseek41.engram.multipliers", GGUF_UINT64, sum(layout["multipliers"], [])),
kv_string("tokenizer.ggml.model", "gpt2"),
kv_string("tokenizer.ggml.pre", "joyai-llm"),
array_record("tokenizer.ggml.tokens", GGUF_STRING, tokens),
array_record("tokenizer.ggml.token_type", 5, [3 if i in special else 1 for i in range(len(tokens))]),
array_record("tokenizer.ggml.merges", GGUF_STRING, merges),
kv_u32("tokenizer.ggml.bos_token_id", config["bos_token_id"]),
kv_u32("tokenizer.ggml.eos_token_id", config["eos_token_id"]),
kv_bool("tokenizer.ggml.add_bos_token", True),
kv_bool("tokenizer.ggml.add_eos_token", False),
]
for key in (
"vocab_size", "hidden_size", "moe_intermediate_size", "num_hidden_layers",
"num_attention_heads", "num_key_value_heads", "head_dim", "qk_rope_head_dim",
"q_lora_rank", "o_lora_rank", "o_groups", "n_routed_experts", "n_shared_experts",
"num_experts_per_tok", "max_position_embeddings", "sliding_window",
"index_n_heads", "index_head_dim", "index_topk", "candidate_source_layer_id",
"candidate_topk_blocks", "candidate_block_size", "hc_mult", "hc_sinkhorn_iters",
"rope_theta", "compress_rope_theta",
):
records.append(kv_u32(f"deepseek41.{key}", text[key]))
for key in ("rms_norm_eps", "hc_eps", "swiglu_limit", "routed_scaling_factor"):
records.append(kv_f32(f"deepseek41.{key}", text[key]))
for key in ("scoring_func", "hidden_act", "topk_method"):
records.append(kv_string(f"deepseek41.{key}", text[key]))
records.append(kv_bool("deepseek41.norm_topk_prob", text["norm_topk_prob"]))
records.append(kv_u32_array("deepseek41.compress_ratios", text["compress_ratios"][:text["num_hidden_layers"]]))
for key in ("kv_source_layer_ids", "index_source_layer_ids"):
records.append(kv_u32_array(f"deepseek41.{key}", text[key]))
for key in ("factor", "beta_fast", "beta_slow", "original_max_position_embeddings"):
records.append(kv_f32(f"deepseek41.rope_scaling.{key}", text["rope_scaling"][key]))
return config, records
gguf-tools/deepseek41_quantize.py
+335
View File
@@ -0,0 +1,335 @@
#!/usr/bin/env python3
"""Convert DeepSeek V4.1 Flash to DwarfStar's Q2 or Q4_K GGUF.
Uses the project's C quantizers. Engram FP8 rows and their scales are packed
losslessly at the end of the file, outside the main model's resident extent.
Vision and DSpark are separate models and are not part of this text GGUF.
"""
import argparse
import concurrent.futures
import dataclasses
import hashlib
import json
import os
import re
import shutil
import struct
import sys
import time
from deepseek41_metadata import GGUF_ALIGNMENT, metadata
from glm53_quantize import (
SourceDB, TensorPlan, Quantizer, Imatrix, QTYPE_F32, QTYPE_F16,
QTYPE_Q8_0, QTYPE_Q2_K, QTYPE_Q4_K, QTYPE_IQ2_XXS, QTYPE_I8, align,
conversion_signature, kv_string, load_resume_state, print_plan,
qtype_nbytes, save_resume_state, tensor_header,
)
QUANTIZATION = {
"q2": "IQ2_XXS gate/up; Q2_K down; Q8_0 attention/shared/head",
"q4": "Q4_K gate/up/down; Q8_0 attention/shared/head",
}
def scale_name(name):
return name.removesuffix(".weight") + ".scale"
def validate_scales(tensors):
for name, info in tensors.items():
if not name.startswith(("layers.", "embed.", "head.", "norm.")):
continue
dtype, shape = info["dtype"], info["shape"]
if dtype not in ("F8_E4M3", "I8") or not name.endswith(".weight"):
continue
if len(shape) != 2:
raise ValueError(f"{name}: expected a matrix")
if dtype == "I8":
expected = [shape[0], shape[1] // 16]
elif ".engram.embed." in name:
expected = [shape[0], shape[1] // 32]
else:
expected = [(dim + 31) // 32 for dim in shape]
scale = tensors.get(scale_name(name))
if not scale or scale["dtype"] != "F8_E8M0" or scale["shape"] != expected:
raise ValueError(f"{name}: expected E8M0 scales {expected}")
def build_plan(db, config, quant="q2"):
if quant not in QUANTIZATION:
raise ValueError(f"unknown quantization recipe: {quant}")
c = config["text_config"]
if config["quantization_config"]["weight_block_size"] != [32, 32]:
raise ValueError("expected native 32x32 FP8 blocks")
dim, inter = c["hidden_size"], c["moe_intermediate_size"]
heads, hd = c["num_attention_heads"], c["head_dim"]
qrank, orank, groups = c["q_lora_rank"], c["o_lora_rank"], c["o_groups"]
hc, experts, vocab = c["hc_mult"], c["n_routed_experts"], c["vocab_size"]
ih, idim = c["index_n_heads"], c["index_head_dim"]
plan, disk, consumed = [], [], set()
def claim(name, expected, dtype=None):
info = db.info(name)
if info["shape"] != list(expected) or (dtype and info["dtype"] != dtype):
raise ValueError(f"{name}: unexpected {info['dtype']} {info['shape']}, expected {dtype} {expected}")
consumed.add(name)
if info["dtype"] in ("I8", "F8_E4M3"):
consumed.add(scale_name(name))
return info
def regular(name, source, shape, qtype, role):
claim(source, shape)
plan.append(TensorPlan(name, tuple(reversed(shape)), qtype, role, source=source))
regular("token_embd.weight", "embed.weight", (vocab, dim), QTYPE_F16, "embedding")
regular("output_norm.weight", "norm.weight", (dim,), QTYPE_F32, "norm")
regular("output.weight", "head.weight", (vocab, dim), QTYPE_Q8_0, "output")
for layer in range(c["num_hidden_layers"]):
src, dst = f"layers.{layer}", f"blk.{layer}"
for site in ("attn", "ffn"):
for part, shape, qt in (("fn", (hc * (hc + 2), hc * dim), QTYPE_F16),
("base", (hc * (hc + 2),), QTYPE_F32),
("scale", (3,), QTYPE_F32)):
regular(f"{dst}.hc_{site}_{part}.weight", f"{src}.hc_{site}_{part}", shape, qt, "mhc")
regular(f"{dst}.{site}_norm.weight", f"{src}.{site}_norm.weight", (dim,), QTYPE_F32, "norm")
for target, source, shape, qt in (
("attn_sinks.weight", "attn_sink", (heads,), QTYPE_F32),
("attn_q_a.weight", "wq_a.weight", (qrank, dim), QTYPE_Q8_0),
("attn_q_b.weight", "wq_b.weight", (heads * hd, qrank), QTYPE_Q8_0),
("attn_q_a_norm.weight", "q_norm.weight", (qrank,), QTYPE_F32),
("attn_kv.weight", "wkv.weight", (hd, dim), QTYPE_Q8_0),
("attn_kv_a_norm.weight", "kv_norm.weight", (hd,), QTYPE_F32),
("attn_output_a.weight", "wo_a.weight", (groups * orank, heads * hd // groups), QTYPE_Q8_0),
("attn_output_b.weight", "wo_b.weight", (dim, groups * orank), QTYPE_Q8_0),
):
regular(f"{dst}.{target}", f"{src}.attn.{source}", shape, qt, "attention")
if layer in c["kv_source_layer_ids"]:
for target, source, shape, qt in (
("attn_compressor_kv.weight", "compressor.wkv.weight", (hd, dim), QTYPE_F16),
("attn_compressor_norm.weight", "compressor.norm.weight", (hd,), QTYPE_F32),
("indexer.attn_k.weight", "indexer.wk.weight", (idim, hd), QTYPE_F16),
("indexer.k_norm.weight", "indexer.k_norm.weight", (idim,), QTYPE_F32),
):
regular(f"{dst}.{target}", f"{src}.attn.{source}", shape, qt, "compressor")
if c["compress_ratios"][layer] > 1:
regular(f"{dst}.attn_compressor_gate.weight", f"{src}.attn.compressor.wgate.weight",
(hd, dim), QTYPE_F16, "compressor")
if layer in c["index_source_layer_ids"]:
regular(f"{dst}.indexer.attn_q_b.weight", f"{src}.attn.indexer.wq_b.weight",
(ih * idim, qrank), QTYPE_F16, "indexer")
regular(f"{dst}.indexer.proj.weight", f"{src}.attn.indexer.weights_proj.weight",
(ih, dim), QTYPE_F16, "indexer")
regular(f"{dst}.ffn_gate_inp.weight", f"{src}.ffn.gate.weight", (experts, dim), QTYPE_F32, "router")
for suffix, target in (("bias", "exp_probs_b.bias"), ("bias_vl", "exp_probs_b_vl.bias")):
regular(f"{dst}.{target}", f"{src}.ffn.gate.{suffix}", (experts,), QTYPE_F32, "router")
for part, source, shape, qt in (("gate", "w1", (inter, dim), QTYPE_IQ2_XXS),
("up", "w3", (inter, dim), QTYPE_IQ2_XXS),
("down", "w2", (dim, inter), QTYPE_Q2_K)):
regular(f"{dst}.ffn_{part}_shexp.weight", f"{src}.ffn.shared_experts.{source}.weight",
shape, QTYPE_Q8_0, "shared")
pattern = f"{src}.ffn.experts.{{expert}}.{source}.weight"
for expert in range(experts):
claim(pattern.format(expert=expert), (shape[0], shape[1] // 2), "I8")
plan.append(TensorPlan(f"{dst}.ffn_{part}_exps.weight", (*reversed(shape), experts),
QTYPE_Q4_K if quant == "q4" else qt,
"experts", source=pattern, expert_layer=layer,
expert_part=part, expert_count=experts))
if layer in c["engram_layer_ids"]:
index = c["engram_layer_ids"].index(layer)
rows = c["engram_num_embeddings"][index]
engram = f"{src}.engram"
claim(f"{engram}.embed.weight", (rows, 256), "F8_E4M3")
claim(f"{engram}.embed.scale", (rows, 8), "F8_E8M0")
disk.append(TensorPlan(f"{dst}.engram_embd.weight", (264, rows), QTYPE_I8,
"engram_disk", source=f"{engram}.embed.weight", raw_copy=True))
for part in ("q", "k"):
regular(f"{dst}.engram_{part}_norm.weight", f"{engram}.{part}_weight", (hc, dim), QTYPE_F32, "engram")
regular(f"{dst}.engram_kv.weight", f"{engram}.wkv.weight", ((hc + 1) * dim, 24 * 256), QTYPE_F16, "engram")
omitted = {name for name in db.tensors if name.startswith(("mtp.", "vision.", "aligner.", "image_"))}
if consumed | omitted != set(db.tensors):
raise ValueError(f"unclaimed source tensors: {sorted(set(db.tensors) - consumed - omitted)[:10]}")
offset = 0
for item in plan + disk:
item.offset = offset
item.nbytes = qtype_nbytes(item.qtype, item.shape)
offset += align(item.nbytes, GGUF_ALIGNMENT)
return plan + disk
class NativeQuantizer(Quantizer):
def to_f32(self, db, name, row_start=0, row_count=None):
if ".engram.embed." in name:
raise ValueError("Engram must never be materialized as a whole float tensor")
np = self.np
info = db.info(name)
if info["dtype"] not in ("I8", "F8_E4M3"):
result = super().to_f32(db, name, row_start, row_count)
else:
shape = info["shape"]
codes = np.frombuffer(db.read(name), dtype=np.uint8).reshape(shape)
sinfo = db.info(scale_name(name))
scales = np.frombuffer(db.read(scale_name(name)), dtype=np.uint8).reshape(sinfo["shape"])
if np.any(scales == 255):
raise ValueError(f"{name}: nonfinite scale")
scales = np.ldexp(np.ones_like(scales, dtype=np.float32), scales.astype(np.int32) - 127)
if info["dtype"] == "I8":
lut = np.array([0, .5, 1, 1.5, 2, 3, 4, 6, -0., -.5, -1, -1.5, -2, -3, -4, -6], np.float32)
result = np.empty((shape[0], shape[1] * 2), dtype=np.float32)
result[:, 0::2] = lut[codes & 15]
result[:, 1::2] = lut[codes >> 4]
result = (result.reshape(shape[0], -1, 32) * scales[:, :, None]).reshape(result.shape)
else:
if np.any((codes & 127) == 127):
raise ValueError(f"{name}: nonfinite FP8 weight")
result = self.fp8_lut[codes]
for row in range(0, shape[0], 32):
for col in range(0, shape[1], 32):
result[row:row + 32, col:col + 32] *= scales[row // 32, col // 32]
if row_count is not None:
result = result[row_start:row_start + row_count]
if not np.all(np.isfinite(result)):
raise ValueError(f"{name}: nonfinite dequantized weight")
return np.ascontiguousarray(result, dtype=np.float32)
def encode(self, array, qtype, imatrix=None):
if qtype == QTYPE_F16 and self.np.any(self.np.abs(array) > 65504):
raise ValueError("F16 tensor would overflow; preserve this family in BF16/F32")
return super().encode(array, qtype, imatrix)
def write_engram(fp, item, db, np):
# Keep weights and scales together without changing either byte. A chunk
# holds 16K rows (4 MiB of weights), independent of the 94 GiB table size.
rows = item.shape[1]
for start in range(0, rows, 16384):
count = min(16384, rows - start)
weights = b"".join(db.iter_read(item.source, start * 256, count * 256))
scales = b"".join(db.iter_read(scale_name(item.source), start * 8, count * 8))
w = np.frombuffer(weights, dtype=np.uint8).reshape(count, 256)
s = np.frombuffer(scales, dtype=np.uint8).reshape(count, 8)
if np.any((w & 127) == 127) or np.any(s == 255):
raise ValueError(f"{item.name}: nonfinite Engram row near {start}")
packed = np.empty((count, 264), dtype=np.uint8)
packed[:, :256], packed[:, 256:] = w, s
fp.write(packed.tobytes())
def write_gguf(args, plan, records, db):
quantizer = NativeQuantizer(args.quants_library)
imatrix = Imatrix(args.imatrix, quantizer.np)
if args.imatrix:
for item in plan:
if item.is_expert and item.name not in imatrix.entries:
raise ValueError(f"missing imatrix tensor {item.name}")
if any(quantizer.np.any(values < 0) for values in imatrix.entries.values()):
raise ValueError("negative importance values")
data_start, data_bytes = print_plan(plan, records, [], GGUF_ALIGNMENT)
partial, journal = args.out + ".partial", args.out + ".partial.json"
signature = conversion_signature(plan, records, [], args.imatrix)
# A metadata/recipe match alone cannot distinguish two source downloads.
source_identity = [(name, db.info(name)) for name in sorted(db.tensors)]
signature = hashlib.sha256((signature + json.dumps(source_identity, sort_keys=True)).encode()).hexdigest()
if os.path.exists(args.out):
raise ValueError(f"refusing to overwrite {args.out}")
completed = 0
if os.path.exists(partial) or os.path.exists(journal):
if not args.resume or not (os.path.exists(partial) and os.path.exists(journal)):
raise ValueError("partial file and journal require --resume")
completed = load_resume_state(journal, signature, plan)
end = data_start + (plan[completed - 1].offset +
align(plan[completed - 1].nbytes, GGUF_ALIGNMENT) if completed else 0)
free = shutil.disk_usage(os.path.dirname(os.path.abspath(args.out))).free
if free < data_start + data_bytes - end + (32 << 30):
raise ValueError("insufficient disk space for remaining output plus 32 GiB reserve")
header = b"GGUF" + struct.pack("<IQQ", 3, len(plan), len(records))
header += b"".join(records) + b"".join(tensor_header(item) for item in plan)
header += bytes(data_start - len(header))
if os.path.exists(partial):
with open(partial, "rb") as fp:
if fp.read(data_start) != header or os.fstat(fp.fileno()).st_size < end:
raise ValueError("partial GGUF is truncated or has a different header")
else:
with open(partial, "xb") as fp:
fp.write(header)
fp.flush()
os.fsync(fp.fileno())
save_resume_state(journal, signature, 0)
with open(partial, "r+b") as fp, concurrent.futures.ThreadPoolExecutor(max_workers=args.threads) as pool:
fp.truncate(end)
fp.seek(end)
for index in range(completed, len(plan)):
item = plan[index]
started = time.monotonic()
missing = 0
if fp.tell() != data_start + item.offset:
raise ValueError(f"incorrect offset for {item.name}")
if item.role == "engram_disk":
write_engram(fp, item, db, quantizer.np)
elif item.is_expert:
def convert(expert):
values = quantizer.to_f32(db, item.source.format(expert=expert))
importance = imatrix.expert(item.name, expert, item.shape[0], item.expert_count)
return quantizer.encode(values, item.qtype, importance), importance is None
for start in range(0, item.expert_count, args.threads):
futures = [pool.submit(convert, e) for e in range(start, min(start + args.threads, item.expert_count))]
for future in futures:
data, fallback = future.result()
if len(data) != item.nbytes // item.expert_count:
raise ValueError("wrong encoded expert size")
fp.write(data)
missing += int(fallback)
else:
fp.write(quantizer.encode(quantizer.to_f32(db, item.source), item.qtype))
if fp.tell() != data_start + item.offset + item.nbytes:
raise ValueError(f"incorrect payload size for {item.name}")
fp.write(bytes(align(item.nbytes, GGUF_ALIGNMENT) - item.nbytes))
fp.flush()
os.fsync(fp.fileno())
save_resume_state(journal, signature, index + 1)
print(f"[{index + 1}/{len(plan)}] {item.name}: {item.nbytes / (1 << 30):.3f} GiB, "
f"{time.monotonic() - started:.1f}s, uncalibrated_experts={missing}", flush=True)
os.rename(partial, args.out)
os.unlink(journal)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--hf", required=True)
parser.add_argument("--out", required=True)
parser.add_argument("--source-revision", required=True)
parser.add_argument("--quant", choices=QUANTIZATION, default="q2")
parser.add_argument("--imatrix")
parser.add_argument("--threads", type=int, default=8)
parser.add_argument("--resume", action="store_true")
parser.add_argument("--dry-run", action="store_true")
suffix = "dylib" if sys.platform == "darwin" else "so"
parser.add_argument("--quants-library", default=os.path.join(os.path.dirname(__file__), f"libds4quants.{suffix}"))
args = parser.parse_args()
if not re.fullmatch(r"[0-9a-f]{40}", args.source_revision):
parser.error("source revision must be a full commit hash")
if not 1 <= args.threads <= 32:
parser.error("threads must be between 1 and 32")
config, records = metadata(args.hf, args.source_revision)
db = SourceDB(args.hf, index_validator=lambda _: None, scale_validator=validate_scales)
try:
plan = build_plan(db, config, args.quant)
records.append(kv_string("deepseek41.quantization", QUANTIZATION[args.quant]))
records.append(kv_string("deepseek41.calibration", "imatrix" if args.imatrix else "weight-energy bootstrap"))
if args.imatrix:
records.append(kv_string("quantize.imatrix.file", os.path.basename(args.imatrix)))
if args.dry_run:
print_plan(plan, records, [], GGUF_ALIGNMENT)
for item in plan:
print(json.dumps(dataclasses.asdict(item), sort_keys=True))
else:
write_gguf(args, plan, records, db)
finally:
db.close()
if __name__ == "__main__":
try:
main()
except (OSError, ValueError) as error:
sys.exit(f"deepseek41-quantize: {error}")
gguf-tools/deepseek41_validate_gguf.py
+122
View File
@@ -0,0 +1,122 @@
#!/usr/bin/env python3
"""Audit a completed V4.1 GGUF and sample its payloads against the source.
Checks the complete tensor layout, then re-encodes selected expert tensors and
samples native Engram rows. This is artifact validation, not inference QA.
"""
import argparse
import json
import os
from pathlib import Path
import random
import sys
from deepseek41_metadata import GGUF_ALIGNMENT
from deepseek41_quantize import (SourceDB, NativeQuantizer, Imatrix, build_plan,
validate_scales, scale_name, QUANTIZATION)
from glm53_quantize import (align, read_exact, read_u32, read_u64,
read_gguf_string, skip_gguf_value)
from glm53_validate_gguf import read_selected_metadata
def check_payload(fp, offset, item, db, quantizer, imatrix):
if item.role == "engram_disk":
rows = item.shape[1]
rng = random.Random(41 + rows)
selected = {0, 1, rows - 1, 16383, 16384, (1 << 32) // 264}
selected.update(rng.randrange(rows) for _ in range(128))
for row in sorted(selected):
if row >= rows:
continue
expected = b"".join(db.iter_read(item.source, row * 256, 256))
expected += b"".join(db.iter_read(scale_name(item.source), row * 8, 8))
fp.seek(offset + row * 264)
if read_exact(fp, 264, item.name) != expected:
raise ValueError(f"{item.name}: Engram row {row} differs from source")
elif item.is_expert:
selected = {0, (item.expert_layer * 17 + 41) % item.expert_count, item.expert_count - 1}
stride = item.nbytes // item.expert_count
for expert in sorted(selected):
values = quantizer.to_f32(db, item.source.format(expert=expert))
importance = imatrix.expert(item.name, expert, item.shape[0], item.expert_count)
expected = quantizer.encode(values, item.qtype, importance)
fp.seek(offset + expert * stride)
if len(expected) != stride or read_exact(fp, stride, item.name) != expected:
raise ValueError(f"{item.name}: encoded expert {expert} differs from source recipe")
else:
values = quantizer.to_f32(db, item.source)
expected = quantizer.encode(values, item.qtype)
fp.seek(offset)
if len(expected) != item.nbytes or read_exact(fp, item.nbytes, item.name) != expected:
raise ValueError(f"{item.name}: payload differs from source recipe")
def validate(args):
config = json.loads((Path(args.hf) / "config.json").read_text())
db = SourceDB(args.hf, index_validator=lambda _: None, scale_validator=validate_scales)
try:
plan = build_plan(db, config, args.quant)
with open(args.gguf, "rb") as fp:
if read_exact(fp, 4, "magic") != b"GGUF" or read_u32(fp, "version") != 3:
raise ValueError("expected GGUF v3")
if read_u64(fp, "tensor count") != len(plan):
raise ValueError("tensor count differs from source plan")
metadata = {}
for _ in range(read_u64(fp, "metadata count")):
key = read_gguf_string(fp, "metadata key")
kind = read_u32(fp, "metadata type")
if key in {"general.architecture", "general.alignment", "general.source.revision",
"deepseek41.calibration", "deepseek41.quantization"}:
if key in metadata:
raise ValueError(f"duplicate metadata: {key}")
metadata[key] = read_selected_metadata(fp, kind)
else:
skip_gguf_value(fp, kind)
expected = {"general.architecture": "deepseek41", "general.alignment": GGUF_ALIGNMENT,
"general.source.revision": args.source_revision,
"deepseek41.quantization": QUANTIZATION[args.quant],
"deepseek41.calibration": "imatrix" if args.imatrix else "weight-energy bootstrap"}
if metadata != expected:
raise ValueError(f"metadata mismatch: {metadata}")
for item in plan:
name = read_gguf_string(fp, "tensor name")
rank = read_u32(fp, "tensor rank")
if rank != len(item.shape):
raise ValueError(f"{name}: unexpected rank {rank}")
shape = tuple(read_u64(fp, "dimension") for _ in range(rank))
kind, offset = read_u32(fp, "tensor type"), read_u64(fp, "tensor offset")
if (name, shape, kind, offset) != (item.name, item.shape, item.qtype, item.offset):
raise ValueError(f"{item.name}: tensor layout mismatch")
start = align(fp.tell(), GGUF_ALIGNMENT)
size = start + plan[-1].offset + align(plan[-1].nbytes, GGUF_ALIGNMENT)
if os.fstat(fp.fileno()).st_size != size:
raise ValueError(f"expected file size {size}")
print(f"PASS: {len(plan)} tensor layouts; complete {size}-byte GGUF", flush=True)
if args.payload:
q = NativeQuantizer(args.quants_library)
imatrix = Imatrix(args.imatrix, q.np)
if args.imatrix and any(t.is_expert and t.name not in imatrix.entries for t in plan):
raise ValueError("imatrix is missing expert tensors")
for index, item in enumerate(plan):
check_payload(fp, start + item.offset, item, db, q, imatrix)
if (index + 1) % 25 == 0 or index + 1 == len(plan):
print(f"PASS: source payload checks {index + 1}/{len(plan)}", flush=True)
finally:
db.close()
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--hf", required=True)
parser.add_argument("--gguf", required=True)
parser.add_argument("--source-revision", required=True)
parser.add_argument("--quant", choices=QUANTIZATION, default="q2")
parser.add_argument("--imatrix")
parser.add_argument("--payload", action="store_true")
suffix = "dylib" if sys.platform == "darwin" else "so"
parser.add_argument("--quants-library", default=str(Path(__file__).with_name(f"libds4quants.{suffix}")))
args = parser.parse_args()
try:
validate(args)
except (OSError, ValueError) as error:
sys.exit(f"deepseek41-validate: {error}")
gguf-tools/deepseek41_vision.py
+143
View File
@@ -0,0 +1,143 @@
#!/usr/bin/env python3
"""Copy the released DeepSeek V4.1 Flash vision encoder into a separate GGUF.
All 266 tensors remain BF16, byte for byte. Visual router biases are already
part of the language GGUF produced by deepseek41_quantize.py.
"""
import argparse
import json
from pathlib import Path
import sys
from deepseek4_vision import create_gguf, print_summary, validate_gguf
from glm53_quantize import (
GGUF_ALIGNMENT, QTYPE_BF16, SourceDB, TensorPlan, align,
kv_f32, kv_string, kv_u32, qtype_nbytes,
)
SOURCE_REVISION = "df42c109f1defefcbfcedbe7d905718a12266e40"
SOURCE_URL = "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash"
def source_shapes():
shapes = {
"vision.patch_embed.proj.weight": (1024, 588),
"vision.patch_embed.proj.bias": (1024,),
"vision.norm.weight": (1024,),
"aligner.w1.weight": (5120, 9216),
"aligner.w1.bias": (5120,),
"aligner.w2.weight": (5120, 5120),
"aligner.w2.bias": (5120,),
"image_start": (5120,),
"image_newline": (5120,),
"image_end": (5120,),
}
for layer in range(32):
for suffix, shape in {
"norm1.weight": (1024,), "norm2.weight": (1024,),
"attn.wqkv.weight": (3072, 1024), "attn.wqkv.bias": (3072,),
"attn.wo.weight": (1024, 1024), "attn.wo.bias": (1024,),
"mlp.w1.weight": (5632, 1024), "mlp.w2.weight": (1024, 2816),
}.items():
shapes[f"vision.blocks.{layer}.{suffix}"] = shape
return shapes
def validate_inventory(tensors):
selected = {name for name in tensors if name.startswith(("vision.", "aligner.", "image_"))}
expected = set(source_shapes())
if selected != expected:
raise ValueError(f"vision inventory mismatch: missing {sorted(expected - selected)}, "
f"unexpected {sorted(selected - expected)}")
def build_plan(db):
validate_inventory(db.tensors)
plan, offset = [], 0
for name, shape in sorted(source_shapes().items()):
info = db.info(name)
if info["dtype"] != "BF16" or tuple(info["shape"]) != shape:
raise ValueError(f"{name}: expected BF16 {shape}, found {info['dtype']} {info['shape']}")
role = "vision" if name.startswith("vision.") else (
"aligner" if name.startswith("aligner.") else "image_embedding")
item = TensorPlan(name, tuple(reversed(shape)), QTYPE_BF16, role,
source=name, raw_copy=True)
item.nbytes = qtype_nbytes(item.qtype, item.shape)
if item.nbytes != info["nbytes"]:
raise ValueError(f"{name}: source byte size mismatch")
item.offset = offset
offset += align(item.nbytes)
plan.append(item)
return plan
def sidecar_metadata(hf_dir):
config = json.loads((Path(hf_dir) / "config.json").read_text())
if config.get("model_type") != "deepseek_v41" or config.get("image_token_id") != 129264:
raise ValueError("expected the DeepSeek V4.1 Flash checkpoint")
for section, expected in {
"text_config": {"hidden_size": 5120, "num_hidden_layers": 40, "n_routed_experts": 384},
"vision_config": {"num_hidden_layers": 32, "hidden_size": 1024,
"num_attention_heads": 16, "intermediate_size": 2816, "patch_size": 14,
"rope_theta": 10000, "downsample_ratio": 3, "max_image_tokens": 1024,
"min_pixels": 295936, "max_wh_ratio": None},
}.items():
for key, value in expected.items():
if key not in config[section] or config[section][key] != value:
raise ValueError(f"unexpected {section}.{key}: {config[section].get(key)!r}")
prefix = "deepseek4-vision"
records = [
kv_string("general.architecture", prefix),
kv_string("general.name", "DeepSeek V4.1 Flash Vision Encoder"),
kv_u32("general.alignment", GGUF_ALIGNMENT),
kv_string("general.source.url", SOURCE_URL),
kv_string("general.source.revision", SOURCE_REVISION),
kv_string(f"{prefix}.checkpoint_variant", "v4.1-flash"),
kv_f32(f"{prefix}.rope.freq_base", 10000),
kv_f32(f"{prefix}.attention.layer_norm_rms_epsilon", 1e-6),
]
for key, value in {
"block_count": 32, "embedding_length": 1024, "feed_forward_length": 2816,
"attention.head_count": 16, "projection_length": 5120, "patch_size": 14,
"downsample_ratio": 3, "image.max_tokens": 1024, "image.min_pixels": 295936,
"image.max_width_height_ratio": 0, "image.token_id": 129264,
"language.block_count": 40, "language.expert_count": 384,
}.items():
records.append(kv_u32(f"{prefix}.{key}", value))
return records
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--hf", required=True, help="official V4.1 Flash snapshot")
output = parser.add_mutually_exclusive_group(required=True)
output.add_argument("--out", help="write this vision GGUF")
output.add_argument("--validate", metavar="GGUF")
output.add_argument("--dry-run", action="store_true")
parser.add_argument("--verify-payload", action="store_true")
parser.add_argument("--expected-sha256")
args = parser.parse_args()
if (args.verify_payload or args.expected_sha256) and not args.validate:
parser.error("--verify-payload and --expected-sha256 require --validate")
records = sidecar_metadata(args.hf)
db = SourceDB(args.hf, validate_inventory, lambda _: None)
try:
plan = build_plan(db)
if args.out:
create_gguf(args.out, plan, records, db, overwrite=False)
elif args.validate:
validate_gguf(args.validate, plan, records, db,
args.verify_payload, args.expected_sha256)
else:
print_summary(plan, records)
finally:
db.close()
if __name__ == "__main__":
try:
main()
except (KeyError, OSError, ValueError) as error:
print(f"deepseek41-vision: {error}", file=sys.stderr)
raise SystemExit(1)
gguf-tools/glm53_quantize.py
+3 -3
View File
@@ -913,7 +913,7 @@ def tensor_header(item):
)
def print_plan(plan, kv_records, tokenizer_records):
def print_plan(plan, kv_records, tokenizer_records, alignment=GGUF_ALIGNMENT):
by_type = {}
by_role = {}
for item in plan:
@@ -921,8 +921,8 @@ def print_plan(plan, kv_records, tokenizer_records):
by_role[item.role] = by_role.get(item.role, 0) + item.nbytes
kv_bytes = sum(map(len, kv_records)) + sum(map(len, tokenizer_records))
tensor_info_bytes = sum(len(tensor_header(item)) for item in plan)
data_offset = align(4 + 4 + 8 + 8 + kv_bytes + tensor_info_bytes)
data_bytes = sum(align(item.nbytes) for item in plan)
data_offset = align(4 + 4 + 8 + 8 + kv_bytes + tensor_info_bytes, alignment)
data_bytes = sum(align(item.nbytes, alignment) for item in plan)
print(f"tensors: {len(plan)}")
print(f"metadata_records: {len(kv_records) + len(tokenizer_records)}")
print(f"metadata_bytes: {data_offset}")
gguf-tools/quality-testing/README.md
+65
View File
@@ -154,6 +154,71 @@ Use `data/flash/manifest.tsv` for Flash GGUFs and
care which model produced the manifest; the manifest path selects the
continuation set.
Add `--continued-prefill N` to process each prompt in two calls, leaving its
last `N` tokens for the second call. This checks continued-prefill quality
against the same official answers. Every prompt must contain more than `N`
tokens; otherwise the scorer fails instead of silently skipping the test.
Add `--session-batch N` (2 to 8) to score an official continuation alongside
unrelated, independently advancing sessions. The scorer rotates their row
order on every step. Compare against a run without this option using the same
manifest, model and context; budget memory for all `N` sessions.
Check session isolation separately on a dedicated Metal host:
```sh
DS4_TEST_MODEL=MODEL.gguf DS4_TEST_BATCH_ISOLATION=1 \
DS4_TEST_SESSION_COUNT=4 DS4_TEST_DECODE_STEPS=32 \
MTL_DEBUG_LAYER=1 ./tests/test_metal_session_batch
```
This requires exact logits when companion prompts and row order change, then
checks a continued prefill and resumed serial decoding. It also checks that
invalid batches leave the target unchanged. It does not replace serial/batch
quality comparisons: different arithmetic can round differently without
sessions contaminating one another. Use `DS4_TEST_PROMPT_FILE` and
`DS4_TEST_CONTEXT_SIZE` to repeat at longer prefixes. For physical TP, the test
also accepts `DS4_TEST_TP_RDMA_DEVICE` and `DS4_TEST_TP_GID_INDEX` alongside
its coordinator/worker settings. Throughput measurements must run separately
without Metal API validation.
For Metal V4.1 prefill scheduling changes, also run:
```sh
make tests/test_deepseek41_prefill
./tests/test_deepseek41_prefill --dispatch
MTL_DEBUG_LAYER=1 ./tests/test_deepseek41_prefill MODEL.gguf speed-bench/promessi_sposi.txt
```
The model test uses SSD streaming and two 128K sessions. Run it alone on a
dedicated host with at least 128 GiB RAM. It alternates small and large appends
through 113K context, checking dispatch, progress, unchanged-prefix reuse,
saved state and subsequent decoding against a control without decoder deferral.
Keep the official continuation checks too: this scheduling test does not judge
quality across different floating-point operation orders. Measure speed without
Metal API validation, separately for initial and continued prefills.
Run the same mixed-prefix test over physical RDMA on two dedicated Metal hosts.
Start the worker with the same model and a 131072-token context, then the test
on the coordinator (replace the host and RDMA device names):
```sh
# Worker
MTL_DEBUG_LAYER=1 ./ds4 -m MODEL.gguf --ctx 131072 \
--tensor-parallel --role worker --coordinator COORDINATOR 19455 \
--transport rdma --rdma-device WORKER_DEVICE --rdma-gid-index 1
# Coordinator
MTL_DEBUG_LAYER=1 ./tests/test_deepseek41_prefill --tensor-parallel \
MODEL.gguf speed-bench/promessi_sposi.txt COORDINATOR 19455 COORDINATOR_DEVICE 1
```
This compares equal TP prefill partitions and queued versus synchronous decode,
including full logits and cache state. TP snapshots rebuild both ranks from
tokens, so restoration is compared with an equivalent fresh replay. Also run
the official scorer in TP mode for initial and continued prompts; scheduling
parity alone does not establish model quality.
Add `--quality` to disable DS4's speed-oriented numerical shortcuts. For an
independent llama.cpp comparison of a DeepSeek V4 GGUF, use the same manifest
and the token-identical DS4 prompt renderer:
gguf-tools/quality-testing/collect_official.py
+12 -2
View File
@@ -5,6 +5,7 @@ from __future__ import annotations
import argparse
import json
import math
import os
import sys
import time
@@ -134,11 +135,12 @@ def request_one(
provider_order: list[str],
provider_allow_fallbacks: bool,
provider_require_parameters: bool,
temperature: float = 0,
) -> dict:
payload = {
"model": model,
"messages": [{"role": "user", "content": prompt}],
"temperature": 0,
"temperature": temperature,
"stream": False,
}
if top_logprobs > 0:
@@ -181,6 +183,7 @@ def fetch_with_retry(
provider_order: list[str],
provider_allow_fallbacks: bool,
provider_require_parameters: bool,
temperature: float = 0,
) -> dict:
delay = 1.0
for attempt in range(6):
@@ -198,6 +201,7 @@ def fetch_with_retry(
provider_order,
provider_allow_fallbacks,
provider_require_parameters,
temperature,
)
except urllib.error.HTTPError as e:
body = e.read().decode("utf-8", "replace")
@@ -223,6 +227,9 @@ def main() -> int:
ap.add_argument("--count", type=int, default=100)
ap.add_argument("--max-tokens", type=int, default=24)
ap.add_argument("--top-logprobs", type=int, default=5)
ap.add_argument("--temperature", type=float, default=0,
help="sampling temperature, 0..2; use 1 for unscaled logprobs "
"from providers reporting probabilities after sampling transforms")
ap.add_argument("--thinking", choices=("disabled", "enabled", "omit"), default="disabled")
ap.add_argument("--reasoning-effort",
choices=("xhigh", "high", "medium", "low", "minimal", "none", "omit"),
@@ -244,6 +251,8 @@ def main() -> int:
args = ap.parse_args()
if args.top_logprobs < 0 or args.top_logprobs > 20:
raise SystemExit("--top-logprobs must be between 0 and 20")
if not math.isfinite(args.temperature) or not 0 <= args.temperature <= 2:
raise SystemExit("--temperature must be between 0 and 2")
openrouter = "openrouter.ai" in args.endpoint
api_key_env = args.api_key_env or ("OPENROUTER_API_KEY" if openrouter else "DEEPSEEK_API_KEY")
@@ -312,6 +321,7 @@ def main() -> int:
provider_order,
args.allow_provider_fallbacks,
provider_require_parameters,
args.temperature,
)
choice = response["choices"][0]
content = choice.get("message", {}).get("content")
@@ -362,7 +372,7 @@ def main() -> int:
"endpoint": args.endpoint,
"observed_models": sorted(observed_models),
"observed_providers": sorted(observed_providers),
"temperature": 0,
"temperature": args.temperature,
"max_tokens": args.max_tokens,
"top_logprobs": args.top_logprobs,
"thinking": thinking,
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/README.md
+23
View File
@@ -0,0 +1,23 @@
# DeepSeek V4.1 Flash batched-prefill continuations
Twelve held-out prompts with approximately 384, 768, 1536 and 4096 tokens,
covering C bounds checking, network-server design and Italian prose. The
archive background varies in length; each size is tested with all three tasks.
These prompts exercise matrix-prefill and sparse attention, unlike the much
shorter prompts in the general set.
Collected from the official DeepSeek API on 2026-09-10 using `deepseek-flash`,
thinking disabled, temperature 1, up to 64 output tokens and top-20 logprobs.
All responses have fingerprint `aeb56401ca74e127821c4f9126dcb669`, matching the
other V4.1 sets. The reference weight revision is
`df42c109f1defefcbfcedbe7d905718a12266e40`.
These are sampled continuations, not greedy answers or full-vocabulary API
logits. Compare teacher-forced likelihood and API token probabilities. Retain
the separate sparse-boundary, snapshot, cancellation and coding-session tests.
```sh
./gguf-tools/quality-testing/score_official MODEL.gguf \
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/manifest.tsv \
scores.tsv 6144 --ssd-streaming --ssd-streaming-cache-experts 64gb
```
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/collection.json
+24
View File
@@ -0,0 +1,24 @@
{
"model": "deepseek-flash",
"endpoint": "https://api.deepseek.com/chat/completions",
"fingerprint": "aeb56401ca74e127821c4f9126dcb669",
"thinking": "disabled",
"temperature": 1,
"max_tokens": 64,
"top_logprobs": 20,
"prompt_tokens": [
388,
389,
389,
772,
773,
772,
1540,
1540,
1541,
4100,
4101,
4101
],
"purpose": "held-out batched prefill quality"
}
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/continuations/case_000.txt
+7
View File
@@ -0,0 +1,7 @@
Looking at this C code, the bounds bug is clear:
```c
int lookup(int *a, int n, int index) { if (index > n) return -1; return a[index]; }
```
**The bug:** The check `index > n` allows `index == n` to pass through
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/continuations/case_001.txt
+1
View File
@@ -0,0 +1 @@
A TCP server must preserve a partially written reply when a client reads slowly because TCP provides a reliable, ordered byte stream with flow control, and the server’s write operation may only accept part of the data if the client’s receive window is full. If the server discards or overwrites the unsent portion instead
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/continuations/case_002.txt
+7
View File
@@ -0,0 +1,7 @@
Ecco un breve racconto in italiano:
---
**Il registro difettoso**
Marta era la nuova tecnica del magazzino, assunta da poco per digitalizzare i vecchi registri cartacei. Una mattina, mentre inseriva i dati nell'archivio, notò qualcosa
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/continuations/case_003.txt
+9
View File
@@ -0,0 +1,9 @@
Looking at this C code:
```c
int lookup(int *a, int n, int index) { if (index > n) return -1; return a[index]; }
```
## The Bug
The bounds check `if (index > n)` is **off by one**. For an array `a`
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/continuations/case_004.txt
+1
View File
@@ -0,0 +1 @@
A TCP server must preserve a partially written reply when a client reads slowly because TCP send buffers are finite and the application cannot assume that a single `write` or `send` call will deliver the entire response at once. If the client’s receive window fills up, the server’s socket buffer may accept only part
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/continuations/case_005.txt
+5
View File
@@ -0,0 +1,5 @@
Ecco un breve racconto in italiano:
---
Il tecnico Elia lavorava da anni nel magazzino sotterraneo, dove i pacchi scorrevano su nastri silenziosi e ogni movimento veniva annotato in un registro infinito. Un pomeriggio, mentre controllava
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/continuations/case_006.txt
+7
View File
@@ -0,0 +1,7 @@
Looking at this code, there's an off-by-one bounds bug:
```c
int lookup(int *a, int n, int index) { if (index > n) return -1; return a[index]; }
```
**The bug:** The check `index > n` rejects only indices *strictly greater
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/continuations/case_007.txt
+1
View File
@@ -0,0 +1 @@
A TCP server must preserve a partially written reply when a client reads slowly because TCP guarantees reliable, in-order delivery of a byte stream, not message boundaries, and the server's send buffer may fill up if the client application isn't reading quickly enough. When a write can only complete partially (for example, a nonblock
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/continuations/case_008.txt
+3
View File
@@ -0,0 +1,3 @@
Ecco un breve racconto in italiano:
Il tecnico anziano, il signor Bruno, passava le sue giornate tra gli scaffali alti del magazzino, con il suo camice grigio e una torcia sempre in tasca. Un pomeriggio, mentre controllava il
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/continuations/case_009.txt
+6
View File
@@ -0,0 +1,6 @@
Looking at the background data, I can see the pattern of entries is useful context — they demonstrate a strong theme of **bounded/checked indexing** (every entry has a "status checked" note), which is exactly the kind of guarantee this C function fails to provide.
## The Bug
```c
int lookup
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/continuations/case_010.txt
+1
View File
@@ -0,0 +1 @@
A TCP server must preserve a partially written reply when a client reads slowly because TCP send buffers are finite, so if the server tries to write more data than the client has acknowledged and the buffer fills, the send operation can only complete part of the reply. Since TCP is a byte-stream protocol with no message boundaries, the
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/continuations/case_011.txt
+5
View File
@@ -0,0 +1,5 @@
Ecco un breve racconto in italiano:
**Il registro difettoso**
Era lunedì mattina quando Elia, tecnico informatico della ditta, ricevette una segnalazione insolita: il registro del magazzino mostrava numeri che non tornavano. I pacchi sped
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/manifest.tsv
+13
View File
@@ -0,0 +1,13 @@
# id prompt_file continuation_file response_file
case_000 gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/prompts/case_000.txt gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/continuations/case_000.txt gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/responses/case_000.json
case_001 gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/prompts/case_001.txt gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/continuations/case_001.txt gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/responses/case_001.json
case_002 gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/prompts/case_002.txt gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/continuations/case_002.txt gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/responses/case_002.json
case_003 gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/prompts/case_003.txt gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/continuations/case_003.txt gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/responses/case_003.json
case_004 gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/prompts/case_004.txt gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/continuations/case_004.txt gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/responses/case_004.json
case_005 gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/prompts/case_005.txt gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/continuations/case_005.txt gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/responses/case_005.json
case_006 gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/prompts/case_006.txt gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/continuations/case_006.txt gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/responses/case_006.json
case_007 gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/prompts/case_007.txt gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/continuations/case_007.txt gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/responses/case_007.json
case_008 gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/prompts/case_008.txt gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/continuations/case_008.txt gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/responses/case_008.json
case_009 gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/prompts/case_009.txt gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/continuations/case_009.txt gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/responses/case_009.json
case_010 gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/prompts/case_010.txt gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/continuations/case_010.txt gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/responses/case_010.json
case_011 gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/prompts/case_011.txt gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/continuations/case_011.txt gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/responses/case_011.json
1 # id prompt_file continuation_file response_file
2 case_000 gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/prompts/case_000.txt gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/continuations/case_000.txt gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/responses/case_000.json
3 case_001 gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/prompts/case_001.txt gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/continuations/case_001.txt gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/responses/case_001.json
4 case_002 gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/prompts/case_002.txt gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/continuations/case_002.txt gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/responses/case_002.json
5 case_003 gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/prompts/case_003.txt gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/continuations/case_003.txt gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/responses/case_003.json
6 case_004 gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/prompts/case_004.txt gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/continuations/case_004.txt gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/responses/case_004.json
7 case_005 gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/prompts/case_005.txt gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/continuations/case_005.txt gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/responses/case_005.json
8 case_006 gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/prompts/case_006.txt gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/continuations/case_006.txt gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/responses/case_006.json
9 case_007 gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/prompts/case_007.txt gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/continuations/case_007.txt gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/responses/case_007.json
10 case_008 gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/prompts/case_008.txt gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/continuations/case_008.txt gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/responses/case_008.json
11 case_009 gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/prompts/case_009.txt gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/continuations/case_009.txt gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/responses/case_009.json
12 case_010 gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/prompts/case_010.txt gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/continuations/case_010.txt gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/responses/case_010.json
13 case_011 gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/prompts/case_011.txt gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/continuations/case_011.txt gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/responses/case_011.json
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/prompts/case_000.txt
+20
View File
@@ -0,0 +1,20 @@
The following is background from an archive. Read it, then answer the final task.
Entry 0000: workshop 000 sent package 0000 to desk 000; status checked.
Entry 0001: workshop 017 sent package 0029 to desk 011; status checked.
Entry 0002: workshop 034 sent package 0058 to desk 022; status checked.
Entry 0003: workshop 051 sent package 0087 to desk 033; status checked.
Entry 0004: workshop 068 sent package 0116 to desk 044; status checked.
Entry 0005: workshop 085 sent package 0145 to desk 055; status checked.
Entry 0006: workshop 102 sent package 0174 to desk 066; status checked.
Entry 0007: workshop 119 sent package 0203 to desk 077; status checked.
Entry 0008: workshop 136 sent package 0232 to desk 088; status checked.
Entry 0009: workshop 153 sent package 0261 to desk 099; status checked.
Entry 0010: workshop 170 sent package 0290 to desk 110; status checked.
Entry 0011: workshop 187 sent package 0319 to desk 121; status checked.
Entry 0012: workshop 204 sent package 0348 to desk 132; status checked.
Entry 0013: workshop 221 sent package 0377 to desk 143; status checked.
Entry 0014: workshop 238 sent package 0406 to desk 154; status checked.
Entry 0015:
Final task:
Review the following C code and show a corrected version, explaining the bounds bug:
int lookup(int *a, int n, int index) { if (index > n) return -1; return a[index]; }
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/prompts/case_001.txt
+20
View File
@@ -0,0 +1,20 @@
The following is background from an archive. Read it, then answer the final task.
Entry 0000: workshop 000 sent package 0000 to desk 000; status checked.
Entry 0001: workshop 017 sent package 0029 to desk 011; status checked.
Entry 0002: workshop 034 sent package 0058 to desk 022; status checked.
Entry 0003: workshop 051 sent package 0087 to desk 033; status checked.
Entry 0004: workshop 068 sent package 0116 to desk 044; status checked.
Entry 0005: workshop 085 sent package 0145 to desk 055; status checked.
Entry 0006: workshop 102 sent package 0174 to desk 066; status checked.
Entry 0007: workshop 119 sent package 0203 to desk 077; status checked.
Entry 0008: workshop 136 sent package 0232 to desk 088; status checked.
Entry 0009: workshop 153 sent package 0261 to desk 099; status checked.
Entry 0010: workshop 170 sent package 0290 to desk 110; status checked.
Entry 0011: workshop 187 sent package 0319 to desk 121; status checked.
Entry 0012: workshop 204 sent package 0348 to desk 132; status checked.
Entry 0013: workshop 221 sent package 0377 to desk 143; status checked.
Entry 0014: workshop 238 sent package 0406 to desk 154; status checked.
Entry 0015: workshop 255 sent package 0435 to desk 165; status checked.
Entry 0016: workshop 272
Final task:
Explain in one paragraph why a TCP server must preserve a partially written reply when a client reads slowly.
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/prompts/case_002.txt
+19
View File
@@ -0,0 +1,19 @@
The following is background from an archive. Read it, then answer the final task.
Entry 0000: workshop 000 sent package 0000 to desk 000; status checked.
Entry 0001: workshop 017 sent package 0029 to desk 011; status checked.
Entry 0002: workshop 034 sent package 0058 to desk 022; status checked.
Entry 0003: workshop 051 sent package 0087 to desk 033; status checked.
Entry 0004: workshop 068 sent package 0116 to desk 044; status checked.
Entry 0005: workshop 085 sent package 0145 to desk 055; status checked.
Entry 0006: workshop 102 sent package 0174 to desk 066; status checked.
Entry 0007: workshop 119 sent package 0203 to desk 077; status checked.
Entry 0008: workshop 136 sent package 0232 to desk 088; status checked.
Entry 0009: workshop 153 sent package 0261 to desk 099; status checked.
Entry 0010: workshop 170 sent package 0290 to desk 110; status checked.
Entry 0011: workshop 187 sent package 0319 to desk 121; status checked.
Entry 0012: workshop 204 sent package 0348 to desk 132; status checked.
Entry 0013: workshop 221 sent package 0377 to desk 143; status checked.
Entry 0014: workshop 238 sent package 0406 to desk 154; status checked.
Entry 0015: workshop 255 sent package 0435 to desk 165; status checked
Final task:
Scrivi in italiano un breve racconto su un tecnico che trova e corregge un errore nel registro del magazzino.
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/prompts/case_003.txt
+38
View File
@@ -0,0 +1,38 @@
The following is background from an archive. Read it, then answer the final task.
Entry 0000: workshop 000 sent package 0000 to desk 000; status checked.
Entry 0001: workshop 017 sent package 0029 to desk 011; status checked.
Entry 0002: workshop 034 sent package 0058 to desk 022; status checked.
Entry 0003: workshop 051 sent package 0087 to desk 033; status checked.
Entry 0004: workshop 068 sent package 0116 to desk 044; status checked.
Entry 0005: workshop 085 sent package 0145 to desk 055; status checked.
Entry 0006: workshop 102 sent package 0174 to desk 066; status checked.
Entry 0007: workshop 119 sent package 0203 to desk 077; status checked.
Entry 0008: workshop 136 sent package 0232 to desk 088; status checked.
Entry 0009: workshop 153 sent package 0261 to desk 099; status checked.
Entry 0010: workshop 170 sent package 0290 to desk 110; status checked.
Entry 0011: workshop 187 sent package 0319 to desk 121; status checked.
Entry 0012: workshop 204 sent package 0348 to desk 132; status checked.
Entry 0013: workshop 221 sent package 0377 to desk 143; status checked.
Entry 0014: workshop 238 sent package 0406 to desk 154; status checked.
Entry 0015: workshop 255 sent package 0435 to desk 165; status checked.
Entry 0016: workshop 272 sent package 0464 to desk 176; status checked.
Entry 0017: workshop 289 sent package 0493 to desk 187; status checked.
Entry 0018: workshop 306 sent package 0522 to desk 198; status checked.
Entry 0019: workshop 323 sent package 0551 to desk 209; status checked.
Entry 0020: workshop 340 sent package 0580 to desk 220; status checked.
Entry 0021: workshop 357 sent package 0609 to desk 231; status checked.
Entry 0022: workshop 374 sent package 0638 to desk 242; status checked.
Entry 0023: workshop 391 sent package 0667 to desk 253; status checked.
Entry 0024: workshop 408 sent package 0696 to desk 007; status checked.
Entry 0025: workshop 425 sent package 0725 to desk 018; status checked.
Entry 0026: workshop 442 sent package 0754 to desk 029; status checked.
Entry 0027: workshop 459 sent package 0783 to desk 040; status checked.
Entry 0028: workshop 476 sent package 0812 to desk 051; status checked.
Entry 0029: workshop 493 sent package 0841 to desk 062; status checked.
Entry 0030: workshop 510 sent package 0870 to desk 073; status checked.
Entry 0031: workshop 527 sent package 0899 to desk 084; status checked.
Entry 0032: workshop 544 sent package 0928 to desk 095; status checked.
Entry 0033: workshop 561 sent package
Final task:
Review the following C code and show a corrected version, explaining the bounds bug:
int lookup(int *a, int n, int index) { if (index > n) return -1; return a[index]; }
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/prompts/case_004.txt
+38
View File
@@ -0,0 +1,38 @@
The following is background from an archive. Read it, then answer the final task.
Entry 0000: workshop 000 sent package 0000 to desk 000; status checked.
Entry 0001: workshop 017 sent package 0029 to desk 011; status checked.
Entry 0002: workshop 034 sent package 0058 to desk 022; status checked.
Entry 0003: workshop 051 sent package 0087 to desk 033; status checked.
Entry 0004: workshop 068 sent package 0116 to desk 044; status checked.
Entry 0005: workshop 085 sent package 0145 to desk 055; status checked.
Entry 0006: workshop 102 sent package 0174 to desk 066; status checked.
Entry 0007: workshop 119 sent package 0203 to desk 077; status checked.
Entry 0008: workshop 136 sent package 0232 to desk 088; status checked.
Entry 0009: workshop 153 sent package 0261 to desk 099; status checked.
Entry 0010: workshop 170 sent package 0290 to desk 110; status checked.
Entry 0011: workshop 187 sent package 0319 to desk 121; status checked.
Entry 0012: workshop 204 sent package 0348 to desk 132; status checked.
Entry 0013: workshop 221 sent package 0377 to desk 143; status checked.
Entry 0014: workshop 238 sent package 0406 to desk 154; status checked.
Entry 0015: workshop 255 sent package 0435 to desk 165; status checked.
Entry 0016: workshop 272 sent package 0464 to desk 176; status checked.
Entry 0017: workshop 289 sent package 0493 to desk 187; status checked.
Entry 0018: workshop 306 sent package 0522 to desk 198; status checked.
Entry 0019: workshop 323 sent package 0551 to desk 209; status checked.
Entry 0020: workshop 340 sent package 0580 to desk 220; status checked.
Entry 0021: workshop 357 sent package 0609 to desk 231; status checked.
Entry 0022: workshop 374 sent package 0638 to desk 242; status checked.
Entry 0023: workshop 391 sent package 0667 to desk 253; status checked.
Entry 0024: workshop 408 sent package 0696 to desk 007; status checked.
Entry 0025: workshop 425 sent package 0725 to desk 018; status checked.
Entry 0026: workshop 442 sent package 0754 to desk 029; status checked.
Entry 0027: workshop 459 sent package 0783 to desk 040; status checked.
Entry 0028: workshop 476 sent package 0812 to desk 051; status checked.
Entry 0029: workshop 493 sent package 0841 to desk 062; status checked.
Entry 0030: workshop 510 sent package 0870 to desk 073; status checked.
Entry 0031: workshop 527 sent package 0899 to desk 084; status checked.
Entry 0032: workshop 544 sent package 0928 to desk 095; status checked.
Entry 0033: workshop 561 sent package 0957 to desk 106; status checked.
Entry 0034: workshop 578 sent package 0986 to
Final task:
Explain in one paragraph why a TCP server must preserve a partially written reply when a client reads slowly.
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/prompts/case_005.txt
+38
View File
@@ -0,0 +1,38 @@
The following is background from an archive. Read it, then answer the final task.
Entry 0000: workshop 000 sent package 0000 to desk 000; status checked.
Entry 0001: workshop 017 sent package 0029 to desk 011; status checked.
Entry 0002: workshop 034 sent package 0058 to desk 022; status checked.
Entry 0003: workshop 051 sent package 0087 to desk 033; status checked.
Entry 0004: workshop 068 sent package 0116 to desk 044; status checked.
Entry 0005: workshop 085 sent package 0145 to desk 055; status checked.
Entry 0006: workshop 102 sent package 0174 to desk 066; status checked.
Entry 0007: workshop 119 sent package 0203 to desk 077; status checked.
Entry 0008: workshop 136 sent package 0232 to desk 088; status checked.
Entry 0009: workshop 153 sent package 0261 to desk 099; status checked.
Entry 0010: workshop 170 sent package 0290 to desk 110; status checked.
Entry 0011: workshop 187 sent package 0319 to desk 121; status checked.
Entry 0012: workshop 204 sent package 0348 to desk 132; status checked.
Entry 0013: workshop 221 sent package 0377 to desk 143; status checked.
Entry 0014: workshop 238 sent package 0406 to desk 154; status checked.
Entry 0015: workshop 255 sent package 0435 to desk 165; status checked.
Entry 0016: workshop 272 sent package 0464 to desk 176; status checked.
Entry 0017: workshop 289 sent package 0493 to desk 187; status checked.
Entry 0018: workshop 306 sent package 0522 to desk 198; status checked.
Entry 0019: workshop 323 sent package 0551 to desk 209; status checked.
Entry 0020: workshop 340 sent package 0580 to desk 220; status checked.
Entry 0021: workshop 357 sent package 0609 to desk 231; status checked.
Entry 0022: workshop 374 sent package 0638 to desk 242; status checked.
Entry 0023: workshop 391 sent package 0667 to desk 253; status checked.
Entry 0024: workshop 408 sent package 0696 to desk 007; status checked.
Entry 0025: workshop 425 sent package 0725 to desk 018; status checked.
Entry 0026: workshop 442 sent package 0754 to desk 029; status checked.
Entry 0027: workshop 459 sent package 0783 to desk 040; status checked.
Entry 0028: workshop 476 sent package 0812 to desk 051; status checked.
Entry 0029: workshop 493 sent package 0841 to desk 062; status checked.
Entry 0030: workshop 510 sent package 0870 to desk 073; status checked.
Entry 0031: workshop 527 sent package 0899 to desk 084; status checked.
Entry 0032: workshop 544 sent package 0928 to desk 095; status checked.
Entry 0033: workshop 561 sent package 0957 to desk 106; status checked.
Entry 0034:
Final task:
Scrivi in italiano un breve racconto su un tecnico che trova e corregge un errore nel registro del magazzino.
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/prompts/case_006.txt
+75
View File
@@ -0,0 +1,75 @@
The following is background from an archive. Read it, then answer the final task.
Entry 0000: workshop 000 sent package 0000 to desk 000; status checked.
Entry 0001: workshop 017 sent package 0029 to desk 011; status checked.
Entry 0002: workshop 034 sent package 0058 to desk 022; status checked.
Entry 0003: workshop 051 sent package 0087 to desk 033; status checked.
Entry 0004: workshop 068 sent package 0116 to desk 044; status checked.
Entry 0005: workshop 085 sent package 0145 to desk 055; status checked.
Entry 0006: workshop 102 sent package 0174 to desk 066; status checked.
Entry 0007: workshop 119 sent package 0203 to desk 077; status checked.
Entry 0008: workshop 136 sent package 0232 to desk 088; status checked.
Entry 0009: workshop 153 sent package 0261 to desk 099; status checked.
Entry 0010: workshop 170 sent package 0290 to desk 110; status checked.
Entry 0011: workshop 187 sent package 0319 to desk 121; status checked.
Entry 0012: workshop 204 sent package 0348 to desk 132; status checked.
Entry 0013: workshop 221 sent package 0377 to desk 143; status checked.
Entry 0014: workshop 238 sent package 0406 to desk 154; status checked.
Entry 0015: workshop 255 sent package 0435 to desk 165; status checked.
Entry 0016: workshop 272 sent package 0464 to desk 176; status checked.
Entry 0017: workshop 289 sent package 0493 to desk 187; status checked.
Entry 0018: workshop 306 sent package 0522 to desk 198; status checked.
Entry 0019: workshop 323 sent package 0551 to desk 209; status checked.
Entry 0020: workshop 340 sent package 0580 to desk 220; status checked.
Entry 0021: workshop 357 sent package 0609 to desk 231; status checked.
Entry 0022: workshop 374 sent package 0638 to desk 242; status checked.
Entry 0023: workshop 391 sent package 0667 to desk 253; status checked.
Entry 0024: workshop 408 sent package 0696 to desk 007; status checked.
Entry 0025: workshop 425 sent package 0725 to desk 018; status checked.
Entry 0026: workshop 442 sent package 0754 to desk 029; status checked.
Entry 0027: workshop 459 sent package 0783 to desk 040; status checked.
Entry 0028: workshop 476 sent package 0812 to desk 051; status checked.
Entry 0029: workshop 493 sent package 0841 to desk 062; status checked.
Entry 0030: workshop 510 sent package 0870 to desk 073; status checked.
Entry 0031: workshop 527 sent package 0899 to desk 084; status checked.
Entry 0032: workshop 544 sent package 0928 to desk 095; status checked.
Entry 0033: workshop 561 sent package 0957 to desk 106; status checked.
Entry 0034: workshop 578 sent package 0986 to desk 117; status checked.
Entry 0035: workshop 595 sent package 0006 to desk 128; status checked.
Entry 0036: workshop 612 sent package 0035 to desk 139; status checked.
Entry 0037: workshop 629 sent package 0064 to desk 150; status checked.
Entry 0038: workshop 646 sent package 0093 to desk 161; status checked.
Entry 0039: workshop 663 sent package 0122 to desk 172; status checked.
Entry 0040: workshop 680 sent package 0151 to desk 183; status checked.
Entry 0041: workshop 697 sent package 0180 to desk 194; status checked.
Entry 0042: workshop 714 sent package 0209 to desk 205; status checked.
Entry 0043: workshop 731 sent package 0238 to desk 216; status checked.
Entry 0044: workshop 748 sent package 0267 to desk 227; status checked.
Entry 0045: workshop 765 sent package 0296 to desk 238; status checked.
Entry 0046: workshop 782 sent package 0325 to desk 249; status checked.
Entry 0047: workshop 799 sent package 0354 to desk 003; status checked.
Entry 0048: workshop 816 sent package 0383 to desk 014; status checked.
Entry 0049: workshop 833 sent package 0412 to desk 025; status checked.
Entry 0050: workshop 850 sent package 0441 to desk 036; status checked.
Entry 0051: workshop 867 sent package 0470 to desk 047; status checked.
Entry 0052: workshop 884 sent package 0499 to desk 058; status checked.
Entry 0053: workshop 901 sent package 0528 to desk 069; status checked.
Entry 0054: workshop 918 sent package 0557 to desk 080; status checked.
Entry 0055: workshop 935 sent package 0586 to desk 091; status checked.
Entry 0056: workshop 952 sent package 0615 to desk 102; status checked.
Entry 0057: workshop 969 sent package 0644 to desk 113; status checked.
Entry 0058: workshop 986 sent package 0673 to desk 124; status checked.
Entry 0059: workshop 006 sent package 0702 to desk 135; status checked.
Entry 0060: workshop 023 sent package 0731 to desk 146; status checked.
Entry 0061: workshop 040 sent package 0760 to desk 157; status checked.
Entry 0062: workshop 057 sent package 0789 to desk 168; status checked.
Entry 0063: workshop 074 sent package 0818 to desk 179; status checked.
Entry 0064: workshop 091 sent package 0847 to desk 190; status checked.
Entry 0065: workshop 108 sent package 0876 to desk 201; status checked.
Entry 0066: workshop 125 sent package 0905 to desk 212; status checked.
Entry 0067: workshop 142 sent package 0934 to desk 223; status checked.
Entry 0068: workshop 159 sent package 0963 to desk 234; status checked.
Entry 0069: workshop 176 sent package 0992 to desk 245; status checked.
Entry
Final task:
Review the following C code and show a corrected version, explaining the bounds bug:
int lookup(int *a, int n, int index) { if (index > n) return -1; return a[index]; }
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/prompts/case_007.txt
+75
View File
@@ -0,0 +1,75 @@
The following is background from an archive. Read it, then answer the final task.
Entry 0000: workshop 000 sent package 0000 to desk 000; status checked.
Entry 0001: workshop 017 sent package 0029 to desk 011; status checked.
Entry 0002: workshop 034 sent package 0058 to desk 022; status checked.
Entry 0003: workshop 051 sent package 0087 to desk 033; status checked.
Entry 0004: workshop 068 sent package 0116 to desk 044; status checked.
Entry 0005: workshop 085 sent package 0145 to desk 055; status checked.
Entry 0006: workshop 102 sent package 0174 to desk 066; status checked.
Entry 0007: workshop 119 sent package 0203 to desk 077; status checked.
Entry 0008: workshop 136 sent package 0232 to desk 088; status checked.
Entry 0009: workshop 153 sent package 0261 to desk 099; status checked.
Entry 0010: workshop 170 sent package 0290 to desk 110; status checked.
Entry 0011: workshop 187 sent package 0319 to desk 121; status checked.
Entry 0012: workshop 204 sent package 0348 to desk 132; status checked.
Entry 0013: workshop 221 sent package 0377 to desk 143; status checked.
Entry 0014: workshop 238 sent package 0406 to desk 154; status checked.
Entry 0015: workshop 255 sent package 0435 to desk 165; status checked.
Entry 0016: workshop 272 sent package 0464 to desk 176; status checked.
Entry 0017: workshop 289 sent package 0493 to desk 187; status checked.
Entry 0018: workshop 306 sent package 0522 to desk 198; status checked.
Entry 0019: workshop 323 sent package 0551 to desk 209; status checked.
Entry 0020: workshop 340 sent package 0580 to desk 220; status checked.
Entry 0021: workshop 357 sent package 0609 to desk 231; status checked.
Entry 0022: workshop 374 sent package 0638 to desk 242; status checked.
Entry 0023: workshop 391 sent package 0667 to desk 253; status checked.
Entry 0024: workshop 408 sent package 0696 to desk 007; status checked.
Entry 0025: workshop 425 sent package 0725 to desk 018; status checked.
Entry 0026: workshop 442 sent package 0754 to desk 029; status checked.
Entry 0027: workshop 459 sent package 0783 to desk 040; status checked.
Entry 0028: workshop 476 sent package 0812 to desk 051; status checked.
Entry 0029: workshop 493 sent package 0841 to desk 062; status checked.
Entry 0030: workshop 510 sent package 0870 to desk 073; status checked.
Entry 0031: workshop 527 sent package 0899 to desk 084; status checked.
Entry 0032: workshop 544 sent package 0928 to desk 095; status checked.
Entry 0033: workshop 561 sent package 0957 to desk 106; status checked.
Entry 0034: workshop 578 sent package 0986 to desk 117; status checked.
Entry 0035: workshop 595 sent package 0006 to desk 128; status checked.
Entry 0036: workshop 612 sent package 0035 to desk 139; status checked.
Entry 0037: workshop 629 sent package 0064 to desk 150; status checked.
Entry 0038: workshop 646 sent package 0093 to desk 161; status checked.
Entry 0039: workshop 663 sent package 0122 to desk 172; status checked.
Entry 0040: workshop 680 sent package 0151 to desk 183; status checked.
Entry 0041: workshop 697 sent package 0180 to desk 194; status checked.
Entry 0042: workshop 714 sent package 0209 to desk 205; status checked.
Entry 0043: workshop 731 sent package 0238 to desk 216; status checked.
Entry 0044: workshop 748 sent package 0267 to desk 227; status checked.
Entry 0045: workshop 765 sent package 0296 to desk 238; status checked.
Entry 0046: workshop 782 sent package 0325 to desk 249; status checked.
Entry 0047: workshop 799 sent package 0354 to desk 003; status checked.
Entry 0048: workshop 816 sent package 0383 to desk 014; status checked.
Entry 0049: workshop 833 sent package 0412 to desk 025; status checked.
Entry 0050: workshop 850 sent package 0441 to desk 036; status checked.
Entry 0051: workshop 867 sent package 0470 to desk 047; status checked.
Entry 0052: workshop 884 sent package 0499 to desk 058; status checked.
Entry 0053: workshop 901 sent package 0528 to desk 069; status checked.
Entry 0054: workshop 918 sent package 0557 to desk 080; status checked.
Entry 0055: workshop 935 sent package 0586 to desk 091; status checked.
Entry 0056: workshop 952 sent package 0615 to desk 102; status checked.
Entry 0057: workshop 969 sent package 0644 to desk 113; status checked.
Entry 0058: workshop 986 sent package 0673 to desk 124; status checked.
Entry 0059: workshop 006 sent package 0702 to desk 135; status checked.
Entry 0060: workshop 023 sent package 0731 to desk 146; status checked.
Entry 0061: workshop 040 sent package 0760 to desk 157; status checked.
Entry 0062: workshop 057 sent package 0789 to desk 168; status checked.
Entry 0063: workshop 074 sent package 0818 to desk 179; status checked.
Entry 0064: workshop 091 sent package 0847 to desk 190; status checked.
Entry 0065: workshop 108 sent package 0876 to desk 201; status checked.
Entry 0066: workshop 125 sent package 0905 to desk 212; status checked.
Entry 0067: workshop 142 sent package 0934 to desk 223; status checked.
Entry 0068: workshop 159 sent package 0963 to desk 234; status checked.
Entry 0069: workshop 176 sent package 0992 to desk 245; status checked.
Entry 0070: workshop 193 sent package 0012 to desk 256; status checked.
Entry 0071:
Final task:
Explain in one paragraph why a TCP server must preserve a partially written reply when a client reads slowly.
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/prompts/case_008.txt
+74
View File
@@ -0,0 +1,74 @@
The following is background from an archive. Read it, then answer the final task.
Entry 0000: workshop 000 sent package 0000 to desk 000; status checked.
Entry 0001: workshop 017 sent package 0029 to desk 011; status checked.
Entry 0002: workshop 034 sent package 0058 to desk 022; status checked.
Entry 0003: workshop 051 sent package 0087 to desk 033; status checked.
Entry 0004: workshop 068 sent package 0116 to desk 044; status checked.
Entry 0005: workshop 085 sent package 0145 to desk 055; status checked.
Entry 0006: workshop 102 sent package 0174 to desk 066; status checked.
Entry 0007: workshop 119 sent package 0203 to desk 077; status checked.
Entry 0008: workshop 136 sent package 0232 to desk 088; status checked.
Entry 0009: workshop 153 sent package 0261 to desk 099; status checked.
Entry 0010: workshop 170 sent package 0290 to desk 110; status checked.
Entry 0011: workshop 187 sent package 0319 to desk 121; status checked.
Entry 0012: workshop 204 sent package 0348 to desk 132; status checked.
Entry 0013: workshop 221 sent package 0377 to desk 143; status checked.
Entry 0014: workshop 238 sent package 0406 to desk 154; status checked.
Entry 0015: workshop 255 sent package 0435 to desk 165; status checked.
Entry 0016: workshop 272 sent package 0464 to desk 176; status checked.
Entry 0017: workshop 289 sent package 0493 to desk 187; status checked.
Entry 0018: workshop 306 sent package 0522 to desk 198; status checked.
Entry 0019: workshop 323 sent package 0551 to desk 209; status checked.
Entry 0020: workshop 340 sent package 0580 to desk 220; status checked.
Entry 0021: workshop 357 sent package 0609 to desk 231; status checked.
Entry 0022: workshop 374 sent package 0638 to desk 242; status checked.
Entry 0023: workshop 391 sent package 0667 to desk 253; status checked.
Entry 0024: workshop 408 sent package 0696 to desk 007; status checked.
Entry 0025: workshop 425 sent package 0725 to desk 018; status checked.
Entry 0026: workshop 442 sent package 0754 to desk 029; status checked.
Entry 0027: workshop 459 sent package 0783 to desk 040; status checked.
Entry 0028: workshop 476 sent package 0812 to desk 051; status checked.
Entry 0029: workshop 493 sent package 0841 to desk 062; status checked.
Entry 0030: workshop 510 sent package 0870 to desk 073; status checked.
Entry 0031: workshop 527 sent package 0899 to desk 084; status checked.
Entry 0032: workshop 544 sent package 0928 to desk 095; status checked.
Entry 0033: workshop 561 sent package 0957 to desk 106; status checked.
Entry 0034: workshop 578 sent package 0986 to desk 117; status checked.
Entry 0035: workshop 595 sent package 0006 to desk 128; status checked.
Entry 0036: workshop 612 sent package 0035 to desk 139; status checked.
Entry 0037: workshop 629 sent package 0064 to desk 150; status checked.
Entry 0038: workshop 646 sent package 0093 to desk 161; status checked.
Entry 0039: workshop 663 sent package 0122 to desk 172; status checked.
Entry 0040: workshop 680 sent package 0151 to desk 183; status checked.
Entry 0041: workshop 697 sent package 0180 to desk 194; status checked.
Entry 0042: workshop 714 sent package 0209 to desk 205; status checked.
Entry 0043: workshop 731 sent package 0238 to desk 216; status checked.
Entry 0044: workshop 748 sent package 0267 to desk 227; status checked.
Entry 0045: workshop 765 sent package 0296 to desk 238; status checked.
Entry 0046: workshop 782 sent package 0325 to desk 249; status checked.
Entry 0047: workshop 799 sent package 0354 to desk 003; status checked.
Entry 0048: workshop 816 sent package 0383 to desk 014; status checked.
Entry 0049: workshop 833 sent package 0412 to desk 025; status checked.
Entry 0050: workshop 850 sent package 0441 to desk 036; status checked.
Entry 0051: workshop 867 sent package 0470 to desk 047; status checked.
Entry 0052: workshop 884 sent package 0499 to desk 058; status checked.
Entry 0053: workshop 901 sent package 0528 to desk 069; status checked.
Entry 0054: workshop 918 sent package 0557 to desk 080; status checked.
Entry 0055: workshop 935 sent package 0586 to desk 091; status checked.
Entry 0056: workshop 952 sent package 0615 to desk 102; status checked.
Entry 0057: workshop 969 sent package 0644 to desk 113; status checked.
Entry 0058: workshop 986 sent package 0673 to desk 124; status checked.
Entry 0059: workshop 006 sent package 0702 to desk 135; status checked.
Entry 0060: workshop 023 sent package 0731 to desk 146; status checked.
Entry 0061: workshop 040 sent package 0760 to desk 157; status checked.
Entry 0062: workshop 057 sent package 0789 to desk 168; status checked.
Entry 0063: workshop 074 sent package 0818 to desk 179; status checked.
Entry 0064: workshop 091 sent package 0847 to desk 190; status checked.
Entry 0065: workshop 108 sent package 0876 to desk 201; status checked.
Entry 0066: workshop 125 sent package 0905 to desk 212; status checked.
Entry 0067: workshop 142 sent package 0934 to desk 223; status checked.
Entry 0068: workshop 159 sent package 0963 to desk 234; status checked.
Entry 0069: workshop 176 sent package 0992 to desk 245; status checked.
Entry 0070: workshop 193 sent package 0012 to desk 256
Final task:
Scrivi in italiano un breve racconto su un tecnico che trova e corregge un errore nel registro del magazzino.
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/prompts/case_009.txt
+197
View File
@@ -0,0 +1,197 @@
The following is background from an archive. Read it, then answer the final task.
Entry 0000: workshop 000 sent package 0000 to desk 000; status checked.
Entry 0001: workshop 017 sent package 0029 to desk 011; status checked.
Entry 0002: workshop 034 sent package 0058 to desk 022; status checked.
Entry 0003: workshop 051 sent package 0087 to desk 033; status checked.
Entry 0004: workshop 068 sent package 0116 to desk 044; status checked.
Entry 0005: workshop 085 sent package 0145 to desk 055; status checked.
Entry 0006: workshop 102 sent package 0174 to desk 066; status checked.
Entry 0007: workshop 119 sent package 0203 to desk 077; status checked.
Entry 0008: workshop 136 sent package 0232 to desk 088; status checked.
Entry 0009: workshop 153 sent package 0261 to desk 099; status checked.
Entry 0010: workshop 170 sent package 0290 to desk 110; status checked.
Entry 0011: workshop 187 sent package 0319 to desk 121; status checked.
Entry 0012: workshop 204 sent package 0348 to desk 132; status checked.
Entry 0013: workshop 221 sent package 0377 to desk 143; status checked.
Entry 0014: workshop 238 sent package 0406 to desk 154; status checked.
Entry 0015: workshop 255 sent package 0435 to desk 165; status checked.
Entry 0016: workshop 272 sent package 0464 to desk 176; status checked.
Entry 0017: workshop 289 sent package 0493 to desk 187; status checked.
Entry 0018: workshop 306 sent package 0522 to desk 198; status checked.
Entry 0019: workshop 323 sent package 0551 to desk 209; status checked.
Entry 0020: workshop 340 sent package 0580 to desk 220; status checked.
Entry 0021: workshop 357 sent package 0609 to desk 231; status checked.
Entry 0022: workshop 374 sent package 0638 to desk 242; status checked.
Entry 0023: workshop 391 sent package 0667 to desk 253; status checked.
Entry 0024: workshop 408 sent package 0696 to desk 007; status checked.
Entry 0025: workshop 425 sent package 0725 to desk 018; status checked.
Entry 0026: workshop 442 sent package 0754 to desk 029; status checked.
Entry 0027: workshop 459 sent package 0783 to desk 040; status checked.
Entry 0028: workshop 476 sent package 0812 to desk 051; status checked.
Entry 0029: workshop 493 sent package 0841 to desk 062; status checked.
Entry 0030: workshop 510 sent package 0870 to desk 073; status checked.
Entry 0031: workshop 527 sent package 0899 to desk 084; status checked.
Entry 0032: workshop 544 sent package 0928 to desk 095; status checked.
Entry 0033: workshop 561 sent package 0957 to desk 106; status checked.
Entry 0034: workshop 578 sent package 0986 to desk 117; status checked.
Entry 0035: workshop 595 sent package 0006 to desk 128; status checked.
Entry 0036: workshop 612 sent package 0035 to desk 139; status checked.
Entry 0037: workshop 629 sent package 0064 to desk 150; status checked.
Entry 0038: workshop 646 sent package 0093 to desk 161; status checked.
Entry 0039: workshop 663 sent package 0122 to desk 172; status checked.
Entry 0040: workshop 680 sent package 0151 to desk 183; status checked.
Entry 0041: workshop 697 sent package 0180 to desk 194; status checked.
Entry 0042: workshop 714 sent package 0209 to desk 205; status checked.
Entry 0043: workshop 731 sent package 0238 to desk 216; status checked.
Entry 0044: workshop 748 sent package 0267 to desk 227; status checked.
Entry 0045: workshop 765 sent package 0296 to desk 238; status checked.
Entry 0046: workshop 782 sent package 0325 to desk 249; status checked.
Entry 0047: workshop 799 sent package 0354 to desk 003; status checked.
Entry 0048: workshop 816 sent package 0383 to desk 014; status checked.
Entry 0049: workshop 833 sent package 0412 to desk 025; status checked.
Entry 0050: workshop 850 sent package 0441 to desk 036; status checked.
Entry 0051: workshop 867 sent package 0470 to desk 047; status checked.
Entry 0052: workshop 884 sent package 0499 to desk 058; status checked.
Entry 0053: workshop 901 sent package 0528 to desk 069; status checked.
Entry 0054: workshop 918 sent package 0557 to desk 080; status checked.
Entry 0055: workshop 935 sent package 0586 to desk 091; status checked.
Entry 0056: workshop 952 sent package 0615 to desk 102; status checked.
Entry 0057: workshop 969 sent package 0644 to desk 113; status checked.
Entry 0058: workshop 986 sent package 0673 to desk 124; status checked.
Entry 0059: workshop 006 sent package 0702 to desk 135; status checked.
Entry 0060: workshop 023 sent package 0731 to desk 146; status checked.
Entry 0061: workshop 040 sent package 0760 to desk 157; status checked.
Entry 0062: workshop 057 sent package 0789 to desk 168; status checked.
Entry 0063: workshop 074 sent package 0818 to desk 179; status checked.
Entry 0064: workshop 091 sent package 0847 to desk 190; status checked.
Entry 0065: workshop 108 sent package 0876 to desk 201; status checked.
Entry 0066: workshop 125 sent package 0905 to desk 212; status checked.
Entry 0067: workshop 142 sent package 0934 to desk 223; status checked.
Entry 0068: workshop 159 sent package 0963 to desk 234; status checked.
Entry 0069: workshop 176 sent package 0992 to desk 245; status checked.
Entry 0070: workshop 193 sent package 0012 to desk 256; status checked.
Entry 0071: workshop 210 sent package 0041 to desk 010; status checked.
Entry 0072: workshop 227 sent package 0070 to desk 021; status checked.
Entry 0073: workshop 244 sent package 0099 to desk 032; status checked.
Entry 0074: workshop 261 sent package 0128 to desk 043; status checked.
Entry 0075: workshop 278 sent package 0157 to desk 054; status checked.
Entry 0076: workshop 295 sent package 0186 to desk 065; status checked.
Entry 0077: workshop 312 sent package 0215 to desk 076; status checked.
Entry 0078: workshop 329 sent package 0244 to desk 087; status checked.
Entry 0079: workshop 346 sent package 0273 to desk 098; status checked.
Entry 0080: workshop 363 sent package 0302 to desk 109; status checked.
Entry 0081: workshop 380 sent package 0331 to desk 120; status checked.
Entry 0082: workshop 397 sent package 0360 to desk 131; status checked.
Entry 0083: workshop 414 sent package 0389 to desk 142; status checked.
Entry 0084: workshop 431 sent package 0418 to desk 153; status checked.
Entry 0085: workshop 448 sent package 0447 to desk 164; status checked.
Entry 0086: workshop 465 sent package 0476 to desk 175; status checked.
Entry 0087: workshop 482 sent package 0505 to desk 186; status checked.
Entry 0088: workshop 499 sent package 0534 to desk 197; status checked.
Entry 0089: workshop 516 sent package 0563 to desk 208; status checked.
Entry 0090: workshop 533 sent package 0592 to desk 219; status checked.
Entry 0091: workshop 550 sent package 0621 to desk 230; status checked.
Entry 0092: workshop 567 sent package 0650 to desk 241; status checked.
Entry 0093: workshop 584 sent package 0679 to desk 252; status checked.
Entry 0094: workshop 601 sent package 0708 to desk 006; status checked.
Entry 0095: workshop 618 sent package 0737 to desk 017; status checked.
Entry 0096: workshop 635 sent package 0766 to desk 028; status checked.
Entry 0097: workshop 652 sent package 0795 to desk 039; status checked.
Entry 0098: workshop 669 sent package 0824 to desk 050; status checked.
Entry 0099: workshop 686 sent package 0853 to desk 061; status checked.
Entry 0100: workshop 703 sent package 0882 to desk 072; status checked.
Entry 0101: workshop 720 sent package 0911 to desk 083; status checked.
Entry 0102: workshop 737 sent package 0940 to desk 094; status checked.
Entry 0103: workshop 754 sent package 0969 to desk 105; status checked.
Entry 0104: workshop 771 sent package 0998 to desk 116; status checked.
Entry 0105: workshop 788 sent package 0018 to desk 127; status checked.
Entry 0106: workshop 805 sent package 0047 to desk 138; status checked.
Entry 0107: workshop 822 sent package 0076 to desk 149; status checked.
Entry 0108: workshop 839 sent package 0105 to desk 160; status checked.
Entry 0109: workshop 856 sent package 0134 to desk 171; status checked.
Entry 0110: workshop 873 sent package 0163 to desk 182; status checked.
Entry 0111: workshop 890 sent package 0192 to desk 193; status checked.
Entry 0112: workshop 907 sent package 0221 to desk 204; status checked.
Entry 0113: workshop 924 sent package 0250 to desk 215; status checked.
Entry 0114: workshop 941 sent package 0279 to desk 226; status checked.
Entry 0115: workshop 958 sent package 0308 to desk 237; status checked.
Entry 0116: workshop 975 sent package 0337 to desk 248; status checked.
Entry 0117: workshop 992 sent package 0366 to desk 002; status checked.
Entry 0118: workshop 012 sent package 0395 to desk 013; status checked.
Entry 0119: workshop 029 sent package 0424 to desk 024; status checked.
Entry 0120: workshop 046 sent package 0453 to desk 035; status checked.
Entry 0121: workshop 063 sent package 0482 to desk 046; status checked.
Entry 0122: workshop 080 sent package 0511 to desk 057; status checked.
Entry 0123: workshop 097 sent package 0540 to desk 068; status checked.
Entry 0124: workshop 114 sent package 0569 to desk 079; status checked.
Entry 0125: workshop 131 sent package 0598 to desk 090; status checked.
Entry 0126: workshop 148 sent package 0627 to desk 101; status checked.
Entry 0127: workshop 165 sent package 0656 to desk 112; status checked.
Entry 0128: workshop 182 sent package 0685 to desk 123; status checked.
Entry 0129: workshop 199 sent package 0714 to desk 134; status checked.
Entry 0130: workshop 216 sent package 0743 to desk 145; status checked.
Entry 0131: workshop 233 sent package 0772 to desk 156; status checked.
Entry 0132: workshop 250 sent package 0801 to desk 167; status checked.
Entry 0133: workshop 267 sent package 0830 to desk 178; status checked.
Entry 0134: workshop 284 sent package 0859 to desk 189; status checked.
Entry 0135: workshop 301 sent package 0888 to desk 200; status checked.
Entry 0136: workshop 318 sent package 0917 to desk 211; status checked.
Entry 0137: workshop 335 sent package 0946 to desk 222; status checked.
Entry 0138: workshop 352 sent package 0975 to desk 233; status checked.
Entry 0139: workshop 369 sent package 1004 to desk 244; status checked.
Entry 0140: workshop 386 sent package 0024 to desk 255; status checked.
Entry 0141: workshop 403 sent package 0053 to desk 009; status checked.
Entry 0142: workshop 420 sent package 0082 to desk 020; status checked.
Entry 0143: workshop 437 sent package 0111 to desk 031; status checked.
Entry 0144: workshop 454 sent package 0140 to desk 042; status checked.
Entry 0145: workshop 471 sent package 0169 to desk 053; status checked.
Entry 0146: workshop 488 sent package 0198 to desk 064; status checked.
Entry 0147: workshop 505 sent package 0227 to desk 075; status checked.
Entry 0148: workshop 522 sent package 0256 to desk 086; status checked.
Entry 0149: workshop 539 sent package 0285 to desk 097; status checked.
Entry 0150: workshop 556 sent package 0314 to desk 108; status checked.
Entry 0151: workshop 573 sent package 0343 to desk 119; status checked.
Entry 0152: workshop 590 sent package 0372 to desk 130; status checked.
Entry 0153: workshop 607 sent package 0401 to desk 141; status checked.
Entry 0154: workshop 624 sent package 0430 to desk 152; status checked.
Entry 0155: workshop 641 sent package 0459 to desk 163; status checked.
Entry 0156: workshop 658 sent package 0488 to desk 174; status checked.
Entry 0157: workshop 675 sent package 0517 to desk 185; status checked.
Entry 0158: workshop 692 sent package 0546 to desk 196; status checked.
Entry 0159: workshop 709 sent package 0575 to desk 207; status checked.
Entry 0160: workshop 726 sent package 0604 to desk 218; status checked.
Entry 0161: workshop 743 sent package 0633 to desk 229; status checked.
Entry 0162: workshop 760 sent package 0662 to desk 240; status checked.
Entry 0163: workshop 777 sent package 0691 to desk 251; status checked.
Entry 0164: workshop 794 sent package 0720 to desk 005; status checked.
Entry 0165: workshop 811 sent package 0749 to desk 016; status checked.
Entry 0166: workshop 828 sent package 0778 to desk 027; status checked.
Entry 0167: workshop 845 sent package 0807 to desk 038; status checked.
Entry 0168: workshop 862 sent package 0836 to desk 049; status checked.
Entry 0169: workshop 879 sent package 0865 to desk 060; status checked.
Entry 0170: workshop 896 sent package 0894 to desk 071; status checked.
Entry 0171: workshop 913 sent package 0923 to desk 082; status checked.
Entry 0172: workshop 930 sent package 0952 to desk 093; status checked.
Entry 0173: workshop 947 sent package 0981 to desk 104; status checked.
Entry 0174: workshop 964 sent package 0001 to desk 115; status checked.
Entry 0175: workshop 981 sent package 0030 to desk 126; status checked.
Entry 0176: workshop 001 sent package 0059 to desk 137; status checked.
Entry 0177: workshop 018 sent package 0088 to desk 148; status checked.
Entry 0178: workshop 035 sent package 0117 to desk 159; status checked.
Entry 0179: workshop 052 sent package 0146 to desk 170; status checked.
Entry 0180: workshop 069 sent package 0175 to desk 181; status checked.
Entry 0181: workshop 086 sent package 0204 to desk 192; status checked.
Entry 0182: workshop 103 sent package 0233 to desk 203; status checked.
Entry 0183: workshop 120 sent package 0262 to desk 214; status checked.
Entry 0184: workshop 137 sent package 0291 to desk 225; status checked.
Entry 0185: workshop 154 sent package 0320 to desk 236; status checked.
Entry 0186: workshop 171 sent package 0349 to desk 247; status checked.
Entry 0187: workshop 188 sent package 0378 to desk 001; status checked.
Entry 0188: workshop 205 sent package 0407 to desk 012; status checked.
Entry 0189: workshop 222 sent package 0436 to desk 023; status checked.
Entry 0190: workshop 239 sent package 0465 to desk 034; status checked.
Entry 0191: workshop 256 sent package 0494 to desk 045; status checked.
Final task:
Review the following C code and show a corrected version, explaining the bounds bug:
int lookup(int *a, int n, int index) { if (index > n) return -1; return a[index]; }
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/prompts/case_010.txt
+197
View File
@@ -0,0 +1,197 @@
The following is background from an archive. Read it, then answer the final task.
Entry 0000: workshop 000 sent package 0000 to desk 000; status checked.
Entry 0001: workshop 017 sent package 0029 to desk 011; status checked.
Entry 0002: workshop 034 sent package 0058 to desk 022; status checked.
Entry 0003: workshop 051 sent package 0087 to desk 033; status checked.
Entry 0004: workshop 068 sent package 0116 to desk 044; status checked.
Entry 0005: workshop 085 sent package 0145 to desk 055; status checked.
Entry 0006: workshop 102 sent package 0174 to desk 066; status checked.
Entry 0007: workshop 119 sent package 0203 to desk 077; status checked.
Entry 0008: workshop 136 sent package 0232 to desk 088; status checked.
Entry 0009: workshop 153 sent package 0261 to desk 099; status checked.
Entry 0010: workshop 170 sent package 0290 to desk 110; status checked.
Entry 0011: workshop 187 sent package 0319 to desk 121; status checked.
Entry 0012: workshop 204 sent package 0348 to desk 132; status checked.
Entry 0013: workshop 221 sent package 0377 to desk 143; status checked.
Entry 0014: workshop 238 sent package 0406 to desk 154; status checked.
Entry 0015: workshop 255 sent package 0435 to desk 165; status checked.
Entry 0016: workshop 272 sent package 0464 to desk 176; status checked.
Entry 0017: workshop 289 sent package 0493 to desk 187; status checked.
Entry 0018: workshop 306 sent package 0522 to desk 198; status checked.
Entry 0019: workshop 323 sent package 0551 to desk 209; status checked.
Entry 0020: workshop 340 sent package 0580 to desk 220; status checked.
Entry 0021: workshop 357 sent package 0609 to desk 231; status checked.
Entry 0022: workshop 374 sent package 0638 to desk 242; status checked.
Entry 0023: workshop 391 sent package 0667 to desk 253; status checked.
Entry 0024: workshop 408 sent package 0696 to desk 007; status checked.
Entry 0025: workshop 425 sent package 0725 to desk 018; status checked.
Entry 0026: workshop 442 sent package 0754 to desk 029; status checked.
Entry 0027: workshop 459 sent package 0783 to desk 040; status checked.
Entry 0028: workshop 476 sent package 0812 to desk 051; status checked.
Entry 0029: workshop 493 sent package 0841 to desk 062; status checked.
Entry 0030: workshop 510 sent package 0870 to desk 073; status checked.
Entry 0031: workshop 527 sent package 0899 to desk 084; status checked.
Entry 0032: workshop 544 sent package 0928 to desk 095; status checked.
Entry 0033: workshop 561 sent package 0957 to desk 106; status checked.
Entry 0034: workshop 578 sent package 0986 to desk 117; status checked.
Entry 0035: workshop 595 sent package 0006 to desk 128; status checked.
Entry 0036: workshop 612 sent package 0035 to desk 139; status checked.
Entry 0037: workshop 629 sent package 0064 to desk 150; status checked.
Entry 0038: workshop 646 sent package 0093 to desk 161; status checked.
Entry 0039: workshop 663 sent package 0122 to desk 172; status checked.
Entry 0040: workshop 680 sent package 0151 to desk 183; status checked.
Entry 0041: workshop 697 sent package 0180 to desk 194; status checked.
Entry 0042: workshop 714 sent package 0209 to desk 205; status checked.
Entry 0043: workshop 731 sent package 0238 to desk 216; status checked.
Entry 0044: workshop 748 sent package 0267 to desk 227; status checked.
Entry 0045: workshop 765 sent package 0296 to desk 238; status checked.
Entry 0046: workshop 782 sent package 0325 to desk 249; status checked.
Entry 0047: workshop 799 sent package 0354 to desk 003; status checked.
Entry 0048: workshop 816 sent package 0383 to desk 014; status checked.
Entry 0049: workshop 833 sent package 0412 to desk 025; status checked.
Entry 0050: workshop 850 sent package 0441 to desk 036; status checked.
Entry 0051: workshop 867 sent package 0470 to desk 047; status checked.
Entry 0052: workshop 884 sent package 0499 to desk 058; status checked.
Entry 0053: workshop 901 sent package 0528 to desk 069; status checked.
Entry 0054: workshop 918 sent package 0557 to desk 080; status checked.
Entry 0055: workshop 935 sent package 0586 to desk 091; status checked.
Entry 0056: workshop 952 sent package 0615 to desk 102; status checked.
Entry 0057: workshop 969 sent package 0644 to desk 113; status checked.
Entry 0058: workshop 986 sent package 0673 to desk 124; status checked.
Entry 0059: workshop 006 sent package 0702 to desk 135; status checked.
Entry 0060: workshop 023 sent package 0731 to desk 146; status checked.
Entry 0061: workshop 040 sent package 0760 to desk 157; status checked.
Entry 0062: workshop 057 sent package 0789 to desk 168; status checked.
Entry 0063: workshop 074 sent package 0818 to desk 179; status checked.
Entry 0064: workshop 091 sent package 0847 to desk 190; status checked.
Entry 0065: workshop 108 sent package 0876 to desk 201; status checked.
Entry 0066: workshop 125 sent package 0905 to desk 212; status checked.
Entry 0067: workshop 142 sent package 0934 to desk 223; status checked.
Entry 0068: workshop 159 sent package 0963 to desk 234; status checked.
Entry 0069: workshop 176 sent package 0992 to desk 245; status checked.
Entry 0070: workshop 193 sent package 0012 to desk 256; status checked.
Entry 0071: workshop 210 sent package 0041 to desk 010; status checked.
Entry 0072: workshop 227 sent package 0070 to desk 021; status checked.
Entry 0073: workshop 244 sent package 0099 to desk 032; status checked.
Entry 0074: workshop 261 sent package 0128 to desk 043; status checked.
Entry 0075: workshop 278 sent package 0157 to desk 054; status checked.
Entry 0076: workshop 295 sent package 0186 to desk 065; status checked.
Entry 0077: workshop 312 sent package 0215 to desk 076; status checked.
Entry 0078: workshop 329 sent package 0244 to desk 087; status checked.
Entry 0079: workshop 346 sent package 0273 to desk 098; status checked.
Entry 0080: workshop 363 sent package 0302 to desk 109; status checked.
Entry 0081: workshop 380 sent package 0331 to desk 120; status checked.
Entry 0082: workshop 397 sent package 0360 to desk 131; status checked.
Entry 0083: workshop 414 sent package 0389 to desk 142; status checked.
Entry 0084: workshop 431 sent package 0418 to desk 153; status checked.
Entry 0085: workshop 448 sent package 0447 to desk 164; status checked.
Entry 0086: workshop 465 sent package 0476 to desk 175; status checked.
Entry 0087: workshop 482 sent package 0505 to desk 186; status checked.
Entry 0088: workshop 499 sent package 0534 to desk 197; status checked.
Entry 0089: workshop 516 sent package 0563 to desk 208; status checked.
Entry 0090: workshop 533 sent package 0592 to desk 219; status checked.
Entry 0091: workshop 550 sent package 0621 to desk 230; status checked.
Entry 0092: workshop 567 sent package 0650 to desk 241; status checked.
Entry 0093: workshop 584 sent package 0679 to desk 252; status checked.
Entry 0094: workshop 601 sent package 0708 to desk 006; status checked.
Entry 0095: workshop 618 sent package 0737 to desk 017; status checked.
Entry 0096: workshop 635 sent package 0766 to desk 028; status checked.
Entry 0097: workshop 652 sent package 0795 to desk 039; status checked.
Entry 0098: workshop 669 sent package 0824 to desk 050; status checked.
Entry 0099: workshop 686 sent package 0853 to desk 061; status checked.
Entry 0100: workshop 703 sent package 0882 to desk 072; status checked.
Entry 0101: workshop 720 sent package 0911 to desk 083; status checked.
Entry 0102: workshop 737 sent package 0940 to desk 094; status checked.
Entry 0103: workshop 754 sent package 0969 to desk 105; status checked.
Entry 0104: workshop 771 sent package 0998 to desk 116; status checked.
Entry 0105: workshop 788 sent package 0018 to desk 127; status checked.
Entry 0106: workshop 805 sent package 0047 to desk 138; status checked.
Entry 0107: workshop 822 sent package 0076 to desk 149; status checked.
Entry 0108: workshop 839 sent package 0105 to desk 160; status checked.
Entry 0109: workshop 856 sent package 0134 to desk 171; status checked.
Entry 0110: workshop 873 sent package 0163 to desk 182; status checked.
Entry 0111: workshop 890 sent package 0192 to desk 193; status checked.
Entry 0112: workshop 907 sent package 0221 to desk 204; status checked.
Entry 0113: workshop 924 sent package 0250 to desk 215; status checked.
Entry 0114: workshop 941 sent package 0279 to desk 226; status checked.
Entry 0115: workshop 958 sent package 0308 to desk 237; status checked.
Entry 0116: workshop 975 sent package 0337 to desk 248; status checked.
Entry 0117: workshop 992 sent package 0366 to desk 002; status checked.
Entry 0118: workshop 012 sent package 0395 to desk 013; status checked.
Entry 0119: workshop 029 sent package 0424 to desk 024; status checked.
Entry 0120: workshop 046 sent package 0453 to desk 035; status checked.
Entry 0121: workshop 063 sent package 0482 to desk 046; status checked.
Entry 0122: workshop 080 sent package 0511 to desk 057; status checked.
Entry 0123: workshop 097 sent package 0540 to desk 068; status checked.
Entry 0124: workshop 114 sent package 0569 to desk 079; status checked.
Entry 0125: workshop 131 sent package 0598 to desk 090; status checked.
Entry 0126: workshop 148 sent package 0627 to desk 101; status checked.
Entry 0127: workshop 165 sent package 0656 to desk 112; status checked.
Entry 0128: workshop 182 sent package 0685 to desk 123; status checked.
Entry 0129: workshop 199 sent package 0714 to desk 134; status checked.
Entry 0130: workshop 216 sent package 0743 to desk 145; status checked.
Entry 0131: workshop 233 sent package 0772 to desk 156; status checked.
Entry 0132: workshop 250 sent package 0801 to desk 167; status checked.
Entry 0133: workshop 267 sent package 0830 to desk 178; status checked.
Entry 0134: workshop 284 sent package 0859 to desk 189; status checked.
Entry 0135: workshop 301 sent package 0888 to desk 200; status checked.
Entry 0136: workshop 318 sent package 0917 to desk 211; status checked.
Entry 0137: workshop 335 sent package 0946 to desk 222; status checked.
Entry 0138: workshop 352 sent package 0975 to desk 233; status checked.
Entry 0139: workshop 369 sent package 1004 to desk 244; status checked.
Entry 0140: workshop 386 sent package 0024 to desk 255; status checked.
Entry 0141: workshop 403 sent package 0053 to desk 009; status checked.
Entry 0142: workshop 420 sent package 0082 to desk 020; status checked.
Entry 0143: workshop 437 sent package 0111 to desk 031; status checked.
Entry 0144: workshop 454 sent package 0140 to desk 042; status checked.
Entry 0145: workshop 471 sent package 0169 to desk 053; status checked.
Entry 0146: workshop 488 sent package 0198 to desk 064; status checked.
Entry 0147: workshop 505 sent package 0227 to desk 075; status checked.
Entry 0148: workshop 522 sent package 0256 to desk 086; status checked.
Entry 0149: workshop 539 sent package 0285 to desk 097; status checked.
Entry 0150: workshop 556 sent package 0314 to desk 108; status checked.
Entry 0151: workshop 573 sent package 0343 to desk 119; status checked.
Entry 0152: workshop 590 sent package 0372 to desk 130; status checked.
Entry 0153: workshop 607 sent package 0401 to desk 141; status checked.
Entry 0154: workshop 624 sent package 0430 to desk 152; status checked.
Entry 0155: workshop 641 sent package 0459 to desk 163; status checked.
Entry 0156: workshop 658 sent package 0488 to desk 174; status checked.
Entry 0157: workshop 675 sent package 0517 to desk 185; status checked.
Entry 0158: workshop 692 sent package 0546 to desk 196; status checked.
Entry 0159: workshop 709 sent package 0575 to desk 207; status checked.
Entry 0160: workshop 726 sent package 0604 to desk 218; status checked.
Entry 0161: workshop 743 sent package 0633 to desk 229; status checked.
Entry 0162: workshop 760 sent package 0662 to desk 240; status checked.
Entry 0163: workshop 777 sent package 0691 to desk 251; status checked.
Entry 0164: workshop 794 sent package 0720 to desk 005; status checked.
Entry 0165: workshop 811 sent package 0749 to desk 016; status checked.
Entry 0166: workshop 828 sent package 0778 to desk 027; status checked.
Entry 0167: workshop 845 sent package 0807 to desk 038; status checked.
Entry 0168: workshop 862 sent package 0836 to desk 049; status checked.
Entry 0169: workshop 879 sent package 0865 to desk 060; status checked.
Entry 0170: workshop 896 sent package 0894 to desk 071; status checked.
Entry 0171: workshop 913 sent package 0923 to desk 082; status checked.
Entry 0172: workshop 930 sent package 0952 to desk 093; status checked.
Entry 0173: workshop 947 sent package 0981 to desk 104; status checked.
Entry 0174: workshop 964 sent package 0001 to desk 115; status checked.
Entry 0175: workshop 981 sent package 0030 to desk 126; status checked.
Entry 0176: workshop 001 sent package 0059 to desk 137; status checked.
Entry 0177: workshop 018 sent package 0088 to desk 148; status checked.
Entry 0178: workshop 035 sent package 0117 to desk 159; status checked.
Entry 0179: workshop 052 sent package 0146 to desk 170; status checked.
Entry 0180: workshop 069 sent package 0175 to desk 181; status checked.
Entry 0181: workshop 086 sent package 0204 to desk 192; status checked.
Entry 0182: workshop 103 sent package 0233 to desk 203; status checked.
Entry 0183: workshop 120 sent package 0262 to desk 214; status checked.
Entry 0184: workshop 137 sent package 0291 to desk 225; status checked.
Entry 0185: workshop 154 sent package 0320 to desk 236; status checked.
Entry 0186: workshop 171 sent package 0349 to desk 247; status checked.
Entry 0187: workshop 188 sent package 0378 to desk 001; status checked.
Entry 0188: workshop 205 sent package 0407 to desk 012; status checked.
Entry 0189: workshop 222 sent package 0436 to desk 023; status checked.
Entry 0190: workshop 239 sent package 0465 to desk 034; status checked.
Entry 0191: workshop 256 sent package 0494 to desk 045; status checked.
Entry 0192: workshop 273 sent package 0523 to desk 056; status checked.
Entry 019
Final task:
Explain in one paragraph why a TCP server must preserve a partially written reply when a client reads slowly.
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/prompts/case_011.txt
+196
View File
@@ -0,0 +1,196 @@
The following is background from an archive. Read it, then answer the final task.
Entry 0000: workshop 000 sent package 0000 to desk 000; status checked.
Entry 0001: workshop 017 sent package 0029 to desk 011; status checked.
Entry 0002: workshop 034 sent package 0058 to desk 022; status checked.
Entry 0003: workshop 051 sent package 0087 to desk 033; status checked.
Entry 0004: workshop 068 sent package 0116 to desk 044; status checked.
Entry 0005: workshop 085 sent package 0145 to desk 055; status checked.
Entry 0006: workshop 102 sent package 0174 to desk 066; status checked.
Entry 0007: workshop 119 sent package 0203 to desk 077; status checked.
Entry 0008: workshop 136 sent package 0232 to desk 088; status checked.
Entry 0009: workshop 153 sent package 0261 to desk 099; status checked.
Entry 0010: workshop 170 sent package 0290 to desk 110; status checked.
Entry 0011: workshop 187 sent package 0319 to desk 121; status checked.
Entry 0012: workshop 204 sent package 0348 to desk 132; status checked.
Entry 0013: workshop 221 sent package 0377 to desk 143; status checked.
Entry 0014: workshop 238 sent package 0406 to desk 154; status checked.
Entry 0015: workshop 255 sent package 0435 to desk 165; status checked.
Entry 0016: workshop 272 sent package 0464 to desk 176; status checked.
Entry 0017: workshop 289 sent package 0493 to desk 187; status checked.
Entry 0018: workshop 306 sent package 0522 to desk 198; status checked.
Entry 0019: workshop 323 sent package 0551 to desk 209; status checked.
Entry 0020: workshop 340 sent package 0580 to desk 220; status checked.
Entry 0021: workshop 357 sent package 0609 to desk 231; status checked.
Entry 0022: workshop 374 sent package 0638 to desk 242; status checked.
Entry 0023: workshop 391 sent package 0667 to desk 253; status checked.
Entry 0024: workshop 408 sent package 0696 to desk 007; status checked.
Entry 0025: workshop 425 sent package 0725 to desk 018; status checked.
Entry 0026: workshop 442 sent package 0754 to desk 029; status checked.
Entry 0027: workshop 459 sent package 0783 to desk 040; status checked.
Entry 0028: workshop 476 sent package 0812 to desk 051; status checked.
Entry 0029: workshop 493 sent package 0841 to desk 062; status checked.
Entry 0030: workshop 510 sent package 0870 to desk 073; status checked.
Entry 0031: workshop 527 sent package 0899 to desk 084; status checked.
Entry 0032: workshop 544 sent package 0928 to desk 095; status checked.
Entry 0033: workshop 561 sent package 0957 to desk 106; status checked.
Entry 0034: workshop 578 sent package 0986 to desk 117; status checked.
Entry 0035: workshop 595 sent package 0006 to desk 128; status checked.
Entry 0036: workshop 612 sent package 0035 to desk 139; status checked.
Entry 0037: workshop 629 sent package 0064 to desk 150; status checked.
Entry 0038: workshop 646 sent package 0093 to desk 161; status checked.
Entry 0039: workshop 663 sent package 0122 to desk 172; status checked.
Entry 0040: workshop 680 sent package 0151 to desk 183; status checked.
Entry 0041: workshop 697 sent package 0180 to desk 194; status checked.
Entry 0042: workshop 714 sent package 0209 to desk 205; status checked.
Entry 0043: workshop 731 sent package 0238 to desk 216; status checked.
Entry 0044: workshop 748 sent package 0267 to desk 227; status checked.
Entry 0045: workshop 765 sent package 0296 to desk 238; status checked.
Entry 0046: workshop 782 sent package 0325 to desk 249; status checked.
Entry 0047: workshop 799 sent package 0354 to desk 003; status checked.
Entry 0048: workshop 816 sent package 0383 to desk 014; status checked.
Entry 0049: workshop 833 sent package 0412 to desk 025; status checked.
Entry 0050: workshop 850 sent package 0441 to desk 036; status checked.
Entry 0051: workshop 867 sent package 0470 to desk 047; status checked.
Entry 0052: workshop 884 sent package 0499 to desk 058; status checked.
Entry 0053: workshop 901 sent package 0528 to desk 069; status checked.
Entry 0054: workshop 918 sent package 0557 to desk 080; status checked.
Entry 0055: workshop 935 sent package 0586 to desk 091; status checked.
Entry 0056: workshop 952 sent package 0615 to desk 102; status checked.
Entry 0057: workshop 969 sent package 0644 to desk 113; status checked.
Entry 0058: workshop 986 sent package 0673 to desk 124; status checked.
Entry 0059: workshop 006 sent package 0702 to desk 135; status checked.
Entry 0060: workshop 023 sent package 0731 to desk 146; status checked.
Entry 0061: workshop 040 sent package 0760 to desk 157; status checked.
Entry 0062: workshop 057 sent package 0789 to desk 168; status checked.
Entry 0063: workshop 074 sent package 0818 to desk 179; status checked.
Entry 0064: workshop 091 sent package 0847 to desk 190; status checked.
Entry 0065: workshop 108 sent package 0876 to desk 201; status checked.
Entry 0066: workshop 125 sent package 0905 to desk 212; status checked.
Entry 0067: workshop 142 sent package 0934 to desk 223; status checked.
Entry 0068: workshop 159 sent package 0963 to desk 234; status checked.
Entry 0069: workshop 176 sent package 0992 to desk 245; status checked.
Entry 0070: workshop 193 sent package 0012 to desk 256; status checked.
Entry 0071: workshop 210 sent package 0041 to desk 010; status checked.
Entry 0072: workshop 227 sent package 0070 to desk 021; status checked.
Entry 0073: workshop 244 sent package 0099 to desk 032; status checked.
Entry 0074: workshop 261 sent package 0128 to desk 043; status checked.
Entry 0075: workshop 278 sent package 0157 to desk 054; status checked.
Entry 0076: workshop 295 sent package 0186 to desk 065; status checked.
Entry 0077: workshop 312 sent package 0215 to desk 076; status checked.
Entry 0078: workshop 329 sent package 0244 to desk 087; status checked.
Entry 0079: workshop 346 sent package 0273 to desk 098; status checked.
Entry 0080: workshop 363 sent package 0302 to desk 109; status checked.
Entry 0081: workshop 380 sent package 0331 to desk 120; status checked.
Entry 0082: workshop 397 sent package 0360 to desk 131; status checked.
Entry 0083: workshop 414 sent package 0389 to desk 142; status checked.
Entry 0084: workshop 431 sent package 0418 to desk 153; status checked.
Entry 0085: workshop 448 sent package 0447 to desk 164; status checked.
Entry 0086: workshop 465 sent package 0476 to desk 175; status checked.
Entry 0087: workshop 482 sent package 0505 to desk 186; status checked.
Entry 0088: workshop 499 sent package 0534 to desk 197; status checked.
Entry 0089: workshop 516 sent package 0563 to desk 208; status checked.
Entry 0090: workshop 533 sent package 0592 to desk 219; status checked.
Entry 0091: workshop 550 sent package 0621 to desk 230; status checked.
Entry 0092: workshop 567 sent package 0650 to desk 241; status checked.
Entry 0093: workshop 584 sent package 0679 to desk 252; status checked.
Entry 0094: workshop 601 sent package 0708 to desk 006; status checked.
Entry 0095: workshop 618 sent package 0737 to desk 017; status checked.
Entry 0096: workshop 635 sent package 0766 to desk 028; status checked.
Entry 0097: workshop 652 sent package 0795 to desk 039; status checked.
Entry 0098: workshop 669 sent package 0824 to desk 050; status checked.
Entry 0099: workshop 686 sent package 0853 to desk 061; status checked.
Entry 0100: workshop 703 sent package 0882 to desk 072; status checked.
Entry 0101: workshop 720 sent package 0911 to desk 083; status checked.
Entry 0102: workshop 737 sent package 0940 to desk 094; status checked.
Entry 0103: workshop 754 sent package 0969 to desk 105; status checked.
Entry 0104: workshop 771 sent package 0998 to desk 116; status checked.
Entry 0105: workshop 788 sent package 0018 to desk 127; status checked.
Entry 0106: workshop 805 sent package 0047 to desk 138; status checked.
Entry 0107: workshop 822 sent package 0076 to desk 149; status checked.
Entry 0108: workshop 839 sent package 0105 to desk 160; status checked.
Entry 0109: workshop 856 sent package 0134 to desk 171; status checked.
Entry 0110: workshop 873 sent package 0163 to desk 182; status checked.
Entry 0111: workshop 890 sent package 0192 to desk 193; status checked.
Entry 0112: workshop 907 sent package 0221 to desk 204; status checked.
Entry 0113: workshop 924 sent package 0250 to desk 215; status checked.
Entry 0114: workshop 941 sent package 0279 to desk 226; status checked.
Entry 0115: workshop 958 sent package 0308 to desk 237; status checked.
Entry 0116: workshop 975 sent package 0337 to desk 248; status checked.
Entry 0117: workshop 992 sent package 0366 to desk 002; status checked.
Entry 0118: workshop 012 sent package 0395 to desk 013; status checked.
Entry 0119: workshop 029 sent package 0424 to desk 024; status checked.
Entry 0120: workshop 046 sent package 0453 to desk 035; status checked.
Entry 0121: workshop 063 sent package 0482 to desk 046; status checked.
Entry 0122: workshop 080 sent package 0511 to desk 057; status checked.
Entry 0123: workshop 097 sent package 0540 to desk 068; status checked.
Entry 0124: workshop 114 sent package 0569 to desk 079; status checked.
Entry 0125: workshop 131 sent package 0598 to desk 090; status checked.
Entry 0126: workshop 148 sent package 0627 to desk 101; status checked.
Entry 0127: workshop 165 sent package 0656 to desk 112; status checked.
Entry 0128: workshop 182 sent package 0685 to desk 123; status checked.
Entry 0129: workshop 199 sent package 0714 to desk 134; status checked.
Entry 0130: workshop 216 sent package 0743 to desk 145; status checked.
Entry 0131: workshop 233 sent package 0772 to desk 156; status checked.
Entry 0132: workshop 250 sent package 0801 to desk 167; status checked.
Entry 0133: workshop 267 sent package 0830 to desk 178; status checked.
Entry 0134: workshop 284 sent package 0859 to desk 189; status checked.
Entry 0135: workshop 301 sent package 0888 to desk 200; status checked.
Entry 0136: workshop 318 sent package 0917 to desk 211; status checked.
Entry 0137: workshop 335 sent package 0946 to desk 222; status checked.
Entry 0138: workshop 352 sent package 0975 to desk 233; status checked.
Entry 0139: workshop 369 sent package 1004 to desk 244; status checked.
Entry 0140: workshop 386 sent package 0024 to desk 255; status checked.
Entry 0141: workshop 403 sent package 0053 to desk 009; status checked.
Entry 0142: workshop 420 sent package 0082 to desk 020; status checked.
Entry 0143: workshop 437 sent package 0111 to desk 031; status checked.
Entry 0144: workshop 454 sent package 0140 to desk 042; status checked.
Entry 0145: workshop 471 sent package 0169 to desk 053; status checked.
Entry 0146: workshop 488 sent package 0198 to desk 064; status checked.
Entry 0147: workshop 505 sent package 0227 to desk 075; status checked.
Entry 0148: workshop 522 sent package 0256 to desk 086; status checked.
Entry 0149: workshop 539 sent package 0285 to desk 097; status checked.
Entry 0150: workshop 556 sent package 0314 to desk 108; status checked.
Entry 0151: workshop 573 sent package 0343 to desk 119; status checked.
Entry 0152: workshop 590 sent package 0372 to desk 130; status checked.
Entry 0153: workshop 607 sent package 0401 to desk 141; status checked.
Entry 0154: workshop 624 sent package 0430 to desk 152; status checked.
Entry 0155: workshop 641 sent package 0459 to desk 163; status checked.
Entry 0156: workshop 658 sent package 0488 to desk 174; status checked.
Entry 0157: workshop 675 sent package 0517 to desk 185; status checked.
Entry 0158: workshop 692 sent package 0546 to desk 196; status checked.
Entry 0159: workshop 709 sent package 0575 to desk 207; status checked.
Entry 0160: workshop 726 sent package 0604 to desk 218; status checked.
Entry 0161: workshop 743 sent package 0633 to desk 229; status checked.
Entry 0162: workshop 760 sent package 0662 to desk 240; status checked.
Entry 0163: workshop 777 sent package 0691 to desk 251; status checked.
Entry 0164: workshop 794 sent package 0720 to desk 005; status checked.
Entry 0165: workshop 811 sent package 0749 to desk 016; status checked.
Entry 0166: workshop 828 sent package 0778 to desk 027; status checked.
Entry 0167: workshop 845 sent package 0807 to desk 038; status checked.
Entry 0168: workshop 862 sent package 0836 to desk 049; status checked.
Entry 0169: workshop 879 sent package 0865 to desk 060; status checked.
Entry 0170: workshop 896 sent package 0894 to desk 071; status checked.
Entry 0171: workshop 913 sent package 0923 to desk 082; status checked.
Entry 0172: workshop 930 sent package 0952 to desk 093; status checked.
Entry 0173: workshop 947 sent package 0981 to desk 104; status checked.
Entry 0174: workshop 964 sent package 0001 to desk 115; status checked.
Entry 0175: workshop 981 sent package 0030 to desk 126; status checked.
Entry 0176: workshop 001 sent package 0059 to desk 137; status checked.
Entry 0177: workshop 018 sent package 0088 to desk 148; status checked.
Entry 0178: workshop 035 sent package 0117 to desk 159; status checked.
Entry 0179: workshop 052 sent package 0146 to desk 170; status checked.
Entry 0180: workshop 069 sent package 0175 to desk 181; status checked.
Entry 0181: workshop 086 sent package 0204 to desk 192; status checked.
Entry 0182: workshop 103 sent package 0233 to desk 203; status checked.
Entry 0183: workshop 120 sent package 0262 to desk 214; status checked.
Entry 0184: workshop 137 sent package 0291 to desk 225; status checked.
Entry 0185: workshop 154 sent package 0320 to desk 236; status checked.
Entry 0186: workshop 171 sent package 0349 to desk 247; status checked.
Entry 0187: workshop 188 sent package 0378 to desk 001; status checked.
Entry 0188: workshop 205 sent package 0407 to desk 012; status checked.
Entry 0189: workshop 222 sent package 0436 to desk 023; status checked.
Entry 0190: workshop 239 sent package 0465 to desk 034; status checked.
Entry 0191: workshop 256 sent package 0494 to desk 045; status checked.
Entry 0192: workshop 273 sent package 0523 to desk
Final task:
Scrivi in italiano un breve racconto su un tecnico che trova e corregge un errore nel registro del magazzino.
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/responses/case_000.json
+14253
View File
File diff suppressed because it is too large. Load diff
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/responses/case_001.json
+16044
View File
File diff suppressed because it is too large. Load diff
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/responses/case_002.json
+13934
View File
File diff suppressed because it is too large. Load diff
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/responses/case_003.json
+14386
View File
File diff suppressed because it is too large. Load diff
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/responses/case_004.json
+15870
View File
File diff suppressed because it is too large. Load diff
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/responses/case_005.json
+13844
View File
File diff suppressed because it is too large. Load diff
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/responses/case_006.json
+14431
View File
File diff suppressed because it is too large. Load diff
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/responses/case_007.json
+15873
View File
File diff suppressed because it is too large. Load diff
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/responses/case_008.json
+13765
View File
File diff suppressed because it is too large. Load diff
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/responses/case_009.json
+16099
View File
File diff suppressed because it is too large. Load diff
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/responses/case_010.json
+16444
View File
File diff suppressed because it is too large. Load diff
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-batch/responses/case_011.json
+13776
View File
File diff suppressed because it is too large. Load diff
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-general/README.md
+26
View File
@@ -0,0 +1,26 @@
# DeepSeek V4.1 Flash general continuations
The 100 prompts in `../prompts.jsonl`, collected from the official DeepSeek API
on 2026-09-10 using `deepseek-flash`. The provider's model table identified this
as V4.1 Flash. All responses have fingerprint
`aeb56401ca74e127821c4f9126dcb669`, matching the
[sparse-boundary set](../deepseek-v4.1-flash-20260910/README.md).
Reference weights: `deepseek-ai/DeepSeek-V4.1-Flash`, revision
`df42c109f1defefcbfcedbe7d905718a12266e40`.
Thinking is disabled, temperature is 1, and each response has at most 32 tokens
with top-20 logprobs. The set contains 2994 API output tokens. These are sampled
continuations, not greedy continuations or full-vocabulary logits. Use
teacher-forced likelihood and API top-token agreement, not greedy prefix length
alone. `collection.json` records the request settings.
```sh
./gguf-tools/quality-testing/score_official MODEL.gguf \
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-general/manifest.tsv \
scores.tsv 4096 --ssd-streaming --ssd-streaming-cache-experts 64gb
```
Use this set alongside the sparse-boundary cases when comparing quantizations.
It broadens the short-prompt comparison; it does not replace long-context,
session-state or actual coding-task tests. None of these prompts occurs verbatim
in the 8192-token corpus used to calibrate the first V4.1 mixed-Q2 candidate.
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-general/collection.json
+22
View File
@@ -0,0 +1,22 @@
{
"schema": "ds4-official-continuations-v1",
"created_at": "2026-09-10T16:46:23Z",
"model": "deepseek-flash",
"endpoint": "https://api.deepseek.com/chat/completions",
"observed_models": [
"deepseek-flash"
],
"observed_providers": [],
"temperature": 1.0,
"max_tokens": 32,
"top_logprobs": 20,
"thinking": "disabled",
"reasoning_effort": "omit",
"token_limit_field": "max_tokens",
"provider_order": [],
"allow_provider_fallbacks": false,
"require_parameters": false,
"required_response_model": "deepseek-flash",
"required_response_provider": null,
"case_count": 100
}
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-general/continuations/case_000.txt
+3
View File
@@ -0,0 +1,3 @@
# B-Tree Insertion
A B-tree is a self-balancing search tree where every node holds **multiple keys** and **multiple children**. A B
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-general/continuations/case_001.txt
+5
View File
@@ -0,0 +1,5 @@
# TCP Echo Server — Design for Slow Clients
## Goal
Accept TCP connections, echo bytes back, and prevent slow or stalled clients from exhausting server resources.
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-general/continuations/case_002.txt
+6
View File
@@ -0,0 +1,6 @@
# mmap vs. Private Buffer Copies for Model Weights on macOS
## How each approach works
**mmap (memory-mapped file)**
-
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-general/continuations/case_003.txt
+3
View File
@@ -0,0 +1,3 @@
# Deriving RMSNorm: Why Sum-of-Squares and a Scaling Pass?
RMSNorm (Zhang & Sennrich, 2019) is defined as
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-general/continuations/case_004.txt
+5
View File
@@ -0,0 +1,5 @@
# Why KV Cache Checkpointing Helps Agentic Sessions
## The core problem: long context, repeated prefix work
An agentic session (ReAct loops
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-general/continuations/case_005.txt
+7
View File
@@ -0,0 +1,7 @@
# Binary Heap Push (Insert) Operation
## Pseudocode
```
function push(heap, value):
// Step 1: Add new element at the
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-general/continuations/case_006.txt
+3
View File
@@ -0,0 +1,3 @@
# Top-p Sampling vs. Temperature-0 (Greedy) Decoding
These are two different approaches to selecting the next token, and they optimize for
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-general/continuations/case_007.txt
+2
View File
@@ -0,0 +1,2 @@
Speculative decoding preserves exact greedy output by using a verification scheme that guarantees the final token sequence is identical to what standard greedy decoding would produce. Here's how:
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-general/continuations/case_008.txt
+1
View File
@@ -0,0 +1 @@
The counterintuitive part is that prefill processes *many* tokens at once while decode processes *one*, yet prefill often achieves far higher throughput (t
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-general/continuations/case_009.txt
+4
View File
@@ -0,0 +1,4 @@
Here are three invariants for a tokenizer special-token table:
**1. Uniqueness of IDs and Surface Forms**
Every special token maps to a
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-general/continuations/case_010.txt
+3
View File
@@ -0,0 +1,3 @@
# Inserimento in un B-tree
Un **B-tree** è un albero di ricerca bilanciato progettato per essere efficiente su memoria
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-general/continuations/case_011.txt
+6
View File
@@ -0,0 +1,6 @@
# Server TCP Echo con Gestione Client Lenti
## 1. Obiettivo
Implementare un server TCP echo che:
- Accetti connessioni
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-general/continuations/case_012.txt
+3
View File
@@ -0,0 +1,3 @@
# Confronto: `mmap` dei pesi vs copia completa dei pesi su macOS
## 1. Panoramica dei due appro
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-general/continuations/case_013.txt
+3
View File
@@ -0,0 +1,3 @@
# Perché RMSNorm richiede somma dei quadrati e riscalamento
RMSNorm (Root Mean Square Normalization) è una semplificazione
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-general/continuations/case_014.txt
+5
View File
@@ -0,0 +1,5 @@
# Perché la cache KV su disco aiuta nelle sessioni agentiche
## Il problema di fondo
In una sessione agentica, l'agente
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-general/continuations/case_015.txt
+5
View File
@@ -0,0 +1,5 @@
# Pseudocodice: Inserimento in un Heap Binario
## Concetto
L'inserimento in un heap binario avviene in
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-general/continuations/case_016.txt
+4
View File
@@ -0,0 +1,4 @@
# Top-p vs Temperatura zero: pro e contro
## Cosa fanno (brevemente)
- **Temperatura = 0**: sampling
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-general/continuations/case_017.txt
+1
View File
@@ -0,0 +1 @@
La decodifica speculativa mantiene l'output esatto perché **non decide mai al posto del modello principale**: usa un modello draft solo per *
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-general/continuations/case_018.txt
+3
View File
@@ -0,0 +1,3 @@
# Perché il prefill GPU può essere più veloce del decode di un singolo token
La differenza fondamentale sta nella **natura del carico
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-general/continuations/case_019.txt
+3
View File
@@ -0,0 +1,3 @@
# Tre invarianti per una tabella di token speciali
Ecco tre invarianti tipici per una tabella che mappa token speciali (es
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-general/continuations/case_020.txt
+3
View File
@@ -0,0 +1,3 @@
# Binary Search for Insertion Point
Binary search finds the insertion point by repeatedly halving the search range until it converges on the correct index where a new value
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-general/continuations/case_021.txt
+6
View File
@@ -0,0 +1,6 @@
# Integer Clamp Function
```c
int clamp(int value, int min, int max) {
if (value < min) return min;
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-general/continuations/case_022.txt
+5
View File
@@ -0,0 +1,5 @@
# Mutex vs. Atomic Counter
Both are used to make concurrent code safe, but they operate at very different levels and are suited to different problems.
##
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-general/continuations/case_023.txt
+3
View File
@@ -0,0 +1,3 @@
# Backpressure in a Network Server
**Backpressure** is a flow-control mechanism that prevents a fast producer from overwhelming a slow consumer. When one part of
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-general/continuations/case_024.txt
+3
View File
@@ -0,0 +1,3 @@
# Safe Format for Storing a Model Checkpoint Header
A checkpoint header should separate **untrusted metadata** from **the data payload**, be parseable
gguf-tools/quality-testing/deepseek-v4.1-flash-20260910-general/continuations/case_025.txt
+3
View File
@@ -0,0 +1,3 @@
# Ring Buffers: Wrapping and Preventing Data Loss
A **ring buffer** (circular buffer) is a fixed-size array used as a FIFO
Loaded 100 of 521 files, more files were not shown because too many files have changed in this diff. Show more