EntrpiandClaude Fable 5
0f62c620e8
Vendors the Entrpi/ds4 fork's cuda/mmq module at its v0.5.1 state (llama.cpp mmq kernels pinned in cuda/mmq/VENDOR.md, MIT, plus the fork's ds4_mmq C ABI, D2R kernels and repack library; fused prefill pipeline portions carry Marco Palaferri's Portions-Copyright headers) and wires the RAW-layout tier only: dense Q8_0 GEMMs and the IQ2_XXS gate/up pair + Q2_K down routed-MoE pipeline at n_tok >= 2. The aligned-SoA / D2R / producer-quantized tiers and the weight server are deliberately not wired (they need the repack artifact pipeline). Decode (n_tok == 1) is untouched. mmq changes FP32 reduction order vs the cublas+dequant pipeline, so prefill logits drift at ULP scale: gated on quality mode off, DS4_CUDA_MMQ=0 restores legacy dispatch. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011CsqQzZEeuz9biUSEkGm8b
42 lines
1.7 KiB
Plaintext
42 lines
1.7 KiB
Plaintext
#pragma once
|
|
|
|
#include "common.cuh"
|
|
#include "mmq.cuh"
|
|
|
|
#include <cstdint>
|
|
|
|
#define CUDA_QUANTIZE_BLOCK_SIZE 256
|
|
#define CUDA_QUANTIZE_BLOCK_SIZE_MMQ 128
|
|
|
|
static_assert(MATRIX_ROW_PADDING % CUDA_QUANTIZE_BLOCK_SIZE == 0, "Risk of out-of-bounds access.");
|
|
static_assert(MATRIX_ROW_PADDING % (4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ) == 0, "Risk of out-of-bounds access.");
|
|
|
|
typedef void (*quantize_cuda_t)(
|
|
const float * x, const int32_t * ids, void * vy,
|
|
ggml_type type_src0, int64_t ne00, int64_t s01, int64_t s02, int64_t s03,
|
|
int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream);
|
|
|
|
void quantize_row_q8_1_cuda(
|
|
const float * x, const int32_t * ids, void * vy,
|
|
ggml_type type_src0, int64_t ne00, int64_t s01, int64_t s02, int64_t s03,
|
|
int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream);
|
|
|
|
void quantize_mmq_q8_1_cuda(
|
|
const float * x, const int32_t * ids, void * vy,
|
|
ggml_type type_src0, int64_t ne00, int64_t s01, int64_t s02, int64_t s03,
|
|
int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream);
|
|
|
|
void quantize_mmq_fp4_cuda(const float * x,
|
|
const int32_t * ids,
|
|
void * vy,
|
|
ggml_type type_src0,
|
|
int64_t ne00,
|
|
int64_t s01,
|
|
int64_t s02,
|
|
int64_t s03,
|
|
int64_t ne0,
|
|
int64_t ne1,
|
|
int64_t ne2,
|
|
int64_t ne3,
|
|
cudaStream_t stream);
|