From f7de7669ad2bba8eff8225a7ea08a6beea2a5bef Mon Sep 17 00:00:00 2001 From: CCLUCKY <2226114701@qq.com> Date: Wed, 28 Oct 2026 13:52:55 +0800 Subject: [PATCH] fix(tokenizer): fold Cyrillic before stemming so Russian keywords match (#21433) --- internal/binding/cpp/rag_analyzer.cpp | 4 +- internal/binding/cpp/string_utils.h | 78 +++++++++++++++++++++++++++ internal/tokenizer/tokenizer_test.go | 39 ++++++++++++++ 3 files changed, 119 insertions(+), 2 deletions(-) diff --git a/internal/binding/cpp/rag_analyzer.cpp b/internal/binding/cpp/rag_analyzer.cpp index 6248287282..530774d6de 100644 --- a/internal/binding/cpp/rag_analyzer.cpp +++ b/internal/binding/cpp/rag_analyzer.cpp @@ -1889,7 +1889,7 @@ std::string RAGAnalyzer::Tokenize(const std::string &line) { for (unsigned i = 0; i < term_list.size(); ++i) { lowercase_string_buffer_.resize(term_list[i].size() + 1); char *lowercase_term = lowercase_string_buffer_.data(); - ToLower(term_list[i].c_str(), term_list[i].size(), lowercase_term, lowercase_string_buffer_.size()); + ToLowerUTF8(term_list[i].c_str(), term_list[i].size(), lowercase_term); std::string term_to_stem; if (use_lemmatizer_) { term_to_stem = wordnet_lemma_->Lemmatize(lowercase_term); @@ -2038,7 +2038,7 @@ std::pair, std::vector>> // Apply lowercase before lemmatization to match Python NLTK behavior lowercase_string_buffer_.resize(term.size() + 1); char *lowercase_term = lowercase_string_buffer_.data(); - ToLower(term.c_str(), term.size(), lowercase_term, lowercase_string_buffer_.size()); + ToLowerUTF8(term.c_str(), term.size(), lowercase_term); std::string term_to_stem; if (use_lemmatizer_) { term_to_stem = wordnet_lemma_->Lemmatize(lowercase_term); diff --git a/internal/binding/cpp/string_utils.h b/internal/binding/cpp/string_utils.h index 8f03b9c4d2..c651071734 100644 --- a/internal/binding/cpp/string_utils.h +++ b/internal/binding/cpp/string_utils.h @@ -113,6 +113,84 @@ inline std::string ToLowerString(std::string_view s) return result; } +// Simple (1:1) lower-case mapping for Cyrillic U+0400-U+04FF. Upper/lower +// pairs are contiguous inside the main blocks; U+0460-U+0481 and +// U+048A-U+04BF interleave them (upper first), while U+04C3-U+04CE interleave +// with the upper case on the odd code point. U+04C0/U+04C1 are the two +// exceptions Unicode defines outside that pattern. +[[nodiscard]] constexpr uint32_t ToLowerCyrillic(uint32_t cp) noexcept +{ + if (cp >= 0x0400 && cp <= 0x040F) + return cp + 0x50; + if (cp >= 0x0410 && cp <= 0x042F) + return cp + 0x20; + if (cp >= 0x0460 && cp <= 0x0481 && (cp & 1) == 0) + return cp + 1; + if (cp >= 0x048A && cp <= 0x04BF && (cp & 1) == 0) + return cp + 1; + if (cp == 0x04C0) + return 0x04CF; + if (cp == 0x04C1) + return 0x04C2; + if (cp >= 0x04C3 && cp <= 0x04CE && (cp & 1) == 1) + return cp + 1; + if (cp >= 0x04D0 && cp <= 0x04FF && (cp & 1) == 0) + return cp + 1; + return cp; +} + +// ToLowerUTF8 folds ASCII A-Z and upper-case Cyrillic U+0400-U+04FF to lower +// case in place. Byte length is preserved so callers that map token offsets +// back to the source string keep their indices valid. The Snowball Russian +// stemmer only accepts lower-case input, so index-side tokens must be folded +// before stemming; ToLower above is deliberately ASCII-only and leaves +// Cyrillic untouched. +inline void ToLowerUTF8(char* data, size_t len) +{ + for (size_t i = 0; i < len;) + { + const unsigned char c = static_cast(data[i]); + if (c < 0x80) + { + if ('A' <= c && c <= 'Z') + data[i] += 32; + i += 1; + continue; + } + // Cyrillic occupies the two-byte leads D0-D3. All foldings stay inside + // the same block, so the encoded length never changes. + if (c >= 0xD0 && c <= 0xD3 && i + 1 < len) + { + const unsigned char t = static_cast(data[i + 1]); + if (t >= 0x80 && t <= 0xBF) + { + const uint32_t cp = ((c - 0xD0u) << 6) + (t - 0x80u) + 0x400u; + const uint32_t folded = ToLowerCyrillic(cp); + if (folded != cp) + { + const uint32_t offset = folded - 0x400u; + data[i] = static_cast(0xD0u + (offset >> 6)); + data[i + 1] = static_cast(0x80u + (offset & 0x3Fu)); + } + i += 2; + continue; + } + } + // Other multi-byte sequences pass through byte for byte. + i += 1; + } +} + +// Copies `len` bytes into `out`, folds them and terminates the result. Callers +// must provide len + 1 bytes: byte length is preserved, so that is the only +// space the fold needs. +inline void ToLowerUTF8(const char* data, size_t len, char* out) +{ + memcpy(out, data, len); + ToLowerUTF8(out, len); + out[len] = '\0'; +} + inline bool IsUTF8Sep(const uint8_t c) { return c < 128 && !std::isalnum(c); } template diff --git a/internal/tokenizer/tokenizer_test.go b/internal/tokenizer/tokenizer_test.go index ade35fdc12..5e1fd263fc 100644 --- a/internal/tokenizer/tokenizer_test.go +++ b/internal/tokenizer/tokenizer_test.go @@ -420,6 +420,45 @@ func TestTokenize_SlovakCzechFoldDiacriticsAndSkipStemming(t *testing.T) { } } +func TestTokenize_RussianFoldsCyrillicBeforeStemming(t *testing.T) { + restore := saveEngineType() + defer restore() + SetEngineType("") + + if err := Init(&PoolConfig{ + DictPath: "", + MinSize: 1, + MaxSize: 1, + IdleTimeout: 5 * time.Second, + AcquireTimeout: 5 * time.Second, + }); err != nil { + t.Fatalf("Failed to initialize pool: %v", err) + } + defer Close() + + // Index-side folding must be Unicode-aware: the Russian stemmer only + // accepts lower-case input, so an upper-case word would otherwise reach + // the suffix table unchanged while the query side folds it with + // strings.ToLower. + var want string + for _, form := range []string{"ДОГОВОРАМИ", "Договора", "договорами", "договор"} { + got, err := New("Russian").Tokenize(form) + if err != nil { + t.Fatalf("Tokenize(Russian, %q) unexpected error: %v", form, err) + } + if want == "" { + want = got + continue + } + if got != want { + t.Errorf("Tokenize(Russian, %q) = %q, want the shared stem %q", form, got, want) + } + } + if want != "договор" { + t.Errorf("Russian stem for the договор forms = %q, want %q", want, "договор") + } +} + func findEnglishDutchDifferentiator(t *testing.T) languageDifferentiator { t.Helper()