mtfyun/ragflow
1
0 Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-11-11 22:01:50 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(tokenizer): fold Cyrillic before stemming so Russian keywords match (#21433)

Browse Source
This commit is contained in:
CCLUCKY authored and GitHub committed 2026-10-28 13:52:55 +08:00
1 parent 1ab7dbf5a8
commit f7de7669ad
3 files changed
+119 -2
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

internal/binding/cpp/rag_analyzer.cpp
+2 -2
View File
@@ -1889,7 +1889,7 @@ std::string RAGAnalyzer::Tokenize(const std::string &line) {
for (unsigned i = 0; i < term_list.size(); ++i) {
lowercase_string_buffer_.resize(term_list[i].size() + 1);
char *lowercase_term = lowercase_string_buffer_.data();
ToLower(term_list[i].c_str(), term_list[i].size(), lowercase_term, lowercase_string_buffer_.size());
ToLowerUTF8(term_list[i].c_str(), term_list[i].size(), lowercase_term);
std::string term_to_stem;
if (use_lemmatizer_) {
term_to_stem = wordnet_lemma_->Lemmatize(lowercase_term);
@@ -2038,7 +2038,7 @@ std::pair<std::vector<std::string>, std::vector<std::pair<unsigned, unsigned>>>
// Apply lowercase before lemmatization to match Python NLTK behavior
lowercase_string_buffer_.resize(term.size() + 1);
char *lowercase_term = lowercase_string_buffer_.data();
ToLower(term.c_str(), term.size(), lowercase_term, lowercase_string_buffer_.size());
ToLowerUTF8(term.c_str(), term.size(), lowercase_term);
std::string term_to_stem;
if (use_lemmatizer_) {
term_to_stem = wordnet_lemma_->Lemmatize(lowercase_term);