From fc7a0cbca351ec8b01e3c3bef7db14ceca48c55d Mon Sep 17 00:00:00 2001
From: Andrew Chen <chuenchen309@gmail.com>
Date: Mon, 21 Sep 2026 14:56:48 +0800
Subject: [PATCH] fix(string_utils): keep spaces between non-Latin words
 (#17015)

`remove_redundant_spaces` (`common/string_utils.py`) deletes the space
between two non-Latin words.

Signed-off-by: chuenchen309 <48723787+chuenchen309@users.noreply.github.com>
---
 common/string_utils.py                     | 14 ++++++++------
 test/unit_test/common/test_string_utils.py | 18 ++++++++++++++++++
 2 files changed, 26 insertions(+), 6 deletions(-)

diff --git a/common/string_utils.py b/common/string_utils.py
index fdc513af28..2bf58799c3 100644
--- a/common/string_utils.py
+++ b/common/string_utils.py
@@ -32,18 +32,20 @@ def remove_redundant_spaces(txt: str):
         str: Text with redundant spaces removed
     """
     # First pass: Remove spaces after left-boundary characters
-    # Matches: [non-alphanumeric-and-specific-right-punctuation] + [non-space]
-    # Removes spaces after characters like '(', '<', and other non-alphanumeric chars
+    # Matches: [non-word-char-and-specific-right-punctuation] + [non-space]
+    # Removes spaces after characters like '(', '<', and other non-word chars.
+    # `\w` (not `a-z0-9`) so that non-ASCII letters count as word chars and the
+    # space between two non-Latin words is preserved.
     # Examples:
     #   "( test" → "(test"
-    txt = re.sub(r"([^a-z0-9.,\)>]) +([^ ])", r"\1\2", txt, flags=re.IGNORECASE)
+    txt = re.sub(r"([^\w.,\)>]) +([^ ])", r"\1\2", txt, flags=re.IGNORECASE)
 
     # Second pass: Remove spaces before right-boundary characters
-    # Matches: [non-space] + [non-alphanumeric-and-specific-left-punctuation]
-    # Removes spaces before characters like non-')', non-',', non-'.', and non-alphanumeric chars
+    # Matches: [non-space] + [non-word-char-and-specific-left-punctuation]
+    # Removes spaces before characters like non-')', non-',', non-'.', and non-word chars
     # Examples:
     #   "world !" → "world!"
-    return re.sub(r"([^ ]) +([^a-z0-9.,\(<])", r"\1\2", txt, flags=re.IGNORECASE)
+    return re.sub(r"([^ ]) +([^\w.,\(<])", r"\1\2", txt, flags=re.IGNORECASE)
 
 
 def clean_markdown_block(text):
diff --git a/test/unit_test/common/test_string_utils.py b/test/unit_test/common/test_string_utils.py
index cb682a53cc..26a6cc93eb 100644
--- a/test/unit_test/common/test_string_utils.py
+++ b/test/unit_test/common/test_string_utils.py
@@ -58,6 +58,24 @@ class TestRemoveRedundantSpaces:
         expected = "This should remain unchanged"
         assert remove_redundant_spaces(input_text) == expected
 
+    @pytest.mark.parametrize(
+        "text",
+        [
+            "привет мир",  # Russian
+            "Γειά σου κόσμε",  # Greek
+            "안녕 세계",  # Korean
+            "مرحبا بالعالم",  # Arabic
+        ],
+    )
+    def test_keep_spaces_between_non_latin_words(self, text):
+        """Regression: spaces between non-Latin words must be preserved.
+
+        The whitelist used ``a-z0-9`` (ASCII only), so a non-ASCII letter fell
+        into the negated class and was treated as punctuation, deleting the
+        space between two words (e.g. "привет мир" -> "приветмир").
+        """
+        assert remove_redundant_spaces(text) == text
+
     @pytest.mark.skip(reason="Failed")
     def test_mixed_punctuation(self):
         """Test mixed punctuation scenarios"""