diff --git a/common/string_utils.py b/common/string_utils.py index fdc513af28..2bf58799c3 100644 --- a/common/string_utils.py +++ b/common/string_utils.py @@ -32,18 +32,20 @@ def remove_redundant_spaces(txt: str): str: Text with redundant spaces removed """ # First pass: Remove spaces after left-boundary characters - # Matches: [non-alphanumeric-and-specific-right-punctuation] + [non-space] - # Removes spaces after characters like '(', '<', and other non-alphanumeric chars + # Matches: [non-word-char-and-specific-right-punctuation] + [non-space] + # Removes spaces after characters like '(', '<', and other non-word chars. + # `\w` (not `a-z0-9`) so that non-ASCII letters count as word chars and the + # space between two non-Latin words is preserved. # Examples: # "( test" → "(test" - txt = re.sub(r"([^a-z0-9.,\)>]) +([^ ])", r"\1\2", txt, flags=re.IGNORECASE) + txt = re.sub(r"([^\w.,\)>]) +([^ ])", r"\1\2", txt, flags=re.IGNORECASE) # Second pass: Remove spaces before right-boundary characters - # Matches: [non-space] + [non-alphanumeric-and-specific-left-punctuation] - # Removes spaces before characters like non-')', non-',', non-'.', and non-alphanumeric chars + # Matches: [non-space] + [non-word-char-and-specific-left-punctuation] + # Removes spaces before characters like non-')', non-',', non-'.', and non-word chars # Examples: # "world !" → "world!" - return re.sub(r"([^ ]) +([^a-z0-9.,\(<])", r"\1\2", txt, flags=re.IGNORECASE) + return re.sub(r"([^ ]) +([^\w.,\(<])", r"\1\2", txt, flags=re.IGNORECASE) def clean_markdown_block(text): diff --git a/test/unit_test/common/test_string_utils.py b/test/unit_test/common/test_string_utils.py index cb682a53cc..26a6cc93eb 100644 --- a/test/unit_test/common/test_string_utils.py +++ b/test/unit_test/common/test_string_utils.py @@ -58,6 +58,24 @@ class TestRemoveRedundantSpaces: expected = "This should remain unchanged" assert remove_redundant_spaces(input_text) == expected + @pytest.mark.parametrize( + "text", + [ + "привет мир", # Russian + "Γειά σου κόσμε", # Greek + "안녕 세계", # Korean + "مرحبا بالعالم", # Arabic + ], + ) + def test_keep_spaces_between_non_latin_words(self, text): + """Regression: spaces between non-Latin words must be preserved. + + The whitelist used ``a-z0-9`` (ASCII only), so a non-ASCII letter fell + into the negated class and was treated as punctuation, deleting the + space between two words (e.g. "привет мир" -> "приветмир"). + """ + assert remove_redundant_spaces(text) == text + @pytest.mark.skip(reason="Failed") def test_mixed_punctuation(self): """Test mixed punctuation scenarios"""