diff --git a/api/core/indexing_runner.py b/api/core/indexing_runner.py index 92246b6614c..63e00f48324 100644 --- a/api/core/indexing_runner.py +++ b/api/core/indexing_runner.py @@ -519,7 +519,7 @@ class IndexingRunner: def filter_string(text): text = re.sub(r"<\|", "<", text) text = re.sub(r"\|>", ">", text) - text = re.sub(r"[\x00-\x08\x0B\x0C\x0E-\x1F\x7F\xEF\xBF\xBE]", "", text) + text = re.sub(r"[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]", "", text) # Unicode U+FFFE text = re.sub("\ufffe", "", text) return text diff --git a/api/core/rag/cleaner/clean_processor.py b/api/core/rag/cleaner/clean_processor.py index 790253053de..452251584e6 100644 --- a/api/core/rag/cleaner/clean_processor.py +++ b/api/core/rag/cleaner/clean_processor.py @@ -9,7 +9,7 @@ class CleanProcessor: # remove invalid symbol text = re.sub(r"<\|", "<", text) text = re.sub(r"\|>", ">", text) - text = re.sub(r"[\x00-\x08\x0B\x0C\x0E-\x1F\x7F\xEF\xBF\xBE]", "", text) + text = re.sub(r"[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]", "", text) # Unicode U+FFFE text = re.sub("\ufffe", "", text) diff --git a/api/tests/unit_tests/core/rag/cleaner/test_clean_processor.py b/api/tests/unit_tests/core/rag/cleaner/test_clean_processor.py index c7a4265a954..2155342ab98 100644 --- a/api/tests/unit_tests/core/rag/cleaner/test_clean_processor.py +++ b/api/tests/unit_tests/core/rag/cleaner/test_clean_processor.py @@ -22,6 +22,23 @@ class TestCleanProcessor: expected = "normalpadding" assert CleanProcessor.clean(text_with_ufffe, None) == expected + def test_clean_preserves_valid_extended_characters(self): + """Default cleaning must not strip valid printable characters. + + The invalid-symbol filter used to include the UTF-8 bytes of U+FFFE + (0xEF 0xBF 0xBE) inside a character class. On a decoded string those + bytes are the code points U+00EF, U+00BF and U+00BE, i.e. the valid + characters 'ï', '¿' and '¾', so words like "naïve" and Spanish + questions like "¿Cómo?" were being silently corrupted on ingest. + """ + assert CleanProcessor.clean("naïve", None) == "naïve" + assert CleanProcessor.clean("¿Cómo estás?", None) == "¿Cómo estás?" + assert CleanProcessor.clean("¾ cup sugar", None) == "¾ cup sugar" + assert CleanProcessor.clean("￾", None) == "￾" + + # The U+FFFE noncharacter is still stripped by its dedicated substitution. + assert CleanProcessor.clean("keep\ufffedrop", None) == "keepdrop" + def test_clean_with_none_process_rule(self): """Test cleaning with None process_rule - only default cleaning applied.""" text = "Hello<|World\x00" diff --git a/api/tests/unit_tests/core/rag/indexing/test_indexing_runner.py b/api/tests/unit_tests/core/rag/indexing/test_indexing_runner.py index 5307da6d343..ef1b38f49d7 100644 --- a/api/tests/unit_tests/core/rag/indexing/test_indexing_runner.py +++ b/api/tests/unit_tests/core/rag/indexing/test_indexing_runner.py @@ -1372,6 +1372,19 @@ class TestIndexingRunnerDocumentCleaning: assert "\ufffe" not in result assert "Text with" in result + def test_filter_string_preserves_valid_extended_characters(self): + """filter_string must keep valid printable characters like 'ï', '¿', '¾'.""" + # Arrange + text = "naïve ¿Cómo? ¾ done" + + # Act + result = IndexingRunner.filter_string(text) + + # Assert + assert result == text + # The U+FFFE noncharacter is still stripped. + assert IndexingRunner.filter_string("keep\ufffedrop") == "keepdrop" + class TestIndexingRunnerSplitter: """Unit tests for text splitter configuration.