fix(rag): stop the document cleaner from stripping valid characters ï, ¿, ¾ (#39215)

api/core/indexing_runner.py

@@ -519,7 +519,7 @@ class IndexingRunner:
     def filter_string(text):
         text = re.sub(r"<\|", "<", text)
         text = re.sub(r"\|>", ">", text)
-        text = re.sub(r"[\x00-\x08\x0B\x0C\x0E-\x1F\x7F\xEF\xBF\xBE]", "", text)
+        text = re.sub(r"[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]", "", text)
         # Unicode  U+FFFE
         text = re.sub("\ufffe", "", text)
         return text

api/core/rag/cleaner/clean_processor.py

@@ -9,7 +9,7 @@ class CleanProcessor:
         # remove invalid symbol
         text = re.sub(r"<\|", "<", text)
         text = re.sub(r"\|>", ">", text)
-        text = re.sub(r"[\x00-\x08\x0B\x0C\x0E-\x1F\x7F\xEF\xBF\xBE]", "", text)
+        text = re.sub(r"[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]", "", text)
         # Unicode  U+FFFE
         text = re.sub("\ufffe", "", text)
 

api/tests/unit_tests/core/rag/cleaner/test_clean_processor.py

@@ -22,6 +22,23 @@ class TestCleanProcessor:
         expected = "normalpadding"
         assert CleanProcessor.clean(text_with_ufffe, None) == expected
 
+    def test_clean_preserves_valid_extended_characters(self):
+        """Default cleaning must not strip valid printable characters.
+
+        The invalid-symbol filter used to include the UTF-8 bytes of U+FFFE
+        (0xEF 0xBF 0xBE) inside a character class. On a decoded string those
+        bytes are the code points U+00EF, U+00BF and U+00BE, i.e. the valid
+        characters 'ï', '¿' and '¾', so words like "naïve" and Spanish
+        questions like "¿Cómo?" were being silently corrupted on ingest.
+        """
+        assert CleanProcessor.clean("naïve", None) == "naïve"
+        assert CleanProcessor.clean("¿Cómo estás?", None) == "¿Cómo estás?"
+        assert CleanProcessor.clean("¾ cup sugar", None) == "¾ cup sugar"
+        assert CleanProcessor.clean("￾", None) == "￾"
+
+        # The U+FFFE noncharacter is still stripped by its dedicated substitution.
+        assert CleanProcessor.clean("keep\ufffedrop", None) == "keepdrop"
+
     def test_clean_with_none_process_rule(self):
         """Test cleaning with None process_rule - only default cleaning applied."""
         text = "Hello<|World\x00"

api/tests/unit_tests/core/rag/indexing/test_indexing_runner.py

@@ -1372,6 +1372,19 @@ class TestIndexingRunnerDocumentCleaning:
         assert "\ufffe" not in result
         assert "Text with" in result
 
+    def test_filter_string_preserves_valid_extended_characters(self):
+        """filter_string must keep valid printable characters like 'ï', '¿', '¾'."""
+        # Arrange
+        text = "naïve ¿Cómo? ¾ done"
+
+        # Act
+        result = IndexingRunner.filter_string(text)
+
+        # Assert
+        assert result == text
+        # The U+FFFE noncharacter is still stripped.
+        assert IndexingRunner.filter_string("keep\ufffedrop") == "keepdrop"
+
 
 class TestIndexingRunnerSplitter:
     """Unit tests for text splitter configuration.