From aeda37db6871b05c9f567e38b9f7e61c642d35fe Mon Sep 17 00:00:00 2001
From: Madan kumar <winklemad@outlook.com>
Date: Mon, 27 Jul 2026 08:23:33 +0530
Subject: [PATCH] =?UTF-8?q?fix(rag):=20stop=20the=20document=20cleaner=20f?=
 =?UTF-8?q?rom=20stripping=20valid=20characters=20=C3=AF,=20=C2=BF,=20?=
 =?UTF-8?q?=C2=BE=20(#39215)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 api/core/indexing_runner.py                     |  2 +-
 api/core/rag/cleaner/clean_processor.py         |  2 +-
 .../core/rag/cleaner/test_clean_processor.py    | 17 +++++++++++++++++
 .../core/rag/indexing/test_indexing_runner.py   | 13 +++++++++++++
 4 files changed, 32 insertions(+), 2 deletions(-)

diff --git a/api/core/indexing_runner.py b/api/core/indexing_runner.py
index 92246b6614c..63e00f48324 100644
--- a/api/core/indexing_runner.py
+++ b/api/core/indexing_runner.py
@@ -519,7 +519,7 @@ class IndexingRunner:
     def filter_string(text):
         text = re.sub(r"<\|", "<", text)
         text = re.sub(r"\|>", ">", text)
-        text = re.sub(r"[\x00-\x08\x0B\x0C\x0E-\x1F\x7F\xEF\xBF\xBE]", "", text)
+        text = re.sub(r"[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]", "", text)
         # Unicode  U+FFFE
         text = re.sub("\ufffe", "", text)
         return text
diff --git a/api/core/rag/cleaner/clean_processor.py b/api/core/rag/cleaner/clean_processor.py
index 790253053de..452251584e6 100644
--- a/api/core/rag/cleaner/clean_processor.py
+++ b/api/core/rag/cleaner/clean_processor.py
@@ -9,7 +9,7 @@ class CleanProcessor:
         # remove invalid symbol
         text = re.sub(r"<\|", "<", text)
         text = re.sub(r"\|>", ">", text)
-        text = re.sub(r"[\x00-\x08\x0B\x0C\x0E-\x1F\x7F\xEF\xBF\xBE]", "", text)
+        text = re.sub(r"[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]", "", text)
         # Unicode  U+FFFE
         text = re.sub("\ufffe", "", text)
 
diff --git a/api/tests/unit_tests/core/rag/cleaner/test_clean_processor.py b/api/tests/unit_tests/core/rag/cleaner/test_clean_processor.py
index c7a4265a954..2155342ab98 100644
--- a/api/tests/unit_tests/core/rag/cleaner/test_clean_processor.py
+++ b/api/tests/unit_tests/core/rag/cleaner/test_clean_processor.py
@@ -22,6 +22,23 @@ class TestCleanProcessor:
         expected = "normalpadding"
         assert CleanProcessor.clean(text_with_ufffe, None) == expected
 
+    def test_clean_preserves_valid_extended_characters(self):
+        """Default cleaning must not strip valid printable characters.
+
+        The invalid-symbol filter used to include the UTF-8 bytes of U+FFFE
+        (0xEF 0xBF 0xBE) inside a character class. On a decoded string those
+        bytes are the code points U+00EF, U+00BF and U+00BE, i.e. the valid
+        characters 'ï', '¿' and '¾', so words like "naïve" and Spanish
+        questions like "¿Cómo?" were being silently corrupted on ingest.
+        """
+        assert CleanProcessor.clean("naïve", None) == "naïve"
+        assert CleanProcessor.clean("¿Cómo estás?", None) == "¿Cómo estás?"
+        assert CleanProcessor.clean("¾ cup sugar", None) == "¾ cup sugar"
+        assert CleanProcessor.clean("￾", None) == "￾"
+
+        # The U+FFFE noncharacter is still stripped by its dedicated substitution.
+        assert CleanProcessor.clean("keep\ufffedrop", None) == "keepdrop"
+
     def test_clean_with_none_process_rule(self):
         """Test cleaning with None process_rule - only default cleaning applied."""
         text = "Hello<|World\x00"
diff --git a/api/tests/unit_tests/core/rag/indexing/test_indexing_runner.py b/api/tests/unit_tests/core/rag/indexing/test_indexing_runner.py
index 5307da6d343..ef1b38f49d7 100644
--- a/api/tests/unit_tests/core/rag/indexing/test_indexing_runner.py
+++ b/api/tests/unit_tests/core/rag/indexing/test_indexing_runner.py
@@ -1372,6 +1372,19 @@ class TestIndexingRunnerDocumentCleaning:
         assert "\ufffe" not in result
         assert "Text with" in result
 
+    def test_filter_string_preserves_valid_extended_characters(self):
+        """filter_string must keep valid printable characters like 'ï', '¿', '¾'."""
+        # Arrange
+        text = "naïve ¿Cómo? ¾ done"
+
+        # Act
+        result = IndexingRunner.filter_string(text)
+
+        # Assert
+        assert result == text
+        # The U+FFFE noncharacter is still stripped.
+        assert IndexingRunner.filter_string("keep\ufffedrop") == "keepdrop"
+
 
 class TestIndexingRunnerSplitter:
     """Unit tests for text splitter configuration.