From f3154e347e5012016e863113efe6bc3295e757d9 Mon Sep 17 00:00:00 2001 From: Sourav Rajvi <144546710+Souravrajvi0@users.noreply.github.com> Date: Tue, 8 Sep 2026 08:47:42 +0000 Subject: [PATCH] fix(rag): preserve CSV cell text during knowledge import (#41922) --- api/core/rag/extractor/csv_extractor.py | 3 ++- .../core/rag/extractor/test_csv_extractor.py | 21 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/api/core/rag/extractor/csv_extractor.py b/api/core/rag/extractor/csv_extractor.py index 778dbd40c68..b836e789c2d 100644 --- a/api/core/rag/extractor/csv_extractor.py +++ b/api/core/rag/extractor/csv_extractor.py @@ -31,7 +31,8 @@ class CSVExtractor(BaseExtractor): self._encoding = encoding self._autodetect_encoding = autodetect_encoding self.source_column = source_column - self.csv_args = csv_args or {} + # Preserve source text for indexing unless the caller requests type or NA conversion. + self.csv_args: dict[str, Any] = {"dtype": str, "keep_default_na": False, **(csv_args or {})} @override def extract(self) -> list[Document]: diff --git a/api/tests/unit_tests/core/rag/extractor/test_csv_extractor.py b/api/tests/unit_tests/core/rag/extractor/test_csv_extractor.py index f82ebb4d75d..6800ab9bb24 100644 --- a/api/tests/unit_tests/core/rag/extractor/test_csv_extractor.py +++ b/api/tests/unit_tests/core/rag/extractor/test_csv_extractor.py @@ -22,6 +22,27 @@ class _ManagedStringIO(io.StringIO): class TestCSVExtractor: + @pytest.mark.parametrize("value", ["00123", "1.00", "1e3", "NA", "NULL", "N/A", "", "hello"]) + def test_extract_preserves_cell_text(self, tmp_path: Path, value: str) -> None: + file_path = tmp_path / "data.csv" + file_path.write_text(f"value,body\n{value},reference\n", encoding="utf-8") + + docs = CSVExtractor(str(file_path), encoding="utf-8", source_column="value").extract() + + assert len(docs) == 1 + assert docs[0].page_content == f"value: {value};body: reference" + assert docs[0].metadata["source"] == value + + def test_extract_honors_explicit_csv_args(self, tmp_path: Path) -> None: + file_path = tmp_path / "data.csv" + file_path.write_text("value;body\n00123;NA\n", encoding="utf-8") + csv_args = {"sep": ";", "dtype": {"value": int}, "keep_default_na": True} + + docs = CSVExtractor(str(file_path), encoding="utf-8", csv_args=csv_args).extract() + + assert docs[0].page_content == "value: 123.0;body: nan" + assert csv_args == {"sep": ";", "dtype": {"value": int}, "keep_default_na": True} + def test_extract_success_with_source_column(self, tmp_path: Path): file_path = tmp_path / "data.csv" file_path.write_text("id,body\nsource-1,hello\n", encoding="utf-8")