From e4e99d198b5461ed43c97934832a035672956686 Mon Sep 17 00:00:00 2001 From: CYJ1226 <68060964+CYJ1226@users.noreply.github.com> Date: Mon, 27 Jul 2026 12:21:26 +0900 Subject: [PATCH] fix: preserve spaces during recursive text splitting (#39498) --- api/core/rag/splitter/fixed_text_splitter.py | 4 ++-- .../core/rag/splitter/test_text_splitter.py | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/api/core/rag/splitter/fixed_text_splitter.py b/api/core/rag/splitter/fixed_text_splitter.py index 98ba4d7fcc7..687f544e874 100644 --- a/api/core/rag/splitter/fixed_text_splitter.py +++ b/api/core/rag/splitter/fixed_text_splitter.py @@ -91,8 +91,8 @@ class FixedRecursiveCharacterTextSplitter(EnhanceRecursiveCharacterTextSplitter) splits = re.split(r" +", text) else: splits = text.split(separator) - if self._keep_separator: - splits = [s + separator for s in splits[:-1]] + splits[-1:] + if self._keep_separator: + splits = [s + separator for s in splits[:-1]] + splits[-1:] else: splits = list(text) if separator == "\n": diff --git a/api/tests/unit_tests/core/rag/splitter/test_text_splitter.py b/api/tests/unit_tests/core/rag/splitter/test_text_splitter.py index 12117241b5d..7a0726d3a9c 100644 --- a/api/tests/unit_tests/core/rag/splitter/test_text_splitter.py +++ b/api/tests/unit_tests/core/rag/splitter/test_text_splitter.py @@ -952,6 +952,21 @@ class TestFixedRecursiveCharacterTextSplitter: assert "word1" in combined assert "word2" in combined + def test_preserves_spaces_when_recursively_splitting_long_paragraph(self): + """Ensure recursive space splitting preserves word boundaries.""" + text = "여름철에는 항상 기상상황에 주목하며 주변 사람들과 함께 정보를 공유합니다." + splitter = FixedRecursiveCharacterTextSplitter( + fixed_separator="\n\n", + chunk_size=20, + chunk_overlap=0, + keep_separator=True, + ) + + result = splitter.split_text(text) + + assert len(result) > 1 + assert " ".join(result) == text + def test_character_level_splitting(self): """Test character-level splitting when no separator works.""" text = "verylongwordwithoutspaces"