diff --git a/api/core/rag/splitter/fixed_text_splitter.py b/api/core/rag/splitter/fixed_text_splitter.py index 98ba4d7fcc7..687f544e874 100644 --- a/api/core/rag/splitter/fixed_text_splitter.py +++ b/api/core/rag/splitter/fixed_text_splitter.py @@ -91,8 +91,8 @@ class FixedRecursiveCharacterTextSplitter(EnhanceRecursiveCharacterTextSplitter) splits = re.split(r" +", text) else: splits = text.split(separator) - if self._keep_separator: - splits = [s + separator for s in splits[:-1]] + splits[-1:] + if self._keep_separator: + splits = [s + separator for s in splits[:-1]] + splits[-1:] else: splits = list(text) if separator == "\n": diff --git a/api/tests/unit_tests/core/rag/splitter/test_text_splitter.py b/api/tests/unit_tests/core/rag/splitter/test_text_splitter.py index 12117241b5d..7a0726d3a9c 100644 --- a/api/tests/unit_tests/core/rag/splitter/test_text_splitter.py +++ b/api/tests/unit_tests/core/rag/splitter/test_text_splitter.py @@ -952,6 +952,21 @@ class TestFixedRecursiveCharacterTextSplitter: assert "word1" in combined assert "word2" in combined + def test_preserves_spaces_when_recursively_splitting_long_paragraph(self): + """Ensure recursive space splitting preserves word boundaries.""" + text = "여름철에는 항상 기상상황에 주목하며 주변 사람들과 함께 정보를 공유합니다." + splitter = FixedRecursiveCharacterTextSplitter( + fixed_separator="\n\n", + chunk_size=20, + chunk_overlap=0, + keep_separator=True, + ) + + result = splitter.split_text(text) + + assert len(result) > 1 + assert " ".join(result) == text + def test_character_level_splitting(self): """Test character-level splitting when no separator works.""" text = "verylongwordwithoutspaces"