diff --git a/api/core/tools/utils/web_reader_tool.py b/api/core/tools/utils/web_reader_tool.py index 2e630b803eb..056703413af 100644 --- a/api/core/tools/utils/web_reader_tool.py +++ b/api/core/tools/utils/web_reader_tool.py @@ -1,6 +1,5 @@ import mimetypes import re -from collections.abc import Sequence from dataclasses import dataclass from typing import Any from urllib.parse import unquote @@ -101,15 +100,24 @@ def get_url(url: str, user_agent: str | None = None) -> str: class Article: title: str author: str - text: Sequence[dict] + text: str def extract_using_readabilipy(html: str): json_article: dict[str, Any] = simple_json_from_html_string(html, use_readability=False) + # readabilipy returns plain_text as a list of dicts whose "text" values + # contain the (possibly HTML-tagged) paragraph content; flatten them into + # a single clean text block. + plain_text = json_article.get("plain_text") or [] + text = "\n".join( + stripped + for item in plain_text + if isinstance(item, dict) and (stripped := re.sub(r"<[^>]+>", "", str(item.get("text") or "")).strip()) + ) article = Article( title=json_article.get("title") or "", author=json_article.get("byline") or "", - text=json_article.get("plain_text") or [], + text=text, ) return article diff --git a/api/tests/unit_tests/core/tools/utils/test_web_reader_tool.py b/api/tests/unit_tests/core/tools/utils/test_web_reader_tool.py index f66a2cc7067..80ab51b73c5 100644 --- a/api/tests/unit_tests/core/tools/utils/test_web_reader_tool.py +++ b/api/tests/unit_tests/core/tools/utils/test_web_reader_tool.py @@ -286,9 +286,31 @@ def test_extract_using_readabilipy_field_mapping_and_defaults(monkeypatch: pytes article = extract_using_readabilipy("...") assert article.title == "Hello" assert article.author == "Alice" - assert isinstance(article.text, list) - assert article.text - assert article.text[0]["text"] == "world" + assert article.text == "world" + + +def test_extract_using_readabilipy_flattens_plain_text_items(monkeypatch: pytest.MonkeyPatch): + """readabilipy returns plain_text as a list of dicts; the article text must be clean joined text, + not the Python repr of that list, and HTML tags inside items must be stripped.""" + + def fake_simple_json_from_html_string(html, use_readability=True): + return { + "title": "T", + "byline": "A", + "plain_text": [ + {"type": "text", "text": "

First paragraph.

"}, + {"type": "text", "text": "Second paragraph."}, + {"type": "text", "text": ""}, + "not-a-dict", + ], + } + + import core.tools.utils.web_reader_tool as mod + + monkeypatch.setattr(mod, "simple_json_from_html_string", fake_simple_json_from_html_string) + + article = extract_using_readabilipy("...") + assert article.text == "First paragraph.\nSecond paragraph." def test_extract_using_readabilipy_defaults_when_missing(monkeypatch: pytest.MonkeyPatch): @@ -302,7 +324,7 @@ def test_extract_using_readabilipy_defaults_when_missing(monkeypatch: pytest.Mon article = extract_using_readabilipy("...") assert article.title == "" assert article.author == "" - assert article.text == [] + assert article.text == "" # ---------------------------