opensource/dify
2
0
Fork 0
mirror of https://github.com/langgenius/dify.git synced 2026-09-09 05:41:00 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(api): flatten readabilipy plain_text items in web reader tool (#41954)

Browse Source
This commit is contained in:
Harsh Kashyap 2026-09-08 10:28:20 +00:00 committed by GitHub
parent 398addf0c1
commit 09ddcbe01e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 37 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
api/core/tools/utils/web_reader_tool.py
View File

@ -1,6 +1,5 @@
import mimetypes
import re
from collections.abc import Sequence
from dataclasses import dataclass
from typing import Any
from urllib.parse import unquote
@ -101,15 +100,24 @@ def get_url(url: str, user_agent: str | None = None) -> str:
class Article:
title: str
author: str
text: Sequence[dict]
text: str
def extract_using_readabilipy(html: str):
json_article: dict[str, Any] = simple_json_from_html_string(html, use_readability=False)
# readabilipy returns plain_text as a list of dicts whose "text" values
# contain the (possibly HTML-tagged) paragraph content; flatten them into
# a single clean text block.
plain_text = json_article.get("plain_text") or []
text = "\n".join(
stripped
for item in plain_text
if isinstance(item, dict) and (stripped := re.sub(r"<[^>]+>", "", str(item.get("text") or "")).strip())
)
article = Article(
title=json_article.get("title") or "",
author=json_article.get("byline") or "",
text=json_article.get("plain_text") or [],
text=text,
)
return article

30
api/tests/unit_tests/core/tools/utils/test_web_reader_tool.py
View File

@ -286,9 +286,31 @@ def test_extract_using_readabilipy_field_mapping_and_defaults(monkeypatch: pytes
article = extract_using_readabilipy("<html>...</html>")
assert article.title == "Hello"
assert article.author == "Alice"
assert isinstance(article.text, list)
assert article.text
assert article.text[0]["text"] == "world"
assert article.text == "world"
def test_extract_using_readabilipy_flattens_plain_text_items(monkeypatch: pytest.MonkeyPatch):
"""readabilipy returns plain_text as a list of dicts; the article text must be clean joined text,
not the Python repr of that list, and HTML tags inside items must be stripped."""
def fake_simple_json_from_html_string(html, use_readability=True):
return {
"title": "T",
"byline": "A",
"plain_text": [
{"type": "text", "text": "<p>First paragraph.</p>"},
{"type": "text", "text": "Second paragraph."},
{"type": "text", "text": ""},
"not-a-dict",
],
}
import core.tools.utils.web_reader_tool as mod
monkeypatch.setattr(mod, "simple_json_from_html_string", fake_simple_json_from_html_string)
article = extract_using_readabilipy("<html>...</html>")
assert article.text == "First paragraph.\nSecond paragraph."
def test_extract_using_readabilipy_defaults_when_missing(monkeypatch: pytest.MonkeyPatch):
@ -302,7 +324,7 @@ def test_extract_using_readabilipy_defaults_when_missing(monkeypatch: pytest.Mon
article = extract_using_readabilipy("<html>...</html>")
assert article.title == ""
assert article.author == ""
assert article.text == []
assert article.text == ""
# ---------------------------