opensource/dify
2
0
Fork 0
mirror of https://github.com/langgenius/dify.git synced 2026-07-21 18:58:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(watercrawl): handle non-json error responses (#37514)

Browse Source
This commit is contained in:
落尘 2026-07-21 14:41:06 +08:00 committed by GitHub
parent 7da6b2ce36
commit f8754286ec
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 47 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
api/core/rag/extractor/watercrawl/client.py
View File

@ -131,7 +131,10 @@ class WaterCrawlAPIClient(BaseAPIClient):
content_type = response.headers.get("Content-Type", "")
media_type = content_type.split(";", 1)[0].strip().lower()
if media_type == "application/json":
return response.json() or {}
try:
return response.json() or {}
except ValueError as exc:
raise ValueError("Invalid JSON response from WaterCrawl") from exc
if media_type == "application/octet-stream":
return response.content

23
api/core/rag/extractor/watercrawl/exceptions.py
View File

@ -1,5 +1,15 @@
"""WaterCrawl domain exceptions.
These exceptions are constructed from upstream HTTP responses, which may be
JSON API errors or plain text/HTML proxy errors. Keep the exception type stable
even when the body is not JSON so callers can handle WaterCrawl failures by
domain type instead of low-level parser errors.
"""
import json
from typing import override
from typing import Any, override
from httpx import Response
class WaterCrawlError(Exception):
@ -7,11 +17,16 @@ class WaterCrawlError(Exception):
class WaterCrawlBadRequestError(WaterCrawlError):
def __init__(self, response):
def __init__(self, response: Response):
self.status_code = response.status_code
self.response = response
data = response.json()
self.message = data.get("message", "Unknown error occurred")
try:
data: Any = response.json()
except ValueError:
data = {}
if not isinstance(data, dict):
data = {}
self.message = data.get("message") or response.text or "Unknown error occurred"
self.errors = data.get("errors", {})
super().__init__(self.message)

24
api/tests/unit_tests/core/rag/extractor/watercrawl/test_watercrawl.py
View File

@ -176,6 +176,22 @@ class TestWaterCrawlAPIClient:
with pytest.raises(expected_exception):
client.process_response(_response(status, {"message": "bad", "errors": {"url": ["x"]}}))
@pytest.mark.parametrize(
("status", "expected_exception"),
[
(401, WaterCrawlAuthenticationError),
(403, WaterCrawlPermissionError),
(422, WaterCrawlBadRequestError),
],
)
def test_process_response_error_statuses_with_non_json_body(self, status: int, expected_exception: type[Exception]):
client = WaterCrawlAPIClient(api_key="k")
response = _response(status, text="<html>upstream error</html>")
response.json.side_effect = json.JSONDecodeError("Expecting value", response.text, 0)
with pytest.raises(expected_exception):
client.process_response(response)
def test_process_response_204_returns_none(self):
client = WaterCrawlAPIClient(api_key="k")
assert client.process_response(_response(204, None)) is None
@ -185,6 +201,14 @@ class TestWaterCrawlAPIClient:
assert client.process_response(_response(200, {"ok": True})) == {"ok": True}
assert client.process_response(_response(200, None)) == {}
def test_process_response_json_payload_with_invalid_body_raises_clear_error(self):
client = WaterCrawlAPIClient(api_key="k")
response = _response(200, text="<html>upstream error</html>")
response.json.side_effect = json.JSONDecodeError("Expecting value", response.text, 0)
with pytest.raises(ValueError, match="Invalid JSON response from WaterCrawl"):
client.process_response(response)
def test_process_response_accepts_json_content_type_parameters(self):
client = WaterCrawlAPIClient(api_key="k")