From f8754286ec07d83a7a3ad4033982bd5a13d41361 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=90=BD=E5=B0=98?= Date: Tue, 21 Jul 2026 14:41:06 +0800 Subject: [PATCH] fix(watercrawl): handle non-json error responses (#37514) --- api/core/rag/extractor/watercrawl/client.py | 5 +++- .../rag/extractor/watercrawl/exceptions.py | 23 ++++++++++++++---- .../extractor/watercrawl/test_watercrawl.py | 24 +++++++++++++++++++ 3 files changed, 47 insertions(+), 5 deletions(-) diff --git a/api/core/rag/extractor/watercrawl/client.py b/api/core/rag/extractor/watercrawl/client.py index 51dd26b1ad0..b069977de9b 100644 --- a/api/core/rag/extractor/watercrawl/client.py +++ b/api/core/rag/extractor/watercrawl/client.py @@ -131,7 +131,10 @@ class WaterCrawlAPIClient(BaseAPIClient): content_type = response.headers.get("Content-Type", "") media_type = content_type.split(";", 1)[0].strip().lower() if media_type == "application/json": - return response.json() or {} + try: + return response.json() or {} + except ValueError as exc: + raise ValueError("Invalid JSON response from WaterCrawl") from exc if media_type == "application/octet-stream": return response.content diff --git a/api/core/rag/extractor/watercrawl/exceptions.py b/api/core/rag/extractor/watercrawl/exceptions.py index fc457697a22..e5b38f8acfe 100644 --- a/api/core/rag/extractor/watercrawl/exceptions.py +++ b/api/core/rag/extractor/watercrawl/exceptions.py @@ -1,5 +1,15 @@ +"""WaterCrawl domain exceptions. + +These exceptions are constructed from upstream HTTP responses, which may be +JSON API errors or plain text/HTML proxy errors. Keep the exception type stable +even when the body is not JSON so callers can handle WaterCrawl failures by +domain type instead of low-level parser errors. +""" + import json -from typing import override +from typing import Any, override + +from httpx import Response class WaterCrawlError(Exception): @@ -7,11 +17,16 @@ class WaterCrawlError(Exception): class WaterCrawlBadRequestError(WaterCrawlError): - def __init__(self, response): + def __init__(self, response: Response): self.status_code = response.status_code self.response = response - data = response.json() - self.message = data.get("message", "Unknown error occurred") + try: + data: Any = response.json() + except ValueError: + data = {} + if not isinstance(data, dict): + data = {} + self.message = data.get("message") or response.text or "Unknown error occurred" self.errors = data.get("errors", {}) super().__init__(self.message) diff --git a/api/tests/unit_tests/core/rag/extractor/watercrawl/test_watercrawl.py b/api/tests/unit_tests/core/rag/extractor/watercrawl/test_watercrawl.py index 673f1e41c5d..bfb3754a7fd 100644 --- a/api/tests/unit_tests/core/rag/extractor/watercrawl/test_watercrawl.py +++ b/api/tests/unit_tests/core/rag/extractor/watercrawl/test_watercrawl.py @@ -176,6 +176,22 @@ class TestWaterCrawlAPIClient: with pytest.raises(expected_exception): client.process_response(_response(status, {"message": "bad", "errors": {"url": ["x"]}})) + @pytest.mark.parametrize( + ("status", "expected_exception"), + [ + (401, WaterCrawlAuthenticationError), + (403, WaterCrawlPermissionError), + (422, WaterCrawlBadRequestError), + ], + ) + def test_process_response_error_statuses_with_non_json_body(self, status: int, expected_exception: type[Exception]): + client = WaterCrawlAPIClient(api_key="k") + response = _response(status, text="upstream error") + response.json.side_effect = json.JSONDecodeError("Expecting value", response.text, 0) + + with pytest.raises(expected_exception): + client.process_response(response) + def test_process_response_204_returns_none(self): client = WaterCrawlAPIClient(api_key="k") assert client.process_response(_response(204, None)) is None @@ -185,6 +201,14 @@ class TestWaterCrawlAPIClient: assert client.process_response(_response(200, {"ok": True})) == {"ok": True} assert client.process_response(_response(200, None)) == {} + def test_process_response_json_payload_with_invalid_body_raises_clear_error(self): + client = WaterCrawlAPIClient(api_key="k") + response = _response(200, text="upstream error") + response.json.side_effect = json.JSONDecodeError("Expecting value", response.text, 0) + + with pytest.raises(ValueError, match="Invalid JSON response from WaterCrawl"): + client.process_response(response) + def test_process_response_accepts_json_content_type_parameters(self): client = WaterCrawlAPIClient(api_key="k")