opensource/dify
2
0
Fork 0
mirror of https://github.com/langgenius/dify.git synced 2026-06-17 06:21:07 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(watercrawl): accept content type parameters (#37500)

Browse Source
This commit is contained in:
落尘 2026-06-16 16:16:24 +08:00 committed by GitHub
parent fe64c5d4a8
commit 431d6bb983
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 13 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
api/core/rag/extractor/watercrawl/client.py
View File

@ -118,16 +118,18 @@ class WaterCrawlAPIClient(BaseAPIClient):
response.raise_for_status()
if response.status_code == 204:
return None
if response.headers.get("Content-Type") == "application/json":
content_type = response.headers.get("Content-Type", "")
media_type = content_type.split(";", 1)[0].strip().lower()
if media_type == "application/json":
return response.json() or {}
if response.headers.get("Content-Type") == "application/octet-stream":
if media_type == "application/octet-stream":
return response.content
if response.headers.get("Content-Type") == "text/event-stream":
if media_type == "text/event-stream":
return self.process_eventstream(response)
raise Exception(f"Unknown response type: {response.headers.get('Content-Type')}")
raise Exception(f"Unknown response type: {content_type}")
def get_crawl_requests_list(self, page: int | None = None, page_size: int | None = None):
query_params = {"page": page or 1, "page_size": page_size or 10}

7
api/tests/unit_tests/core/rag/extractor/watercrawl/test_watercrawl.py
View File

@ -168,6 +168,13 @@ class TestWaterCrawlAPIClient:
assert client.process_response(_response(200, {"ok": True})) == {"ok": True}
assert client.process_response(_response(200, None)) == {}
def test_process_response_accepts_json_content_type_parameters(self):
client = WaterCrawlAPIClient(api_key="k")
response = _response(200, {"ok": True}, content_type="application/json; charset=utf-8")
assert client.process_response(response) == {"ok": True}
def test_process_response_octet_stream_returns_bytes(self):
client = WaterCrawlAPIClient(api_key="k")
assert (