opensource/dify
2
0
Fork 0
mirror of https://github.com/langgenius/dify.git synced 2026-08-01 18:30:48 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(rag): add bounded timeout to Firecrawl extractor requests (#39831)

Browse Source
This commit is contained in:
Chester 2026-07-31 21:41:19 +08:00 committed by GitHub
parent 33518407c9
commit 074811106d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 18 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
api/core/rag/extractor/firecrawl/firecrawl_app.py
View File

@ -6,6 +6,10 @@ import httpx
from extensions.ext_storage import storage
# Bounded connect/read timeout so a slow or hanging Firecrawl endpoint cannot
# block extraction indefinitely (mirrors the WaterCrawl extractor client).
_REQUEST_TIMEOUT = httpx.Timeout(30.0, connect=5.0)
class FirecrawlDocumentData(TypedDict):
title: str | None
@ -176,7 +180,7 @@ class FirecrawlApp:
def _post_request(self, url, data, headers, retries=3, backoff_factor=0.5) -> httpx.Response:
response: httpx.Response | None = None
for attempt in range(retries):
response = httpx.post(url, headers=headers, json=data)
response = httpx.post(url, headers=headers, json=data, timeout=_REQUEST_TIMEOUT)
if response.status_code == 502:
time.sleep(backoff_factor * (2**attempt))
else:
@ -187,7 +191,7 @@ class FirecrawlApp:
def _get_request(self, url, headers, retries=3, backoff_factor=0.5) -> httpx.Response:
response: httpx.Response | None = None
for attempt in range(retries):
response = httpx.get(url, headers=headers)
response = httpx.get(url, headers=headers, timeout=_REQUEST_TIMEOUT)
if response.status_code == 502:
time.sleep(backoff_factor * (2**attempt))
else:

12
api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.py
View File

@ -35,6 +35,18 @@ class TestFirecrawlApp:
}
assert app._build_url("/v2/crawl") == "https://custom.firecrawl.dev/v2/crawl"
def test_requests_use_bounded_timeout(self, mocker: MockerFixture):
"""Outbound requests must carry a bounded timeout so a hanging endpoint cannot block extraction."""
app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")
mock_post = mocker.patch("httpx.post", return_value=_response(200, {"id": "job-1"}))
mock_get = mocker.patch("httpx.get", return_value=_response(200, {"status": "completed"}))
app._post_request("https://custom.firecrawl.dev/v1/crawl", {}, app._prepare_headers())
app._get_request("https://custom.firecrawl.dev/v1/crawl/job-1", app._prepare_headers())
assert mock_post.call_args.kwargs["timeout"] == firecrawl_module._REQUEST_TIMEOUT
assert mock_get.call_args.kwargs["timeout"] == firecrawl_module._REQUEST_TIMEOUT
def test_scrape_url_success(self, mocker: MockerFixture):
app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev")
mocker.patch(