From 074811106dc81c37deeae26a165306018c65bef2 Mon Sep 17 00:00:00 2001 From: Chester Date: Fri, 31 Jul 2026 21:41:19 +0800 Subject: [PATCH] fix(rag): add bounded timeout to Firecrawl extractor requests (#39831) --- api/core/rag/extractor/firecrawl/firecrawl_app.py | 8 ++++++-- .../core/rag/extractor/firecrawl/test_firecrawl.py | 12 ++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/api/core/rag/extractor/firecrawl/firecrawl_app.py b/api/core/rag/extractor/firecrawl/firecrawl_app.py index 556158cf00a..77edffa77cd 100644 --- a/api/core/rag/extractor/firecrawl/firecrawl_app.py +++ b/api/core/rag/extractor/firecrawl/firecrawl_app.py @@ -6,6 +6,10 @@ import httpx from extensions.ext_storage import storage +# Bounded connect/read timeout so a slow or hanging Firecrawl endpoint cannot +# block extraction indefinitely (mirrors the WaterCrawl extractor client). +_REQUEST_TIMEOUT = httpx.Timeout(30.0, connect=5.0) + class FirecrawlDocumentData(TypedDict): title: str | None @@ -176,7 +180,7 @@ class FirecrawlApp: def _post_request(self, url, data, headers, retries=3, backoff_factor=0.5) -> httpx.Response: response: httpx.Response | None = None for attempt in range(retries): - response = httpx.post(url, headers=headers, json=data) + response = httpx.post(url, headers=headers, json=data, timeout=_REQUEST_TIMEOUT) if response.status_code == 502: time.sleep(backoff_factor * (2**attempt)) else: @@ -187,7 +191,7 @@ class FirecrawlApp: def _get_request(self, url, headers, retries=3, backoff_factor=0.5) -> httpx.Response: response: httpx.Response | None = None for attempt in range(retries): - response = httpx.get(url, headers=headers) + response = httpx.get(url, headers=headers, timeout=_REQUEST_TIMEOUT) if response.status_code == 502: time.sleep(backoff_factor * (2**attempt)) else: diff --git a/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.py b/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.py index db49221583f..3bf1441c0af 100644 --- a/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.py +++ b/api/tests/unit_tests/core/rag/extractor/firecrawl/test_firecrawl.py @@ -35,6 +35,18 @@ class TestFirecrawlApp: } assert app._build_url("/v2/crawl") == "https://custom.firecrawl.dev/v2/crawl" + def test_requests_use_bounded_timeout(self, mocker: MockerFixture): + """Outbound requests must carry a bounded timeout so a hanging endpoint cannot block extraction.""" + app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev") + mock_post = mocker.patch("httpx.post", return_value=_response(200, {"id": "job-1"})) + mock_get = mocker.patch("httpx.get", return_value=_response(200, {"status": "completed"})) + + app._post_request("https://custom.firecrawl.dev/v1/crawl", {}, app._prepare_headers()) + app._get_request("https://custom.firecrawl.dev/v1/crawl/job-1", app._prepare_headers()) + + assert mock_post.call_args.kwargs["timeout"] == firecrawl_module._REQUEST_TIMEOUT + assert mock_get.call_args.kwargs["timeout"] == firecrawl_module._REQUEST_TIMEOUT + def test_scrape_url_success(self, mocker: MockerFixture): app = FirecrawlApp(api_key="fc-key", base_url="https://custom.firecrawl.dev") mocker.patch(