diff --git a/api/services/website_service.py b/api/services/website_service.py index ea584088bbc..b833c146216 100644 --- a/api/services/website_service.py +++ b/api/services/website_service.py @@ -17,13 +17,22 @@ from extensions.ext_storage import storage from services.datasource_provider_service import DatasourceProviderService # Reuse pooled HTTP clients to avoid creating new connections per request and ease testing. +# Both clients carry a bounded read/connect timeout so a stalled Jina or +# adaptive-crawl endpoint fails fast instead of pinning a worker. The values +# match the floor used in the WaterCrawl PR (#37512). See #39859. _jina_http_client: httpx.Client = get_pooled_http_client( "website:jinareader", - lambda: httpx.Client(limits=httpx.Limits(max_keepalive_connections=50, max_connections=100)), + lambda: httpx.Client( + timeout=httpx.Timeout(30.0, connect=5.0), + limits=httpx.Limits(max_keepalive_connections=50, max_connections=100), + ), ) _adaptive_http_client: httpx.Client = get_pooled_http_client( "website:adaptivecrawl", - lambda: httpx.Client(limits=httpx.Limits(max_keepalive_connections=50, max_connections=100)), + lambda: httpx.Client( + timeout=httpx.Timeout(30.0, connect=5.0), + limits=httpx.Limits(max_keepalive_connections=50, max_connections=100), + ), ) diff --git a/api/tests/unit_tests/services/test_website_service.py b/api/tests/unit_tests/services/test_website_service.py index 2024aec13a3..571e8e99903 100644 --- a/api/tests/unit_tests/services/test_website_service.py +++ b/api/tests/unit_tests/services/test_website_service.py @@ -724,3 +724,23 @@ def test_scrape_with_watercrawl_calls_provider(monkeypatch: pytest.MonkeyPatch) ) assert result == {"markdown": "m"} provider_instance.scrape_url.assert_called_once_with("u") + + +def test_pooled_clients_carry_bounded_timeouts() -> None: + """Regression for #39859: the Jina and adaptive-crawl pooled clients + must carry a read/connect timeout so a stalled endpoint fails fast + instead of pinning a worker. Same shape as the WaterCrawl hardening + that landed in PR #37512. + """ + jina = website_service_module._jina_http_client + adaptive = website_service_module._adaptive_http_client + + # Read and connect bounds are set. + assert jina.timeout is not None + assert adaptive.timeout is not None + + # The values match the documented floor (read 30.0 / connect 5.0). + assert jina.timeout.read == 30.0 + assert jina.timeout.connect == 5.0 + assert adaptive.timeout.read == 30.0 + assert adaptive.timeout.connect == 5.0