fix(api): bound JinaReader and adaptive-crawl HTTP timeouts (#39859) (#39860)

Co-authored-by: Harsh Kashyap <Harsh23Kashyap@users.noreply.github.com>

api/services/website_service.py

@@ -17,13 +17,22 @@ from extensions.ext_storage import storage
 from services.datasource_provider_service import DatasourceProviderService
 
 # Reuse pooled HTTP clients to avoid creating new connections per request and ease testing.
+# Both clients carry a bounded read/connect timeout so a stalled Jina or
+# adaptive-crawl endpoint fails fast instead of pinning a worker. The values
+# match the floor used in the WaterCrawl PR (#37512). See #39859.
 _jina_http_client: httpx.Client = get_pooled_http_client(
     "website:jinareader",
-    lambda: httpx.Client(limits=httpx.Limits(max_keepalive_connections=50, max_connections=100)),
+    lambda: httpx.Client(
+        timeout=httpx.Timeout(30.0, connect=5.0),
+        limits=httpx.Limits(max_keepalive_connections=50, max_connections=100),
+    ),
 )
 _adaptive_http_client: httpx.Client = get_pooled_http_client(
     "website:adaptivecrawl",
-    lambda: httpx.Client(limits=httpx.Limits(max_keepalive_connections=50, max_connections=100)),
+    lambda: httpx.Client(
+        timeout=httpx.Timeout(30.0, connect=5.0),
+        limits=httpx.Limits(max_keepalive_connections=50, max_connections=100),
+    ),
 )
 
 

api/tests/unit_tests/services/test_website_service.py

@@ -724,3 +724,23 @@ def test_scrape_with_watercrawl_calls_provider(monkeypatch: pytest.MonkeyPatch)
     )
     assert result == {"markdown": "m"}
     provider_instance.scrape_url.assert_called_once_with("u")
+
+
+def test_pooled_clients_carry_bounded_timeouts() -> None:
+    """Regression for #39859: the Jina and adaptive-crawl pooled clients
+    must carry a read/connect timeout so a stalled endpoint fails fast
+    instead of pinning a worker. Same shape as the WaterCrawl hardening
+    that landed in PR #37512.
+    """
+    jina = website_service_module._jina_http_client
+    adaptive = website_service_module._adaptive_http_client
+
+    # Read and connect bounds are set.
+    assert jina.timeout is not None
+    assert adaptive.timeout is not None
+
+    # The values match the documented floor (read 30.0 / connect 5.0).
+    assert jina.timeout.read == 30.0
+    assert jina.timeout.connect == 5.0
+    assert adaptive.timeout.read == 30.0
+    assert adaptive.timeout.connect == 5.0