feat: http

api/core/helper/ssrf_proxy.py

@@ -26,6 +26,7 @@ httpx_proxies = {
 } if SSRF_PROXY_HTTP_URL and SSRF_PROXY_HTTPS_URL else None
 
 def get(url, *args, **kwargs):
+    print(url, kwargs)
     return _get(url=url, *args, proxies=httpx_proxies, **kwargs)
 
 def post(url, *args, **kwargs):

api/core/workflow/nodes/http_request/entities.py

@@ -1,4 +1,4 @@
-from typing import Literal, Union
+from typing import Literal, Optional, Union
 
 from pydantic import BaseModel
 
@@ -29,4 +29,4 @@ class HttpRequestNodeData(BaseNodeData):
     authorization: Authorization
     headers: str
     params: str
-    body: Body
+    body: Optional[Body]

api/core/workflow/nodes/http_request/http_executor.py

@@ -76,11 +76,17 @@ class HttpExecutor:
         # fill in params
         kv_paris = original_params.split('\n')
         for kv in kv_paris:
+            if not kv.strip():
+                continue
+
             kv = kv.split(':')
-            if len(kv) != 2:
+            if len(kv) == 2:
+                k, v = kv
+            elif len(kv) == 1:
+                k, v = kv[0], ''
+            else:
                 raise ValueError(f'Invalid params {kv}')
             
-            k, v = kv
             self.params[k] = v
 
         # extract all template in headers
@@ -96,51 +102,61 @@ class HttpExecutor:
         # fill in headers
         kv_paris = original_headers.split('\n')
         for kv in kv_paris:
+            if not kv.strip():
+                continue
+
             kv = kv.split(':')
-            if len(kv) != 2:
+            if len(kv) == 2:
+                k, v = kv
+            elif len(kv) == 1:
+                k, v = kv[0], ''
+            else:
                 raise ValueError(f'Invalid headers {kv}')
             
-            k, v = kv
             self.headers[k] = v
 
         # extract all template in body
-        body_template = re.findall(r'{{(.*?)}}', node_data.body.data or '') or []
+        if node_data.body:
-        body_template = list(set(body_template))
+            body_template = re.findall(r'{{(.*?)}}', node_data.body.data or '') or []
-        original_body = node_data.body.data or ''
+            body_template = list(set(body_template))
-        for body in body_template:
+            original_body = node_data.body.data or ''
-            if not body:
+            for body in body_template:
-                continue
+                if not body:
+                    continue
 
-            original_body = original_body.replace(f'{{{{{body}}}}}', str(variables.get(body, '')))
+                original_body = original_body.replace(f'{{{{{body}}}}}', str(variables.get(body, '')))
 
-        if node_data.body.type == 'json':
+            if node_data.body.type == 'json':
-            self.headers['Content-Type'] = 'application/json'
+                self.headers['Content-Type'] = 'application/json'
-        elif node_data.body.type == 'x-www-form-urlencoded':
+            elif node_data.body.type == 'x-www-form-urlencoded':
-            self.headers['Content-Type'] = 'application/x-www-form-urlencoded'
+                self.headers['Content-Type'] = 'application/x-www-form-urlencoded'
-        # elif node_data.body.type == 'form-data':
+            # elif node_data.body.type == 'form-data':
-        #    self.headers['Content-Type'] = 'multipart/form-data'
+            #    self.headers['Content-Type'] = 'multipart/form-data'
 
-        if node_data.body.type in ['form-data', 'x-www-form-urlencoded']:
+            if node_data.body.type in ['form-data', 'x-www-form-urlencoded']:
-            body = {}
+                body = {}
-            kv_paris = original_body.split('\n')
+                kv_paris = original_body.split('\n')
-            for kv in kv_paris:
+                for kv in kv_paris:
-                kv = kv.split(':')
+                    kv = kv.split(':')
-                if len(kv) != 2:
+                    if len(kv) == 2:
-                    raise ValueError(f'Invalid body {kv}')
+                        body[kv[0]] = kv[1]
-                body[kv[0]] = kv[1]
+                    elif len(kv) == 1:
+                        body[kv[0]] = ''
+                    else:
+                        raise ValueError(f'Invalid body {kv}')
 
-            if node_data.body.type == 'form-data':
+                if node_data.body.type == 'form-data':
-                self.files = {
+                    self.files = {
-                    k: ('', v) for k, v in body.items()
+                        k: ('', v) for k, v in body.items()
-                }
+                    }
+                else:
+                    self.body = urlencode(body)
             else:
-                self.body = urlencode(body)
+                self.body = original_body
-        else:
-            self.body = original_body
                 
     def _assembling_headers(self) -> dict[str, Any]:
         authorization = deepcopy(self.authorization)
-        headers = deepcopy(self.headers) or []
+        headers = deepcopy(self.headers) or {}
         if self.authorization.type == 'api-key':
             if self.authorization.config.api_key is None:
                 raise ValueError('api_key is required')

api/core/workflow/nodes/http_request/http_request_node.py

@@ -24,10 +24,12 @@ class HttpRequestNode(BaseNode):
         # init http executor
         try:
             http_executor = HttpExecutor(node_data=node_data, variables=variables)
-            # invoke http executor
 
+            # invoke http executor
             response = http_executor.invoke()
         except Exception as e:
+            import traceback
+            print(traceback.format_exc())
             return NodeRunResult(
                 status=WorkflowNodeExecutionStatus.FAILED,
                 inputs=variables,

api/tests/integration_tests/workflow/nodes/__mock/http.py

@@ -0,0 +1,82 @@
+import os
+import pytest
+import requests.api as requests
+import httpx._api as httpx
+from requests import Response as RequestsResponse
+from yarl import URL
+
+from typing import Literal
+from _pytest.monkeypatch import MonkeyPatch
+from json import dumps
+
+MOCK = os.getenv('MOCK_SWITCH', 'false') == 'true'
+
+class MockedHttp:
+    def requests_request(self, method: Literal['GET', 'POST', 'PUT', 'DELETE', 'PATCH', 'OPTIONS'], 
+                         url: str, **kwargs) -> RequestsResponse:
+        """
+        Mocked requests.request
+        """
+        response = RequestsResponse()
+        response.url = str(URL(url) % kwargs.get('params', {}))
+        response.headers = kwargs.get('headers', {})
+
+        if url == 'http://404.com':
+            response.status_code = 404
+            response._content = b'Not Found'
+            return response
+        
+        # get data, files
+        data = kwargs.get('data', None)
+        files = kwargs.get('files', None)
+
+        if data is not None:
+            resp = dumps(data).encode('utf-8')
+        if files is not None:
+            resp = dumps(files).encode('utf-8')
+        else:
+            resp = b'OK'
+
+        response.status_code = 200
+        response._content = resp
+        return response
+
+    def httpx_request(self, method: Literal['GET', 'POST', 'PUT', 'DELETE', 'PATCH', 'OPTIONS'], 
+                      url: str, **kwargs) -> httpx.Response:
+        """
+        Mocked httpx.request
+        """
+        response = httpx.Response()
+        response.url = str(URL(url) % kwargs.get('params', {}))
+        response.headers = kwargs.get('headers', {})
+
+        if url == 'http://404.com':
+            response.status_code = 404
+            response.content = b'Not Found'
+            return response
+        
+        # get data, files
+        data = kwargs.get('data', None)
+        files = kwargs.get('files', None)
+
+        if data is not None:
+            resp = dumps(data).encode('utf-8')
+        if files is not None:
+            resp = dumps(files).encode('utf-8')
+        else:
+            resp = b'OK'
+
+        response.status_code = 200
+        response.content = resp
+        return response
+
+@pytest.fixture
+def setup_http_mock(request, monkeypatch: MonkeyPatch):
+    if not MOCK:
+        yield
+        return
+
+    monkeypatch.setattr(requests, "request", MockedHttp.requests_request)
+    monkeypatch.setattr(httpx, "request", MockedHttp.httpx_request)
+    yield
+    monkeypatch.undo()

api/tests/integration_tests/workflow/nodes/test_http.py

@@ -0,0 +1,51 @@
+from calendar import c
+import pytest
+from core.app.entities.app_invoke_entities import InvokeFrom
+from core.workflow.entities.variable_pool import VariablePool
+from core.workflow.nodes.http_request.entities import HttpRequestNodeData
+from core.workflow.nodes.http_request.http_request_node import HttpRequestNode
+
+from tests.integration_tests.workflow.nodes.__mock.http import setup_http_mock
+
+BASIC_NODE_DATA = {
+    'tenant_id': '1',
+    'app_id': '1',
+    'workflow_id': '1',
+    'user_id': '1',
+    'user_from': InvokeFrom.WEB_APP,
+}
+
+# construct variable pool
+pool = VariablePool(system_variables={}, user_inputs={})
+pool.append_variable(node_id='1', variable_key_list=['123', 'args1'], value=1)
+pool.append_variable(node_id='1', variable_key_list=['123', 'args2'], value=2)
+
+@pytest.mark.parametrize('setup_http_mock', [['none']], indirect=True)
+def test_get_param(setup_http_mock):
+    node = HttpRequestNode(config={
+        'id': '1',
+        'data': {
+            'title': 'http',
+            'desc': '',
+            'variables': [],
+            'method': 'get',
+            'url': 'http://example.com',
+            'authorization': {
+                'type': 'api-key',
+                'config': {
+                    'type': 'basic',
+                    'api_key':'ak-xxx',
+                    'header': 'api-key',
+                }
+            },
+            'headers': '',
+            'params': '',
+            'body': None,
+        }
+    }, **BASIC_NODE_DATA)
+
+    result = node.run(pool)
+
+    print(result)
+
+    assert 1==2