Merge remote-tracking branch 'origin/feat/workflow-backend' into feat/workflow-backend

api/core/workflow/nodes/http_request/entities.py

@@ -1,4 +1,4 @@
-from typing import Literal, Union
+from typing import Literal, Optional, Union
 
 from pydantic import BaseModel
 
@@ -29,4 +29,4 @@ class HttpRequestNodeData(BaseNodeData):
     authorization: Authorization
     headers: str
     params: str
-    body: Body
+    body: Optional[Body]

api/core/workflow/nodes/http_request/http_executor.py

@@ -43,6 +43,7 @@ class HttpExecutor:
         self.params = {}
         self.headers = {}
         self.body = None
+        self.files = None
 
         # init template
         self._init_template(node_data, variables)
@@ -76,11 +77,17 @@ class HttpExecutor:
         # fill in params
         kv_paris = original_params.split('\n')
         for kv in kv_paris:
+            if not kv.strip():
+                continue
+
             kv = kv.split(':')
-            if len(kv) != 2:
+            if len(kv) == 2:
+                k, v = kv
+            elif len(kv) == 1:
+                k, v = kv[0], ''
+            else:
                 raise ValueError(f'Invalid params {kv}')
             
-            k, v = kv
             self.params[k] = v
 
         # extract all template in headers
@@ -96,51 +103,61 @@ class HttpExecutor:
         # fill in headers
         kv_paris = original_headers.split('\n')
         for kv in kv_paris:
+            if not kv.strip():
+                continue
+
             kv = kv.split(':')
-            if len(kv) != 2:
+            if len(kv) == 2:
+                k, v = kv
+            elif len(kv) == 1:
+                k, v = kv[0], ''
+            else:
                 raise ValueError(f'Invalid headers {kv}')
             
-            k, v = kv
             self.headers[k] = v
 
         # extract all template in body
-        body_template = re.findall(r'{{(.*?)}}', node_data.body.data or '') or []
+        if node_data.body:
-        body_template = list(set(body_template))
+            body_template = re.findall(r'{{(.*?)}}', node_data.body.data or '') or []
-        original_body = node_data.body.data or ''
+            body_template = list(set(body_template))
-        for body in body_template:
+            original_body = node_data.body.data or ''
-            if not body:
+            for body in body_template:
-                continue
+                if not body:
+                    continue
 
-            original_body = original_body.replace(f'{{{{{body}}}}}', str(variables.get(body, '')))
+                original_body = original_body.replace(f'{{{{{body}}}}}', str(variables.get(body, '')))
 
-        if node_data.body.type == 'json':
+            if node_data.body.type == 'json':
-            self.headers['Content-Type'] = 'application/json'
+                self.headers['Content-Type'] = 'application/json'
-        elif node_data.body.type == 'x-www-form-urlencoded':
+            elif node_data.body.type == 'x-www-form-urlencoded':
-            self.headers['Content-Type'] = 'application/x-www-form-urlencoded'
+                self.headers['Content-Type'] = 'application/x-www-form-urlencoded'
-        # elif node_data.body.type == 'form-data':
+            # elif node_data.body.type == 'form-data':
-        #    self.headers['Content-Type'] = 'multipart/form-data'
+            #    self.headers['Content-Type'] = 'multipart/form-data'
 
-        if node_data.body.type in ['form-data', 'x-www-form-urlencoded']:
+            if node_data.body.type in ['form-data', 'x-www-form-urlencoded']:
-            body = {}
+                body = {}
-            kv_paris = original_body.split('\n')
+                kv_paris = original_body.split('\n')
-            for kv in kv_paris:
+                for kv in kv_paris:
-                kv = kv.split(':')
+                    kv = kv.split(':')
-                if len(kv) != 2:
+                    if len(kv) == 2:
-                    raise ValueError(f'Invalid body {kv}')
+                        body[kv[0]] = kv[1]
-                body[kv[0]] = kv[1]
+                    elif len(kv) == 1:
+                        body[kv[0]] = ''
+                    else:
+                        raise ValueError(f'Invalid body {kv}')
 
-            if node_data.body.type == 'form-data':
+                if node_data.body.type == 'form-data':
-                self.files = {
+                    self.files = {
-                    k: ('', v) for k, v in body.items()
+                        k: ('', v) for k, v in body.items()
-                }
+                    }
+                else:
+                    self.body = urlencode(body)
             else:
-                self.body = urlencode(body)
+                self.body = original_body
-        else:
-            self.body = original_body
                 
     def _assembling_headers(self) -> dict[str, Any]:
         authorization = deepcopy(self.authorization)
-        headers = deepcopy(self.headers) or []
+        headers = deepcopy(self.headers) or {}
         if self.authorization.type == 'api-key':
             if self.authorization.config.api_key is None:
                 raise ValueError('api_key is required')
@@ -232,10 +249,24 @@ class HttpExecutor:
             server_url += f'?{urlencode(self.params)}'
 
         raw_request = f'{self.method.upper()} {server_url} HTTP/1.1\n'
-        for k, v in self.headers.items():
+
+        headers = self._assembling_headers()
+        for k, v in headers.items():
             raw_request += f'{k}: {v}\n'
 
         raw_request += '\n'
-        raw_request += self.body or ''
+
+        # if files, use multipart/form-data with boundary
+        if self.files:
+            boundary = '----WebKitFormBoundary7MA4YWxkTrZu0gW'
+            raw_request = f'--{boundary}\n' + raw_request
+            for k, v in self.files.items():
+                raw_request += f'Content-Disposition: form-data; name="{k}"; filename="{v[0]}"\n'
+                raw_request += f'Content-Type: {v[1]}\n\n'
+                raw_request += v[1] + '\n'
+                raw_request += f'--{boundary}\n'
+            raw_request += '--\n'
+        else:
+            raw_request += self.body or ''
 
         return raw_request

api/core/workflow/nodes/http_request/http_request_node.py

@@ -24,15 +24,17 @@ class HttpRequestNode(BaseNode):
         # init http executor
         try:
             http_executor = HttpExecutor(node_data=node_data, variables=variables)
-            # invoke http executor
 
+            # invoke http executor
             response = http_executor.invoke()
         except Exception as e:
             return NodeRunResult(
                 status=WorkflowNodeExecutionStatus.FAILED,
                 inputs=variables,
                 error=str(e),
-                process_data=http_executor.to_raw_request()
+                process_data={
+                    'request': http_executor.to_raw_request()
+                }
             )
 
         return NodeRunResult(
@@ -43,7 +45,9 @@ class HttpRequestNode(BaseNode):
                 'body': response,
                 'headers': response.headers
             },
-            process_data=http_executor.to_raw_request()
+            process_data={
+                'request': http_executor.to_raw_request(),
+            }
         )
 
 

api/tests/integration_tests/workflow/nodes/__mock/http.py

@@ -0,0 +1,85 @@
+import os
+import pytest
+import requests.api as requests
+import httpx._api as httpx
+from requests import Response as RequestsResponse
+from httpx import Request as HttpxRequest
+from yarl import URL
+
+from typing import Literal
+from _pytest.monkeypatch import MonkeyPatch
+from json import dumps
+
+MOCK = os.getenv('MOCK_SWITCH', 'false') == 'true'
+
+class MockedHttp:
+    def requests_request(method: Literal['GET', 'POST', 'PUT', 'DELETE', 'PATCH', 'OPTIONS'], url: str,
+                        **kwargs) -> RequestsResponse:
+        """
+        Mocked requests.request
+        """
+        response = RequestsResponse()
+        response.url = str(URL(url) % kwargs.get('params', {}))
+        response.headers = kwargs.get('headers', {})
+
+        if url == 'http://404.com':
+            response.status_code = 404
+            response._content = b'Not Found'
+            return response
+        
+        # get data, files
+        data = kwargs.get('data', None)
+        files = kwargs.get('files', None)
+
+        if data is not None:
+            resp = dumps(data).encode('utf-8')
+        if files is not None:
+            resp = dumps(files).encode('utf-8')
+        else:
+            resp = b'OK'
+
+        response.status_code = 200
+        response._content = resp
+        return response
+
+    def httpx_request(method: Literal['GET', 'POST', 'PUT', 'DELETE', 'PATCH', 'OPTIONS'], 
+                      url: str, **kwargs) -> httpx.Response:
+        """
+        Mocked httpx.request
+        """
+        response = httpx.Response(
+            status_code=200,
+            request=HttpxRequest(method, url)
+        )
+        response.headers = kwargs.get('headers', {})
+
+        if url == 'http://404.com':
+            response.status_code = 404
+            response.content = b'Not Found'
+            return response
+        
+        # get data, files
+        data = kwargs.get('data', None)
+        files = kwargs.get('files', None)
+
+        if data is not None:
+            resp = dumps(data).encode('utf-8')
+        if files is not None:
+            resp = dumps(files).encode('utf-8')
+        else:
+            resp = b'OK'
+
+        response.status_code = 200
+        response._content = resp
+        return response
+
+@pytest.fixture
+def setup_http_mock(request, monkeypatch: MonkeyPatch):
+    if not MOCK:
+        yield
+        return
+
+    monkeypatch.setattr(requests, "request", MockedHttp.requests_request)
+    monkeypatch.setattr(httpx, "request", MockedHttp.httpx_request)
+    yield
+    monkeypatch.undo()

api/tests/integration_tests/workflow/nodes/test_http.py

@@ -0,0 +1,209 @@
+from calendar import c
+import pytest
+from core.app.entities.app_invoke_entities import InvokeFrom
+from core.workflow.entities.variable_pool import VariablePool
+from core.workflow.nodes.http_request.http_request_node import HttpRequestNode
+
+from tests.integration_tests.workflow.nodes.__mock.http import setup_http_mock
+
+BASIC_NODE_DATA = {
+    'tenant_id': '1',
+    'app_id': '1',
+    'workflow_id': '1',
+    'user_id': '1',
+    'user_from': InvokeFrom.WEB_APP,
+}
+
+# construct variable pool
+pool = VariablePool(system_variables={}, user_inputs={})
+pool.append_variable(node_id='1', variable_key_list=['123', 'args1'], value=1)
+pool.append_variable(node_id='1', variable_key_list=['123', 'args2'], value=2)
+
+@pytest.mark.parametrize('setup_http_mock', [['none']], indirect=True)
+def test_get(setup_http_mock):
+    node = HttpRequestNode(config={
+        'id': '1',
+        'data': {
+            'title': 'http',
+            'desc': '',
+            'variables': [{
+                'variable': 'args1',
+                'value_selector': ['1', '123', 'args1'],
+            }],
+            'method': 'get',
+            'url': 'http://example.com',
+            'authorization': {
+                'type': 'api-key',
+                'config': {
+                    'type': 'basic',
+                    'api_key':'ak-xxx',
+                    'header': 'api-key',
+                }
+            },
+            'headers': 'X-Header:123',
+            'params': 'A:b',
+            'body': None,
+        }
+    }, **BASIC_NODE_DATA)
+
+    result = node.run(pool)
+
+    data = result.process_data.get('request', '')
+
+    assert '?A=b' in data
+    assert 'api-key: Basic ak-xxx' in data
+    assert 'X-Header: 123' in data
+
+@pytest.mark.parametrize('setup_http_mock', [['none']], indirect=True)
+def test_template(setup_http_mock):
+    node = HttpRequestNode(config={
+        'id': '1',
+        'data': {
+            'title': 'http',
+            'desc': '',
+            'variables': [{
+                'variable': 'args1',
+                'value_selector': ['1', '123', 'args2'],
+            }],
+            'method': 'get',
+            'url': 'http://example.com/{{args1}}',
+            'authorization': {
+                'type': 'api-key',
+                'config': {
+                    'type': 'basic',
+                    'api_key':'ak-xxx',
+                    'header': 'api-key',
+                }
+            },
+            'headers': 'X-Header:123\nX-Header2:{{args1}}',
+            'params': 'A:b\nTemplate:{{args1}}',
+            'body': None,
+        }
+    }, **BASIC_NODE_DATA)
+
+    result = node.run(pool)
+    data = result.process_data.get('request', '')
+
+    assert '?A=b' in data
+    assert 'Template=2' in data
+    assert 'api-key: Basic ak-xxx' in data
+    assert 'X-Header: 123' in data
+    assert 'X-Header2: 2' in data
+
+@pytest.mark.parametrize('setup_http_mock', [['none']], indirect=True)
+def test_json(setup_http_mock):
+    node = HttpRequestNode(config={
+        'id': '1',
+        'data': {
+            'title': 'http',
+            'desc': '',
+            'variables': [{
+                'variable': 'args1',
+                'value_selector': ['1', '123', 'args1'],
+            }],
+            'method': 'post',
+            'url': 'http://example.com',
+            'authorization': {
+                'type': 'api-key',
+                'config': {
+                    'type': 'basic',
+                    'api_key':'ak-xxx',
+                    'header': 'api-key',
+                }
+            },
+            'headers': 'X-Header:123',
+            'params': 'A:b',
+            'body': {
+                'type': 'json',
+                'data': '{"a": "{{args1}}"}'
+            },
+        }
+    }, **BASIC_NODE_DATA)
+
+    result = node.run(pool)
+    data = result.process_data.get('request', '')
+
+    assert '{"a": "1"}' in data
+    assert 'api-key: Basic ak-xxx' in data
+    assert 'X-Header: 123' in data
+
+def test_x_www_form_urlencoded(setup_http_mock):
+    node = HttpRequestNode(config={
+        'id': '1',
+        'data': {
+            'title': 'http',
+            'desc': '',
+            'variables': [{
+                'variable': 'args1',
+                'value_selector': ['1', '123', 'args1'],
+            }, {
+                'variable': 'args2',
+                'value_selector': ['1', '123', 'args2'],
+            }],
+            'method': 'post',
+            'url': 'http://example.com',
+            'authorization': {
+                'type': 'api-key',
+                'config': {
+                    'type': 'basic',
+                    'api_key':'ak-xxx',
+                    'header': 'api-key',
+                }
+            },
+            'headers': 'X-Header:123',
+            'params': 'A:b',
+            'body': {
+                'type': 'x-www-form-urlencoded',
+                'data': 'a:{{args1}}\nb:{{args2}}'
+            },
+        }
+    }, **BASIC_NODE_DATA)
+
+    result = node.run(pool)
+    data = result.process_data.get('request', '')
+
+    assert 'a=1&b=2' in data
+    assert 'api-key: Basic ak-xxx' in data
+    assert 'X-Header: 123' in data
+
+def test_form_data(setup_http_mock):
+    node = HttpRequestNode(config={
+        'id': '1',
+        'data': {
+            'title': 'http',
+            'desc': '',
+            'variables': [{
+                'variable': 'args1',
+                'value_selector': ['1', '123', 'args1'],
+            }, {
+                'variable': 'args2',
+                'value_selector': ['1', '123', 'args2'],
+            }],
+            'method': 'post',
+            'url': 'http://example.com',
+            'authorization': {
+                'type': 'api-key',
+                'config': {
+                    'type': 'basic',
+                    'api_key':'ak-xxx',
+                    'header': 'api-key',
+                }
+            },
+            'headers': 'X-Header:123',
+            'params': 'A:b',
+            'body': {
+                'type': 'form-data',
+                'data': 'a:{{args1}}\nb:{{args2}}'
+            },
+        }
+    }, **BASIC_NODE_DATA)
+
+    result = node.run(pool)
+    data = result.process_data.get('request', '')
+
+    assert 'form-data; name="a"' in data
+    assert '1' in data
+    assert 'form-data; name="b"' in data
+    assert '2' in data
+    assert 'api-key: Basic ak-xxx' in data
+    assert 'X-Header: 123' in data