diff --git a/api/core/workflow/nodes/http_request/entities.py b/api/core/workflow/nodes/http_request/entities.py index 1e906cbaa4..ce806b6bdb 100644 --- a/api/core/workflow/nodes/http_request/entities.py +++ b/api/core/workflow/nodes/http_request/entities.py @@ -1,4 +1,4 @@ -from typing import Literal, Union +from typing import Literal, Optional, Union from pydantic import BaseModel @@ -29,4 +29,4 @@ class HttpRequestNodeData(BaseNodeData): authorization: Authorization headers: str params: str - body: Body \ No newline at end of file + body: Optional[Body] \ No newline at end of file diff --git a/api/core/workflow/nodes/http_request/http_executor.py b/api/core/workflow/nodes/http_request/http_executor.py index 82d879a89c..c96d5f07d1 100644 --- a/api/core/workflow/nodes/http_request/http_executor.py +++ b/api/core/workflow/nodes/http_request/http_executor.py @@ -43,6 +43,7 @@ class HttpExecutor: self.params = {} self.headers = {} self.body = None + self.files = None # init template self._init_template(node_data, variables) @@ -76,11 +77,17 @@ class HttpExecutor: # fill in params kv_paris = original_params.split('\n') for kv in kv_paris: + if not kv.strip(): + continue + kv = kv.split(':') - if len(kv) != 2: + if len(kv) == 2: + k, v = kv + elif len(kv) == 1: + k, v = kv[0], '' + else: raise ValueError(f'Invalid params {kv}') - k, v = kv self.params[k] = v # extract all template in headers @@ -96,51 +103,61 @@ class HttpExecutor: # fill in headers kv_paris = original_headers.split('\n') for kv in kv_paris: + if not kv.strip(): + continue + kv = kv.split(':') - if len(kv) != 2: + if len(kv) == 2: + k, v = kv + elif len(kv) == 1: + k, v = kv[0], '' + else: raise ValueError(f'Invalid headers {kv}') - k, v = kv self.headers[k] = v # extract all template in body - body_template = re.findall(r'{{(.*?)}}', node_data.body.data or '') or [] - body_template = list(set(body_template)) - original_body = node_data.body.data or '' - for body in body_template: - if not body: - continue + if node_data.body: + body_template = re.findall(r'{{(.*?)}}', node_data.body.data or '') or [] + body_template = list(set(body_template)) + original_body = node_data.body.data or '' + for body in body_template: + if not body: + continue - original_body = original_body.replace(f'{{{{{body}}}}}', str(variables.get(body, ''))) + original_body = original_body.replace(f'{{{{{body}}}}}', str(variables.get(body, ''))) - if node_data.body.type == 'json': - self.headers['Content-Type'] = 'application/json' - elif node_data.body.type == 'x-www-form-urlencoded': - self.headers['Content-Type'] = 'application/x-www-form-urlencoded' - # elif node_data.body.type == 'form-data': - # self.headers['Content-Type'] = 'multipart/form-data' + if node_data.body.type == 'json': + self.headers['Content-Type'] = 'application/json' + elif node_data.body.type == 'x-www-form-urlencoded': + self.headers['Content-Type'] = 'application/x-www-form-urlencoded' + # elif node_data.body.type == 'form-data': + # self.headers['Content-Type'] = 'multipart/form-data' - if node_data.body.type in ['form-data', 'x-www-form-urlencoded']: - body = {} - kv_paris = original_body.split('\n') - for kv in kv_paris: - kv = kv.split(':') - if len(kv) != 2: - raise ValueError(f'Invalid body {kv}') - body[kv[0]] = kv[1] + if node_data.body.type in ['form-data', 'x-www-form-urlencoded']: + body = {} + kv_paris = original_body.split('\n') + for kv in kv_paris: + kv = kv.split(':') + if len(kv) == 2: + body[kv[0]] = kv[1] + elif len(kv) == 1: + body[kv[0]] = '' + else: + raise ValueError(f'Invalid body {kv}') - if node_data.body.type == 'form-data': - self.files = { - k: ('', v) for k, v in body.items() - } + if node_data.body.type == 'form-data': + self.files = { + k: ('', v) for k, v in body.items() + } + else: + self.body = urlencode(body) else: - self.body = urlencode(body) - else: - self.body = original_body + self.body = original_body def _assembling_headers(self) -> dict[str, Any]: authorization = deepcopy(self.authorization) - headers = deepcopy(self.headers) or [] + headers = deepcopy(self.headers) or {} if self.authorization.type == 'api-key': if self.authorization.config.api_key is None: raise ValueError('api_key is required') @@ -232,10 +249,24 @@ class HttpExecutor: server_url += f'?{urlencode(self.params)}' raw_request = f'{self.method.upper()} {server_url} HTTP/1.1\n' - for k, v in self.headers.items(): + + headers = self._assembling_headers() + for k, v in headers.items(): raw_request += f'{k}: {v}\n' raw_request += '\n' - raw_request += self.body or '' + + # if files, use multipart/form-data with boundary + if self.files: + boundary = '----WebKitFormBoundary7MA4YWxkTrZu0gW' + raw_request = f'--{boundary}\n' + raw_request + for k, v in self.files.items(): + raw_request += f'Content-Disposition: form-data; name="{k}"; filename="{v[0]}"\n' + raw_request += f'Content-Type: {v[1]}\n\n' + raw_request += v[1] + '\n' + raw_request += f'--{boundary}\n' + raw_request += '--\n' + else: + raw_request += self.body or '' return raw_request \ No newline at end of file diff --git a/api/core/workflow/nodes/http_request/http_request_node.py b/api/core/workflow/nodes/http_request/http_request_node.py index 853f8fe5e3..c83e331fa8 100644 --- a/api/core/workflow/nodes/http_request/http_request_node.py +++ b/api/core/workflow/nodes/http_request/http_request_node.py @@ -24,15 +24,17 @@ class HttpRequestNode(BaseNode): # init http executor try: http_executor = HttpExecutor(node_data=node_data, variables=variables) - # invoke http executor + # invoke http executor response = http_executor.invoke() except Exception as e: return NodeRunResult( status=WorkflowNodeExecutionStatus.FAILED, inputs=variables, error=str(e), - process_data=http_executor.to_raw_request() + process_data={ + 'request': http_executor.to_raw_request() + } ) return NodeRunResult( @@ -43,7 +45,9 @@ class HttpRequestNode(BaseNode): 'body': response, 'headers': response.headers }, - process_data=http_executor.to_raw_request() + process_data={ + 'request': http_executor.to_raw_request(), + } ) diff --git a/api/tests/integration_tests/workflow/nodes/__mock/http.py b/api/tests/integration_tests/workflow/nodes/__mock/http.py new file mode 100644 index 0000000000..9cc43031f3 --- /dev/null +++ b/api/tests/integration_tests/workflow/nodes/__mock/http.py @@ -0,0 +1,85 @@ +import os +import pytest +import requests.api as requests +import httpx._api as httpx +from requests import Response as RequestsResponse +from httpx import Request as HttpxRequest +from yarl import URL + +from typing import Literal +from _pytest.monkeypatch import MonkeyPatch +from json import dumps + +MOCK = os.getenv('MOCK_SWITCH', 'false') == 'true' + +class MockedHttp: + def requests_request(method: Literal['GET', 'POST', 'PUT', 'DELETE', 'PATCH', 'OPTIONS'], url: str, + **kwargs) -> RequestsResponse: + """ + Mocked requests.request + """ + response = RequestsResponse() + response.url = str(URL(url) % kwargs.get('params', {})) + response.headers = kwargs.get('headers', {}) + + if url == 'http://404.com': + response.status_code = 404 + response._content = b'Not Found' + return response + + # get data, files + data = kwargs.get('data', None) + files = kwargs.get('files', None) + + if data is not None: + resp = dumps(data).encode('utf-8') + if files is not None: + resp = dumps(files).encode('utf-8') + else: + resp = b'OK' + + response.status_code = 200 + response._content = resp + return response + + def httpx_request(method: Literal['GET', 'POST', 'PUT', 'DELETE', 'PATCH', 'OPTIONS'], + url: str, **kwargs) -> httpx.Response: + """ + Mocked httpx.request + """ + response = httpx.Response( + status_code=200, + request=HttpxRequest(method, url) + ) + response.headers = kwargs.get('headers', {}) + + if url == 'http://404.com': + response.status_code = 404 + response.content = b'Not Found' + return response + + # get data, files + data = kwargs.get('data', None) + files = kwargs.get('files', None) + + if data is not None: + resp = dumps(data).encode('utf-8') + if files is not None: + resp = dumps(files).encode('utf-8') + else: + resp = b'OK' + + response.status_code = 200 + response._content = resp + return response + +@pytest.fixture +def setup_http_mock(request, monkeypatch: MonkeyPatch): + if not MOCK: + yield + return + + monkeypatch.setattr(requests, "request", MockedHttp.requests_request) + monkeypatch.setattr(httpx, "request", MockedHttp.httpx_request) + yield + monkeypatch.undo() \ No newline at end of file diff --git a/api/tests/integration_tests/workflow/nodes/test_http.py b/api/tests/integration_tests/workflow/nodes/test_http.py new file mode 100644 index 0000000000..6df8f6b673 --- /dev/null +++ b/api/tests/integration_tests/workflow/nodes/test_http.py @@ -0,0 +1,209 @@ +from calendar import c +import pytest +from core.app.entities.app_invoke_entities import InvokeFrom +from core.workflow.entities.variable_pool import VariablePool +from core.workflow.nodes.http_request.http_request_node import HttpRequestNode + +from tests.integration_tests.workflow.nodes.__mock.http import setup_http_mock + +BASIC_NODE_DATA = { + 'tenant_id': '1', + 'app_id': '1', + 'workflow_id': '1', + 'user_id': '1', + 'user_from': InvokeFrom.WEB_APP, +} + +# construct variable pool +pool = VariablePool(system_variables={}, user_inputs={}) +pool.append_variable(node_id='1', variable_key_list=['123', 'args1'], value=1) +pool.append_variable(node_id='1', variable_key_list=['123', 'args2'], value=2) + +@pytest.mark.parametrize('setup_http_mock', [['none']], indirect=True) +def test_get(setup_http_mock): + node = HttpRequestNode(config={ + 'id': '1', + 'data': { + 'title': 'http', + 'desc': '', + 'variables': [{ + 'variable': 'args1', + 'value_selector': ['1', '123', 'args1'], + }], + 'method': 'get', + 'url': 'http://example.com', + 'authorization': { + 'type': 'api-key', + 'config': { + 'type': 'basic', + 'api_key':'ak-xxx', + 'header': 'api-key', + } + }, + 'headers': 'X-Header:123', + 'params': 'A:b', + 'body': None, + } + }, **BASIC_NODE_DATA) + + result = node.run(pool) + + data = result.process_data.get('request', '') + + assert '?A=b' in data + assert 'api-key: Basic ak-xxx' in data + assert 'X-Header: 123' in data + +@pytest.mark.parametrize('setup_http_mock', [['none']], indirect=True) +def test_template(setup_http_mock): + node = HttpRequestNode(config={ + 'id': '1', + 'data': { + 'title': 'http', + 'desc': '', + 'variables': [{ + 'variable': 'args1', + 'value_selector': ['1', '123', 'args2'], + }], + 'method': 'get', + 'url': 'http://example.com/{{args1}}', + 'authorization': { + 'type': 'api-key', + 'config': { + 'type': 'basic', + 'api_key':'ak-xxx', + 'header': 'api-key', + } + }, + 'headers': 'X-Header:123\nX-Header2:{{args1}}', + 'params': 'A:b\nTemplate:{{args1}}', + 'body': None, + } + }, **BASIC_NODE_DATA) + + result = node.run(pool) + data = result.process_data.get('request', '') + + assert '?A=b' in data + assert 'Template=2' in data + assert 'api-key: Basic ak-xxx' in data + assert 'X-Header: 123' in data + assert 'X-Header2: 2' in data + +@pytest.mark.parametrize('setup_http_mock', [['none']], indirect=True) +def test_json(setup_http_mock): + node = HttpRequestNode(config={ + 'id': '1', + 'data': { + 'title': 'http', + 'desc': '', + 'variables': [{ + 'variable': 'args1', + 'value_selector': ['1', '123', 'args1'], + }], + 'method': 'post', + 'url': 'http://example.com', + 'authorization': { + 'type': 'api-key', + 'config': { + 'type': 'basic', + 'api_key':'ak-xxx', + 'header': 'api-key', + } + }, + 'headers': 'X-Header:123', + 'params': 'A:b', + 'body': { + 'type': 'json', + 'data': '{"a": "{{args1}}"}' + }, + } + }, **BASIC_NODE_DATA) + + result = node.run(pool) + data = result.process_data.get('request', '') + + assert '{"a": "1"}' in data + assert 'api-key: Basic ak-xxx' in data + assert 'X-Header: 123' in data + +def test_x_www_form_urlencoded(setup_http_mock): + node = HttpRequestNode(config={ + 'id': '1', + 'data': { + 'title': 'http', + 'desc': '', + 'variables': [{ + 'variable': 'args1', + 'value_selector': ['1', '123', 'args1'], + }, { + 'variable': 'args2', + 'value_selector': ['1', '123', 'args2'], + }], + 'method': 'post', + 'url': 'http://example.com', + 'authorization': { + 'type': 'api-key', + 'config': { + 'type': 'basic', + 'api_key':'ak-xxx', + 'header': 'api-key', + } + }, + 'headers': 'X-Header:123', + 'params': 'A:b', + 'body': { + 'type': 'x-www-form-urlencoded', + 'data': 'a:{{args1}}\nb:{{args2}}' + }, + } + }, **BASIC_NODE_DATA) + + result = node.run(pool) + data = result.process_data.get('request', '') + + assert 'a=1&b=2' in data + assert 'api-key: Basic ak-xxx' in data + assert 'X-Header: 123' in data + +def test_form_data(setup_http_mock): + node = HttpRequestNode(config={ + 'id': '1', + 'data': { + 'title': 'http', + 'desc': '', + 'variables': [{ + 'variable': 'args1', + 'value_selector': ['1', '123', 'args1'], + }, { + 'variable': 'args2', + 'value_selector': ['1', '123', 'args2'], + }], + 'method': 'post', + 'url': 'http://example.com', + 'authorization': { + 'type': 'api-key', + 'config': { + 'type': 'basic', + 'api_key':'ak-xxx', + 'header': 'api-key', + } + }, + 'headers': 'X-Header:123', + 'params': 'A:b', + 'body': { + 'type': 'form-data', + 'data': 'a:{{args1}}\nb:{{args2}}' + }, + } + }, **BASIC_NODE_DATA) + + result = node.run(pool) + data = result.process_data.get('request', '') + + assert 'form-data; name="a"' in data + assert '1' in data + assert 'form-data; name="b"' in data + assert '2' in data + assert 'api-key: Basic ak-xxx' in data + assert 'X-Header: 123' in data