From 1a57951d72360b3effdb898605f501f775e5ab67 Mon Sep 17 00:00:00 2001 From: Yeuoly Date: Mon, 11 Mar 2024 18:02:20 +0800 Subject: [PATCH 1/2] feat: http --- api/core/helper/ssrf_proxy.py | 1 + .../workflow/nodes/http_request/entities.py | 4 +- .../nodes/http_request/http_executor.py | 82 +++++++++++-------- .../nodes/http_request/http_request_node.py | 4 +- .../workflow/nodes/__mock/http.py | 82 +++++++++++++++++++ .../workflow/nodes/test_http.py | 51 ++++++++++++ 6 files changed, 188 insertions(+), 36 deletions(-) create mode 100644 api/tests/integration_tests/workflow/nodes/__mock/http.py create mode 100644 api/tests/integration_tests/workflow/nodes/test_http.py diff --git a/api/core/helper/ssrf_proxy.py b/api/core/helper/ssrf_proxy.py index c44d4717e6..22f5fe57e0 100644 --- a/api/core/helper/ssrf_proxy.py +++ b/api/core/helper/ssrf_proxy.py @@ -26,6 +26,7 @@ httpx_proxies = { } if SSRF_PROXY_HTTP_URL and SSRF_PROXY_HTTPS_URL else None def get(url, *args, **kwargs): + print(url, kwargs) return _get(url=url, *args, proxies=httpx_proxies, **kwargs) def post(url, *args, **kwargs): diff --git a/api/core/workflow/nodes/http_request/entities.py b/api/core/workflow/nodes/http_request/entities.py index 1e906cbaa4..ce806b6bdb 100644 --- a/api/core/workflow/nodes/http_request/entities.py +++ b/api/core/workflow/nodes/http_request/entities.py @@ -1,4 +1,4 @@ -from typing import Literal, Union +from typing import Literal, Optional, Union from pydantic import BaseModel @@ -29,4 +29,4 @@ class HttpRequestNodeData(BaseNodeData): authorization: Authorization headers: str params: str - body: Body \ No newline at end of file + body: Optional[Body] \ No newline at end of file diff --git a/api/core/workflow/nodes/http_request/http_executor.py b/api/core/workflow/nodes/http_request/http_executor.py index 82d879a89c..6134a7d780 100644 --- a/api/core/workflow/nodes/http_request/http_executor.py +++ b/api/core/workflow/nodes/http_request/http_executor.py @@ -76,11 +76,17 @@ class HttpExecutor: # fill in params kv_paris = original_params.split('\n') for kv in kv_paris: + if not kv.strip(): + continue + kv = kv.split(':') - if len(kv) != 2: + if len(kv) == 2: + k, v = kv + elif len(kv) == 1: + k, v = kv[0], '' + else: raise ValueError(f'Invalid params {kv}') - k, v = kv self.params[k] = v # extract all template in headers @@ -96,51 +102,61 @@ class HttpExecutor: # fill in headers kv_paris = original_headers.split('\n') for kv in kv_paris: + if not kv.strip(): + continue + kv = kv.split(':') - if len(kv) != 2: + if len(kv) == 2: + k, v = kv + elif len(kv) == 1: + k, v = kv[0], '' + else: raise ValueError(f'Invalid headers {kv}') - k, v = kv self.headers[k] = v # extract all template in body - body_template = re.findall(r'{{(.*?)}}', node_data.body.data or '') or [] - body_template = list(set(body_template)) - original_body = node_data.body.data or '' - for body in body_template: - if not body: - continue + if node_data.body: + body_template = re.findall(r'{{(.*?)}}', node_data.body.data or '') or [] + body_template = list(set(body_template)) + original_body = node_data.body.data or '' + for body in body_template: + if not body: + continue - original_body = original_body.replace(f'{{{{{body}}}}}', str(variables.get(body, ''))) + original_body = original_body.replace(f'{{{{{body}}}}}', str(variables.get(body, ''))) - if node_data.body.type == 'json': - self.headers['Content-Type'] = 'application/json' - elif node_data.body.type == 'x-www-form-urlencoded': - self.headers['Content-Type'] = 'application/x-www-form-urlencoded' - # elif node_data.body.type == 'form-data': - # self.headers['Content-Type'] = 'multipart/form-data' + if node_data.body.type == 'json': + self.headers['Content-Type'] = 'application/json' + elif node_data.body.type == 'x-www-form-urlencoded': + self.headers['Content-Type'] = 'application/x-www-form-urlencoded' + # elif node_data.body.type == 'form-data': + # self.headers['Content-Type'] = 'multipart/form-data' - if node_data.body.type in ['form-data', 'x-www-form-urlencoded']: - body = {} - kv_paris = original_body.split('\n') - for kv in kv_paris: - kv = kv.split(':') - if len(kv) != 2: - raise ValueError(f'Invalid body {kv}') - body[kv[0]] = kv[1] + if node_data.body.type in ['form-data', 'x-www-form-urlencoded']: + body = {} + kv_paris = original_body.split('\n') + for kv in kv_paris: + kv = kv.split(':') + if len(kv) == 2: + body[kv[0]] = kv[1] + elif len(kv) == 1: + body[kv[0]] = '' + else: + raise ValueError(f'Invalid body {kv}') - if node_data.body.type == 'form-data': - self.files = { - k: ('', v) for k, v in body.items() - } + if node_data.body.type == 'form-data': + self.files = { + k: ('', v) for k, v in body.items() + } + else: + self.body = urlencode(body) else: - self.body = urlencode(body) - else: - self.body = original_body + self.body = original_body def _assembling_headers(self) -> dict[str, Any]: authorization = deepcopy(self.authorization) - headers = deepcopy(self.headers) or [] + headers = deepcopy(self.headers) or {} if self.authorization.type == 'api-key': if self.authorization.config.api_key is None: raise ValueError('api_key is required') diff --git a/api/core/workflow/nodes/http_request/http_request_node.py b/api/core/workflow/nodes/http_request/http_request_node.py index 853f8fe5e3..1ef6f4b66d 100644 --- a/api/core/workflow/nodes/http_request/http_request_node.py +++ b/api/core/workflow/nodes/http_request/http_request_node.py @@ -24,10 +24,12 @@ class HttpRequestNode(BaseNode): # init http executor try: http_executor = HttpExecutor(node_data=node_data, variables=variables) - # invoke http executor + # invoke http executor response = http_executor.invoke() except Exception as e: + import traceback + print(traceback.format_exc()) return NodeRunResult( status=WorkflowNodeExecutionStatus.FAILED, inputs=variables, diff --git a/api/tests/integration_tests/workflow/nodes/__mock/http.py b/api/tests/integration_tests/workflow/nodes/__mock/http.py new file mode 100644 index 0000000000..3c2b0cebfc --- /dev/null +++ b/api/tests/integration_tests/workflow/nodes/__mock/http.py @@ -0,0 +1,82 @@ +import os +import pytest +import requests.api as requests +import httpx._api as httpx +from requests import Response as RequestsResponse +from yarl import URL + +from typing import Literal +from _pytest.monkeypatch import MonkeyPatch +from json import dumps + +MOCK = os.getenv('MOCK_SWITCH', 'false') == 'true' + +class MockedHttp: + def requests_request(self, method: Literal['GET', 'POST', 'PUT', 'DELETE', 'PATCH', 'OPTIONS'], + url: str, **kwargs) -> RequestsResponse: + """ + Mocked requests.request + """ + response = RequestsResponse() + response.url = str(URL(url) % kwargs.get('params', {})) + response.headers = kwargs.get('headers', {}) + + if url == 'http://404.com': + response.status_code = 404 + response._content = b'Not Found' + return response + + # get data, files + data = kwargs.get('data', None) + files = kwargs.get('files', None) + + if data is not None: + resp = dumps(data).encode('utf-8') + if files is not None: + resp = dumps(files).encode('utf-8') + else: + resp = b'OK' + + response.status_code = 200 + response._content = resp + return response + + def httpx_request(self, method: Literal['GET', 'POST', 'PUT', 'DELETE', 'PATCH', 'OPTIONS'], + url: str, **kwargs) -> httpx.Response: + """ + Mocked httpx.request + """ + response = httpx.Response() + response.url = str(URL(url) % kwargs.get('params', {})) + response.headers = kwargs.get('headers', {}) + + if url == 'http://404.com': + response.status_code = 404 + response.content = b'Not Found' + return response + + # get data, files + data = kwargs.get('data', None) + files = kwargs.get('files', None) + + if data is not None: + resp = dumps(data).encode('utf-8') + if files is not None: + resp = dumps(files).encode('utf-8') + else: + resp = b'OK' + + response.status_code = 200 + response.content = resp + return response + +@pytest.fixture +def setup_http_mock(request, monkeypatch: MonkeyPatch): + if not MOCK: + yield + return + + monkeypatch.setattr(requests, "request", MockedHttp.requests_request) + monkeypatch.setattr(httpx, "request", MockedHttp.httpx_request) + yield + monkeypatch.undo() \ No newline at end of file diff --git a/api/tests/integration_tests/workflow/nodes/test_http.py b/api/tests/integration_tests/workflow/nodes/test_http.py new file mode 100644 index 0000000000..25c293d563 --- /dev/null +++ b/api/tests/integration_tests/workflow/nodes/test_http.py @@ -0,0 +1,51 @@ +from calendar import c +import pytest +from core.app.entities.app_invoke_entities import InvokeFrom +from core.workflow.entities.variable_pool import VariablePool +from core.workflow.nodes.http_request.entities import HttpRequestNodeData +from core.workflow.nodes.http_request.http_request_node import HttpRequestNode + +from tests.integration_tests.workflow.nodes.__mock.http import setup_http_mock + +BASIC_NODE_DATA = { + 'tenant_id': '1', + 'app_id': '1', + 'workflow_id': '1', + 'user_id': '1', + 'user_from': InvokeFrom.WEB_APP, +} + +# construct variable pool +pool = VariablePool(system_variables={}, user_inputs={}) +pool.append_variable(node_id='1', variable_key_list=['123', 'args1'], value=1) +pool.append_variable(node_id='1', variable_key_list=['123', 'args2'], value=2) + +@pytest.mark.parametrize('setup_http_mock', [['none']], indirect=True) +def test_get_param(setup_http_mock): + node = HttpRequestNode(config={ + 'id': '1', + 'data': { + 'title': 'http', + 'desc': '', + 'variables': [], + 'method': 'get', + 'url': 'http://example.com', + 'authorization': { + 'type': 'api-key', + 'config': { + 'type': 'basic', + 'api_key':'ak-xxx', + 'header': 'api-key', + } + }, + 'headers': '', + 'params': '', + 'body': None, + } + }, **BASIC_NODE_DATA) + + result = node.run(pool) + + print(result) + + assert 1==2 \ No newline at end of file From 2008986f83fe4dabf20c03ceddbcb55b37e86cb3 Mon Sep 17 00:00:00 2001 From: Yeuoly Date: Mon, 11 Mar 2024 19:51:31 +0800 Subject: [PATCH 2/2] feat --- api/core/helper/ssrf_proxy.py | 1 - .../nodes/http_request/http_executor.py | 19 +- .../nodes/http_request/http_request_node.py | 10 +- .../workflow/nodes/__mock/http.py | 15 +- .../workflow/nodes/test_http.py | 172 +++++++++++++++++- 5 files changed, 197 insertions(+), 20 deletions(-) diff --git a/api/core/helper/ssrf_proxy.py b/api/core/helper/ssrf_proxy.py index 22f5fe57e0..c44d4717e6 100644 --- a/api/core/helper/ssrf_proxy.py +++ b/api/core/helper/ssrf_proxy.py @@ -26,7 +26,6 @@ httpx_proxies = { } if SSRF_PROXY_HTTP_URL and SSRF_PROXY_HTTPS_URL else None def get(url, *args, **kwargs): - print(url, kwargs) return _get(url=url, *args, proxies=httpx_proxies, **kwargs) def post(url, *args, **kwargs): diff --git a/api/core/workflow/nodes/http_request/http_executor.py b/api/core/workflow/nodes/http_request/http_executor.py index 6134a7d780..c96d5f07d1 100644 --- a/api/core/workflow/nodes/http_request/http_executor.py +++ b/api/core/workflow/nodes/http_request/http_executor.py @@ -43,6 +43,7 @@ class HttpExecutor: self.params = {} self.headers = {} self.body = None + self.files = None # init template self._init_template(node_data, variables) @@ -248,10 +249,24 @@ class HttpExecutor: server_url += f'?{urlencode(self.params)}' raw_request = f'{self.method.upper()} {server_url} HTTP/1.1\n' - for k, v in self.headers.items(): + + headers = self._assembling_headers() + for k, v in headers.items(): raw_request += f'{k}: {v}\n' raw_request += '\n' - raw_request += self.body or '' + + # if files, use multipart/form-data with boundary + if self.files: + boundary = '----WebKitFormBoundary7MA4YWxkTrZu0gW' + raw_request = f'--{boundary}\n' + raw_request + for k, v in self.files.items(): + raw_request += f'Content-Disposition: form-data; name="{k}"; filename="{v[0]}"\n' + raw_request += f'Content-Type: {v[1]}\n\n' + raw_request += v[1] + '\n' + raw_request += f'--{boundary}\n' + raw_request += '--\n' + else: + raw_request += self.body or '' return raw_request \ No newline at end of file diff --git a/api/core/workflow/nodes/http_request/http_request_node.py b/api/core/workflow/nodes/http_request/http_request_node.py index 1ef6f4b66d..c83e331fa8 100644 --- a/api/core/workflow/nodes/http_request/http_request_node.py +++ b/api/core/workflow/nodes/http_request/http_request_node.py @@ -28,13 +28,13 @@ class HttpRequestNode(BaseNode): # invoke http executor response = http_executor.invoke() except Exception as e: - import traceback - print(traceback.format_exc()) return NodeRunResult( status=WorkflowNodeExecutionStatus.FAILED, inputs=variables, error=str(e), - process_data=http_executor.to_raw_request() + process_data={ + 'request': http_executor.to_raw_request() + } ) return NodeRunResult( @@ -45,7 +45,9 @@ class HttpRequestNode(BaseNode): 'body': response, 'headers': response.headers }, - process_data=http_executor.to_raw_request() + process_data={ + 'request': http_executor.to_raw_request(), + } ) diff --git a/api/tests/integration_tests/workflow/nodes/__mock/http.py b/api/tests/integration_tests/workflow/nodes/__mock/http.py index 3c2b0cebfc..9cc43031f3 100644 --- a/api/tests/integration_tests/workflow/nodes/__mock/http.py +++ b/api/tests/integration_tests/workflow/nodes/__mock/http.py @@ -3,6 +3,7 @@ import pytest import requests.api as requests import httpx._api as httpx from requests import Response as RequestsResponse +from httpx import Request as HttpxRequest from yarl import URL from typing import Literal @@ -12,8 +13,8 @@ from json import dumps MOCK = os.getenv('MOCK_SWITCH', 'false') == 'true' class MockedHttp: - def requests_request(self, method: Literal['GET', 'POST', 'PUT', 'DELETE', 'PATCH', 'OPTIONS'], - url: str, **kwargs) -> RequestsResponse: + def requests_request(method: Literal['GET', 'POST', 'PUT', 'DELETE', 'PATCH', 'OPTIONS'], url: str, + **kwargs) -> RequestsResponse: """ Mocked requests.request """ @@ -41,13 +42,15 @@ class MockedHttp: response._content = resp return response - def httpx_request(self, method: Literal['GET', 'POST', 'PUT', 'DELETE', 'PATCH', 'OPTIONS'], + def httpx_request(method: Literal['GET', 'POST', 'PUT', 'DELETE', 'PATCH', 'OPTIONS'], url: str, **kwargs) -> httpx.Response: """ Mocked httpx.request """ - response = httpx.Response() - response.url = str(URL(url) % kwargs.get('params', {})) + response = httpx.Response( + status_code=200, + request=HttpxRequest(method, url) + ) response.headers = kwargs.get('headers', {}) if url == 'http://404.com': @@ -67,7 +70,7 @@ class MockedHttp: resp = b'OK' response.status_code = 200 - response.content = resp + response._content = resp return response @pytest.fixture diff --git a/api/tests/integration_tests/workflow/nodes/test_http.py b/api/tests/integration_tests/workflow/nodes/test_http.py index 25c293d563..6df8f6b673 100644 --- a/api/tests/integration_tests/workflow/nodes/test_http.py +++ b/api/tests/integration_tests/workflow/nodes/test_http.py @@ -2,7 +2,6 @@ from calendar import c import pytest from core.app.entities.app_invoke_entities import InvokeFrom from core.workflow.entities.variable_pool import VariablePool -from core.workflow.nodes.http_request.entities import HttpRequestNodeData from core.workflow.nodes.http_request.http_request_node import HttpRequestNode from tests.integration_tests.workflow.nodes.__mock.http import setup_http_mock @@ -21,13 +20,16 @@ pool.append_variable(node_id='1', variable_key_list=['123', 'args1'], value=1) pool.append_variable(node_id='1', variable_key_list=['123', 'args2'], value=2) @pytest.mark.parametrize('setup_http_mock', [['none']], indirect=True) -def test_get_param(setup_http_mock): +def test_get(setup_http_mock): node = HttpRequestNode(config={ 'id': '1', 'data': { 'title': 'http', 'desc': '', - 'variables': [], + 'variables': [{ + 'variable': 'args1', + 'value_selector': ['1', '123', 'args1'], + }], 'method': 'get', 'url': 'http://example.com', 'authorization': { @@ -38,14 +40,170 @@ def test_get_param(setup_http_mock): 'header': 'api-key', } }, - 'headers': '', - 'params': '', + 'headers': 'X-Header:123', + 'params': 'A:b', 'body': None, } }, **BASIC_NODE_DATA) result = node.run(pool) - print(result) + data = result.process_data.get('request', '') - assert 1==2 \ No newline at end of file + assert '?A=b' in data + assert 'api-key: Basic ak-xxx' in data + assert 'X-Header: 123' in data + +@pytest.mark.parametrize('setup_http_mock', [['none']], indirect=True) +def test_template(setup_http_mock): + node = HttpRequestNode(config={ + 'id': '1', + 'data': { + 'title': 'http', + 'desc': '', + 'variables': [{ + 'variable': 'args1', + 'value_selector': ['1', '123', 'args2'], + }], + 'method': 'get', + 'url': 'http://example.com/{{args1}}', + 'authorization': { + 'type': 'api-key', + 'config': { + 'type': 'basic', + 'api_key':'ak-xxx', + 'header': 'api-key', + } + }, + 'headers': 'X-Header:123\nX-Header2:{{args1}}', + 'params': 'A:b\nTemplate:{{args1}}', + 'body': None, + } + }, **BASIC_NODE_DATA) + + result = node.run(pool) + data = result.process_data.get('request', '') + + assert '?A=b' in data + assert 'Template=2' in data + assert 'api-key: Basic ak-xxx' in data + assert 'X-Header: 123' in data + assert 'X-Header2: 2' in data + +@pytest.mark.parametrize('setup_http_mock', [['none']], indirect=True) +def test_json(setup_http_mock): + node = HttpRequestNode(config={ + 'id': '1', + 'data': { + 'title': 'http', + 'desc': '', + 'variables': [{ + 'variable': 'args1', + 'value_selector': ['1', '123', 'args1'], + }], + 'method': 'post', + 'url': 'http://example.com', + 'authorization': { + 'type': 'api-key', + 'config': { + 'type': 'basic', + 'api_key':'ak-xxx', + 'header': 'api-key', + } + }, + 'headers': 'X-Header:123', + 'params': 'A:b', + 'body': { + 'type': 'json', + 'data': '{"a": "{{args1}}"}' + }, + } + }, **BASIC_NODE_DATA) + + result = node.run(pool) + data = result.process_data.get('request', '') + + assert '{"a": "1"}' in data + assert 'api-key: Basic ak-xxx' in data + assert 'X-Header: 123' in data + +def test_x_www_form_urlencoded(setup_http_mock): + node = HttpRequestNode(config={ + 'id': '1', + 'data': { + 'title': 'http', + 'desc': '', + 'variables': [{ + 'variable': 'args1', + 'value_selector': ['1', '123', 'args1'], + }, { + 'variable': 'args2', + 'value_selector': ['1', '123', 'args2'], + }], + 'method': 'post', + 'url': 'http://example.com', + 'authorization': { + 'type': 'api-key', + 'config': { + 'type': 'basic', + 'api_key':'ak-xxx', + 'header': 'api-key', + } + }, + 'headers': 'X-Header:123', + 'params': 'A:b', + 'body': { + 'type': 'x-www-form-urlencoded', + 'data': 'a:{{args1}}\nb:{{args2}}' + }, + } + }, **BASIC_NODE_DATA) + + result = node.run(pool) + data = result.process_data.get('request', '') + + assert 'a=1&b=2' in data + assert 'api-key: Basic ak-xxx' in data + assert 'X-Header: 123' in data + +def test_form_data(setup_http_mock): + node = HttpRequestNode(config={ + 'id': '1', + 'data': { + 'title': 'http', + 'desc': '', + 'variables': [{ + 'variable': 'args1', + 'value_selector': ['1', '123', 'args1'], + }, { + 'variable': 'args2', + 'value_selector': ['1', '123', 'args2'], + }], + 'method': 'post', + 'url': 'http://example.com', + 'authorization': { + 'type': 'api-key', + 'config': { + 'type': 'basic', + 'api_key':'ak-xxx', + 'header': 'api-key', + } + }, + 'headers': 'X-Header:123', + 'params': 'A:b', + 'body': { + 'type': 'form-data', + 'data': 'a:{{args1}}\nb:{{args2}}' + }, + } + }, **BASIC_NODE_DATA) + + result = node.run(pool) + data = result.process_data.get('request', '') + + assert 'form-data; name="a"' in data + assert '1' in data + assert 'form-data; name="b"' in data + assert '2' in data + assert 'api-key: Basic ak-xxx' in data + assert 'X-Header: 123' in data