From e6f3e896c43aa73236d4541c1193486e7176ef05 Mon Sep 17 00:00:00 2001 From: Nathan Buckner Date: Sat, 22 Aug 2015 01:44:47 -0500 Subject: [PATCH] moved request creator code to http client fixed flake8 issues made datagen a Mixin --- syntribos/__init__.py | 1 - syntribos/clients/__init__.py | 1 - syntribos/clients/http/__init__.py | 18 ++ syntribos/clients/http/client.py | 23 ++ syntribos/clients/{http.py => http/models.py} | 119 +--------- syntribos/clients/http/parser.py | 108 +++++++++ syntribos/extensions/__init__.py | 1 - syntribos/extensions/identity/__init__.py | 1 - .../extensions/identity/models/__init__.py | 1 - syntribos/extensions/identity/models/v2.py | 4 +- syntribos/extensions/random_data/__init__.py | 1 - syntribos/runner.py | 8 +- syntribos/tests/__init__.py | 1 - syntribos/tests/fuzz/__init__.py | 1 - syntribos/tests/fuzz/base_fuzz.py | 58 +++-- syntribos/tests/fuzz/config.py | 4 - syntribos/tests/fuzz/datagen.py | 44 +++- syntribos/tests/request_creator.py | 210 ------------------ 18 files changed, 230 insertions(+), 374 deletions(-) create mode 100644 syntribos/clients/http/__init__.py create mode 100644 syntribos/clients/http/client.py rename syntribos/clients/{http.py => http/models.py} (54%) create mode 100644 syntribos/clients/http/parser.py delete mode 100644 syntribos/tests/request_creator.py diff --git a/syntribos/__init__.py b/syntribos/__init__.py index 94d152e4..71a5888d 100644 --- a/syntribos/__init__.py +++ b/syntribos/__init__.py @@ -13,4 +13,3 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ - diff --git a/syntribos/clients/__init__.py b/syntribos/clients/__init__.py index 94d152e4..71a5888d 100644 --- a/syntribos/clients/__init__.py +++ b/syntribos/clients/__init__.py @@ -13,4 +13,3 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ - diff --git a/syntribos/clients/http/__init__.py b/syntribos/clients/http/__init__.py new file mode 100644 index 00000000..617b99d6 --- /dev/null +++ b/syntribos/clients/http/__init__.py @@ -0,0 +1,18 @@ +""" +Copyright 2015 Rackspace + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +# flake8: noqa +from syntribos.clients.http.parser import RequestCreator as parser +from syntribos.clients.http.client import SynHTTPClient as client diff --git a/syntribos/clients/http/client.py b/syntribos/clients/http/client.py new file mode 100644 index 00000000..fd91f17c --- /dev/null +++ b/syntribos/clients/http/client.py @@ -0,0 +1,23 @@ +""" +Copyright 2015 Rackspace + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from cafe.engine.http.client import HTTPClient + + +class SynHTTPClient(HTTPClient): + def send_request(self, r): + return self.request( + method=r.method, url=r.url, headers=r.headers, params=r.params, + data=r.data) diff --git a/syntribos/clients/http.py b/syntribos/clients/http/models.py similarity index 54% rename from syntribos/clients/http.py rename to syntribos/clients/http/models.py index 6c60d0e1..1e8a494e 100644 --- a/syntribos/clients/http.py +++ b/syntribos/clients/http/models.py @@ -13,28 +13,14 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ - from copy import deepcopy -from importlib import import_module -from uuid import uuid4 from xml.etree import ElementTree import json -import re -import types -import urlparse -from cafe.engine.http.client import HTTPClient _iterators = {} -class SynHTTPClient(HTTPClient): - def send_request(self, r): - return self.request( - method=r.method, url=r.url, headers=r.headers, params=r.params, - data=r.data) - - class RequestHelperMixin(object): @classmethod def _run_iters(cls, data, action_field): @@ -42,6 +28,8 @@ class RequestHelperMixin(object): return cls._run_iters_dict(data, action_field) elif isinstance(data, ElementTree.Element): return cls._run_iters_xml(data, action_field) + elif isinstance(data, basestring): + return cls._run_iters_xml(data, action_field) else: return data @@ -97,18 +85,6 @@ class RequestHelperMixin(object): string = string.replace(k, v.next()) return string - -class RequestObject(RequestHelperMixin): - def __init__( - self, method, url, action_field=None, headers=None, params=None, - data=None): - self.method = method - self.url = url - self.headers = headers or {} - self.params = params or {} - self.data = data - self.action_field = action_field - def prepare_request(self): """ it should be noted this function does not make a request copy destroying iterators in request. A copy should be made if making @@ -127,84 +103,13 @@ class RequestObject(RequestHelperMixin): return deepcopy(self) -class RequestCreator(object): - ACTION_FIELD = "ACTION_FIELD:" - EXTERNAL = r"CALL_EXTERNAL\|([^|]+?):([^|]+?):([^|]+?)\|" - - @classmethod - def create_request(cls, string, endpoint): - string = cls.call_external_functions(string) - action_field = str(uuid4()).replace("-", "") - string = string.replace(cls.ACTION_FIELD, action_field) - lines = string.splitlines() - for index, line in enumerate(lines): - if line == "": - break - method, url, params, version = cls._parse_url_line(lines[0], endpoint) - headers = cls._parse_headers(lines[1:index]) - data = cls._parse_data(lines[index+1:]) - return RequestObject( - method=method, url=url, headers=headers, params=params, data=data, - action_field=action_field) - - @classmethod - def _parse_url_line(cls, line, endpoint): - params = {} - method, url, version = line.split() - url = url.split("?", 1) - if len(url) == 2: - for param in url[1].split("&"): - param = param.split("=", 1) - if len(param) > 1: - params[param[0]] = param[1] - else: - params[param[0]] = "" - url = url[0] - url = urlparse.urljoin(endpoint, url) - return method, url, params, version - - @classmethod - def _parse_headers(cls, lines): - headers = {} - for line in lines: - key, value = line.split(":", 1) - headers[key] = value.strip() - return headers - - @classmethod - def _parse_data(cls, lines): - data = "\n".join(lines).strip() - if not data: - return "" - try: - data = json.loads(data) - except: - try: - data = ElementTree.fromstring(data) - except: - raise Exception("Unknown Data format") - return data - - @classmethod - def call_external_functions(cls, string): - if not isinstance(string, basestring): - return string - - while True: - match = re.search(cls.EXTERNAL, string) - if not match: - break - dot_path = match.group(1) - func_name = match.group(2) - arg_list = match.group(3) - mod = import_module(dot_path) - func = getattr(mod, func_name) - args = json.loads(arg_list) - val = func(*args) - if isinstance(val, types.GeneratorType): - uuid = str(uuid4()).replace("-", "") - string = re.sub(cls.EXTERNAL, uuid, string, count=1) - _iterators[uuid] = val - else: - string = re.sub(cls.EXTERNAL, str(val), string, count=1) - return string +class RequestObject(object): + def __init__( + self, method, url, action_field=None, headers=None, params=None, + data=None): + self.method = method + self.url = url + self.headers = headers or {} + self.params = params or {} + self.data = data + self.action_field = action_field diff --git a/syntribos/clients/http/parser.py b/syntribos/clients/http/parser.py new file mode 100644 index 00000000..e716fd37 --- /dev/null +++ b/syntribos/clients/http/parser.py @@ -0,0 +1,108 @@ +""" +Copyright 2015 Rackspace + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from importlib import import_module +from uuid import uuid4 +from xml.etree import ElementTree +import json +import re +import types +import urlparse + +from syntribos.clients.http.models import RequestObject, _iterators + + +class RequestCreator(object): + ACTION_FIELD = "ACTION_FIELD:" + EXTERNAL = r"CALL_EXTERNAL\|([^:]+?):([^:]+?):([^|]+?)\|" + request_model_type = RequestObject + + @classmethod + def create_request(cls, string, endpoint): + string = cls.call_external_functions(string) + action_field = str(uuid4()).replace("-", "") + string = string.replace(cls.ACTION_FIELD, action_field) + lines = string.splitlines() + for index, line in enumerate(lines): + if line == "": + break + method, url, params, version = cls._parse_url_line(lines[0], endpoint) + headers = cls._parse_headers(lines[1:index]) + data = cls._parse_data(lines[index+1:]) + return cls.request_model_type( + method=method, url=url, headers=headers, params=params, data=data, + action_field=action_field) + + @classmethod + def _parse_url_line(cls, line, endpoint): + params = {} + method, url, version = line.split() + url = url.split("?", 1) + if len(url) == 2: + for param in url[1].split("&"): + param = param.split("=", 1) + if len(param) > 1: + params[param[0]] = param[1] + else: + params[param[0]] = "" + url = url[0] + url = urlparse.urljoin(endpoint, url) + return method, url, params, version + + @classmethod + def _parse_headers(cls, lines): + headers = {} + for line in lines: + key, value = line.split(":", 1) + headers[key] = value.strip() + return headers + + @classmethod + def _parse_data(cls, lines): + data = "\n".join(lines).strip() + if not data: + return "" + try: + data = json.loads(data) + except: + try: + data = ElementTree.fromstring(data) + except: + raise Exception("Unknown Data format") + return data + + @classmethod + def call_external_functions(cls, string): + if not isinstance(string, basestring): + return string + + while True: + match = re.search(cls.EXTERNAL, string) + if not match: + break + dot_path = match.group(1) + func_name = match.group(2) + arg_list = match.group(3) + mod = import_module(dot_path) + func = getattr(mod, func_name) + args = json.loads(arg_list) + val = func(*args) + if isinstance(val, types.GeneratorType): + uuid = str(uuid4()).replace("-", "") + string = re.sub(cls.EXTERNAL, uuid, string, count=1) + _iterators[uuid] = val + else: + string = re.sub(cls.EXTERNAL, str(val), string, count=1) + return string diff --git a/syntribos/extensions/__init__.py b/syntribos/extensions/__init__.py index 94d152e4..71a5888d 100644 --- a/syntribos/extensions/__init__.py +++ b/syntribos/extensions/__init__.py @@ -13,4 +13,3 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ - diff --git a/syntribos/extensions/identity/__init__.py b/syntribos/extensions/identity/__init__.py index 94d152e4..71a5888d 100644 --- a/syntribos/extensions/identity/__init__.py +++ b/syntribos/extensions/identity/__init__.py @@ -13,4 +13,3 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ - diff --git a/syntribos/extensions/identity/models/__init__.py b/syntribos/extensions/identity/models/__init__.py index 94d152e4..71a5888d 100644 --- a/syntribos/extensions/identity/models/__init__.py +++ b/syntribos/extensions/identity/models/__init__.py @@ -13,4 +13,3 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ - diff --git a/syntribos/extensions/identity/models/v2.py b/syntribos/extensions/identity/models/v2.py index 1473e240..437358c6 100644 --- a/syntribos/extensions/identity/models/v2.py +++ b/syntribos/extensions/identity/models/v2.py @@ -91,10 +91,12 @@ class Token(BaseIdentityModel): @classmethod def _dict_to_obj(cls, data): + if data is None: + return None return cls(id_=data.get('id'), expires=data.get('expires'), issued_at=data.get('issued_at'), - tenant=Tenant._dict_to_obj(data.get('tenant'))) + tenant=Tenant._dict_to_obj(data.get('tenant', {}))) @classmethod def _xml_ele_to_obj(cls, data): diff --git a/syntribos/extensions/random_data/__init__.py b/syntribos/extensions/random_data/__init__.py index 94d152e4..71a5888d 100644 --- a/syntribos/extensions/random_data/__init__.py +++ b/syntribos/extensions/random_data/__init__.py @@ -13,4 +13,3 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ - diff --git a/syntribos/runner.py b/syntribos/runner.py index ba2ea2be..cec4c4ef 100644 --- a/syntribos/runner.py +++ b/syntribos/runner.py @@ -34,6 +34,8 @@ from syntribos.tests.base import test_table from syntribos.config import MainConfig from syntribos.arguments import SyntribosCLI +result = None + class Runner(object): @classmethod @@ -54,11 +56,6 @@ class Runner(object): if any([True for t in test_types if t in k]): yield k, v - @classmethod - def print_tests(cls): - for name, test in cls.get_tests(): - print(name) - @staticmethod def print_symbol(): """ Syntribos radiation symbol """ @@ -96,6 +93,7 @@ class Runner(object): @classmethod def run(cls): + global result requests.packages.urllib3.disable_warnings() try: cls.print_symbol() diff --git a/syntribos/tests/__init__.py b/syntribos/tests/__init__.py index 94d152e4..71a5888d 100644 --- a/syntribos/tests/__init__.py +++ b/syntribos/tests/__init__.py @@ -13,4 +13,3 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ - diff --git a/syntribos/tests/fuzz/__init__.py b/syntribos/tests/fuzz/__init__.py index 94d152e4..71a5888d 100644 --- a/syntribos/tests/fuzz/__init__.py +++ b/syntribos/tests/fuzz/__init__.py @@ -13,4 +13,3 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ - diff --git a/syntribos/tests/fuzz/base_fuzz.py b/syntribos/tests/fuzz/base_fuzz.py index f420f662..a314c645 100644 --- a/syntribos/tests/fuzz/base_fuzz.py +++ b/syntribos/tests/fuzz/base_fuzz.py @@ -16,10 +16,10 @@ limitations under the License. import os -from syntribos.clients.http import SynHTTPClient, RequestCreator +from syntribos.clients.http import client from syntribos.tests import base from syntribos.tests.fuzz.config import BaseFuzzConfig -from syntribos.tests.fuzz.datagen import FuzzBehavior +from syntribos.tests.fuzz.datagen import FuzzParser data_dir = os.environ.get("CAFE_DATA_DIR_PATH") @@ -27,7 +27,9 @@ data_dir = os.environ.get("CAFE_DATA_DIR_PATH") class BaseFuzzTestCase(base.BaseTestCase): config = BaseFuzzConfig() - client = SynHTTPClient() + client = client() + failure_keys = None + success_keys = None @classmethod def validate_length(cls): @@ -39,7 +41,7 @@ class BaseFuzzTestCase(base.BaseTestCase): resp_len = len(cls.resp.content or "") request_diff = req_len - init_req_len response_diff = resp_len - init_resp_len - percent_diff = abs(float(response_diff) / init_resp_len) * 100 + percent_diff = abs(float(response_diff) / (init_resp_len + 1)) * 100 msg = ( "Validate Length:\n" "\tInitial request length: {0}\n" @@ -68,6 +70,19 @@ class BaseFuzzTestCase(base.BaseTestCase): with open(path, "rb") as fp: return fp.read().splitlines() + @classmethod + def data_driven_failure_cases(cls): + if cls.failure_keys is None: + return + for line in cls.failure_keys: + cls.assertNotIn(line, cls.resp.content) + + @classmethod + def data_driven_pass_cases(cls): + if cls.success_keys is None: + return True + assert any(True for s in cls.success_keys if s in cls.resp.content) + @classmethod def setUpClass(cls): """being used as a setup test not""" @@ -80,39 +95,20 @@ class BaseFuzzTestCase(base.BaseTestCase): def test_case(self): self.assertTrue(self.resp.status_code < 500) self.assertTrue(self.validate_length()) + self.data_driven_failure_cases() + self.data_driven_pass_cases() @classmethod def get_test_cases(cls, filename, file_content): # maybe move this block to base.py - request_obj = RequestCreator.create_request( + request_obj = FuzzParser.create_request( file_content, os.environ.get("SYNTRIBOS_ENDPOINT")) prepared_copy = request_obj.get_prepared_copy() cls.init_response = cls.client.send_request(prepared_copy) # end block - for fuzz_name, request in cls._get_fuzz_requests( - request_obj, cls._get_strings()): - full_name = "({filename})_{fuzz_name}".format( - filename=filename, fuzz_name=fuzz_name) - yield cls.extend_class(full_name, {"request": request}) - - @classmethod - def _get_fuzz_requests(cls, request, strings): - prefix_name = "({test_name})_({fuzz_file})_".format( - test_name=cls.test_name, - fuzz_file=cls.data_key) - for name, data in FuzzBehavior.fuzz_data( - strings, getattr(request, cls.test_type), - request.action_field, prefix_name, - cls.config.string_fuzz_name): - request_copy = request.get_copy() - setattr(request_copy, cls.test_type, data) - request_copy.prepare_request() - yield name, request_copy - - -class BaseFuzzDataDrivenValidatorTestCase(BaseFuzzTestCase): - def test_case(self): - super(BaseFuzzDataDrivenValidatorTestCase, self).test_case() - for line in self._get_strings(self.detect_key): - self.assertNotIn(line, self.resp.content) + prefix_name = "{filename}_{test_name}_{fuzz_file}_".format( + filename=filename, test_name=cls.test_name, fuzz_file=cls.data_key) + for fuzz_name, request in request_obj.fuzz_request( + cls._get_strings(), cls.test_type, prefix_name): + yield cls.extend_class(fuzz_name, {"request": request}) diff --git a/syntribos/tests/fuzz/config.py b/syntribos/tests/fuzz/config.py index b38f21d6..609030fb 100644 --- a/syntribos/tests/fuzz/config.py +++ b/syntribos/tests/fuzz/config.py @@ -23,7 +23,3 @@ class BaseFuzzConfig(ConfigSectionInterface): @property def percent(self): return float(self.get("percent", 200.0)) - - @property - def string_fuzz_name(self): - return self.get("string_fuzz_name", "FUZZ") diff --git a/syntribos/tests/fuzz/datagen.py b/syntribos/tests/fuzz/datagen.py index c03d7d14..67ba9cf4 100644 --- a/syntribos/tests/fuzz/datagen.py +++ b/syntribos/tests/fuzz/datagen.py @@ -13,11 +13,14 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ - +import re from xml.etree import ElementTree +from syntribos.clients.http import parser +from syntribos.clients.http.models import RequestObject, RequestHelperMixin -class FuzzBehavior(object): + +class FuzzMixin(object): """ FuzzBehavior provides the fuzz_data function which yields a test name and all iterations of a given piece of data (currently supports dict, @@ -25,7 +28,7 @@ class FuzzBehavior(object): """ @classmethod - def fuzz_data(cls, strings, data, skip_var, name_prefix, string_fuzz_name): + def _fuzz_data(cls, strings, data, skip_var, name_prefix): for str_num, stri in enumerate(strings, 1): if isinstance(data, dict): model_iter = cls._build_combinations(stri, data, skip_var) @@ -41,11 +44,12 @@ class FuzzBehavior(object): yield (name, model) @classmethod - def _build_str_combinations(cls, name, string, url): - rep_str = "{{{0}}}".format(name) - if rep_str in url: - string = url.format(**{name: string}) - yield string + def _build_str_combinations(cls, string, data): + for match in re.finditer(r"{[^}]*}", data): + start, stop = match.span() + yield "{0}{1}{2}".format( + cls.remove_braces(data[:start]), + string, cls.remove_braces(data[stop+1:])) @classmethod def _build_combinations(cls, stri, dic, skip_var): @@ -119,3 +123,27 @@ class FuzzBehavior(object): for i, v in enumerate(list_): ret[i] = v return ret + + @staticmethod + def remove_braces(string): + return string.replace("}", "").replace("{", "") + + +class FuzzRequest(RequestObject, FuzzMixin, RequestHelperMixin): + def fuzz_request(self, strings, fuzz_type, name_prefix): + for name, data in self._fuzz_data( + strings, getattr(self, fuzz_type), self.action_field, + name_prefix): + request_copy = self.get_copy() + setattr(request_copy, fuzz_type, data) + request_copy.prepare_request(fuzz_type) + yield name, request_copy + + def prepare_request(self, fuzz_type=None): + super(FuzzRequest, self).prepare_request() + if fuzz_type != "url": + self.url = self.remove_braces(self.url) + + +class FuzzParser(parser): + request_model_type = FuzzRequest diff --git a/syntribos/tests/request_creator.py b/syntribos/tests/request_creator.py deleted file mode 100644 index bd116e83..00000000 --- a/syntribos/tests/request_creator.py +++ /dev/null @@ -1,210 +0,0 @@ -""" -Copyright 2015 Rackspace - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - -import re -from copy import deepcopy -from importlib import import_module -from uuid import uuid4 -from xml.etree import ElementTree -import json -import types -import urlparse - -_iterators = {} - - -class RequestHelperMixin(object): - """ - Provides functionality to dynamically update iterator objects with the next - returned object. - """ - - @classmethod - def _run_iters(cls, data, action_field): - if isinstance(data, dict): - return cls._run_iters_dict(data, action_field) - elif isinstance(data, ElementTree.Element): - return cls._run_iters_xml(data, action_field) - else: - return data - - @classmethod - def _run_iters_dict(cls, dic, action_field=""): - for key, val in dic.iteritems(): - dic[key] = val = cls._replace_iter(val) - if isinstance(val, basestring): - val = val.replace(action_field, "") - elif isinstance(val, dict): - cls._run_iters_dict(val, action_field) - elif isinstance(val, list): - cls._run_iters_list(val, action_field) - - if isinstance(key, basestring): - new_key = cls._replace_iter(key).replace(action_field, "") - if new_key != key: - del dic[key] - dic[new_key] = val - return dic - - @classmethod - def _run_iters_list(cls, val, action_field=""): - for i, v in enumerate(val): - if isinstance(v, basestring): - val[i] = v = cls._replace_iter(v).replace(action_field, "") - elif isinstance(v, dict): - val[i] = cls._run_iters_dict(v, action_field) - elif isinstance(v, list): - cls._run_iters_list(v, action_field) - - @classmethod - def _run_iters_xml(cls, ele, action_field=""): - if isinstance(ele.text, basestring): - ele.text = cls._replace_iter(ele.text).replace(action_field, "") - cls._run_iters_dict(ele.attrib, action_field) - for i, v in enumerate(list(ele)): - ele[i] = cls._run_iters_xml(v, action_field) - return ele - - @staticmethod - def _string_data(data): - if isinstance(data, dict): - return json.dumps(data) - elif isinstance(data, ElementTree.Element): - return ElementTree.tostring(data) - else: - return data - - @staticmethod - def _replace_iter(string): - if not isinstance(string, basestring): - return string - for k, v in _iterators.items(): - if k in string: - string = string.replace(k, v.next()) - return string - - -class RequestObject(RequestHelperMixin): - def __init__( - self, method, url, action_field=None, headers=None, params=None, - data=None): - self.method = method - self.url = url - self.headers = headers or {} - self.params = params or {} - self.data = data or "" - self.action_field = action_field - - def prepare_request(self): - """ it should be noted this function does not make a request copy - destroying iterators in request. A copy should be made if making - multiple requests""" - self.data = self._run_iters(self.data, self.action_field) - self.headers = self._run_iters(self.headers, self.action_field) - self.params = self._run_iters(self.params, self.action_field) - self.data = self._string_data(self.data) - self.url = self._replace_iter(self.url) - - def get_prepared_copy(self): - copy = deepcopy(self) - copy.prepare_request() - return copy - - def get_copy(self): - return deepcopy(self) - - -class RequestCreator(object): - ACTION_FIELD = "ACTION_FIELD:" - EXTERNAL = r"CALL_EXTERNAL\|([^|]+?):([^|]+?):([^|]+?)\|" - - @classmethod - def create_request(cls, string, endpoint): - string = cls.call_external_functions(string) - action_field = str(uuid4()).replace("-", "") - string = string.replace(cls.ACTION_FIELD, action_field) - lines = string.splitlines() - for index, line in enumerate(lines): - if line == "": - break - method, url, params, version = cls._parse_url_line(lines[0], endpoint) - headers = cls._parse_headers(lines[1:index]) - data = cls._parse_data(lines[index+1:]) - return RequestObject( - method=method, url=url, headers=headers, params=params, data=data, - action_field=action_field) - - @classmethod - def _parse_url_line(cls, line, endpoint): - params = {} - method, url, version = line.split() - url = url.split("?", 1) - if len(url) == 2: - for param in url[1].split("&"): - param = param.split("=", 1) - if len(param) > 1: - params[param[0]] = param[1] - else: - params[param[0]] = "" - url = url[0] - url = urlparse.urljoin(endpoint, url) - return method, url, params, version - - @classmethod - def _parse_headers(cls, lines): - headers = {} - for line in lines: - key, value = line.split(":", 1) - headers[key] = value.strip() - return headers - - @classmethod - def _parse_data(cls, lines): - data = "\n".join(lines).strip() - if not data: - return "" - try: - data = json.loads(data) - except: - try: - data = ElementTree.fromstring(data) - except: - raise Exception("Unknown Data format") - return data - - @classmethod - def call_external_functions(cls, string): - if not isinstance(string, basestring): - return string - - while True: - match = re.search(cls.EXTERNAL, string) - if not match: - break - dot_path = match.group(1) - func_name = match.group(2) - arg_list = match.group(3) - mod = import_module(dot_path) - func = getattr(mod, func_name) - args = json.loads(arg_list) - val = func(*args) - if isinstance(val, types.GeneratorType): - uuid = str(uuid4()).replace("-", "") - string = re.sub(cls.EXTERNAL, uuid, string, count=1) - _iterators[uuid] = val - else: - string = re.sub(cls.EXTERNAL, str(val), string, count=1) - return string