From c8548fc93c7c0e9f48a0d1a8a20d8e9e96c6a071 Mon Sep 17 00:00:00 2001 From: Stephen Lowrie Date: Tue, 24 May 2016 15:57:35 -0500 Subject: [PATCH] Add subunit-describe-calls Adds new command subunit-describe-calls, documentation, and unittests. subunit-describe-calls is a parser for subunit v1 & v2 streams to determine what REST API calls are made inside of a test and in what order they are called. Later commits will add additional functionality relating to request & response headers & body data along with a stdout output option and stdin input. Change-Id: I468d0d3e3b6098da95a81cc86d9bdd1b47ee1f03 --- doc/source/index.rst | 1 + doc/source/subunit_describe_calls.rst | 5 + ...bunit_describe_calls-5498a37e6cd66c4b.yaml | 4 + setup.cfg | 1 + tempest/cmd/subunit_describe_calls.py | 259 ++++++++++++++++++ .../tests/cmd/sample_streams/calls.subunit | Bin 0 -> 7267 bytes .../tests/cmd/test_subunit_describe_calls.py | 83 ++++++ 7 files changed, 353 insertions(+) create mode 100644 doc/source/subunit_describe_calls.rst create mode 100644 releasenotes/notes/add_subunit_describe_calls-5498a37e6cd66c4b.yaml create mode 100644 tempest/cmd/subunit_describe_calls.py create mode 100644 tempest/tests/cmd/sample_streams/calls.subunit create mode 100644 tempest/tests/cmd/test_subunit_describe_calls.py diff --git a/doc/source/index.rst b/doc/source/index.rst index 98b006d482..1faf711a5d 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -51,6 +51,7 @@ Command Documentation account_generator cleanup javelin + subunit_describe_calls workspace ================== diff --git a/doc/source/subunit_describe_calls.rst b/doc/source/subunit_describe_calls.rst new file mode 100644 index 0000000000..2bda50c235 --- /dev/null +++ b/doc/source/subunit_describe_calls.rst @@ -0,0 +1,5 @@ +------------------------------ +Subunit Describe Calls Utility +------------------------------ + +.. automodule:: tempest.cmd.subunit_describe_calls diff --git a/releasenotes/notes/add_subunit_describe_calls-5498a37e6cd66c4b.yaml b/releasenotes/notes/add_subunit_describe_calls-5498a37e6cd66c4b.yaml new file mode 100644 index 0000000000..b457dddab3 --- /dev/null +++ b/releasenotes/notes/add_subunit_describe_calls-5498a37e6cd66c4b.yaml @@ -0,0 +1,4 @@ +--- +features: + - Adds subunit-describe-calls. A parser for subunit streams to determine what + REST API calls are made inside of a test and in what order they are called. diff --git a/setup.cfg b/setup.cfg index 0bf493c76a..56344b07cb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -34,6 +34,7 @@ console_scripts = tempest = tempest.cmd.main:main skip-tracker = tempest.lib.cmd.skip_tracker:main check-uuid = tempest.lib.cmd.check_uuid:run + subunit-describe-calls = tempest.cmd.subunit_describe_calls:entry_point tempest.cm = account-generator = tempest.cmd.account_generator:TempestAccountGenerator init = tempest.cmd.init:TempestInit diff --git a/tempest/cmd/subunit_describe_calls.py b/tempest/cmd/subunit_describe_calls.py new file mode 100644 index 0000000000..c990adde32 --- /dev/null +++ b/tempest/cmd/subunit_describe_calls.py @@ -0,0 +1,259 @@ +# Copyright 2016 Rackspace +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +subunit-describe-calls is a parser for subunit streams to determine what REST +API calls are made inside of a test and in what order they are called. + +Runtime Arguments +----------------- + +**--subunit, -s**: (Required) The path to the subunit file being parsed + +**--non-subunit-name, -n**: (Optional) The file_name that the logs are being +stored in + +**--output-file, -o**: (Required) The path where the JSON output will be +written to + +**--ports, -p**: (Optional) The path to a JSON file describing the ports being +used by different services + +Usage +----- + +subunit-describe-calls will take in a file path via the --subunit parameter +which contains either a subunit v1 or v2 stream. This is then parsed checking +for details contained in the file_bytes of the --non-subunit-name parameter +(the default is pythonlogging which is what Tempest uses to store logs). By +default the OpenStack Kilo release port defaults (http://bit.ly/22jpF5P) +are used unless a file is provided via the --ports option. The resulting output +is dumped in JSON output to the path provided in the --output-file option. + +Ports file JSON structure +^^^^^^^^^^^^^^^^^^^^^^^^^ + + { + "": "", + ... + } + + +Output file JSON structure +^^^^^^^^^^^^^^^^^^^^^^^^^^ + { + "full_test_name[with_id_and_tags]": [ + { + "name": "The ClassName.MethodName that made the call", + "verb": "HTTP Verb", + "service": "Name of the service", + "url": "A shortened version of the URL called", + "status_code": "The status code of the response" + } + ] + } +""" +import argparse +import collections +import io +import json +import os +import re + +import subunit +import testtools + + +class UrlParser(testtools.TestResult): + uuid_re = re.compile(r'(^|[^0-9a-f])[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-' + '[0-9a-f]{4}-[0-9a-f]{12}([^0-9a-f]|$)') + id_re = re.compile(r'(^|[^0-9a-z])[0-9a-z]{8}[0-9a-z]{4}[0-9a-z]{4}' + '[0-9a-z]{4}[0-9a-z]{12}([^0-9a-z]|$)') + ip_re = re.compile(r'(^|[^0-9])[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]' + '{1,3}([^0-9]|$)') + url_re = re.compile(r'.*INFO.*Request \((?P.*)\): (?P[\d]{3}) ' + '(?P\w*) (?P.*) .*') + port_re = re.compile(r'.*:(?P\d+).*') + path_re = re.compile(r'http[s]?://[^/]*/(?P.*)') + + # Based on mitaka defaults: + # http://docs.openstack.org/mitaka/config-reference/ + # firewalls-default-ports.html + services = { + "8776": "Block Storage", + "8774": "Nova", + "8773": "Nova-API", "8775": "Nova-API", + "8386": "Sahara", + "35357": "Keystone", "5000": "Keystone", + "9292": "Glance", "9191": "Glance", + "9696": "Neutron", + "6000": "Swift", "6001": "Swift", "6002": "Swift", + "8004": "Heat", "8000": "Heat", "8003": "Heat", + "8777": "Ceilometer", + "80": "Horizon", + "8080": "Swift", + "443": "SSL", + "873": "rsync", + "3260": "iSCSI", + "3306": "MySQL", + "5672": "AMQP"} + + def __init__(self, services=None): + super(UrlParser, self).__init__() + self.test_logs = {} + self.services = services or self.services + + def addSuccess(self, test, details=None): + output = test.shortDescription() or test.id() + calls = self.parse_details(details) + self.test_logs.update({output: calls}) + + def addSkip(self, test, err, details=None): + output = test.shortDescription() or test.id() + calls = self.parse_details(details) + self.test_logs.update({output: calls}) + + def addError(self, test, err, details=None): + output = test.shortDescription() or test.id() + calls = self.parse_details(details) + self.test_logs.update({output: calls}) + + def addFailure(self, test, err, details=None): + output = test.shortDescription() or test.id() + calls = self.parse_details(details) + self.test_logs.update({output: calls}) + + def stopTestRun(self): + super(UrlParser, self).stopTestRun() + + def startTestRun(self): + super(UrlParser, self).startTestRun() + + def parse_details(self, details): + if details is None: + return + + calls = [] + for _, detail in details.items(): + for line in detail.as_text().split("\n"): + match = self.url_re.match(line) + if match is not None: + calls.append({ + "name": match.group("name"), + "verb": match.group("verb"), + "status_code": match.group("code"), + "service": self.get_service(match.group("url")), + "url": self.url_path(match.group("url"))}) + + return calls + + def get_service(self, url): + match = self.port_re.match(url) + if match is not None: + return self.services.get(match.group("port"), "Unknown") + return "Unknown" + + def url_path(self, url): + match = self.path_re.match(url) + if match is not None: + path = match.group("path") + path = self.uuid_re.sub(r'\1\2', path) + path = self.ip_re.sub(r'\1\2', path) + path = self.id_re.sub(r'\1\2', path) + return path + return url + + +class FileAccumulator(testtools.StreamResult): + + def __init__(self, non_subunit_name='pythonlogging'): + super(FileAccumulator, self).__init__() + self.route_codes = collections.defaultdict(io.BytesIO) + self.non_subunit_name = non_subunit_name + + def status(self, **kwargs): + if kwargs.get('file_name') != self.non_subunit_name: + return + file_bytes = kwargs.get('file_bytes') + if not file_bytes: + return + route_code = kwargs.get('route_code') + stream = self.route_codes[route_code] + stream.write(file_bytes) + + +class ArgumentParser(argparse.ArgumentParser): + def __init__(self): + desc = "Outputs all HTTP calls a given test made that were logged." + super(ArgumentParser, self).__init__(description=desc) + + self.prog = "Argument Parser" + + self.add_argument( + "-s", "--subunit", metavar="", required=True, + default=None, help="The path to the subunit output file.") + + self.add_argument( + "-n", "--non-subunit-name", metavar="", + default="pythonlogging", + help="The name used in subunit to describe the file contents.") + + self.add_argument( + "-o", "--output-file", metavar="", default=None, + help="The output file name for the json.", required=True) + + self.add_argument( + "-p", "--ports", metavar="", default=None, + help="A JSON file describing the ports for each service.") + + +def parse(subunit_file, non_subunit_name, ports): + if ports is not None and os.path.exists(ports): + ports = json.loads(open(ports).read()) + + url_parser = UrlParser(ports) + stream = open(subunit_file, 'rb') + suite = subunit.ByteStreamToStreamResult( + stream, non_subunit_name=non_subunit_name) + result = testtools.StreamToExtendedDecorator(url_parser) + accumulator = FileAccumulator(non_subunit_name) + result = testtools.StreamResultRouter(result) + result.add_rule(accumulator, 'test_id', test_id=None) + result.startTestRun() + suite.run(result) + + for bytes_io in accumulator.route_codes.values(): # v1 processing + bytes_io.seek(0) + suite = subunit.ProtocolTestCase(bytes_io) + suite.run(url_parser) + result.stopTestRun() + + return url_parser + + +def output(url_parser, output_file): + with open(output_file, "w") as outfile: + outfile.write(json.dumps(url_parser.test_logs)) + + +def entry_point(): + cl_args = ArgumentParser().parse_args() + parser = parse(cl_args.subunit, cl_args.non_subunit_name, cl_args.ports) + output(parser, cl_args.output_file) + + +if __name__ == "__main__": + entry_point() diff --git a/tempest/tests/cmd/sample_streams/calls.subunit b/tempest/tests/cmd/sample_streams/calls.subunit new file mode 100644 index 0000000000000000000000000000000000000000..d5b479036eba61a413f3ffa8f12d7249a9e4c09e GIT binary patch literal 7267 zcmeHM%Wm676gAMDi!QS7g78KyqLJps;ZSV@IJKRoXzUucA(Y5lRb_Fh(& zKmPiA^!&QH^Yfx*-MfkJRzsB+A5_PFP!_}T;20#2QF^O18 zX>@ycXDw#(rXA%)9~!>A zzqi|KoPSV#+>0p2(cQiK{b=BQ)$4XC>(H2W646PMo=g(nJ)#}bHJQ*PWh`^C=84F( zR$5E=AuQ1;<=wJDYI>Up!v&IN{iPtbwr@VT^+q62^o3K_)lDz@zV&%ocz}fZ$CYdK zqL!+vVQ!Sq%cA>jQx>gu)Y>%0RsO7_57Fk>4^Y26bVca%X*tThclJi>+4^*LVRN&z z$8fieDXxw1Yg`ZEdt)-0z)Zn*5;oy~qaAGwU@-Mj-jsFdGkYMRePrW-Y6hWESu3=T zr6VlWj$py>Y1mf9o z8u(4bLPgQ7JN@}>ru^||@#pH0An+&XazD4g3IRS9f( zVq&Qcj-?DDx$K%gc!>DzWkf6uIN zo5gsg>Nr>5M)5LPoB(l&WO3nr8>bw@y1{7hcyHOa(b!t09F!Pb200+M1W8IIic@U` zSBXnveR1@RUsunU&%B>Wg44c@EcI=;9gEdgg{HpE)DDdFDc`p83K5peBh3QP)sHlX zN2BvD4S zjxhh>!#^Ea=oyhEN#d2}E#57%`cQISbyiP8A&8LWL1lsE5(SmFPzZ7^#c^f@Qn3b! zTE&qxEI~07%xD^Go7$@&(Itf-+)m|cE1vuHxPUh=P8H*-Ij3qTybc1AFSV{G0R2yL rf8q^=O9?f~U?iBsh>IjeN=St?yttCq8ikV!RH%7_O!vhPPhb2AX^@m> literal 0 HcmV?d00001 diff --git a/tempest/tests/cmd/test_subunit_describe_calls.py b/tempest/tests/cmd/test_subunit_describe_calls.py new file mode 100644 index 0000000000..43b417ab94 --- /dev/null +++ b/tempest/tests/cmd/test_subunit_describe_calls.py @@ -0,0 +1,83 @@ +# Copyright 2016 Rackspace +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import subprocess +import tempfile + +from tempest.cmd import subunit_describe_calls +from tempest.tests import base + + +class TestSubunitDescribeCalls(base.TestCase): + def test_return_code(self): + subunit_file = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + 'sample_streams/calls.subunit') + p = subprocess.Popen([ + 'subunit-describe-calls', '-s', subunit_file, + '-o', tempfile.mkstemp()[1]], stdin=subprocess.PIPE) + p.communicate() + self.assertEqual(0, p.returncode) + + def test_parse(self): + subunit_file = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + 'sample_streams/calls.subunit') + parser = subunit_describe_calls.parse( + subunit_file, "pythonlogging", None) + expected_result = { + 'bar': [{'name': 'AgentsAdminTestJSON:setUp', + 'service': 'Nova', + 'status_code': '200', + 'url': 'v2.1//os-agents', + 'verb': 'POST'}, + {'name': 'AgentsAdminTestJSON:test_create_agent', + 'service': 'Nova', + 'status_code': '200', + 'url': 'v2.1//os-agents', + 'verb': 'POST'}, + {'name': 'AgentsAdminTestJSON:tearDown', + 'service': 'Nova', + 'status_code': '200', + 'url': 'v2.1//os-agents/1', + 'verb': 'DELETE'}, + {'name': 'AgentsAdminTestJSON:_run_cleanups', + 'service': 'Nova', + 'status_code': '200', + 'url': 'v2.1//os-agents/2', + 'verb': 'DELETE'}], + 'foo': [{'name': 'AgentsAdminTestJSON:setUp', + 'service': 'Nova', + 'status_code': '200', + 'url': 'v2.1//os-agents', + 'verb': 'POST'}, + {'name': 'AgentsAdminTestJSON:test_delete_agent', + 'service': 'Nova', + 'status_code': '200', + 'url': 'v2.1//os-agents/3', + 'verb': 'DELETE'}, + {'name': 'AgentsAdminTestJSON:test_delete_agent', + 'service': 'Nova', + 'status_code': '200', + 'url': 'v2.1//os-agents', + 'verb': 'GET'}, + {'name': 'AgentsAdminTestJSON:tearDown', + 'service': 'Nova', + 'status_code': '404', + 'url': 'v2.1//os-agents/3', + 'verb': 'DELETE'}]} + self.assertEqual(expected_result, parser.test_logs)