From f0e4e094c3e4f2120b37da4e6343241beea7d968 Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Fri, 28 Sep 2018 11:55:08 -0700 Subject: [PATCH] Treat subunit as binary under python3 The subunit stream is a binary protocol and can't be encoded to utf8. Unfortunately under python3 the default behavior when reading from stdin or opened files is to read them encoded as your platform dependent encoding. For linux users this is typically utf8 and subunit isn't valid utf8. Fix this by reading the subunit streams as binary under python3 as we do on python2. Change-Id: I1ddfe514c219c3bbfcd69f6808faa2331933acee --- requirements.txt | 1 + stackviz/parser/tempest_subunit.py | 14 ++++++++-- stackviz/tests/fixtures/tempest.subunit | 11 ++++++++ stackviz/tests/test_stackviz.py | 37 +++++++++++++++++++++++-- test-requirements.txt | 1 + 5 files changed, 60 insertions(+), 4 deletions(-) create mode 100644 stackviz/tests/fixtures/tempest.subunit diff --git a/requirements.txt b/requirements.txt index 7771f11..efa25c1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ python-subunit>=0.0.18 testtools>=0.9.30 testrepository>=0.0.18 subunit2sql>=0.8.0 +six diff --git a/stackviz/parser/tempest_subunit.py b/stackviz/parser/tempest_subunit.py index 8ac6933..6dd2882 100644 --- a/stackviz/parser/tempest_subunit.py +++ b/stackviz/parser/tempest_subunit.py @@ -15,6 +15,7 @@ import os import re import shutil +import six import subunit import sys @@ -134,13 +135,22 @@ class FileProvider(SubunitProvider): if index != 0: raise IndexError("Index out of bounds: %d" % index) - return open(self.path, "r") + # Subunit is a binary protocol we need to ensure we read + # the contents as binary. On python3 this requires we open + # the file in binary mode otherwise it will be encoded. + return open(self.path, "rb") class StandardInputProvider(SubunitProvider): def __init__(self): self.buffer = BytesIO() - shutil.copyfileobj(sys.stdin, self.buffer) + # Subunit is a binary protocol we need to ensure we read + # the contents as binary. On python3 this requires we use + # the stdin.buffer object as stdin is encoded otherwise. + if six.PY3: + shutil.copyfileobj(sys.stdin.buffer, self.buffer) + else: + shutil.copyfileobj(sys.stdin, self.buffer) self.buffer.seek(0) @property diff --git a/stackviz/tests/fixtures/tempest.subunit b/stackviz/tests/fixtures/tempest.subunit new file mode 100644 index 0000000..9caff94 --- /dev/null +++ b/stackviz/tests/fixtures/tempest.subunit @@ -0,0 +1,11 @@ ++@[ \5@ttempest.api.compute.admin.test_agents.AgentsAdminTestJSON.test_create_agent[id-1fc6bdc8-0b6d-4cc7-9f30-9b04fabe5b90]2h-+pI[ \bH@ttempest.api.compute.admin.test_agents.AgentsAdminTestJSON.test_create_agent[id-1fc6bdc8-0b6d-4cc7-9f30-9b04fabe5b90]text/plain;charset=utf8pythonlogging:''H2018-09-28 02:46:35,903 26441 INFO [tempest.lib.common.rest_client] Request (AgentsAdminTestJSON:test_create_agent): 200 POST https://10.210.196.239/compute/v2.1/os-agents 0.275s +2018-09-28 02:46:35,903 26441 DEBUG [tempest.lib.common.rest_client] Request - Headers: {'Accept': 'application/json', 'Content-Type': 'application/json', 'X-Auth-Token': ''} + Body: {"agent": {"md5hash": "add6bb58e139be103324d04d82d8f545", "url": "xxx://xxxx/xxx/xxx", "version": "7.0", "architecture": "tempest-x86-1736555698", "hypervisor": "kvm", "os": "win"}} + Response - Headers: {'server': 'Apache/2.4.18 (Ubuntu)', 'date': 'Fri, 28 Sep 2018 02:46:35 GMT', 'content-location': 'https://10.210.196.239/compute/v2.1/os-agents', 'connection': 'close', 'openstack-api-version': 'compute 2.1', 'x-openstack-nova-api-version': '2.1', 'status': '200', 'content-length': '196', 'vary': 'OpenStack-API-Version,X-OpenStack-Nova-API-Version', 'x-compute-request-id': 'req-bc16c3df-13bf-4b96-bf7c-fcc243ee6a35', 'x-openstack-request-id': 'req-bc16c3df-13bf-4b96-bf7c-fcc243ee6a35', 'content-type': 'application/json'} + Body: b'{"agent": {"md5hash": "add6bb58e139be103324d04d82d8f545", "hypervisor": "kvm", "url": "xxx://xxxx/xxx/xxx", "os": "win", "version": "7.0", "architecture": "tempest-x86-1736555698", "agent_id": 1}}' +2018-09-28 02:46:35,977 26441 INFO [tempest.lib.common.rest_client] Request (AgentsAdminTestJSON:_run_cleanups): 200 DELETE https://10.210.196.239/compute/v2.1/os-agents/1 0.068s +2018-09-28 02:46:35,978 26441 DEBUG [tempest.lib.common.rest_client] Request - Headers: {'Accept': 'application/json', 'Content-Type': 'application/json', 'X-Auth-Token': ''} + Body: None + Response - Headers: {'server': 'Apache/2.4.18 (Ubuntu)', 'date': 'Fri, 28 Sep 2018 02:46:35 GMT', 'content-location': 'https://10.210.196.239/compute/v2.1/os-agents/1', 'connection': 'close', 'openstack-api-version': 'compute 2.1', 'x-openstack-nova-api-version': '2.1', 'status': '200', 'content-length': '0', 'vary': 'OpenStack-API-Version,X-OpenStack-Nova-API-Version', 'x-compute-request-id': 'req-78d28bbe-3990-44d9-a117-dc8512f2a62c', 'x-openstack-request-id': 'req-78d28bbe-3990-44d9-a117-dc8512f2a62c', 'content-type': 'application/json'} + Body: b'' +N+@[ \bH@ttempest.api.compute.admin.test_agents.AgentsAdminTestJSON.test_create_agent[id-1fc6bdc8-0b6d-4cc7-9f30-9b04fabe5b90]worker-36 \ No newline at end of file diff --git a/stackviz/tests/test_stackviz.py b/stackviz/tests/test_stackviz.py index 5503415..b37c4e4 100644 --- a/stackviz/tests/test_stackviz.py +++ b/stackviz/tests/test_stackviz.py @@ -21,10 +21,43 @@ test_stackviz Tests for `stackviz` module. """ +import json +import os.path +import sys # noqa for monkeypatching below + +import fixtures + +import stackviz.export as export +from stackviz.parser import tempest_subunit from stackviz.tests import base class TestStackviz(base.TestCase): - def test_something(self): - pass + def test_export_file(self): + tmp_fixture = self.useFixture(fixtures.TempDir()) + output_dir = tmp_fixture.path + subunit_path = os.path.join(os.path.dirname(__file__), + 'fixtures', 'tempest.subunit') + providers = tempest_subunit.get_providers(None, [subunit_path], None) + export.export_tempest(list(providers.values())[0], output_dir, False) + output_file = os.path.join(output_dir, + 'tempest.subunit-0-details.json') + j = json.load(open(output_file)) + assert "tempest.api.compute.admin" \ + ".test_agents.AgentsAdminTestJSON.test_create_agent" in j + + def test_export_stdin(self): + tmp_fixture = self.useFixture(fixtures.TempDir()) + output_dir = tmp_fixture.path + subunit_path = os.path.join(os.path.dirname(__file__), + 'fixtures', 'tempest.subunit') + subunit_stream = open(subunit_path) + with fixtures.MonkeyPatch('sys.stdin', subunit_stream): + providers = tempest_subunit.get_providers(None, None, True) + export.export_tempest(list(providers.values())[0], + output_dir, False) + output_file = os.path.join(output_dir, 'stdin-0-details.json') + j = json.load(open(output_file)) + assert "tempest.api.compute.admin" \ + ".test_agents.AgentsAdminTestJSON.test_create_agent" in j diff --git a/test-requirements.txt b/test-requirements.txt index 7e02908..71fa392 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -10,3 +10,4 @@ discover sphinx>=1.6.2 # BSD oslotest>=1.2.0 # Apache-2.0 openstackdocstheme>=1.17.0 # Apache-2.0 +fixtures