Enable py35 functional test env

This patch also adds the necessary changes to fix py35 compatibility
and ensure that the current functional/unit tests pass.

Change-Id: Iaced98e98db29882a35bea1abd98e50dd167b511

refstack/api/controllers/products.py

@@ -176,7 +176,7 @@ class ProductsController(validation.BaseRestControllerWithValidation):
                     if _id not in result:
                         result[_id] = s
                     result[_id]['can_manage'] = True
-                products = result.values()
+                products = list(result.values())
         except Exception as ex:
             LOG.exception('An error occurred during '
                           'operation with database: %s' % ex)
@@ -201,7 +201,7 @@ class ProductsController(validation.BaseRestControllerWithValidation):
         if not is_admin:
             admin_only_keys = ['created_by_user', 'created_at', 'updated_at',
                                'properties']
-            for key in product.keys():
+            for key in list(product):
                 if key in admin_only_keys:
                     product.pop(key)
 

refstack/api/controllers/validation.py

@@ -57,7 +57,7 @@ class BaseRestControllerWithValidation(rest.RestController):
     def post(self, ):
         """POST handler."""
         self.validator.validate(pecan.request)
-        item = json.loads(pecan.request.body)
+        item = json.loads(pecan.request.body.decode('utf-8'))
         item_id = self.store_item(item)
         pecan.response.status = 201
         return item_id

refstack/api/controllers/vendors.py

@@ -166,12 +166,11 @@ class VendorsController(validation.BaseRestControllerWithValidation):
                     if _id not in result:
                         result[_id] = vendor
                     result[_id]['can_manage'] = True
-                vendors = result.values()
+                vendors = list(result.values())
         except Exception as ex:
             LOG.exception('An error occurred during '
                           'operation with database: %s' % ex)
             pecan.abort(400)
-
         return {'vendors': vendors}
 
     @pecan.expose('json')

refstack/api/validators.py

@@ -65,7 +65,7 @@ class BaseValidator(object):
     def validate(self, request):
         """Validate request."""
         try:
-            body = json.loads(request.body)
+            body = json.loads(request.body.decode('utf-8'))
         except (ValueError, TypeError) as e:
             raise api_exc.ValidationError('Malformed request', e)
 
@@ -135,7 +135,7 @@ class TestResultValidator(BaseValidator):
                 raise api_exc.ValidationError('Malformed public key', e)
 
             verifier = key.verifier(sign, padding.PKCS1v15(), hashes.SHA256())
-            verifier.update(request.body.encode('utf-8'))
+            verifier.update(request.body)
             try:
                 verifier.verify()
             except InvalidSignature:
@@ -146,7 +146,7 @@ class TestResultValidator(BaseValidator):
 
     def _is_empty_result(self, request):
         """Check if the test results list is empty."""
-        body = json.loads(request.body)
+        body = json.loads(request.body.decode('utf-8'))
         if len(body['results']) != 0:
             return False
         return True
@@ -173,7 +173,7 @@ class PubkeyValidator(BaseValidator):
     def validate(self, request):
         """Validate uploaded test results."""
         super(PubkeyValidator, self).validate(request)
-        body = json.loads(request.body)
+        body = json.loads(request.body.decode('utf-8'))
         key_format = body['raw_key'].strip().split()[0]
 
         if key_format not in ('ssh-dss', 'ssh-rsa',
@@ -215,7 +215,7 @@ class VendorValidator(BaseValidator):
     def validate(self, request):
         """Validate uploaded vendor data."""
         super(VendorValidator, self).validate(request)
-        body = json.loads(request.body)
+        body = json.loads(request.body.decode('utf-8'))
 
         self.check_emptyness(body, ['name'])
 
@@ -239,7 +239,7 @@ class ProductValidator(BaseValidator):
     def validate(self, request):
         """Validate uploaded test results."""
         super(ProductValidator, self).validate(request)
-        body = json.loads(request.body)
+        body = json.loads(request.body.decode('utf-8'))
 
         self.check_emptyness(body, ['name', 'product_type'])
 
@@ -260,6 +260,6 @@ class ProductVersionValidator(BaseValidator):
     def validate(self, request):
         """Validate product version data."""
         super(ProductVersionValidator, self).validate(request)
-        body = json.loads(request.body)
+        body = json.loads(request.body.decode('utf-8'))
 
         self.check_emptyness(body, ['version'])

refstack/tests/api/test_profile.py

@@ -62,9 +62,9 @@ class TestProfileEndpoint(api.FunctionalTest):
         pubkey = key.public_key().public_bytes(
             serialization.Encoding.OpenSSH,
             serialization.PublicFormat.OpenSSH
-        )
+        ).decode('utf-8')
         body = {'raw_key': pubkey,
-                'self_signature': binascii.b2a_hex(sign)}
+                'self_signature': binascii.b2a_hex(sign).decode('utf-8')}
         json_params = json.dumps(body)
 
         # POST endpoint

refstack/tests/unit/test_api.py

@@ -147,7 +147,7 @@ class ResultsControllerTestCase(BaseControllerTestCase):
 
     @mock.patch('refstack.db.store_results')
     def test_post(self, mock_store_results):
-        self.mock_request.body = '{"answer": 42}'
+        self.mock_request.body = b'{"answer": 42}'
         self.mock_request.headers = {}
         mock_store_results.return_value = 'fake_test_id'
         result = self.controller.post()
@@ -167,7 +167,7 @@ class ResultsControllerTestCase(BaseControllerTestCase):
     @mock.patch('refstack.db.get_pubkey')
     def test_post_with_sign(self, mock_get_pubkey, mock_store_results,
                             mock_get_version, mock_check, mock_foundation):
-        self.mock_request.body = '{"answer": 42, "cpid": "123"}'
+        self.mock_request.body = b'{"answer": 42, "cpid": "123"}'
         self.mock_request.headers = {
             'X-Signature': 'fake-sign',
             'X-Public-Key': 'ssh-rsa Zm9vIGJhcg=='
@@ -438,7 +438,7 @@ class BaseRestControllerWithValidationTestCase(BaseControllerTestCase):
     @mock.patch('pecan.response')
     @mock.patch('pecan.request')
     def test_post(self, mock_request, mock_response):
-        mock_request.body = '[42]'
+        mock_request.body = b'[42]'
         self.controller.store_item = mock.Mock(return_value='fake_id')
 
         result = self.controller.post()
@@ -724,7 +724,9 @@ class PublicKeysControllerTestCase(BaseControllerTestCase):
             'comment': 'Don\'t_Panic.',
             'openid': 'fake_id'
         }
-        self.mock_request.body = json.dumps({'raw_key': raw_key})
+        self.mock_request.body = json.dumps(
+            {'raw_key': raw_key}
+        ).encode('utf-8')
         self.controller.post()
         self.assertEqual(201, self.mock_response.status)
         mock_store_pubkey.assert_called_once_with(fake_pubkey)
@@ -737,7 +739,9 @@ class PublicKeysControllerTestCase(BaseControllerTestCase):
             'comment': '',
             'openid': 'fake_id'
         }
-        self.mock_request.body = json.dumps({'raw_key': raw_key})
+        self.mock_request.body = json.dumps(
+            {'raw_key': raw_key}
+        ).encode('utf-8')
         self.controller.post()
         mock_store_pubkey.assert_called_once_with(fake_pubkey)
 

refstack/tests/unit/test_validators.py

@@ -97,7 +97,7 @@ class TestResultValidatorTestCase(base.BaseTestCase):
     def test_validation(self):
         with mock.patch('jsonschema.validate') as mock_validate:
             request = mock.Mock()
-            request.body = json.dumps(self.FAKE_JSON)
+            request.body = json.dumps(self.FAKE_JSON).encode('utf-8')
             request.headers = {}
             self.validator.validate(request)
             mock_validate.assert_called_once_with(self.FAKE_JSON,
@@ -105,7 +105,7 @@ class TestResultValidatorTestCase(base.BaseTestCase):
 
     def test_validation_with_signature(self):
         request = mock.Mock()
-        request.body = json.dumps(self.FAKE_JSON)
+        request.body = json.dumps(self.FAKE_JSON).encode('utf-8')
 
         key = rsa.generate_private_key(
             public_exponent=65537,
@@ -113,7 +113,7 @@ class TestResultValidatorTestCase(base.BaseTestCase):
             backend=default_backend()
         )
         signer = key.signer(padding.PKCS1v15(), hashes.SHA256())
-        signer.update(request.body.encode('utf-8'))
+        signer.update(request.body)
         sign = signer.finalize()
         pubkey = key.public_key().public_bytes(
             serialization.Encoding.OpenSSH,
@@ -127,7 +127,7 @@ class TestResultValidatorTestCase(base.BaseTestCase):
 
     def test_validation_fail_no_json(self):
         wrong_request = mock.Mock()
-        wrong_request.body = 'foo'
+        wrong_request.body = b'foo'
         self.assertRaises(api_exc.ValidationError,
                           self.validator.validate,
                           wrong_request)
@@ -140,7 +140,7 @@ class TestResultValidatorTestCase(base.BaseTestCase):
         wrong_request = mock.Mock()
         wrong_request.body = json.dumps({
             'foo': 'bar'
-        })
+        }).encode('utf-8')
         self.assertRaises(api_exc.ValidationError,
                           self.validator.validate,
                           wrong_request)
@@ -151,7 +151,9 @@ class TestResultValidatorTestCase(base.BaseTestCase):
 
     def test_validation_fail_with_empty_result(self):
         wrong_request = mock.Mock()
-        wrong_request.body = json.dumps(self.FAKE_JSON_WITH_EMPTY_RESULTS)
+        wrong_request.body = json.dumps(
+            self.FAKE_JSON_WITH_EMPTY_RESULTS
+        ).encode('utf-8')
         self.assertRaises(api_exc.ValidationError,
                           self.validator.validate,
                           wrong_request)
@@ -160,7 +162,7 @@ class TestResultValidatorTestCase(base.BaseTestCase):
     def test_validation_with_broken_signature(self, mock_validate):
 
         request = mock.Mock()
-        request.body = json.dumps(self.FAKE_JSON)
+        request.body = json.dumps(self.FAKE_JSON).encode('utf-8')
         key = rsa.generate_private_key(
             public_exponent=65537,
             key_size=2048,
@@ -224,12 +226,12 @@ class PubkeyValidatorTestCase(base.BaseTestCase):
 
     def test_validation(self):
         request = mock.Mock()
-        request.body = json.dumps(self.FAKE_JSON)
+        request.body = json.dumps(self.FAKE_JSON).encode('utf-8')
         self.validator.validate(request)
 
     def test_validation_fail_no_json(self):
         wrong_request = mock.Mock()
-        wrong_request.body = 'foo'
+        wrong_request.body = b'foo'
         self.assertRaises(api_exc.ValidationError,
                           self.validator.validate,
                           wrong_request)
@@ -242,7 +244,7 @@ class PubkeyValidatorTestCase(base.BaseTestCase):
         wrong_request = mock.Mock()
         wrong_request.body = json.dumps({
             'foo': 'bar'
-        })
+        }).encode('utf-8')
         self.assertRaises(api_exc.ValidationError,
                           self.validator.validate,
                           wrong_request)
@@ -257,7 +259,7 @@ class PubkeyValidatorTestCase(base.BaseTestCase):
         body['self_signature'] = 'deadbeef'
 
         request = mock.Mock()
-        request.body = json.dumps(body)
+        request.body = json.dumps(body).encode('utf-8')
         try:
             self.validator.validate(request)
         except api_exc.ValidationError as e:
@@ -269,7 +271,7 @@ class PubkeyValidatorTestCase(base.BaseTestCase):
             'self_signature': 'deadbeef'
         }
         request = mock.Mock()
-        request.body = json.dumps(body)
+        request.body = json.dumps(body).encode('utf-8')
         try:
             self.validator.validate(request)
         except api_exc.ValidationError as e:
@@ -281,7 +283,7 @@ class PubkeyValidatorTestCase(base.BaseTestCase):
             'self_signature': 'deadbeef?'
         }
         request = mock.Mock()
-        request.body = json.dumps(body)
+        request.body = json.dumps(body).encode('utf-8')
         try:
             self.validator.validate(request)
         except api_exc.ValidationError as e:
@@ -293,7 +295,7 @@ class PubkeyValidatorTestCase(base.BaseTestCase):
             'self_signature': 'deadbeef'
         }
         request = mock.Mock()
-        request.body = json.dumps(body)
+        request.body = json.dumps(body).encode('utf-8')
         try:
             self.validator.validate(request)
         except api_exc.ValidationError as e:

tox.ini

@@ -1,8 +1,8 @@
 [tox]
-# py34 tests should be run before py27
+# py3* tests should be run before py27
 # it is a workaround for testr bug
 # https://bugs.launchpad.net/testrepository/+bug/1229445
-envlist = py34,py27,pep8,pip-check-reqs
+envlist = py35,py27,pep8,pip-check-reqs
 minversion = 1.6
 skipsdist = True
 
@@ -27,6 +27,16 @@ setenv = SUBUNIT_TEST_PATH=./refstack/tests/api
 # require cleanup of database
 commands = {toxinidir}/setup-mysql-tests.sh python setup.py testr --slowest --testr-args='{posargs:--concurrency=1}'
 
+[testenv:py35-func-mysql]
+basepython = python3.5
+setenv = SUBUNIT_TEST_PATH=./refstack/tests/api
+# Integration/functional tests
+# must not be run in parallel (--concurrency=1),
+# because each of these tests
+# require cleanup of database
+commands = {toxinidir}/setup-mysql-tests.sh python setup.py testr --slowest --testr-args='{posargs:--concurrency=1}'
+
+
 [testenv:pep8]
 commands =
   flake8 {posargs}