Python3 support

In order to achieve py3-first goals, upgrading code base that will be
compatible with py3 as well as py2.

Change-Id: I562c63c576cc2f5dc5d93b5ec6741152e1f80466

tacker/alarm_receiver.py

@@ -74,7 +74,7 @@ class AlarmReceiver(wsgi.Middleware):
             # Update policy and action
             body_dict['trigger']['policy_name'] = info[4]
             body_dict['trigger']['action_name'] = info[5]
-            req.body = jsonutils.dumps(body_dict)
+            req.body = jsonutils.dump_as_bytes(body_dict)
             LOG.debug('Body alarm: %s', req.body)
         # Need to change url because of mandatory
         req.environ['PATH_INFO'] = prefix + 'triggers'

tacker/tests/common/helpers.py

@@ -15,6 +15,7 @@
 #    under the License.
 
 import six
+import sys
 import testtools
 
 
@@ -24,3 +25,10 @@ def requires_py2(testcase):
 
 def requires_py3(testcase):
     return testtools.skipUnless(six.PY3, "requires python 3.x")(testcase)
+
+if sys.version_info < (3,):
+    def compact_byte(x):
+        return x
+else:
+    def compact_byte(x):
+        return bytes(x, 'utf-8')

tacker/tests/functional/vnfm/test_tosca_vnf_alarm.py

@@ -11,9 +11,10 @@
 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 #    License for the specific language governing permissions and limitations
 #    under the License.
-import json
 import time
 
+from oslo_serialization import jsonutils
+
 from tacker.plugins.common import constants as evt_constants
 from tacker.tests import constants
 from tacker.tests.functional import base
@@ -51,7 +52,7 @@ class VnfTestAlarmMonitor(base.BaseTackerTest):
                 constants.ACTIVE_SLEEP_TIME)
             vnf = self.client.show_vnf(vnf_id)['vnf']
             # {"VDU1": ["10.0.0.14", "10.0.0.5"]}
-            self.assertEqual(count, len(json.loads(vnf[
+            self.assertEqual(count, len(jsonutils.loads(vnf[
                 'mgmt_ip_address'])['VDU1']))
 
         def trigger_vnf(vnf, policy_name, policy_action):

tacker/tests/functional/vnfm/test_tosca_vnf_scale.py

@@ -11,11 +11,11 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import json
 import time
 import yaml
 
 from oslo_config import cfg
+from oslo_serialization import jsonutils
 
 from tacker.plugins.common import constants as evt_constants
 from tacker.tests import constants
@@ -59,7 +59,7 @@ class VnfTestToscaScale(base.BaseTackerTest):
             vnf = self.client.show_vnf(vnf_id)['vnf']
 
             # {"VDU1": ["10.0.0.14", "10.0.0.5"]}
-            self.assertEqual(count, len(json.loads(vnf[
+            self.assertEqual(count, len(jsonutils.loads(vnf[
                 'mgmt_ip_address'])['VDU1']))
 
         _wait(2)

tacker/tests/unit/agent/linux/test_utils.py

@@ -14,12 +14,12 @@
 
 
 import mock
-import six
 
 import oslo_i18n
 
 from tacker.agent.linux import utils
 from tacker.tests import base
+from tacker.tests.common import helpers
 
 
 _marker = object()
@@ -123,19 +123,12 @@ class AgentUtilsExecuteTest(base.BaseTestCase):
             self.assertTrue(log.error.called)
 
     def test_encode_process_input(self):
-        str_idata = "%s\n" % self.test_file[:-1]
-        str_odata = "%s\n" % self.test_file
-        if six.PY3:
-            bytes_idata = str_idata.encode(encoding='utf-8')
-            bytes_odata = str_odata.encode(encoding='utf-8')
-            self.mock_popen.return_value = [bytes_odata, b'']
-            result = utils.execute(['cat'], process_input=str_idata)
-            self.mock_popen.assert_called_once_with(bytes_idata)
-        else:
-            self.mock_popen.return_value = [str_odata, '']
-            result = utils.execute(['cat'], process_input=str_idata)
-            self.mock_popen.assert_called_once_with(str_idata)
-        self.assertEqual(str_odata, result)
+        bytes_idata = helpers.compact_byte("%s\n" % self.test_file[:-1])
+        bytes_odata = helpers.compact_byte("%s\n" % self.test_file)
+        self.mock_popen.return_value = [bytes_odata, b'']
+        result = utils.execute(['cat'], process_input=bytes_idata)
+        self.mock_popen.assert_called_once_with(bytes_idata)
+        self.assertEqual(bytes_odata, result)
 
     def test_return_str_data(self):
         str_data = "%s\n" % self.test_file
@@ -151,6 +144,6 @@ class AgentUtilsExecuteEncodeTest(base.BaseTestCase):
         open(self.test_file, 'w').close()
 
     def test_decode_return_data(self):
-        str_data = "%s\n" % self.test_file
+        str_data = helpers.compact_byte("%s\n" % self.test_file)
         result = utils.execute(['ls', self.test_file], return_stderr=True)
-        self.assertEqual((str_data, ''), result)
+        self.assertEqual((str_data, helpers.compact_byte('')), result)

tacker/tests/unit/api/test_api_common.py

@@ -13,13 +13,13 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import json
 import netaddr
 
 from testtools import matchers
 from webob import exc
 
 from oslo_policy import policy as oslo_policy
+from oslo_serialization import jsonutils
 from tacker.api import api_common as common
 from tacker.api.v1 import resource as wsgi_resource
 from tacker.common import exceptions
@@ -111,7 +111,8 @@ class APICommonTestCase(base.BaseTestCase):
                       'type': 'HTTPClientError',
                       'detail': ''}
 
-        self.assertEqual(except_res, json.loads(result.body)["TackerError"])
+        self.assertEqual(
+            except_res, jsonutils.loads(result.body)["TackerError"])
         self.assertEqual(400, result.code)
 
     def test_http_exception(self):
@@ -125,7 +126,8 @@ class APICommonTestCase(base.BaseTestCase):
                       "type": "HTTPInternalServerError",
                       "detail": ""}
 
-        self.assertEqual(except_res, json.loads(result.body)["TackerError"])
+        self.assertEqual(
+            except_res, jsonutils.loads(result.body)["TackerError"])
         self.assertEqual(500, result.code)
 
     def test_tacker_exception(self):
@@ -138,7 +140,8 @@ class APICommonTestCase(base.BaseTestCase):
                       'type': 'TackerException',
                       'detail': ''}
 
-        self.assertEqual(except_res, json.loads(result.body)["TackerError"])
+        self.assertEqual(
+            except_res, jsonutils.loads(result.body)["TackerError"])
         self.assertEqual(500, result.code)
 
     def test_addr_format_error_exception(self):
@@ -151,7 +154,8 @@ class APICommonTestCase(base.BaseTestCase):
                       'type': 'AddrFormatError',
                       'detail': ''}
 
-        self.assertEqual(except_res, json.loads(result.body)["TackerError"])
+        self.assertEqual(
+            except_res, jsonutils.loads(result.body)["TackerError"])
         self.assertEqual(500, result.code)
 
     def test_policy_not_authorized_exception(self):
@@ -164,7 +168,8 @@ class APICommonTestCase(base.BaseTestCase):
                       'type': 'PolicyNotAuthorized',
                       'detail': ''}
 
-        self.assertEqual(except_res, json.loads(result.body)["TackerError"])
+        self.assertEqual(
+            except_res, jsonutils.loads(result.body)["TackerError"])
         self.assertEqual(500, result.code)
 
     def test_not_implemented_error_exception(self):
@@ -177,7 +182,7 @@ class APICommonTestCase(base.BaseTestCase):
                                               'type': 'NotImplementedError',
                                               'detail': ''}}
 
-        self.assertEqual(except_res, json.loads(result.body))
+        self.assertEqual(except_res, jsonutils.loads(result.body))
         self.assertEqual(501, result.code)
 
     def test_get_exception_data(self):

tacker/tests/unit/api/test_extensions.py

@@ -338,7 +338,7 @@ class ActionExtensionTest(base.BaseTestCase):
     def test_extended_action_for_adding_extra_data(self):
         action_name = 'FOXNSOX:add_tweedle'
         action_params = dict(name='Beetle')
-        req_body = jsonutils.dumps({action_name: action_params})
+        req_body = jsonutils.dump_as_bytes({action_name: action_params})
         response = self.extension_app.post('/dummy_resources/1/action',
                                            req_body,
                                            content_type='application/json')
@@ -347,7 +347,7 @@ class ActionExtensionTest(base.BaseTestCase):
     def test_extended_action_for_deleting_extra_data(self):
         action_name = 'FOXNSOX:delete_tweedle'
         action_params = dict(name='Bailey')
-        req_body = jsonutils.dumps({action_name: action_params})
+        req_body = jsonutils.dump_as_bytes({action_name: action_params})
         response = self.extension_app.post("/dummy_resources/1/action",
                                            req_body,
                                            content_type='application/json')
@@ -356,7 +356,8 @@ class ActionExtensionTest(base.BaseTestCase):
     def test_returns_404_for_non_existent_action(self):
         non_existent_action = 'blah_action'
         action_params = dict(name="test")
-        req_body = jsonutils.dumps({non_existent_action: action_params})
+        req_body = jsonutils.dump_as_bytes(
+            {non_existent_action: action_params})
 
         response = self.extension_app.post("/dummy_resources/1/action",
                                            req_body,
@@ -368,7 +369,7 @@ class ActionExtensionTest(base.BaseTestCase):
     def test_returns_404_for_non_existent_resource(self):
         action_name = 'add_tweedle'
         action_params = dict(name='Beetle')
-        req_body = jsonutils.dumps({action_name: action_params})
+        req_body = jsonutils.dump_as_bytes({action_name: action_params})
 
         response = self.extension_app.post("/asdf/1/action", req_body,
                                            content_type='application/json',

tacker/tests/unit/api/v1/test_router.py

@@ -13,9 +13,10 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import json
 import mock
 
+from oslo_serialization import jsonutils
+
 from tacker.api.v1.router import APIRouter
 from tacker.api.v1.router import Index
 from tacker.tests import base
@@ -38,7 +39,7 @@ class TestIndex(base.BaseTestCase):
                   'rel': 'self'}],
              'name': 'version'}]}
 
-        self.assertEqual(expect_body, json.loads(result.body))
+        self.assertEqual(expect_body, jsonutils.loads(result.body))
         self.assertEqual('application/json', result.content_type)
 
 

tacker/tests/unit/extensions/foxinsocks.py

@@ -80,7 +80,7 @@ class Foxinsocks(object):
             # You can use content type header to test for XML.
             data = jsonutils.loads(res.body)
             data['FOXNSOX:googoose'] = req.GET.get('chewing')
-            res.body = jsonutils.dumps(data)
+            res.body = jsonutils.dump_as_bytes(data)
             return res
 
         req_ext1 = extensions.RequestExtension('GET', '/dummy_resources/:(id)',
@@ -92,7 +92,7 @@ class Foxinsocks(object):
             # You can use content type header to test for XML.
             data = jsonutils.loads(res.body)
             data['FOXNSOX:big_bands'] = 'Pig Bands!'
-            res.body = jsonutils.dumps(data)
+            res.body = jsonutils.dump_as_bytes(data)
             return res
 
         req_ext2 = extensions.RequestExtension('GET', '/dummy_resources/:(id)',

tacker/tests/unit/test_alarm_receiver.py

@@ -82,7 +82,7 @@ class TestAlarmReceiver(base.TestCase):
         req = Request.blank(self.ordered_url)
         req.method = 'POST'
         old_body = {'fake_key': 'fake_value'}
-        req.body = jsonutils.dumps(old_body)
+        req.body = jsonutils.dump_as_bytes(old_body)
 
         self.alarmrc.process_request(req)
 

tacker/tests/unit/vnfm/infra_drivers/openstack/test_openstack.py

@@ -14,14 +14,16 @@
 #    under the License.
 
 import codecs
-import json
 import mock
 import os
 import yaml
 
+from oslo_serialization import jsonutils
+
 from tacker import context
 from tacker.db.common_services import common_services_db_plugin
 from tacker.extensions import vnfm
+from tacker.tests.common import helpers
 from tacker.tests.unit import base
 from tacker.tests.unit.db import utils
 from tacker.vnfm.infra_drivers.openstack import openstack
@@ -291,29 +293,31 @@ class TestOpenStack(base.TestCase):
         if is_monitor:
             if multi_vdus:
                 dvc['attributes'].update(
-                    {'monitoring_policy': '{"vdus": {"VDU1": {"ping": '
-                                          '{"name": "ping", "actions": '
-                                          '{"failure": "respawn"}, '
-                                          '"parameters": {"count": 3, '
-                                          '"interval": 10}, '
-                                          '"monitoring_params": '
-                                          '{"count": 3, "interval": 10}}}, '
-                                          '"VDU2": {"ping": {"name": "ping", '
-                                          '"actions": {"failure": "respawn"}, '
-                                          '"parameters": {"count": 3, '
-                                          '"interval": 10}, '
-                                          '"monitoring_params": {"count": 3, '
-                                          '"interval": 10}}}}}'})
+                    {'monitoring_policy': helpers.compact_byte(
+                     '{"vdus": {"VDU1": {"ping": '
+                     '{"name": "ping", "actions": '
+                     '{"failure": "respawn"}, '
+                     '"parameters": {"count": 3, '
+                     '"interval": 10}, '
+                     '"monitoring_params": '
+                     '{"count": 3, "interval": 10}}}, '
+                     '"VDU2": {"ping": {"name": "ping", '
+                     '"actions": {"failure": "respawn"}, '
+                     '"parameters": {"count": 3, '
+                     '"interval": 10}, '
+                     '"monitoring_params": {"count": 3, '
+                     '"interval": 10}}}}}')})
             else:
                 dvc['attributes'].update(
-                    {'monitoring_policy': '{"vdus": {"VDU1": {"ping": '
-                                          '{"name": "ping", "actions": '
-                                          '{"failure": "respawn"}, '
-                                          '"parameters": {"count": 3, '
-                                          '"interval": 10}, '
-                                          '"monitoring_params": '
-                                          '{"count": 3, '
-                                          '"interval": 10}}}}}'})
+                    {'monitoring_policy': helpers.compact_byte(
+                     '{"vdus": {"VDU1": {"ping": '
+                     '{"name": "ping", "actions": '
+                     '{"failure": "respawn"}, '
+                     '"parameters": {"count": 3, '
+                     '"interval": 10}, '
+                     '"monitoring_params": '
+                     '{"count": 3, '
+                     '"interval": 10}}}}}')})
 
         return dvc
 
@@ -374,7 +378,7 @@ class TestOpenStack(base.TestCase):
                     vnf["attributes"][k])
             expected_vnf["attributes"]['scaling_group_names'] = {
                 'SP1': 'SP1_group'}
-            vnf["attributes"]['scaling_group_names'] = json.loads(
+            vnf["attributes"]['scaling_group_names'] = jsonutils.loads(
                 vnf["attributes"]['scaling_group_names']
             )
         self.assertEqual(expected_vnf, vnf)

tacker/tests/unit/vnfm/test_plugin.py

@@ -125,7 +125,7 @@ class TestVNFMPluginMonitor(db_base.SqlTestCase):
         monitor.VNFMonitor._hosting_vnfs = dict()
         vnfm_plugin = plugin.VNFMPlugin()
         hosting_vnfs = vnfm_plugin._vnf_monitor._hosting_vnfs.values()
-        hosting_vnf = hosting_vnfs[0]['vnf']
+        hosting_vnf = list(hosting_vnfs)[0]['vnf']
         self.assertEqual('{"VDU1": "a.b.c.d"}', hosting_vnf['mgmt_ip_address'])
         self.assertEqual(1, len(hosting_vnfs))
 

tacker/vnfm/infra_drivers/kubernetes/kubernetes_driver.py

@@ -163,7 +163,8 @@ class Kubernetes(abstract_driver.VnfAbstractDriver,
                         get("vdu_name").split("-")[1]
                     mgmt_ip = service_info.spec.cluster_ip
                     mgmt_ips.update({vdu_name: mgmt_ip})
-                    vnf_dict['mgmt_ip_address'] = jsonutils.dumps(mgmt_ips)
+                    vnf_dict['mgmt_ip_address'] = jsonutils.dump_as_bytes(
+                        mgmt_ips)
         except Exception as e:
             LOG.error('Creating wait VNF got an error due to %s', e)
             raise

tacker/vnfm/infra_drivers/openstack/openstack.py

@@ -159,7 +159,7 @@ class OpenStack(abstract_driver.VnfAbstractDriver,
             mgmt_ips = self._find_mgmt_ips(stack.outputs)
 
         if mgmt_ips:
-            vnf_dict['mgmt_ip_address'] = jsonutils.dumps(mgmt_ips)
+            vnf_dict['mgmt_ip_address'] = jsonutils.dump_as_bytes(mgmt_ips)
 
     def _wait_until_stack_ready(self, vnf_id, auth_attr, wait_status,
                                 region_name=None):
@@ -253,7 +253,7 @@ class OpenStack(abstract_driver.VnfAbstractDriver,
         mgmt_ips = self._find_mgmt_ips(stack.outputs)
 
         if mgmt_ips:
-            vnf_dict['mgmt_ip_address'] = jsonutils.dumps(mgmt_ips)
+            vnf_dict['mgmt_ip_address'] = jsonutils.dump_as_bytes(mgmt_ips)
 
     @log.log
     def delete(self, plugin, context, vnf_id, auth_attr, region_name=None):
@@ -393,7 +393,7 @@ class OpenStack(abstract_driver.VnfAbstractDriver,
                                                    policy['instance_id'],
                                                    [policy['group_name']])
 
-        return jsonutils.dumps(mgmt_ips)
+        return jsonutils.dump_as_bytes(mgmt_ips)
 
     @log.log
     def get_resource_info(self, plugin, context, vnf_info, auth_attr,

tacker/vnfm/infra_drivers/openstack/translate_template.py

@@ -79,11 +79,12 @@ class TOSCAToHOT(object):
         if not self.vnf['attributes'].get('heat_template'):
             self.vnf['attributes']['heat_template'] = self.fields['template']
         if self.monitoring_dict:
-            self.vnf['attributes']['monitoring_policy'] = jsonutils.dumps(
+            self.vnf['attributes'][
+                'monitoring_policy'] = jsonutils.dump_as_bytes(
                 self.monitoring_dict)
         if self.appmonitoring_dict:
             self.vnf['attributes']['app_monitoring_policy'] = \
-                jsonutils.dumps(self.appmonitoring_dict)
+                jsonutils.dump_as_bytes(self.appmonitoring_dict)
 
     @log.log
     def _get_vnfd(self):
@@ -318,7 +319,7 @@ class TOSCAToHOT(object):
             scaling_group_dict = toscautils.get_scaling_group_dict(
                 heat_template_yaml, scaling_policy_names)
             self.vnf['attributes']['scaling_group_names'] =\
-                jsonutils.dumps(scaling_group_dict)
+                jsonutils.dump_as_bytes(scaling_group_dict)
 
         heat_template_yaml = toscautils.post_process_heat_template(
             heat_template_yaml, mgmt_ports, metadata, alarm_resources,

tacker/vnfm/monitor_drivers/zabbix/zabbix.py

@@ -13,13 +13,13 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import json
 import netaddr
 import requests
 import time
 
 import copy
 from oslo_log import log as logging
+from oslo_serialization import jsonutils
 from tacker.vnfm.monitor_drivers import abstract_driver
 from tacker.vnfm.monitor_drivers.zabbix import zabbix_api as zapi
 
@@ -78,7 +78,7 @@ class VNFMonitorZabbix(abstract_driver.VNFMonitorAbstractDriver):
 
     def send_post(self, query):
         response = requests.post(self.URL, headers=zapi.HEADERS,
-                                 data=json.dumps(query))
+                                 data=jsonutils.dump_as_bytes(query))
         return dict(response.json())
 
     @staticmethod
@@ -367,7 +367,7 @@ class VNFMonitorZabbix(abstract_driver.VNFMonitorAbstractDriver):
         response = requests.post(
             self.URL,
             headers=zapi.HEADERS,
-            data=json.dumps(temp_auth_api)
+            data=jsonutils.dump_as_bytes(temp_auth_api)
         )
         response_dict = dict(response.json())
         VNFMonitorZabbix.check_error(response_dict)

tacker/wsgi.py

@@ -409,7 +409,7 @@ class JSONDictSerializer(DictSerializer):
     def default(self, data):
         def sanitizer(obj):
             return six.text_type(obj)
-        return encode_body(jsonutils.dumps(data, default=sanitizer))
+        return encode_body(jsonutils.dump_as_bytes(data, default=sanitizer))
 
 
 class ResponseHeaderSerializer(ActionDispatcher):