diff --git a/octavia/amphorae/backends/agent/api_server/amphora_info.py b/octavia/amphorae/backends/agent/api_server/amphora_info.py index 27ed98d23c..adbfe61c44 100644 --- a/octavia/amphorae/backends/agent/api_server/amphora_info.py +++ b/octavia/amphorae/backends/agent/api_server/amphora_info.py @@ -105,7 +105,7 @@ class AmphoraInfo: def _get_version_of_installed_package(self, name): cmd = self._osutils.cmd_get_version_of_installed_package(name) - version = subprocess.check_output(cmd.split()) + version = subprocess.check_output(cmd.split(), encoding='utf-8') return version def _count_haproxy_processes(self, lb_list): diff --git a/octavia/amphorae/backends/agent/api_server/keepalived.py b/octavia/amphorae/backends/agent/api_server/keepalived.py index cdfc36d93a..081870f5fb 100644 --- a/octavia/amphorae/backends/agent/api_server/keepalived.py +++ b/octavia/amphorae/backends/agent/api_server/keepalived.py @@ -121,7 +121,8 @@ class Keepalived: if init_enable_cmd is not None: try: subprocess.check_output(init_enable_cmd.split(), - stderr=subprocess.STDOUT) + stderr=subprocess.STDOUT, + encoding='utf-8') except subprocess.CalledProcessError as e: LOG.debug('Failed to enable octavia-keepalived service: ' '%(err)s %(output)s', {'err': e, 'output': e.output}) @@ -160,7 +161,8 @@ class Keepalived: cmd = f"/usr/sbin/service octavia-keepalived {action}" try: - subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT) + subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT, + encoding='utf-8') except subprocess.CalledProcessError as e: LOG.debug('Failed to %s octavia-keepalived service: %s %s', action, e, e.output) diff --git a/octavia/amphorae/backends/agent/api_server/keepalivedlvs.py b/octavia/amphorae/backends/agent/api_server/keepalivedlvs.py index 4bb6915930..2f9e11f3e9 100644 --- a/octavia/amphorae/backends/agent/api_server/keepalivedlvs.py +++ b/octavia/amphorae/backends/agent/api_server/keepalivedlvs.py @@ -143,7 +143,8 @@ class KeepalivedLvs(lvs_listener_base.LvsListenerApiServerBase): init_enable_cmd = f"insserv {file_path}" try: subprocess.check_output(init_enable_cmd.split(), - stderr=subprocess.STDOUT) + stderr=subprocess.STDOUT, + encoding='utf-8') except subprocess.CalledProcessError as e: LOG.debug('Failed to enable ' 'octavia-keepalivedlvs service: ' @@ -220,7 +221,8 @@ class KeepalivedLvs(lvs_listener_base.LvsListenerApiServerBase): f"octavia-keepalivedlvs-{listener_id} {action}") try: - subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT) + subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT, + encoding='utf-8') except subprocess.CalledProcessError as e: LOG.debug('Failed to %s keepalivedlvs listener %s', listener_id + ' : ' + action, e) @@ -281,7 +283,8 @@ class KeepalivedLvs(lvs_listener_base.LvsListenerApiServerBase): cmd = (f"/usr/sbin/service " f"octavia-keepalivedlvs-{listener_id} stop") try: - subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT) + subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT, + encoding='utf-8') except subprocess.CalledProcessError as e: LOG.error("Failed to stop keepalivedlvs service: %s", e) return webob.Response(json={ @@ -309,7 +312,8 @@ class KeepalivedLvs(lvs_listener_base.LvsListenerApiServerBase): init_disable_cmd = f"insserv -r {init_path}" try: subprocess.check_output(init_disable_cmd.split(), - stderr=subprocess.STDOUT) + stderr=subprocess.STDOUT, + encoding='utf-8') except subprocess.CalledProcessError as e: LOG.error("Failed to disable " "octavia-keepalivedlvs-%(list)s service: " diff --git a/octavia/amphorae/backends/agent/api_server/loadbalancer.py b/octavia/amphorae/backends/agent/api_server/loadbalancer.py index f2a9287b6c..d7ef0d0b86 100644 --- a/octavia/amphorae/backends/agent/api_server/loadbalancer.py +++ b/octavia/amphorae/backends/agent/api_server/loadbalancer.py @@ -134,7 +134,8 @@ class Loadbalancer: haproxy_ug=consts.HAPROXY_USER_GROUP_CFG) try: - subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT) + subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT, + encoding='utf-8') except subprocess.CalledProcessError as e: LOG.error("Failed to verify haproxy file: %s %s", e, e.output) # Save the last config that failed validation for debugging @@ -213,7 +214,8 @@ class Loadbalancer: elif init_system == consts.INIT_SYSVINIT: try: subprocess.check_output(init_enable_cmd.split(), - stderr=subprocess.STDOUT) + stderr=subprocess.STDOUT, + encoding='utf-8') except subprocess.CalledProcessError as e: LOG.error("Failed to enable haproxy-%(lb_id)s service: " "%(err)s %(out)s", {'lb_id': lb_id, 'err': e, @@ -286,11 +288,12 @@ class Loadbalancer: lb_id=lb_id, action=action)) try: - subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT) + subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT, + encoding='utf-8') except subprocess.CalledProcessError as e: # Mitigation for # https://bugs.launchpad.net/octavia/+bug/2054666 - if (b'is not active, cannot reload.' in e.output and + if ('is not active, cannot reload.' in e.output and action == consts.AMP_ACTION_RELOAD): saved_exc = e @@ -312,20 +315,21 @@ class Loadbalancer: "was reloaded, check the haproxy logs for " "more details.") break - if b'Job is already running' not in e.output: + if 'Job is already running' not in e.output: LOG.debug( "Failed to %(action)s haproxy-%(lb_id)s service: " "%(err)s %(out)s", {'action': action, 'lb_id': lb_id, 'err': e, 'out': e.output}) return webob.Response(json={ 'message': f"Error {action}ing haproxy", - 'details': e.output}, status=500) + 'details': e.output + }, status=500) break else: # no break, we reach the retry limit for reloads return webob.Response(json={ 'message': f"Error {action}ing haproxy", - 'details': saved_exc.output if saved_exc else ''}, status=500) + 'details': saved_exc.output}, status=500) # If we are not in active/standby we need to send an IP # advertisement (GARP or NA). Keepalived handles this for @@ -360,7 +364,8 @@ class Loadbalancer: os.path.join('/proc', util.get_haproxy_pid(lb_id))): cmd = f"/usr/sbin/service haproxy-{lb_id} stop" try: - subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT) + subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT, + encoding='utf-8') except subprocess.CalledProcessError as e: LOG.error("Failed to stop haproxy-%s service: %s %s", lb_id, e, e.output) @@ -401,7 +406,8 @@ class Loadbalancer: init_disable_cmd = f"insserv -r {init_path}" try: subprocess.check_output(init_disable_cmd.split(), - stderr=subprocess.STDOUT) + stderr=subprocess.STDOUT, + encoding='utf-8') except subprocess.CalledProcessError as e: LOG.error("Failed to disable haproxy-%(lb_id)s service: " "%(err)s %(out)s", {'lb_id': lb_id, 'err': e, diff --git a/octavia/amphorae/backends/agent/api_server/osutils.py b/octavia/amphorae/backends/agent/api_server/osutils.py index a1e55584e6..169aa38cae 100644 --- a/octavia/amphorae/backends/agent/api_server/osutils.py +++ b/octavia/amphorae/backends/agent/api_server/osutils.py @@ -89,8 +89,9 @@ class BaseOS: LOG.debug("Executing: %s", cmd) try: out = subprocess.check_output(cmd.split(), - stderr=subprocess.STDOUT) - for line in out.decode('utf-8').split('\n'): + stderr=subprocess.STDOUT, + encoding='utf-8') + for line in out.split('\n'): LOG.debug(line) except subprocess.CalledProcessError as e: LOG.error('Failed to set up %s due to error: %s %s', interface, diff --git a/octavia/amphorae/backends/agent/api_server/util.py b/octavia/amphorae/backends/agent/api_server/util.py index 77500700ea..aff55d47ed 100644 --- a/octavia/amphorae/backends/agent/api_server/util.py +++ b/octavia/amphorae/backends/agent/api_server/util.py @@ -269,7 +269,8 @@ def install_netns_systemd_service(): def run_systemctl_command(command, service): cmd = f"systemctl {command} {service}" try: - subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT) + subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT, + encoding='utf-8') except subprocess.CalledProcessError as e: LOG.error("Failed to %(cmd)s %(srvc)s service: " "%(err)s %(out)s", {'cmd': command, 'srvc': service, diff --git a/octavia/amphorae/backends/health_daemon/health_daemon.py b/octavia/amphorae/backends/health_daemon/health_daemon.py index 50774ac526..9d64e94355 100644 --- a/octavia/amphorae/backends/health_daemon/health_daemon.py +++ b/octavia/amphorae/backends/health_daemon/health_daemon.py @@ -15,6 +15,7 @@ # under the License. import errno +import json import os import queue import stat @@ -22,7 +23,6 @@ import time from oslo_config import cfg from oslo_log import log as logging -import simplejson from octavia.amphorae.backends.agent.api_server import util from octavia.amphorae.backends.health_daemon import health_sender @@ -73,8 +73,8 @@ def get_counters(): global COUNTERS if COUNTERS is None: try: - COUNTERS = simplejson.load(get_counters_file()) or {} - except (simplejson.JSONDecodeError, AttributeError): + COUNTERS = json.load(get_counters_file()) or {} + except (json.JSONDecodeError, AttributeError): COUNTERS = {} return COUNTERS @@ -84,7 +84,7 @@ def persist_counters(): if COUNTERS is None: return try: - stats = simplejson.dumps(COUNTERS) + stats = json.dumps(COUNTERS) counters_file = get_counters_file() counters_file.truncate(0) counters_file.write(stats) diff --git a/octavia/amphorae/backends/utils/interface_file.py b/octavia/amphorae/backends/utils/interface_file.py index 1da9bc0347..63a01b8aee 100644 --- a/octavia/amphorae/backends/utils/interface_file.py +++ b/octavia/amphorae/backends/utils/interface_file.py @@ -13,11 +13,11 @@ # under the License. import ipaddress +import json import os import stat from oslo_config import cfg -import simplejson from octavia.common import constants as consts @@ -45,11 +45,11 @@ class InterfaceFile: @classmethod def load(cls, fp): - return simplejson.load(fp) + return json.load(fp) @classmethod def dump(cls, obj): - return simplejson.dumps(obj) + return json.dumps(obj) @classmethod def from_file(cls, filename): diff --git a/octavia/amphorae/drivers/haproxy/rest_api_driver.py b/octavia/amphorae/drivers/haproxy/rest_api_driver.py index a1d85fc35c..ae23c95712 100644 --- a/octavia/amphorae/drivers/haproxy/rest_api_driver.py +++ b/octavia/amphorae/drivers/haproxy/rest_api_driver.py @@ -14,6 +14,7 @@ # under the License. import functools import hashlib +import json import os import ssl import time @@ -24,7 +25,6 @@ from oslo_context import context as oslo_context from oslo_log import log as logging from oslo_utils.secretutils import md5 import requests -import simplejson from stevedore import driver as stevedore_driver from octavia.amphorae.driver_exceptions import exceptions as driver_except @@ -723,7 +723,7 @@ class AmphoraAPIClientBase: if 'No suitable network interface found' in json_data: LOG.debug("Amphora network interface not found.") raise requests.ConnectionError - except simplejson.JSONDecodeError: # if r.json() fails + except json.JSONDecodeError: # if r.json() fails pass # TODO(rm_work) Should we do something? return r except (requests.ConnectionError, requests.Timeout) as e: diff --git a/octavia/tests/functional/amphorae/backend/agent/api_server/test_keepalivedlvs.py b/octavia/tests/functional/amphorae/backend/agent/api_server/test_keepalivedlvs.py index 9094cb6e61..a3be326cd6 100644 --- a/octavia/tests/functional/amphorae/backend/agent/api_server/test_keepalivedlvs.py +++ b/octavia/tests/functional/amphorae/backend/agent/api_server/test_keepalivedlvs.py @@ -314,7 +314,8 @@ class KeepalivedLvsTestCase(base.TestCase): cmd = ("/usr/sbin/service octavia-keepalivedlvs-{listener_id}" " {action}".format(listener_id=self.FAKE_ID, action='start')) mock_check_output.assert_called_once_with(cmd.split(), - stderr=subprocess.STDOUT) + stderr=subprocess.STDOUT, + encoding='utf-8') self.assertEqual(202, res.status_code) res = self.test_keepalivedlvs.manage_lvs_listener(self.FAKE_ID, @@ -347,8 +348,10 @@ class KeepalivedLvsTestCase(base.TestCase): cmd2 = ("systemctl disable " "octavia-keepalivedlvs-{list}".format(list=self.FAKE_ID)) calls = [ - mock.call(cmd1.split(), stderr=subprocess.STDOUT), - mock.call(cmd2.split(), stderr=subprocess.STDOUT) + mock.call(cmd1.split(), stderr=subprocess.STDOUT, + encoding='utf-8'), + mock.call(cmd2.split(), stderr=subprocess.STDOUT, + encoding='utf-8') ] m_check_output.assert_has_calls(calls) self.assertEqual(200, res.status_code) diff --git a/octavia/tests/functional/amphorae/backend/agent/api_server/test_server.py b/octavia/tests/functional/amphorae/backend/agent/api_server/test_server.py index f5f3ad31ae..52bc8dc57f 100644 --- a/octavia/tests/functional/amphorae/backend/agent/api_server/test_server.py +++ b/octavia/tests/functional/amphorae/backend/agent/api_server/test_server.py @@ -38,7 +38,7 @@ import octavia.tests.unit.base as base AMP_AGENT_CONF_PATH = '/etc/octavia/amphora-agent.conf' -RANDOM_ERROR = b'random error' +RANDOM_ERROR = 'random error' OK = dict(message='OK') FAKE_INTERFACE = 'eth33' @@ -137,7 +137,7 @@ class TestServerTestCase(base.TestCase): haproxy_ug=consts.HAPROXY_USER_GROUP_CFG, peer=(octavia_utils. base64_sha1_string('amp_123').rstrip('='))).split(), - stderr=-2) + stderr=subprocess.STDOUT, encoding='utf-8') mock_rename.assert_called_with( '/var/lib/octavia/123/haproxy.cfg.new', '/var/lib/octavia/123/haproxy.cfg') @@ -145,11 +145,11 @@ class TestServerTestCase(base.TestCase): if init_system == consts.INIT_SYSTEMD: mock_subprocess.assert_any_call( "systemctl enable haproxy-123".split(), - stderr=subprocess.STDOUT) + stderr=subprocess.STDOUT, encoding='utf-8') elif init_system == consts.INIT_SYSVINIT: mock_subprocess.assert_any_call( "insserv /etc/init.d/haproxy-123".split(), - stderr=subprocess.STDOUT) + stderr=subprocess.STDOUT, encoding='utf-8') else: self.assertIn(init_system, consts.VALID_INIT_SYSTEMS) @@ -243,7 +243,7 @@ class TestServerTestCase(base.TestCase): haproxy_ug=consts.HAPROXY_USER_GROUP_CFG, peer=(octavia_utils. base64_sha1_string('amp_123').rstrip('='))).split(), - stderr=-2) + stderr=subprocess.STDOUT, encoding='utf-8') mock_rename.assert_called_with( '/var/lib/octavia/123/haproxy.cfg.new', '/var/lib/octavia/123/haproxy.cfg.new-failed') @@ -321,7 +321,8 @@ class TestServerTestCase(base.TestCase): ' 123 started'}, jsonutils.loads(rv.data.decode('utf-8'))) mock_subprocess.assert_called_with( - ['/usr/sbin/service', 'haproxy-123', 'start'], stderr=-2) + ['/usr/sbin/service', 'haproxy-123', 'start'], + stderr=subprocess.STDOUT, encoding='utf-8') mock_exists.return_value = True mock_subprocess.side_effect = subprocess.CalledProcessError( @@ -336,10 +337,11 @@ class TestServerTestCase(base.TestCase): self.assertEqual( { 'message': 'Error starting haproxy', - 'details': RANDOM_ERROR.decode('utf-8'), + 'details': RANDOM_ERROR, }, jsonutils.loads(rv.data.decode('utf-8'))) mock_subprocess.assert_called_with( - ['/usr/sbin/service', 'haproxy-123', 'start'], stderr=-2) + ['/usr/sbin/service', 'haproxy-123', 'start'], + stderr=subprocess.STDOUT, encoding='utf-8') def test_ubuntu_reload(self): self._test_reload(consts.UBUNTU) @@ -377,7 +379,8 @@ class TestServerTestCase(base.TestCase): 'details': 'Listener 123 reloaded'}, jsonutils.loads(rv.data.decode('utf-8'))) mock_subprocess.assert_called_with( - ['/usr/sbin/service', 'haproxy-123', 'reload'], stderr=-2) + ['/usr/sbin/service', 'haproxy-123', 'reload'], + stderr=subprocess.STDOUT, encoding='utf-8') # Process not running so start mock_exists.return_value = True @@ -395,7 +398,8 @@ class TestServerTestCase(base.TestCase): ' 123 started'}, jsonutils.loads(rv.data.decode('utf-8'))) mock_subprocess.assert_called_with( - ['/usr/sbin/service', 'haproxy-123', 'start'], stderr=-2) + ['/usr/sbin/service', 'haproxy-123', 'start'], + stderr=subprocess.STDOUT, encoding='utf-8') def test_ubuntu_info(self): self._test_info(consts.UBUNTU) @@ -604,19 +608,22 @@ class TestServerTestCase(base.TestCase): jsonutils.loads(rv.data.decode('utf-8'))) mock_pid.assert_called_once_with('123') mock_check_output.assert_any_call( - ['/usr/sbin/service', 'haproxy-123', 'stop'], stderr=-2) + ['/usr/sbin/service', 'haproxy-123', 'stop'], + stderr=subprocess.STDOUT, encoding='utf-8') if init_system == consts.INIT_SYSTEMD: mock_check_output.assert_any_call( "systemctl disable haproxy-123".split(), - stderr=subprocess.STDOUT) + stderr=subprocess.STDOUT, + encoding='utf-8') elif init_system == consts.INIT_UPSTART: mock_remove.assert_any_call(consts.UPSTART_DIR + '/haproxy-123.conf') elif init_system == consts.INIT_SYSVINIT: mock_check_output.assert_any_call( "insserv -r /etc/init.d/haproxy-123".split(), - stderr=subprocess.STDOUT) + stderr=subprocess.STDOUT, + encoding='utf-8') else: self.assertIn(init_system, consts.VALID_INIT_SYSTEMS) @@ -634,19 +641,22 @@ class TestServerTestCase(base.TestCase): jsonutils.loads(rv.data.decode('utf-8'))) mock_pid.assert_called_with('123') mock_check_output.assert_any_call( - ['/usr/sbin/service', 'haproxy-123', 'stop'], stderr=-2) + ['/usr/sbin/service', 'haproxy-123', 'stop'], + stderr=subprocess.STDOUT, encoding='utf-8') if init_system == consts.INIT_SYSTEMD: mock_check_output.assert_any_call( "systemctl disable haproxy-123".split(), - stderr=subprocess.STDOUT) + stderr=subprocess.STDOUT, + encoding='utf-8') elif init_system == consts.INIT_UPSTART: mock_remove.assert_any_call(consts.UPSTART_DIR + '/haproxy-123.conf') elif init_system == consts.INIT_SYSVINIT: mock_check_output.assert_any_call( "insserv -r /etc/init.d/haproxy-123".split(), - stderr=subprocess.STDOUT) + stderr=subprocess.STDOUT, + encoding='utf-8') else: self.assertIn(init_system, consts.VALID_INIT_SYSTEMS) @@ -1014,7 +1024,7 @@ class TestServerTestCase(base.TestCase): for idx in range(test_int_num)] mock_isfile.return_value = True - mock_check_output.return_value = b"1\n2\n3\n" + mock_check_output.return_value = "1\n2\n3\n" test_int_num = str(test_int_num) @@ -1145,7 +1155,8 @@ class TestServerTestCase(base.TestCase): mock_check_output.assert_called_with( ['ip', 'netns', 'exec', consts.AMPHORA_NAMESPACE, - 'amphora-interface', 'up', 'eth' + test_int_num], stderr=-2) + 'amphora-interface', 'up', 'eth' + test_int_num], + stderr=subprocess.STDOUT, encoding='utf-8') # fixed IPs happy path port_info = {'mac_address': '123', 'mtu': 1450, 'fixed_ips': [ @@ -1215,7 +1226,8 @@ class TestServerTestCase(base.TestCase): mock_check_output.assert_called_with( ['ip', 'netns', 'exec', consts.AMPHORA_NAMESPACE, - 'amphora-interface', 'up', 'eth' + test_int_num], stderr=-2) + 'amphora-interface', 'up', 'eth' + test_int_num], + stderr=subprocess.STDOUT, encoding='utf-8') # fixed IPs happy path IPv6 port_info = {'mac_address': '123', 'mtu': 1450, 'fixed_ips': [ @@ -1285,7 +1297,8 @@ class TestServerTestCase(base.TestCase): mock_check_output.assert_called_with( ['ip', 'netns', 'exec', consts.AMPHORA_NAMESPACE, - 'amphora-interface', 'up', 'eth' + test_int_num], stderr=-2) + 'amphora-interface', 'up', 'eth' + test_int_num], + stderr=subprocess.STDOUT, encoding='utf-8') # fixed IPs, bogus IP port_info = {'mac_address': '123', 'fixed_ips': [ @@ -1314,9 +1327,10 @@ class TestServerTestCase(base.TestCase): # same as above but ifup fails port_info = {'mac_address': '123', 'fixed_ips': [ {'ip_address': '10.0.0.5', 'subnet_cidr': '10.0.0.0/24'}]} - mock_check_output.side_effect = [subprocess.CalledProcessError( - 7, 'test', RANDOM_ERROR), subprocess.CalledProcessError( - 7, 'test', RANDOM_ERROR)] + mock_check_output.side_effect = [ + subprocess.CalledProcessError(7, 'test', RANDOM_ERROR), + subprocess.CalledProcessError(7, 'test', RANDOM_ERROR) + ] m = self.useFixture(test_utils.OpenFixture(file_name)).mock_open with mock.patch('os.open'), mock.patch.object(os, 'fdopen', m): @@ -1332,7 +1346,7 @@ class TestServerTestCase(base.TestCase): data=jsonutils.dumps(port_info)) self.assertEqual(500, rv.status_code) self.assertEqual( - {'details': RANDOM_ERROR.decode('utf-8'), + {'details': RANDOM_ERROR, 'message': 'Error plugging network'}, jsonutils.loads(rv.data.decode('utf-8'))) @@ -1477,7 +1491,8 @@ class TestServerTestCase(base.TestCase): mock_check_output.assert_called_with( ['ip', 'netns', 'exec', consts.AMPHORA_NAMESPACE, - 'amphora-interface', 'up', 'eth3'], stderr=-2) + 'amphora-interface', 'up', 'eth3'], stderr=subprocess.STDOUT, + encoding='utf-8') def test_ubuntu_plug_VIP4(self): self._test_plug_VIP4(consts.UBUNTU) @@ -1732,7 +1747,8 @@ class TestServerTestCase(base.TestCase): mock_check_output.assert_called_with( ['ip', 'netns', 'exec', consts.AMPHORA_NAMESPACE, 'amphora-interface', 'up', - consts.NETNS_PRIMARY_INTERFACE], stderr=-2) + consts.NETNS_PRIMARY_INTERFACE], stderr=subprocess.STDOUT, + encoding='utf-8') # One Interface down, Happy Path IPv4 mode = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH @@ -1832,12 +1848,13 @@ class TestServerTestCase(base.TestCase): mock_check_output.assert_called_with( ['ip', 'netns', 'exec', consts.AMPHORA_NAMESPACE, 'amphora-interface', 'up', - consts.NETNS_PRIMARY_INTERFACE], stderr=-2) + consts.NETNS_PRIMARY_INTERFACE], stderr=subprocess.STDOUT, + encoding='utf-8') mock_check_output.side_effect = [ - subprocess.CalledProcessError( - 7, 'test', RANDOM_ERROR), subprocess.CalledProcessError( - 7, 'test', RANDOM_ERROR)] + subprocess.CalledProcessError(7, 'test', RANDOM_ERROR), + subprocess.CalledProcessError(7, 'test', RANDOM_ERROR) + ] m = self.useFixture(test_utils.OpenFixture(file_name)).mock_open with mock.patch('os.open'), mock.patch.object(os, 'fdopen', m): @@ -1853,7 +1870,7 @@ class TestServerTestCase(base.TestCase): data=jsonutils.dumps(subnet_info)) self.assertEqual(500, rv.status_code) self.assertEqual( - {'details': RANDOM_ERROR.decode('utf-8'), + {'details': RANDOM_ERROR, 'message': 'Error plugging VIP'}, jsonutils.loads(rv.data.decode('utf-8'))) @@ -2086,9 +2103,14 @@ class TestServerTestCase(base.TestCase): self, args[0], expected_dict) mock_check_output.assert_called_with( - ['ip', 'netns', 'exec', consts.AMPHORA_NAMESPACE, + [ + 'ip', 'netns', 'exec', consts.AMPHORA_NAMESPACE, 'amphora-interface', 'up', '{netns_int}'.format( - netns_int=consts.NETNS_PRIMARY_INTERFACE)], stderr=-2) + netns_int=consts.NETNS_PRIMARY_INTERFACE + ) + ], + stderr=subprocess.STDOUT, + encoding='utf-8') # One Interface down, Happy Path IPv6 flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC @@ -2181,14 +2203,14 @@ class TestServerTestCase(base.TestCase): test_utils.assert_interface_files_equal( self, args[0], expected_dict) - mock_check_output.assert_called_with( - ['ip', 'netns', 'exec', consts.AMPHORA_NAMESPACE, - 'amphora-interface', 'up', '{netns_int}'.format( - netns_int=consts.NETNS_PRIMARY_INTERFACE)], stderr=-2) + mock_check_output.assert_called_with([ + 'ip', 'netns', 'exec', consts.AMPHORA_NAMESPACE, + 'amphora-interface', 'up', consts.NETNS_PRIMARY_INTERFACE + ], stderr=subprocess.STDOUT, encoding='utf-8') mock_check_output.side_effect = [ - subprocess.CalledProcessError( - 7, 'test', RANDOM_ERROR), subprocess.CalledProcessError( - 7, 'test', RANDOM_ERROR)] + subprocess.CalledProcessError(7, 'test', RANDOM_ERROR), + subprocess.CalledProcessError(7, 'test', RANDOM_ERROR) + ] m = self.useFixture(test_utils.OpenFixture(file_name)).mock_open with mock.patch('os.open'), mock.patch.object(os, 'fdopen', m): @@ -2204,7 +2226,7 @@ class TestServerTestCase(base.TestCase): data=jsonutils.dumps(subnet_info)) self.assertEqual(500, rv.status_code) self.assertEqual( - {'details': RANDOM_ERROR.decode('utf-8'), + {'details': RANDOM_ERROR, 'message': 'Error plugging VIP'}, jsonutils.loads(rv.data.decode('utf-8'))) @@ -2421,10 +2443,11 @@ class TestServerTestCase(base.TestCase): test_utils.assert_interface_files_equal( self, args[0], expected_dict) - mock_check_output.assert_called_with( - ['ip', 'netns', 'exec', consts.AMPHORA_NAMESPACE, - 'amphora-interface', 'up', - consts.NETNS_PRIMARY_INTERFACE], stderr=-2) + mock_check_output.assert_called_with([ + 'ip', 'netns', 'exec', consts.AMPHORA_NAMESPACE, + 'amphora-interface', 'up', + consts.NETNS_PRIMARY_INTERFACE + ], stderr=subprocess.STDOUT, encoding='utf-8') def test_ubuntu_plug_VIP6_with_additional_VIP(self): self._test_plug_VIP6_with_additional_VIP(consts.UBUNTU) @@ -2638,10 +2661,10 @@ class TestServerTestCase(base.TestCase): test_utils.assert_interface_files_equal( self, args[0], expected_dict) - mock_check_output.assert_called_with( - ['ip', 'netns', 'exec', consts.AMPHORA_NAMESPACE, - 'amphora-interface', 'up', - consts.NETNS_PRIMARY_INTERFACE], stderr=-2) + mock_check_output.assert_called_with([ + 'ip', 'netns', 'exec', consts.AMPHORA_NAMESPACE, + 'amphora-interface', 'up', consts.NETNS_PRIMARY_INTERFACE + ], stderr=subprocess.STDOUT, encoding='utf-8') def test_ubuntu_get_interface(self): self._test_get_interface(consts.UBUNTU) diff --git a/octavia/tests/unit/amphorae/backends/agent/api_server/test_keepalived.py b/octavia/tests/unit/amphorae/backends/agent/api_server/test_keepalived.py index 8f6c86a60f..af9cdc4b3b 100644 --- a/octavia/tests/unit/amphorae/backends/agent/api_server/test_keepalived.py +++ b/octavia/tests/unit/amphorae/backends/agent/api_server/test_keepalived.py @@ -35,7 +35,8 @@ class KeepalivedTestCase(base.TestCase): cmd = ("/usr/sbin/service octavia-keepalived {action}".format( action='start')) mock_check_output.assert_called_once_with(cmd.split(), - stderr=subprocess.STDOUT) + stderr=subprocess.STDOUT, + encoding='utf-8') self.assertEqual(202, res.status_code) res = self.test_keepalived.manager_keepalived_service('restart') diff --git a/octavia/tests/unit/amphorae/backends/agent/api_server/test_loadbalancer.py b/octavia/tests/unit/amphorae/backends/agent/api_server/test_loadbalancer.py index 1adbbb3c15..afa7fc0b4b 100644 --- a/octavia/tests/unit/amphorae/backends/agent/api_server/test_loadbalancer.py +++ b/octavia/tests/unit/amphorae/backends/agent/api_server/test_loadbalancer.py @@ -86,7 +86,8 @@ class ListenerTestCase(base.TestCase): listener_id, consts.AMP_ACTION_START) mock_check_output.assert_called_once_with(ref_command_split, - stderr=subprocess.STDOUT) + stderr=subprocess.STDOUT, + encoding='utf-8') mock_lb_exists.assert_called_once_with(listener_id) mock_vrrp_update.assert_not_called() self.assertEqual(202, result.status_code) @@ -111,7 +112,8 @@ class ListenerTestCase(base.TestCase): listener_id, consts.AMP_ACTION_RELOAD) mock_check_output.assert_called_once_with(ref_command_split, - stderr=subprocess.STDOUT) + stderr=subprocess.STDOUT, + encoding='utf-8') mock_lb_exists.assert_called_once_with(listener_id) mock_vrrp_update.assert_called_once_with(listener_id, consts.AMP_ACTION_RELOAD) @@ -134,7 +136,8 @@ class ListenerTestCase(base.TestCase): listener_id, consts.AMP_ACTION_RELOAD) mock_check_output.assert_called_once_with(ref_command_split, - stderr=subprocess.STDOUT) + stderr=subprocess.STDOUT, + encoding='utf-8') mock_lb_exists.assert_called_once_with(listener_id) mock_vrrp_update.assert_called_once_with(listener_id, consts.AMP_ACTION_RELOAD) @@ -157,13 +160,14 @@ class ListenerTestCase(base.TestCase): ref_command_split.append(consts.AMP_ACTION_START) mock_check_output.side_effect = subprocess.CalledProcessError( - output=b'bogus', returncode=-2, cmd='sit') + output='bogus', returncode=-2, cmd='sit') result = self.test_loadbalancer.start_stop_lb( listener_id, consts.AMP_ACTION_START) mock_check_output.assert_called_once_with(ref_command_split, - stderr=subprocess.STDOUT) + stderr=subprocess.STDOUT, + encoding='utf-8') mock_lb_exists.assert_called_once_with(listener_id) mock_vrrp_update.assert_not_called() self.assertEqual(500, result.status_code) @@ -181,13 +185,14 @@ class ListenerTestCase(base.TestCase): ref_command_split.append(consts.AMP_ACTION_START) mock_check_output.side_effect = subprocess.CalledProcessError( - output=b'Job is already running', returncode=-2, cmd='sit') + output='Job is already running', returncode=-2, cmd='sit') result = self.test_loadbalancer.start_stop_lb( listener_id, consts.AMP_ACTION_START) mock_check_output.assert_called_once_with(ref_command_split, - stderr=subprocess.STDOUT) + stderr=subprocess.STDOUT, + encoding='utf-8') mock_lb_exists.assert_called_once_with(listener_id) mock_vrrp_update.assert_not_called() self.assertEqual(202, result.status_code) @@ -220,7 +225,7 @@ class ListenerTestCase(base.TestCase): mock_check_output.side_effect = [ subprocess.CalledProcessError( - output=b'haproxy.service is not active, cannot reload.', + output='haproxy.service is not active, cannot reload.', returncode=-2, cmd='service'), None] mock_check_status.return_value = 'ACTIVE' @@ -248,13 +253,13 @@ class ListenerTestCase(base.TestCase): mock_check_output.side_effect = [ subprocess.CalledProcessError( - output=b'haproxy.service is not active, cannot reload.', + output='haproxy.service is not active, cannot reload.', returncode=-2, cmd='service'), subprocess.CalledProcessError( - output=b'haproxy.service is not active, cannot reload.', + output='haproxy.service is not active, cannot reload.', returncode=-2, cmd='service'), subprocess.CalledProcessError( - output=b'haproxy.service is not active, cannot reload.', + output='haproxy.service is not active, cannot reload.', returncode=-2, cmd='service')] mock_check_status.return_value = 'ACTIVE' mock_check_status.side_effect = None diff --git a/octavia/tests/unit/amphorae/backends/agent/api_server/test_util.py b/octavia/tests/unit/amphorae/backends/agent/api_server/test_util.py index 551f4d06fb..150cd1a3f6 100644 --- a/octavia/tests/unit/amphorae/backends/agent/api_server/test_util.py +++ b/octavia/tests/unit/amphorae/backends/agent/api_server/test_util.py @@ -162,7 +162,8 @@ class TestUtil(base.TestCase): util.run_systemctl_command('test', 'world') mock_check_output.assert_called_once_with( - ['systemctl', 'test', 'world'], stderr=subprocess.STDOUT) + ['systemctl', 'test', 'world'], stderr=subprocess.STDOUT, + encoding='utf-8') mock_check_output.side_effect = subprocess.CalledProcessError(1, 'boom') diff --git a/octavia/tests/unit/amphorae/backends/health_daemon/test_health_daemon.py b/octavia/tests/unit/amphorae/backends/health_daemon/test_health_daemon.py index e7af2aea20..8397c16a9d 100644 --- a/octavia/tests/unit/amphorae/backends/health_daemon/test_health_daemon.py +++ b/octavia/tests/unit/amphorae/backends/health_daemon/test_health_daemon.py @@ -12,6 +12,7 @@ # License for the specific language governing permissions and limitations # under the License. # +import json import os import queue from unittest import mock @@ -19,7 +20,6 @@ from unittest import mock from oslo_config import cfg from oslo_config import fixture as oslo_fixture from oslo_utils import uuidutils -import simplejson from octavia.amphorae.backends.health_daemon import health_daemon from octavia.common import constants @@ -345,7 +345,7 @@ class TestHealthDaemon(base.TestCase): with mock.patch('os.open'), mock.patch.object( os, 'fdopen', self.mock_open) as mock_fdopen: - mock_fdopen().read.return_value = simplejson.dumps({ + mock_fdopen().read.return_value = json.dumps({ LISTENER_ID1: {'bin': 1, 'bout': 2}, }) msg = health_daemon.build_stats_message() @@ -353,7 +353,7 @@ class TestHealthDaemon(base.TestCase): self.assertEqual(SAMPLE_STATS_MSG, msg) mock_get_stats.assert_any_call(lb1_stats_socket) - mock_fdopen().write.assert_called_once_with(simplejson.dumps({ + mock_fdopen().write.assert_called_once_with(json.dumps({ LISTENER_ID1: { 'bin': int(FRONTEND_STATS['bin']), 'bout': int(FRONTEND_STATS['bout']), @@ -450,14 +450,14 @@ class TestHealthDaemon(base.TestCase): with mock.patch('os.open'), mock.patch.object( os, 'fdopen', self.mock_open) as mock_fdopen: - mock_fdopen().read.return_value = simplejson.dumps({ + mock_fdopen().read.return_value = json.dumps({ udp_listener_id1: { 'bin': 1, 'bout': 2, "ereq": 0, "stot": 0} }) msg = health_daemon.build_stats_message() self.assertEqual(expected, msg) - mock_fdopen().write.assert_called_once_with(simplejson.dumps({ + mock_fdopen().write.assert_called_once_with(json.dumps({ udp_listener_id1: {'bin': 5, 'bout': 10, 'ereq': 0, 'stot': 5}, udp_listener_id3: {'bin': 0, 'bout': 0, 'ereq': 0, 'stot': 0}, })) @@ -480,7 +480,7 @@ class TestHealthDaemon(base.TestCase): with mock.patch('os.open'), mock.patch.object( os, 'fdopen', self.mock_open) as mock_fdopen: - mock_fdopen().read.return_value = simplejson.dumps({ + mock_fdopen().read.return_value = json.dumps({ LISTENER_ID1: {'bin': 15, 'bout': 20}, }) msg = health_daemon.build_stats_message() @@ -488,7 +488,7 @@ class TestHealthDaemon(base.TestCase): self.assertEqual(SAMPLE_MSG_HAPROXY_RESTART, msg) mock_get_stats.assert_any_call(lb1_stats_socket) - mock_fdopen().write.assert_called_once_with(simplejson.dumps({ + mock_fdopen().write.assert_called_once_with(json.dumps({ LISTENER_ID1: { 'bin': int(FRONTEND_STATS['bin']), 'bout': int(FRONTEND_STATS['bout']), diff --git a/octavia/tests/unit/amphorae/backends/utils/test_interface.py b/octavia/tests/unit/amphorae/backends/utils/test_interface.py index 24753e7216..18c6bd1c4b 100644 --- a/octavia/tests/unit/amphorae/backends/utils/test_interface.py +++ b/octavia/tests/unit/amphorae/backends/utils/test_interface.py @@ -278,7 +278,7 @@ class TestInterface(base.TestCase): iface), "-pf", f"/run/dhclient-{iface}.pid", - iface], stderr=-2) + iface], stderr=subprocess.STDOUT) @mock.patch('subprocess.check_output') def test__dhclient_down(self, mock_check_output): @@ -295,7 +295,7 @@ class TestInterface(base.TestCase): iface), "-pf", f"/run/dhclient-{iface}.pid", - iface], stderr=-2) + iface], stderr=subprocess.STDOUT) @mock.patch('subprocess.check_output') def test__ipv6auto_up(self, mock_check_output): @@ -306,9 +306,11 @@ class TestInterface(base.TestCase): mock_check_output.assert_has_calls([ mock.call(["/sbin/sysctl", "-w", - "net.ipv6.conf.iface2.accept_ra=2"], stderr=-2), + "net.ipv6.conf.iface2.accept_ra=2"], + stderr=subprocess.STDOUT), mock.call(["/sbin/sysctl", "-w", - "net.ipv6.conf.iface2.autoconf=1"], stderr=-2)]) + "net.ipv6.conf.iface2.autoconf=1"], + stderr=subprocess.STDOUT)]) @mock.patch('subprocess.check_output') def test__ipv6auto_down(self, mock_check_output): @@ -319,9 +321,11 @@ class TestInterface(base.TestCase): mock_check_output.assert_has_calls([ mock.call(["/sbin/sysctl", "-w", - "net.ipv6.conf.iface2.accept_ra=0"], stderr=-2), + "net.ipv6.conf.iface2.accept_ra=0"], + stderr=subprocess.STDOUT), mock.call(["/sbin/sysctl", "-w", - "net.ipv6.conf.iface2.autoconf=0"], stderr=-2)]) + "net.ipv6.conf.iface2.autoconf=0"], + stderr=subprocess.STDOUT)]) @mock.patch('pyroute2.IPRoute.rule') @mock.patch('pyroute2.IPRoute.route') @@ -579,15 +583,16 @@ class TestInterface(base.TestCase): mock_check_output.assert_has_calls([ mock.call([consts.NFT_CMD, consts.NFT_ADD, 'table', - consts.NFT_FAMILY, consts.NFT_VIP_TABLE], stderr=-2), + consts.NFT_FAMILY, consts.NFT_VIP_TABLE], + stderr=subprocess.STDOUT), mock.call([consts.NFT_CMD, consts.NFT_ADD, 'chain', consts.NFT_FAMILY, consts.NFT_VIP_TABLE, consts.NFT_VIP_CHAIN, '{', 'type', 'filter', 'hook', 'ingress', 'device', 'fake-eth1', 'priority', consts.NFT_SRIOV_PRIORITY, ';', 'policy', 'drop', ';', - '}'], stderr=-2), + '}'], stderr=subprocess.STDOUT), mock.call([consts.NFT_CMD, '-o', '-f', consts.NFT_VIP_RULES_FILE], - stderr=-2), + stderr=subprocess.STDOUT), mock.call(["post-up", "fake-eth1"]) ]) @@ -925,13 +930,13 @@ class TestInterface(base.TestCase): iface.name), "-pf", f"/run/dhclient-{iface.name}.pid", - iface.name], stderr=-2), + iface.name], stderr=subprocess.STDOUT), mock.call(["/sbin/sysctl", "-w", f"net.ipv6.conf.{iface.name}.accept_ra=2"], - stderr=-2), + stderr=subprocess.STDOUT), mock.call(["/sbin/sysctl", "-w", f"net.ipv6.conf.{iface.name}.autoconf=1"], - stderr=-2), + stderr=subprocess.STDOUT), mock.call(["post-up", iface.name]) ]) @@ -1342,13 +1347,13 @@ class TestInterface(base.TestCase): iface.name), "-pf", f"/run/dhclient-{iface.name}.pid", - iface.name], stderr=-2), + iface.name], stderr=subprocess.STDOUT), mock.call(["/sbin/sysctl", "-w", f"net.ipv6.conf.{iface.name}.accept_ra=0"], - stderr=-2), + stderr=subprocess.STDOUT), mock.call(["/sbin/sysctl", "-w", f"net.ipv6.conf.{iface.name}.autoconf=0"], - stderr=-2), + stderr=subprocess.STDOUT), mock.call(["post-down", iface.name]) ]) diff --git a/requirements.txt b/requirements.txt index d3acbd82d8..66020e6418 100644 --- a/requirements.txt +++ b/requirements.txt @@ -46,7 +46,6 @@ tenacity>=5.0.4 # Apache-2.0 distro>=1.2.0 # Apache-2.0 jsonschema>=3.2.0 # MIT octavia-lib>=3.3.0 # Apache-2.0 -simplejson>=3.13.2 # MIT setproctitle>=1.1.10 # BSD python-dateutil>=2.7.0 # BSD