From bdbce00e31c96e2d8f3e3de3f1fb24bf9fdc34ae Mon Sep 17 00:00:00 2001 From: Alex Schultz Date: Wed, 12 Dec 2018 13:38:14 -0700 Subject: [PATCH] Address python3 string issues with subprocess This patch updates our Popen calls to enable universal newlines for calls that we parse or consume the output for. Without univeral_newlines=True, the output is treated as bytes under python3 which leads to issues later where we are using it as strings. See https://docs.python.org/3/glossary.html#term-universal-newlines Change-Id: I1a9edd0ebac6f0539ac5c33056c442cfcfafb7f3 Related-Blueprint: python3-support --- scripts/container-update.py | 18 ++++++++++++------ tripleo_common/actions/ansible.py | 3 ++- tripleo_common/image/image_uploader.py | 6 ++++-- tripleo_common/image/kolla_builder.py | 3 ++- .../tests/image/test_image_uploader.py | 2 +- .../tests/image/test_kolla_builder.py | 8 ++++---- 6 files changed, 25 insertions(+), 15 deletions(-) diff --git a/scripts/container-update.py b/scripts/container-update.py index 448f68d96..800ab938c 100755 --- a/scripts/container-update.py +++ b/scripts/container-update.py @@ -54,7 +54,8 @@ def rm_container(name): log.info('Removing container: %s' % name) subproc = subprocess.Popen(['/usr/bin/docker', 'rm', name], stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + stderr=subprocess.PIPE, + universal_newlines=True) cmd_stdout, cmd_stderr = subproc.communicate() if cmd_stdout: log.debug(cmd_stdout) @@ -76,7 +77,8 @@ def populate_container_rpms_list(container): log.info('Running docker command: %s' % ' '.join(dcmd)) subproc = subprocess.Popen(dcmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + stderr=subprocess.PIPE, + universal_newlines=True) cmd_stdout, cmd_stderr = subproc.communicate() if subproc.returncode != 0: log.error('Failed running rpm -qa for %s' % container) @@ -99,7 +101,8 @@ def yum_update_container(container, name, packages): '--format', '{{json .Config.Cmd}}', container] subproc = subprocess.Popen(dcmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + stderr=subprocess.PIPE, + universal_newlines=True) cmd_stdout, cmd_stderr = subproc.communicate() container_cmd = cmd_stdout log.info('Original container command: %s' % container_cmd) @@ -122,7 +125,8 @@ def yum_update_container(container, name, packages): subproc = subprocess.Popen(dcmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + stderr=subprocess.PIPE, + universal_newlines=True) cmd_stdout, cmd_stderr = subproc.communicate() log.info(cmd_stdout) if subproc.returncode != 0: @@ -146,7 +150,8 @@ def yum_update_container(container, name, packages): log.info('Running docker command: %s' % ' '.join(dcmd)) subproc = subprocess.Popen(dcmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + stderr=subprocess.PIPE, + universal_newlines=True) cmd_stdout, cmd_stderr = subproc.communicate() if subproc.returncode != 0: log.error('Failed running docker commit for %s' % container) @@ -157,7 +162,8 @@ def yum_update_container(container, name, packages): log.info('Running docker command: %s' % ' '.join(dcmd)) subproc = subprocess.Popen(dcmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + stderr=subprocess.PIPE, + universal_newlines=True) cmd_stdout, cmd_stderr = subproc.communicate() if subproc.returncode != 0: log.error('Failed running docker push for %s' % container) diff --git a/tripleo_common/actions/ansible.py b/tripleo_common/actions/ansible.py index f870f778d..709e57793 100644 --- a/tripleo_common/actions/ansible.py +++ b/tripleo_common/actions/ansible.py @@ -557,7 +557,8 @@ class AnsiblePlaybookAction(base.TripleOAction): stderr=subprocess.STDOUT, shell=False, bufsize=1, cwd=self.work_dir, - env=env_variables) + env=env_variables, + universal_newlines=True) start = time.time() stdout = [] lines = [] diff --git a/tripleo_common/image/image_uploader.py b/tripleo_common/image/image_uploader.py index 392a45067..1a3b67b7c 100644 --- a/tripleo_common/image/image_uploader.py +++ b/tripleo_common/image/image_uploader.py @@ -791,7 +791,8 @@ class SkopeoImageUploader(BaseImageUploader): cmd.append(target) LOG.info('Running %s' % ' '.join(cmd)) env = os.environ.copy() - process = subprocess.Popen(cmd, env=env, stdout=subprocess.PIPE) + process = subprocess.Popen(cmd, env=env, stdout=subprocess.PIPE, + universal_newlines=True) out, err = process.communicate() LOG.info(out) @@ -811,7 +812,8 @@ class SkopeoImageUploader(BaseImageUploader): cmd.append(image) LOG.info('Running %s' % ' '.join(cmd)) env = os.environ.copy() - process = subprocess.Popen(cmd, env=env, stdout=subprocess.PIPE) + process = subprocess.Popen(cmd, env=env, stdout=subprocess.PIPE, + universal_newlines=True) out, err = process.communicate() LOG.info(out) diff --git a/tripleo_common/image/kolla_builder.py b/tripleo_common/image/kolla_builder.py index 8d7cd3d6b..111115560 100644 --- a/tripleo_common/image/kolla_builder.py +++ b/tripleo_common/image/kolla_builder.py @@ -453,7 +453,8 @@ class KollaImageBuilder(base.BaseImageManager): self.logger.info('Running %s' % ' '.join(cmd)) env = os.environ.copy() - process = subprocess.Popen(cmd, env=env, stdout=subprocess.PIPE) + process = subprocess.Popen(cmd, env=env, stdout=subprocess.PIPE, + universal_newlines=True) out, err = process.communicate() if process.returncode != 0: raise subprocess.CalledProcessError(process.returncode, cmd, err) diff --git a/tripleo_common/tests/image/test_image_uploader.py b/tripleo_common/tests/image/test_image_uploader.py index bb76baa2b..06740ccec 100644 --- a/tripleo_common/tests/image/test_image_uploader.py +++ b/tripleo_common/tests/image/test_image_uploader.py @@ -1005,7 +1005,7 @@ class TestSkopeoImageUploader(base.TestCase): 'copy', 'docker://docker.io/t/nova-api:latest', 'docker://localhost:8787/t/nova-api:latest'], - env={}, stdout=-1 + env={}, stdout=-1, universal_newlines=True ) @mock.patch('tripleo_common.image.image_uploader.' diff --git a/tripleo_common/tests/image/test_kolla_builder.py b/tripleo_common/tests/image/test_kolla_builder.py index 95aad669f..cd861e6fc 100644 --- a/tripleo_common/tests/image/test_kolla_builder.py +++ b/tripleo_common/tests/image/test_kolla_builder.py @@ -122,7 +122,7 @@ class TestKollaImageBuilder(base.TestCase): 'nova-libvirt', 'heat-docker-agents-centos', 'image-with-missing-tag', - ], env=env, stdout=-1) + ], env=env, stdout=-1, universal_newlines=True) @mock.patch('subprocess.Popen') def test_build_images_no_conf(self, mock_popen): @@ -136,7 +136,7 @@ class TestKollaImageBuilder(base.TestCase): env = os.environ.copy() mock_popen.assert_called_once_with([ 'kolla-build', - ], env=env, stdout=-1) + ], env=env, stdout=-1, universal_newlines=True) @mock.patch('tripleo_common.image.base.open', mock.mock_open(read_data=filedata), create=True) @@ -159,7 +159,7 @@ class TestKollaImageBuilder(base.TestCase): 'nova-libvirt', 'heat-docker-agents-centos', 'image-with-missing-tag', - ], env=env, stdout=-1) + ], env=env, stdout=-1, universal_newlines=True) @mock.patch('subprocess.Popen') def test_build_images_fail(self, mock_popen): @@ -175,7 +175,7 @@ class TestKollaImageBuilder(base.TestCase): env = os.environ.copy() mock_popen.assert_called_once_with([ 'kolla-build', - ], env=env, stdout=-1) + ], env=env, stdout=-1, universal_newlines=True) class TestKollaImageBuilderTemplate(base.TestCase):