openstack/tripleo-common
Code Issues Proposed changes

Address python3 string issues with subprocess

Browse Source 
This patch updates our Popen calls to enable universal newlines for
calls that we parse or consume the output for. Without
univeral_newlines=True, the output is treated as bytes under python3
which leads to issues later where we are using it as strings.

See https://docs.python.org/3/glossary.html#term-universal-newlines

Change-Id: I1a9edd0ebac6f0539ac5c33056c442cfcfafb7f3
Related-Blueprint: python3-support
This commit is contained in:
Alex Schultz 2018-12-12 13:38:14 -07:00
parent d8de81fadd
commit bdbce00e31
6 changed files with 25 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
scripts/container-update.py
View File

@ -54,7 +54,8 @@ def rm_container(name):
log.info('Removing container: %s' % name) log.info('Removing container: %s' % name)
subproc = subprocess.Popen(['/usr/bin/docker', 'rm', name], subproc = subprocess.Popen(['/usr/bin/docker', 'rm', name],
stdout=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE) stderr=subprocess.PIPE,
universal_newlines=True)
cmd_stdout, cmd_stderr = subproc.communicate() cmd_stdout, cmd_stderr = subproc.communicate()
if cmd_stdout: if cmd_stdout:
log.debug(cmd_stdout) log.debug(cmd_stdout)
@ -76,7 +77,8 @@ def populate_container_rpms_list(container):
log.info('Running docker command: %s' % ' '.join(dcmd)) log.info('Running docker command: %s' % ' '.join(dcmd))
subproc = subprocess.Popen(dcmd, stdout=subprocess.PIPE, subproc = subprocess.Popen(dcmd, stdout=subprocess.PIPE,
stderr=subprocess.PIPE) stderr=subprocess.PIPE,
universal_newlines=True)
cmd_stdout, cmd_stderr = subproc.communicate() cmd_stdout, cmd_stderr = subproc.communicate()
if subproc.returncode != 0: if subproc.returncode != 0:
log.error('Failed running rpm -qa for %s' % container) log.error('Failed running rpm -qa for %s' % container)
@ -99,7 +101,8 @@ def yum_update_container(container, name, packages):
'--format', '{{json .Config.Cmd}}', '--format', '{{json .Config.Cmd}}',
container] container]
subproc = subprocess.Popen(dcmd, stdout=subprocess.PIPE, subproc = subprocess.Popen(dcmd, stdout=subprocess.PIPE,
stderr=subprocess.PIPE) stderr=subprocess.PIPE,
universal_newlines=True)
cmd_stdout, cmd_stderr = subproc.communicate() cmd_stdout, cmd_stderr = subproc.communicate()
container_cmd = cmd_stdout container_cmd = cmd_stdout
log.info('Original container command: %s' % container_cmd) log.info('Original container command: %s' % container_cmd)
@ -122,7 +125,8 @@ def yum_update_container(container, name, packages):
subproc = subprocess.Popen(dcmd, subproc = subprocess.Popen(dcmd,
stdout=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE) stderr=subprocess.PIPE,
universal_newlines=True)
cmd_stdout, cmd_stderr = subproc.communicate() cmd_stdout, cmd_stderr = subproc.communicate()
log.info(cmd_stdout) log.info(cmd_stdout)
if subproc.returncode != 0: if subproc.returncode != 0:
@ -146,7 +150,8 @@ def yum_update_container(container, name, packages):
log.info('Running docker command: %s' % ' '.join(dcmd)) log.info('Running docker command: %s' % ' '.join(dcmd))
subproc = subprocess.Popen(dcmd, stdout=subprocess.PIPE, subproc = subprocess.Popen(dcmd, stdout=subprocess.PIPE,
stderr=subprocess.PIPE) stderr=subprocess.PIPE,
universal_newlines=True)
cmd_stdout, cmd_stderr = subproc.communicate() cmd_stdout, cmd_stderr = subproc.communicate()
if subproc.returncode != 0: if subproc.returncode != 0:
log.error('Failed running docker commit for %s' % container) log.error('Failed running docker commit for %s' % container)
@ -157,7 +162,8 @@ def yum_update_container(container, name, packages):
log.info('Running docker command: %s' % ' '.join(dcmd)) log.info('Running docker command: %s' % ' '.join(dcmd))
subproc = subprocess.Popen(dcmd, stdout=subprocess.PIPE, subproc = subprocess.Popen(dcmd, stdout=subprocess.PIPE,
stderr=subprocess.PIPE) stderr=subprocess.PIPE,
universal_newlines=True)
cmd_stdout, cmd_stderr = subproc.communicate() cmd_stdout, cmd_stderr = subproc.communicate()
if subproc.returncode != 0: if subproc.returncode != 0:
log.error('Failed running docker push for %s' % container) log.error('Failed running docker push for %s' % container)

3
tripleo_common/actions/ansible.py
View File

@ -557,7 +557,8 @@ class AnsiblePlaybookAction(base.TripleOAction):
stderr=subprocess.STDOUT, stderr=subprocess.STDOUT,
shell=False, bufsize=1, shell=False, bufsize=1,
cwd=self.work_dir, cwd=self.work_dir,
env=env_variables) env=env_variables,
universal_newlines=True)
start = time.time() start = time.time()
stdout = [] stdout = []
lines = [] lines = []

6
tripleo_common/image/image_uploader.py
View File

@ -791,7 +791,8 @@ class SkopeoImageUploader(BaseImageUploader):
cmd.append(target) cmd.append(target)
LOG.info('Running %s' % ' '.join(cmd)) LOG.info('Running %s' % ' '.join(cmd))
env = os.environ.copy() env = os.environ.copy()
process = subprocess.Popen(cmd, env=env, stdout=subprocess.PIPE) process = subprocess.Popen(cmd, env=env, stdout=subprocess.PIPE,
universal_newlines=True)
out, err = process.communicate() out, err = process.communicate()
LOG.info(out) LOG.info(out)
@ -811,7 +812,8 @@ class SkopeoImageUploader(BaseImageUploader):
cmd.append(image) cmd.append(image)
LOG.info('Running %s' % ' '.join(cmd)) LOG.info('Running %s' % ' '.join(cmd))
env = os.environ.copy() env = os.environ.copy()
process = subprocess.Popen(cmd, env=env, stdout=subprocess.PIPE) process = subprocess.Popen(cmd, env=env, stdout=subprocess.PIPE,
universal_newlines=True)
out, err = process.communicate() out, err = process.communicate()
LOG.info(out) LOG.info(out)

3
tripleo_common/image/kolla_builder.py
View File

@ -453,7 +453,8 @@ class KollaImageBuilder(base.BaseImageManager):
self.logger.info('Running %s' % ' '.join(cmd)) self.logger.info('Running %s' % ' '.join(cmd))
env = os.environ.copy() env = os.environ.copy()
process = subprocess.Popen(cmd, env=env, stdout=subprocess.PIPE) process = subprocess.Popen(cmd, env=env, stdout=subprocess.PIPE,
universal_newlines=True)
out, err = process.communicate() out, err = process.communicate()
if process.returncode != 0: if process.returncode != 0:
raise subprocess.CalledProcessError(process.returncode, cmd, err) raise subprocess.CalledProcessError(process.returncode, cmd, err)

2
tripleo_common/tests/image/test_image_uploader.py
View File

@ -1005,7 +1005,7 @@ class TestSkopeoImageUploader(base.TestCase):
'copy', 'copy',
'docker://docker.io/t/nova-api:latest', 'docker://docker.io/t/nova-api:latest',
'docker://localhost:8787/t/nova-api:latest'], 'docker://localhost:8787/t/nova-api:latest'],
env={}, stdout=-1 env={}, stdout=-1, universal_newlines=True
) )
@mock.patch('tripleo_common.image.image_uploader.' @mock.patch('tripleo_common.image.image_uploader.'

8
tripleo_common/tests/image/test_kolla_builder.py
View File

@ -122,7 +122,7 @@ class TestKollaImageBuilder(base.TestCase):
'nova-libvirt', 'nova-libvirt',
'heat-docker-agents-centos', 'heat-docker-agents-centos',
'image-with-missing-tag', 'image-with-missing-tag',
], env=env, stdout=-1) ], env=env, stdout=-1, universal_newlines=True)
@mock.patch('subprocess.Popen') @mock.patch('subprocess.Popen')
def test_build_images_no_conf(self, mock_popen): def test_build_images_no_conf(self, mock_popen):
@ -136,7 +136,7 @@ class TestKollaImageBuilder(base.TestCase):
env = os.environ.copy() env = os.environ.copy()
mock_popen.assert_called_once_with([ mock_popen.assert_called_once_with([
'kolla-build', 'kolla-build',
], env=env, stdout=-1) ], env=env, stdout=-1, universal_newlines=True)
@mock.patch('tripleo_common.image.base.open', @mock.patch('tripleo_common.image.base.open',
mock.mock_open(read_data=filedata), create=True) mock.mock_open(read_data=filedata), create=True)
@ -159,7 +159,7 @@ class TestKollaImageBuilder(base.TestCase):
'nova-libvirt', 'nova-libvirt',
'heat-docker-agents-centos', 'heat-docker-agents-centos',
'image-with-missing-tag', 'image-with-missing-tag',
], env=env, stdout=-1) ], env=env, stdout=-1, universal_newlines=True)
@mock.patch('subprocess.Popen') @mock.patch('subprocess.Popen')
def test_build_images_fail(self, mock_popen): def test_build_images_fail(self, mock_popen):
@ -175,7 +175,7 @@ class TestKollaImageBuilder(base.TestCase):
env = os.environ.copy() env = os.environ.copy()
mock_popen.assert_called_once_with([ mock_popen.assert_called_once_with([
'kolla-build', 'kolla-build',
], env=env, stdout=-1) ], env=env, stdout=-1, universal_newlines=True)
class TestKollaImageBuilderTemplate(base.TestCase): class TestKollaImageBuilderTemplate(base.TestCase):