starlingx
/
config
Code Issues Proposed changes

Specify encoding of the file for yaml load 

yaml.load might not properly detect the encoding.

This error was observed during bootstrap when the first app
(nginx-ingress) had the values.yaml file loaded as yaml.
The encoding was detected as being ascii when it was not.
A UnicodeDecodeError is thrown: 'ascii' codec can't decode
byte 0xe2 in position.

The fix is to specifically tell yaml the encoding of the file.
As 'encoding' parameter exists only for open in Python3, but not
Python2, switch to io.open to be backward compatible.
As a precaution, did the same changed to ruamel yaml.

Story: 2006796
Task: 42798
Signed-off-by: Dan Voiculeasa <dan.voiculeasa@windriver.com>
Change-Id: Ib101c95392cb453121baecadebb4f75b81216477
(cherry picked from commit 156cc123e4)
This commit is contained in:
Dan Voiculeasa 2021-07-08 10:51:38 +03:00 committed by Chuck Short
parent 1518543bf7
commit 75081afa1f
7 changed files with 33 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
sysinv/sysinv/sysinv/sysinv/common/utils.py
View File

@ -40,6 +40,7 @@ import fcntl
import glob
import grp
import hashlib
import io
import itertools as it
import json
import keyring
@ -2034,7 +2035,7 @@ def find_metadata_file(path, metadata_file, upgrade_from_release=None):
patches = []
metadata_path = os.path.join(path, metadata_file)
if os.path.isfile(metadata_path):
with open(metadata_path, 'r') as f:
with io.open(metadata_path, 'r', encoding='utf-8') as f:
try:
doc = yaml.safe_load(f)
app_name = doc['app_name']
@ -2311,7 +2312,7 @@ def find_metadata_file(path, metadata_file, upgrade_from_release=None):
def find_manifest_file(path):
""" Find all manifest files in a given directory. """
def _is_manifest(yaml_file):
with open(yaml_file, 'r') as f:
with io.open(yaml_file, 'r', encoding='utf-8') as f:
docs = yaml.load_all(f)
for doc in docs:
try:
@ -2498,7 +2499,7 @@ def get_app_supported_kube_version(app_name, app_version):
kube_max_version = None
if (os.path.exists(app_metadata_path) and
os.path.getsize(app_metadata_path) > 0):
with open(app_metadata_path, 'r') as f:
with io.open(app_metadata_path, 'r', encoding='utf-8') as f:
y = yaml.safe_load(f)
supported_kube_version = y.get('supported_k8s_version', {})
kube_min_version = supported_kube_version.get('minimum', None)

19
sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py
View File

@ -15,6 +15,7 @@ from eventlet.green import subprocess
import glob
import grp
import functools
import io
import os
import pkg_resources
import pwd
@ -521,11 +522,11 @@ class AppOperator(object):
manifest_update_required = False
if os.path.exists(app_images_file):
with open(app_images_file, 'r') as f:
with io.open(app_images_file, 'r', encoding='utf-8') as f:
images_file = yaml.safe_load(f)
if os.path.exists(app_manifest_file):
with open(app_manifest_file, 'r') as f:
with io.open(app_manifest_file, 'r', encoding='utf-8') as f:
# The RoundTripLoader removes the superfluous quotes by default,
# resulting the dumped out charts not readable in Armada.
# Set preserve_quotes=True to preserve all the quotes.
@ -548,7 +549,7 @@ class AppOperator(object):
app_overrides_file = os.path.join(overrides_dir, overrides)
overrides_file = {}
if os.path.exists(app_overrides_file):
with open(app_overrides_file, 'r') as f:
with io.open(app_overrides_file, 'r', encoding='utf-8') as f:
overrides_file = yaml.safe_load(f)
override_imgs = self._image.find_images_in_dict(
@ -686,7 +687,7 @@ class AppOperator(object):
chart_path = os.path.join(chart_name, 'values.yaml')
if os.path.exists(chart_path):
with open(chart_path, 'r') as f:
with io.open(chart_path, 'r', encoding='utf-8') as f:
y = yaml.safe_load(f)
chart_images = self._image.find_images_in_dict(y)
@ -698,7 +699,7 @@ class AppOperator(object):
default_flow_style=False)
def _retrieve_images_list(self, app_images_file):
with open(app_images_file, 'r') as f:
with io.open(app_images_file, 'r', encoding='utf-8') as f:
images_list = yaml.safe_load(f)
return images_list
@ -805,7 +806,7 @@ class AppOperator(object):
lfile = os.path.join(app.inst_path, constants.APP_METADATA_FILE)
if os.path.exists(lfile) and os.path.getsize(lfile) > 0:
with open(lfile, 'r') as f:
with io.open(lfile, 'r', encoding='utf-8') as f:
try:
y = yaml.safe_load(f)
repo = y.get('helm_repo', common.HELM_REPO_FOR_APPS)
@ -1077,7 +1078,7 @@ class AppOperator(object):
chart_groups = []
armada_charts = {}
with open(manifest_file, 'r') as f:
with io.open(manifest_file, 'r', encoding='utf-8') as f:
docs = yaml.safe_load_all(f)
for doc in docs:
# iterative docs in the manifest file to get required
@ -1301,7 +1302,7 @@ class AppOperator(object):
metadata_file = os.path.join(app.inst_path,
constants.APP_METADATA_FILE)
if os.path.exists(metadata_file) and os.path.getsize(metadata_file) > 0:
with open(metadata_file, 'r') as f:
with io.open(metadata_file, 'r', encoding='utf-8') as f:
try:
metadata = yaml.safe_load(f) or {}
value = cutils.deep_get(metadata, keys, default=default)
@ -2223,7 +2224,7 @@ class AppOperator(object):
metadata = {}
if os.path.exists(app.sync_metadata_file):
with open(app.sync_metadata_file, 'r') as f:
with io.open(app.sync_metadata_file, 'r', encoding='utf-8') as f:
# The RoundTripLoader removes the superfluous quotes by default.
# Set preserve_quotes=True to preserve all the quotes.
# The assumption here: there is just one yaml section

2
sysinv/sysinv/sysinv/sysinv/conductor/manager.py
View File

@ -5982,7 +5982,7 @@ class ConductorManager(service.PeriodicService):
metadata_file = os.path.join(app_path,
constants.APP_METADATA_FILE)
if os.path.exists(metadata_file):
with open(metadata_file, 'r') as f:
with io.open(metadata_file, 'r', encoding='utf-8') as f:
# The RoundTripLoader removes the superfluous quotes by default.
# Set preserve_quotes=True to preserve all the quotes.
# The assumption here: there is just one yaml section

5
sysinv/sysinv/sysinv/sysinv/helm/manifest_base.py
View File

@ -10,6 +10,7 @@
""" System inventory Armada manifest operator."""
import abc
import io
import os
import json
import ruamel.yaml as yaml
@ -89,7 +90,7 @@ class ArmadaManifestOperator(object):
summary_fqpn = os.path.join(path, SUMMARY_FILE)
if os.path.exists(summary_fqpn):
self.manifest_path = os.path.dirname(summary_fqpn)
with open(summary_fqpn, 'r') as f:
with io.open(summary_fqpn, 'r', encoding='utf-8') as f:
# The RoundTripLoader removes the superfluous quotes by default,
# resulting the dumped out charts not readable in Armada.
# Set preserve_quotes=True to preserve all the quotes.
@ -109,7 +110,7 @@ class ArmadaManifestOperator(object):
# Save the name for a delete manifest
self.delete_manifest = "%s-del%s" % os.path.splitext(manifest_fqpn)
with open(manifest_fqpn, 'r') as f:
with io.open(manifest_fqpn, 'r', encoding='utf-8') as f:
# The RoundTripLoader removes the superfluous quotes by default,
# resulting the dumped out charts not readable in Armada.
# Set preserve_quotes=True to preserve all the quotes.

4
sysinv/sysinv/sysinv/sysinv/puppet/puppet.py
View File

@ -9,6 +9,7 @@
from __future__ import absolute_import
import eventlet
import io
import os
import tempfile
import yaml
@ -172,7 +173,8 @@ class PuppetOperator(object):
target_load,
'hieradata')
with open(os.path.join(path, filename), 'r') as yaml_file:
with io.open(os.path.join(path, filename), 'r',
encoding='utf-8') as yaml_file:
host_config = yaml.load(yaml_file)
host_config.update(config)

3
sysinv/sysinv/sysinv/sysinv/tests/conductor/test_kube_app_image_parser.py
View File

@ -9,6 +9,7 @@
"""Test class for Sysinv Kube App Image Parser."""
import copy
import io
import ruamel.yaml as yaml
import os
@ -61,7 +62,7 @@ class TestKubeAppImageParser(base.TestCase):
def test_find_images_in_dict(self):
yaml_file = os.path.join(os.path.dirname(__file__),
"data", "chart_values_sample.yaml")
with open(yaml_file, 'r') as f:
with io.open(yaml_file, 'r', encoding='utf-8') as f:
values = yaml.safe_load(f)
expected = copy.deepcopy(IMAGES_RESOURCE)

19
sysinv/sysinv/sysinv/sysinv/tests/conductor/test_kube_app_metadata.py
View File

@ -7,6 +7,7 @@
"""Test class for Sysinv Kube App Metadata operations."""
import copy
import io
import os
import ruamel.yaml as yaml
@ -30,7 +31,7 @@ class TestKubeAppMetadata(base.TestCase):
yaml_file = os.path.join(os.path.dirname(__file__),
"data", "metadata_app_reapply_1.yaml")
with open(yaml_file, 'r') as f:
with io.open(yaml_file, 'r', encoding='utf-8') as f:
metadata_collection = yaml.safe_load_all(f)
for metadata in metadata_collection:
@ -62,7 +63,7 @@ class TestKubeAppMetadata(base.TestCase):
yaml_file = os.path.join(os.path.dirname(__file__),
"data", "metadata_app_reapply_non_existing_1.yaml")
with open(yaml_file, 'r') as f:
with io.open(yaml_file, 'r', encoding='utf-8') as f:
metadata_collection = yaml.safe_load_all(f)
for metadata in metadata_collection:
@ -100,7 +101,7 @@ class TestKubeAppMetadata(base.TestCase):
yaml_file = os.path.join(os.path.dirname(__file__),
"data", "metadata_app_reapply_non_existing_2.yaml")
with open(yaml_file, 'r') as f:
with io.open(yaml_file, 'r', encoding='utf-8') as f:
metadata_collection = yaml.safe_load_all(f)
for metadata in metadata_collection:
@ -135,7 +136,7 @@ class TestKubeAppMetadata(base.TestCase):
yaml_file = os.path.join(os.path.dirname(__file__),
"data", "metadata_app_reapply_non_managed_1.yaml")
with open(yaml_file, 'r') as f:
with io.open(yaml_file, 'r', encoding='utf-8') as f:
metadata_collection = yaml.safe_load_all(f)
for metadata in metadata_collection:
@ -173,7 +174,7 @@ class TestKubeAppMetadata(base.TestCase):
yaml_file = os.path.join(os.path.dirname(__file__),
"data", "metadata_app_reapply_not_cycle_1_non_managed.yaml")
with open(yaml_file, 'r') as f:
with io.open(yaml_file, 'r', encoding='utf-8') as f:
metadata_collection = yaml.safe_load_all(f)
for metadata in metadata_collection:
@ -211,7 +212,7 @@ class TestKubeAppMetadata(base.TestCase):
yaml_file = os.path.join(os.path.dirname(__file__),
"data", "metadata_app_reapply_not_cycle_2.yaml")
with open(yaml_file, 'r') as f:
with io.open(yaml_file, 'r', encoding='utf-8') as f:
metadata_collection = yaml.safe_load_all(f)
for metadata in metadata_collection:
@ -265,7 +266,7 @@ class TestKubeAppMetadata(base.TestCase):
yaml_file = os.path.join(os.path.dirname(__file__),
"data", "metadata_app_reapply_cycle_1.yaml")
with open(yaml_file, 'r') as f:
with io.open(yaml_file, 'r', encoding='utf-8') as f:
metadata_collection = yaml.safe_load_all(f)
for metadata in metadata_collection:
@ -283,7 +284,7 @@ class TestKubeAppMetadata(base.TestCase):
yaml_file = os.path.join(os.path.dirname(__file__),
"data", "metadata_app_reapply_cycle_2.yaml")
with open(yaml_file, 'r') as f:
with io.open(yaml_file, 'r', encoding='utf-8') as f:
metadata_collection = yaml.safe_load_all(f)
for metadata in metadata_collection:
@ -301,7 +302,7 @@ class TestKubeAppMetadata(base.TestCase):
yaml_file = os.path.join(os.path.dirname(__file__),
"data", "metadata_app_reapply_cycle_3.yaml")
with open(yaml_file, 'r') as f:
with io.open(yaml_file, 'r', encoding='utf-8') as f:
metadata_collection = yaml.safe_load_all(f)
for metadata in metadata_collection: