Specify encoding of the file for yaml load
yaml.load might not properly detect the encoding.
This error was observed during bootstrap when the first app
(nginx-ingress) had the values.yaml file loaded as yaml.
The encoding was detected as being ascii when it was not.
A UnicodeDecodeError is thrown: 'ascii' codec can't decode
byte 0xe2 in position.
The fix is to specifically tell yaml the encoding of the file.
As 'encoding' parameter exists only for open in Python3, but not
Python2, switch to io.open to be backward compatible.
As a precaution, did the same changed to ruamel yaml.
Story: 2006796
Task: 42798
Signed-off-by: Dan Voiculeasa <dan.voiculeasa@windriver.com>
Change-Id: Ib101c95392cb453121baecadebb4f75b81216477
(cherry picked from commit 156cc123e4
)
This commit is contained in:
parent
1518543bf7
commit
75081afa1f
@ -40,6 +40,7 @@ import fcntl
|
||||
import glob
|
||||
import grp
|
||||
import hashlib
|
||||
import io
|
||||
import itertools as it
|
||||
import json
|
||||
import keyring
|
||||
@ -2034,7 +2035,7 @@ def find_metadata_file(path, metadata_file, upgrade_from_release=None):
|
||||
patches = []
|
||||
metadata_path = os.path.join(path, metadata_file)
|
||||
if os.path.isfile(metadata_path):
|
||||
with open(metadata_path, 'r') as f:
|
||||
with io.open(metadata_path, 'r', encoding='utf-8') as f:
|
||||
try:
|
||||
doc = yaml.safe_load(f)
|
||||
app_name = doc['app_name']
|
||||
@ -2311,7 +2312,7 @@ def find_metadata_file(path, metadata_file, upgrade_from_release=None):
|
||||
def find_manifest_file(path):
|
||||
""" Find all manifest files in a given directory. """
|
||||
def _is_manifest(yaml_file):
|
||||
with open(yaml_file, 'r') as f:
|
||||
with io.open(yaml_file, 'r', encoding='utf-8') as f:
|
||||
docs = yaml.load_all(f)
|
||||
for doc in docs:
|
||||
try:
|
||||
@ -2498,7 +2499,7 @@ def get_app_supported_kube_version(app_name, app_version):
|
||||
kube_max_version = None
|
||||
if (os.path.exists(app_metadata_path) and
|
||||
os.path.getsize(app_metadata_path) > 0):
|
||||
with open(app_metadata_path, 'r') as f:
|
||||
with io.open(app_metadata_path, 'r', encoding='utf-8') as f:
|
||||
y = yaml.safe_load(f)
|
||||
supported_kube_version = y.get('supported_k8s_version', {})
|
||||
kube_min_version = supported_kube_version.get('minimum', None)
|
||||
|
@ -15,6 +15,7 @@ from eventlet.green import subprocess
|
||||
import glob
|
||||
import grp
|
||||
import functools
|
||||
import io
|
||||
import os
|
||||
import pkg_resources
|
||||
import pwd
|
||||
@ -521,11 +522,11 @@ class AppOperator(object):
|
||||
manifest_update_required = False
|
||||
|
||||
if os.path.exists(app_images_file):
|
||||
with open(app_images_file, 'r') as f:
|
||||
with io.open(app_images_file, 'r', encoding='utf-8') as f:
|
||||
images_file = yaml.safe_load(f)
|
||||
|
||||
if os.path.exists(app_manifest_file):
|
||||
with open(app_manifest_file, 'r') as f:
|
||||
with io.open(app_manifest_file, 'r', encoding='utf-8') as f:
|
||||
# The RoundTripLoader removes the superfluous quotes by default,
|
||||
# resulting the dumped out charts not readable in Armada.
|
||||
# Set preserve_quotes=True to preserve all the quotes.
|
||||
@ -548,7 +549,7 @@ class AppOperator(object):
|
||||
app_overrides_file = os.path.join(overrides_dir, overrides)
|
||||
overrides_file = {}
|
||||
if os.path.exists(app_overrides_file):
|
||||
with open(app_overrides_file, 'r') as f:
|
||||
with io.open(app_overrides_file, 'r', encoding='utf-8') as f:
|
||||
overrides_file = yaml.safe_load(f)
|
||||
|
||||
override_imgs = self._image.find_images_in_dict(
|
||||
@ -686,7 +687,7 @@ class AppOperator(object):
|
||||
|
||||
chart_path = os.path.join(chart_name, 'values.yaml')
|
||||
if os.path.exists(chart_path):
|
||||
with open(chart_path, 'r') as f:
|
||||
with io.open(chart_path, 'r', encoding='utf-8') as f:
|
||||
y = yaml.safe_load(f)
|
||||
|
||||
chart_images = self._image.find_images_in_dict(y)
|
||||
@ -698,7 +699,7 @@ class AppOperator(object):
|
||||
default_flow_style=False)
|
||||
|
||||
def _retrieve_images_list(self, app_images_file):
|
||||
with open(app_images_file, 'r') as f:
|
||||
with io.open(app_images_file, 'r', encoding='utf-8') as f:
|
||||
images_list = yaml.safe_load(f)
|
||||
return images_list
|
||||
|
||||
@ -805,7 +806,7 @@ class AppOperator(object):
|
||||
lfile = os.path.join(app.inst_path, constants.APP_METADATA_FILE)
|
||||
|
||||
if os.path.exists(lfile) and os.path.getsize(lfile) > 0:
|
||||
with open(lfile, 'r') as f:
|
||||
with io.open(lfile, 'r', encoding='utf-8') as f:
|
||||
try:
|
||||
y = yaml.safe_load(f)
|
||||
repo = y.get('helm_repo', common.HELM_REPO_FOR_APPS)
|
||||
@ -1077,7 +1078,7 @@ class AppOperator(object):
|
||||
chart_groups = []
|
||||
armada_charts = {}
|
||||
|
||||
with open(manifest_file, 'r') as f:
|
||||
with io.open(manifest_file, 'r', encoding='utf-8') as f:
|
||||
docs = yaml.safe_load_all(f)
|
||||
for doc in docs:
|
||||
# iterative docs in the manifest file to get required
|
||||
@ -1301,7 +1302,7 @@ class AppOperator(object):
|
||||
metadata_file = os.path.join(app.inst_path,
|
||||
constants.APP_METADATA_FILE)
|
||||
if os.path.exists(metadata_file) and os.path.getsize(metadata_file) > 0:
|
||||
with open(metadata_file, 'r') as f:
|
||||
with io.open(metadata_file, 'r', encoding='utf-8') as f:
|
||||
try:
|
||||
metadata = yaml.safe_load(f) or {}
|
||||
value = cutils.deep_get(metadata, keys, default=default)
|
||||
@ -2223,7 +2224,7 @@ class AppOperator(object):
|
||||
metadata = {}
|
||||
|
||||
if os.path.exists(app.sync_metadata_file):
|
||||
with open(app.sync_metadata_file, 'r') as f:
|
||||
with io.open(app.sync_metadata_file, 'r', encoding='utf-8') as f:
|
||||
# The RoundTripLoader removes the superfluous quotes by default.
|
||||
# Set preserve_quotes=True to preserve all the quotes.
|
||||
# The assumption here: there is just one yaml section
|
||||
|
@ -5982,7 +5982,7 @@ class ConductorManager(service.PeriodicService):
|
||||
metadata_file = os.path.join(app_path,
|
||||
constants.APP_METADATA_FILE)
|
||||
if os.path.exists(metadata_file):
|
||||
with open(metadata_file, 'r') as f:
|
||||
with io.open(metadata_file, 'r', encoding='utf-8') as f:
|
||||
# The RoundTripLoader removes the superfluous quotes by default.
|
||||
# Set preserve_quotes=True to preserve all the quotes.
|
||||
# The assumption here: there is just one yaml section
|
||||
|
@ -10,6 +10,7 @@
|
||||
""" System inventory Armada manifest operator."""
|
||||
|
||||
import abc
|
||||
import io
|
||||
import os
|
||||
import json
|
||||
import ruamel.yaml as yaml
|
||||
@ -89,7 +90,7 @@ class ArmadaManifestOperator(object):
|
||||
summary_fqpn = os.path.join(path, SUMMARY_FILE)
|
||||
if os.path.exists(summary_fqpn):
|
||||
self.manifest_path = os.path.dirname(summary_fqpn)
|
||||
with open(summary_fqpn, 'r') as f:
|
||||
with io.open(summary_fqpn, 'r', encoding='utf-8') as f:
|
||||
# The RoundTripLoader removes the superfluous quotes by default,
|
||||
# resulting the dumped out charts not readable in Armada.
|
||||
# Set preserve_quotes=True to preserve all the quotes.
|
||||
@ -109,7 +110,7 @@ class ArmadaManifestOperator(object):
|
||||
# Save the name for a delete manifest
|
||||
self.delete_manifest = "%s-del%s" % os.path.splitext(manifest_fqpn)
|
||||
|
||||
with open(manifest_fqpn, 'r') as f:
|
||||
with io.open(manifest_fqpn, 'r', encoding='utf-8') as f:
|
||||
# The RoundTripLoader removes the superfluous quotes by default,
|
||||
# resulting the dumped out charts not readable in Armada.
|
||||
# Set preserve_quotes=True to preserve all the quotes.
|
||||
|
@ -9,6 +9,7 @@
|
||||
from __future__ import absolute_import
|
||||
|
||||
import eventlet
|
||||
import io
|
||||
import os
|
||||
import tempfile
|
||||
import yaml
|
||||
@ -172,7 +173,8 @@ class PuppetOperator(object):
|
||||
target_load,
|
||||
'hieradata')
|
||||
|
||||
with open(os.path.join(path, filename), 'r') as yaml_file:
|
||||
with io.open(os.path.join(path, filename), 'r',
|
||||
encoding='utf-8') as yaml_file:
|
||||
host_config = yaml.load(yaml_file)
|
||||
|
||||
host_config.update(config)
|
||||
|
@ -9,6 +9,7 @@
|
||||
"""Test class for Sysinv Kube App Image Parser."""
|
||||
|
||||
import copy
|
||||
import io
|
||||
import ruamel.yaml as yaml
|
||||
import os
|
||||
|
||||
@ -61,7 +62,7 @@ class TestKubeAppImageParser(base.TestCase):
|
||||
def test_find_images_in_dict(self):
|
||||
yaml_file = os.path.join(os.path.dirname(__file__),
|
||||
"data", "chart_values_sample.yaml")
|
||||
with open(yaml_file, 'r') as f:
|
||||
with io.open(yaml_file, 'r', encoding='utf-8') as f:
|
||||
values = yaml.safe_load(f)
|
||||
|
||||
expected = copy.deepcopy(IMAGES_RESOURCE)
|
||||
|
@ -7,6 +7,7 @@
|
||||
"""Test class for Sysinv Kube App Metadata operations."""
|
||||
|
||||
import copy
|
||||
import io
|
||||
import os
|
||||
import ruamel.yaml as yaml
|
||||
|
||||
@ -30,7 +31,7 @@ class TestKubeAppMetadata(base.TestCase):
|
||||
|
||||
yaml_file = os.path.join(os.path.dirname(__file__),
|
||||
"data", "metadata_app_reapply_1.yaml")
|
||||
with open(yaml_file, 'r') as f:
|
||||
with io.open(yaml_file, 'r', encoding='utf-8') as f:
|
||||
metadata_collection = yaml.safe_load_all(f)
|
||||
|
||||
for metadata in metadata_collection:
|
||||
@ -62,7 +63,7 @@ class TestKubeAppMetadata(base.TestCase):
|
||||
|
||||
yaml_file = os.path.join(os.path.dirname(__file__),
|
||||
"data", "metadata_app_reapply_non_existing_1.yaml")
|
||||
with open(yaml_file, 'r') as f:
|
||||
with io.open(yaml_file, 'r', encoding='utf-8') as f:
|
||||
metadata_collection = yaml.safe_load_all(f)
|
||||
|
||||
for metadata in metadata_collection:
|
||||
@ -100,7 +101,7 @@ class TestKubeAppMetadata(base.TestCase):
|
||||
|
||||
yaml_file = os.path.join(os.path.dirname(__file__),
|
||||
"data", "metadata_app_reapply_non_existing_2.yaml")
|
||||
with open(yaml_file, 'r') as f:
|
||||
with io.open(yaml_file, 'r', encoding='utf-8') as f:
|
||||
metadata_collection = yaml.safe_load_all(f)
|
||||
|
||||
for metadata in metadata_collection:
|
||||
@ -135,7 +136,7 @@ class TestKubeAppMetadata(base.TestCase):
|
||||
|
||||
yaml_file = os.path.join(os.path.dirname(__file__),
|
||||
"data", "metadata_app_reapply_non_managed_1.yaml")
|
||||
with open(yaml_file, 'r') as f:
|
||||
with io.open(yaml_file, 'r', encoding='utf-8') as f:
|
||||
metadata_collection = yaml.safe_load_all(f)
|
||||
|
||||
for metadata in metadata_collection:
|
||||
@ -173,7 +174,7 @@ class TestKubeAppMetadata(base.TestCase):
|
||||
|
||||
yaml_file = os.path.join(os.path.dirname(__file__),
|
||||
"data", "metadata_app_reapply_not_cycle_1_non_managed.yaml")
|
||||
with open(yaml_file, 'r') as f:
|
||||
with io.open(yaml_file, 'r', encoding='utf-8') as f:
|
||||
metadata_collection = yaml.safe_load_all(f)
|
||||
|
||||
for metadata in metadata_collection:
|
||||
@ -211,7 +212,7 @@ class TestKubeAppMetadata(base.TestCase):
|
||||
|
||||
yaml_file = os.path.join(os.path.dirname(__file__),
|
||||
"data", "metadata_app_reapply_not_cycle_2.yaml")
|
||||
with open(yaml_file, 'r') as f:
|
||||
with io.open(yaml_file, 'r', encoding='utf-8') as f:
|
||||
metadata_collection = yaml.safe_load_all(f)
|
||||
|
||||
for metadata in metadata_collection:
|
||||
@ -265,7 +266,7 @@ class TestKubeAppMetadata(base.TestCase):
|
||||
|
||||
yaml_file = os.path.join(os.path.dirname(__file__),
|
||||
"data", "metadata_app_reapply_cycle_1.yaml")
|
||||
with open(yaml_file, 'r') as f:
|
||||
with io.open(yaml_file, 'r', encoding='utf-8') as f:
|
||||
metadata_collection = yaml.safe_load_all(f)
|
||||
|
||||
for metadata in metadata_collection:
|
||||
@ -283,7 +284,7 @@ class TestKubeAppMetadata(base.TestCase):
|
||||
|
||||
yaml_file = os.path.join(os.path.dirname(__file__),
|
||||
"data", "metadata_app_reapply_cycle_2.yaml")
|
||||
with open(yaml_file, 'r') as f:
|
||||
with io.open(yaml_file, 'r', encoding='utf-8') as f:
|
||||
metadata_collection = yaml.safe_load_all(f)
|
||||
|
||||
for metadata in metadata_collection:
|
||||
@ -301,7 +302,7 @@ class TestKubeAppMetadata(base.TestCase):
|
||||
|
||||
yaml_file = os.path.join(os.path.dirname(__file__),
|
||||
"data", "metadata_app_reapply_cycle_3.yaml")
|
||||
with open(yaml_file, 'r') as f:
|
||||
with io.open(yaml_file, 'r', encoding='utf-8') as f:
|
||||
metadata_collection = yaml.safe_load_all(f)
|
||||
|
||||
for metadata in metadata_collection:
|
||||
|
Loading…
Reference in New Issue
Block a user