diff --git a/sysinv/sysinv/sysinv/sysinv/common/utils.py b/sysinv/sysinv/sysinv/sysinv/common/utils.py index 2fc58ceb3b..866e72fafd 100644 --- a/sysinv/sysinv/sysinv/sysinv/common/utils.py +++ b/sysinv/sysinv/sysinv/sysinv/common/utils.py @@ -40,6 +40,7 @@ import fcntl import glob import grp import hashlib +import io import itertools as it import json import keyring @@ -2034,7 +2035,7 @@ def find_metadata_file(path, metadata_file, upgrade_from_release=None): patches = [] metadata_path = os.path.join(path, metadata_file) if os.path.isfile(metadata_path): - with open(metadata_path, 'r') as f: + with io.open(metadata_path, 'r', encoding='utf-8') as f: try: doc = yaml.safe_load(f) app_name = doc['app_name'] @@ -2311,7 +2312,7 @@ def find_metadata_file(path, metadata_file, upgrade_from_release=None): def find_manifest_file(path): """ Find all manifest files in a given directory. """ def _is_manifest(yaml_file): - with open(yaml_file, 'r') as f: + with io.open(yaml_file, 'r', encoding='utf-8') as f: docs = yaml.load_all(f) for doc in docs: try: @@ -2498,7 +2499,7 @@ def get_app_supported_kube_version(app_name, app_version): kube_max_version = None if (os.path.exists(app_metadata_path) and os.path.getsize(app_metadata_path) > 0): - with open(app_metadata_path, 'r') as f: + with io.open(app_metadata_path, 'r', encoding='utf-8') as f: y = yaml.safe_load(f) supported_kube_version = y.get('supported_k8s_version', {}) kube_min_version = supported_kube_version.get('minimum', None) diff --git a/sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py b/sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py index 370f815427..398e610067 100644 --- a/sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py +++ b/sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py @@ -15,6 +15,7 @@ from eventlet.green import subprocess import glob import grp import functools +import io import os import pkg_resources import pwd @@ -521,11 +522,11 @@ class AppOperator(object): manifest_update_required = False if os.path.exists(app_images_file): - with open(app_images_file, 'r') as f: + with io.open(app_images_file, 'r', encoding='utf-8') as f: images_file = yaml.safe_load(f) if os.path.exists(app_manifest_file): - with open(app_manifest_file, 'r') as f: + with io.open(app_manifest_file, 'r', encoding='utf-8') as f: # The RoundTripLoader removes the superfluous quotes by default, # resulting the dumped out charts not readable in Armada. # Set preserve_quotes=True to preserve all the quotes. @@ -548,7 +549,7 @@ class AppOperator(object): app_overrides_file = os.path.join(overrides_dir, overrides) overrides_file = {} if os.path.exists(app_overrides_file): - with open(app_overrides_file, 'r') as f: + with io.open(app_overrides_file, 'r', encoding='utf-8') as f: overrides_file = yaml.safe_load(f) override_imgs = self._image.find_images_in_dict( @@ -686,7 +687,7 @@ class AppOperator(object): chart_path = os.path.join(chart_name, 'values.yaml') if os.path.exists(chart_path): - with open(chart_path, 'r') as f: + with io.open(chart_path, 'r', encoding='utf-8') as f: y = yaml.safe_load(f) chart_images = self._image.find_images_in_dict(y) @@ -698,7 +699,7 @@ class AppOperator(object): default_flow_style=False) def _retrieve_images_list(self, app_images_file): - with open(app_images_file, 'r') as f: + with io.open(app_images_file, 'r', encoding='utf-8') as f: images_list = yaml.safe_load(f) return images_list @@ -805,7 +806,7 @@ class AppOperator(object): lfile = os.path.join(app.inst_path, constants.APP_METADATA_FILE) if os.path.exists(lfile) and os.path.getsize(lfile) > 0: - with open(lfile, 'r') as f: + with io.open(lfile, 'r', encoding='utf-8') as f: try: y = yaml.safe_load(f) repo = y.get('helm_repo', common.HELM_REPO_FOR_APPS) @@ -1077,7 +1078,7 @@ class AppOperator(object): chart_groups = [] armada_charts = {} - with open(manifest_file, 'r') as f: + with io.open(manifest_file, 'r', encoding='utf-8') as f: docs = yaml.safe_load_all(f) for doc in docs: # iterative docs in the manifest file to get required @@ -1301,7 +1302,7 @@ class AppOperator(object): metadata_file = os.path.join(app.inst_path, constants.APP_METADATA_FILE) if os.path.exists(metadata_file) and os.path.getsize(metadata_file) > 0: - with open(metadata_file, 'r') as f: + with io.open(metadata_file, 'r', encoding='utf-8') as f: try: metadata = yaml.safe_load(f) or {} value = cutils.deep_get(metadata, keys, default=default) @@ -2223,7 +2224,7 @@ class AppOperator(object): metadata = {} if os.path.exists(app.sync_metadata_file): - with open(app.sync_metadata_file, 'r') as f: + with io.open(app.sync_metadata_file, 'r', encoding='utf-8') as f: # The RoundTripLoader removes the superfluous quotes by default. # Set preserve_quotes=True to preserve all the quotes. # The assumption here: there is just one yaml section diff --git a/sysinv/sysinv/sysinv/sysinv/conductor/manager.py b/sysinv/sysinv/sysinv/sysinv/conductor/manager.py index 937f7d650a..c3e1629944 100644 --- a/sysinv/sysinv/sysinv/sysinv/conductor/manager.py +++ b/sysinv/sysinv/sysinv/sysinv/conductor/manager.py @@ -5982,7 +5982,7 @@ class ConductorManager(service.PeriodicService): metadata_file = os.path.join(app_path, constants.APP_METADATA_FILE) if os.path.exists(metadata_file): - with open(metadata_file, 'r') as f: + with io.open(metadata_file, 'r', encoding='utf-8') as f: # The RoundTripLoader removes the superfluous quotes by default. # Set preserve_quotes=True to preserve all the quotes. # The assumption here: there is just one yaml section diff --git a/sysinv/sysinv/sysinv/sysinv/helm/manifest_base.py b/sysinv/sysinv/sysinv/sysinv/helm/manifest_base.py index d3f1dc9e15..1f578872b0 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/manifest_base.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/manifest_base.py @@ -10,6 +10,7 @@ """ System inventory Armada manifest operator.""" import abc +import io import os import json import ruamel.yaml as yaml @@ -89,7 +90,7 @@ class ArmadaManifestOperator(object): summary_fqpn = os.path.join(path, SUMMARY_FILE) if os.path.exists(summary_fqpn): self.manifest_path = os.path.dirname(summary_fqpn) - with open(summary_fqpn, 'r') as f: + with io.open(summary_fqpn, 'r', encoding='utf-8') as f: # The RoundTripLoader removes the superfluous quotes by default, # resulting the dumped out charts not readable in Armada. # Set preserve_quotes=True to preserve all the quotes. @@ -109,7 +110,7 @@ class ArmadaManifestOperator(object): # Save the name for a delete manifest self.delete_manifest = "%s-del%s" % os.path.splitext(manifest_fqpn) - with open(manifest_fqpn, 'r') as f: + with io.open(manifest_fqpn, 'r', encoding='utf-8') as f: # The RoundTripLoader removes the superfluous quotes by default, # resulting the dumped out charts not readable in Armada. # Set preserve_quotes=True to preserve all the quotes. diff --git a/sysinv/sysinv/sysinv/sysinv/puppet/puppet.py b/sysinv/sysinv/sysinv/sysinv/puppet/puppet.py index 8def9ae7aa..159e0b8909 100644 --- a/sysinv/sysinv/sysinv/sysinv/puppet/puppet.py +++ b/sysinv/sysinv/sysinv/sysinv/puppet/puppet.py @@ -9,6 +9,7 @@ from __future__ import absolute_import import eventlet +import io import os import tempfile import yaml @@ -172,7 +173,8 @@ class PuppetOperator(object): target_load, 'hieradata') - with open(os.path.join(path, filename), 'r') as yaml_file: + with io.open(os.path.join(path, filename), 'r', + encoding='utf-8') as yaml_file: host_config = yaml.load(yaml_file) host_config.update(config) diff --git a/sysinv/sysinv/sysinv/sysinv/tests/conductor/test_kube_app_image_parser.py b/sysinv/sysinv/sysinv/sysinv/tests/conductor/test_kube_app_image_parser.py index eed533091c..dff824fb58 100644 --- a/sysinv/sysinv/sysinv/sysinv/tests/conductor/test_kube_app_image_parser.py +++ b/sysinv/sysinv/sysinv/sysinv/tests/conductor/test_kube_app_image_parser.py @@ -9,6 +9,7 @@ """Test class for Sysinv Kube App Image Parser.""" import copy +import io import ruamel.yaml as yaml import os @@ -61,7 +62,7 @@ class TestKubeAppImageParser(base.TestCase): def test_find_images_in_dict(self): yaml_file = os.path.join(os.path.dirname(__file__), "data", "chart_values_sample.yaml") - with open(yaml_file, 'r') as f: + with io.open(yaml_file, 'r', encoding='utf-8') as f: values = yaml.safe_load(f) expected = copy.deepcopy(IMAGES_RESOURCE) diff --git a/sysinv/sysinv/sysinv/sysinv/tests/conductor/test_kube_app_metadata.py b/sysinv/sysinv/sysinv/sysinv/tests/conductor/test_kube_app_metadata.py index 8ca287ce9f..9a73c289fc 100644 --- a/sysinv/sysinv/sysinv/sysinv/tests/conductor/test_kube_app_metadata.py +++ b/sysinv/sysinv/sysinv/sysinv/tests/conductor/test_kube_app_metadata.py @@ -7,6 +7,7 @@ """Test class for Sysinv Kube App Metadata operations.""" import copy +import io import os import ruamel.yaml as yaml @@ -30,7 +31,7 @@ class TestKubeAppMetadata(base.TestCase): yaml_file = os.path.join(os.path.dirname(__file__), "data", "metadata_app_reapply_1.yaml") - with open(yaml_file, 'r') as f: + with io.open(yaml_file, 'r', encoding='utf-8') as f: metadata_collection = yaml.safe_load_all(f) for metadata in metadata_collection: @@ -62,7 +63,7 @@ class TestKubeAppMetadata(base.TestCase): yaml_file = os.path.join(os.path.dirname(__file__), "data", "metadata_app_reapply_non_existing_1.yaml") - with open(yaml_file, 'r') as f: + with io.open(yaml_file, 'r', encoding='utf-8') as f: metadata_collection = yaml.safe_load_all(f) for metadata in metadata_collection: @@ -100,7 +101,7 @@ class TestKubeAppMetadata(base.TestCase): yaml_file = os.path.join(os.path.dirname(__file__), "data", "metadata_app_reapply_non_existing_2.yaml") - with open(yaml_file, 'r') as f: + with io.open(yaml_file, 'r', encoding='utf-8') as f: metadata_collection = yaml.safe_load_all(f) for metadata in metadata_collection: @@ -135,7 +136,7 @@ class TestKubeAppMetadata(base.TestCase): yaml_file = os.path.join(os.path.dirname(__file__), "data", "metadata_app_reapply_non_managed_1.yaml") - with open(yaml_file, 'r') as f: + with io.open(yaml_file, 'r', encoding='utf-8') as f: metadata_collection = yaml.safe_load_all(f) for metadata in metadata_collection: @@ -173,7 +174,7 @@ class TestKubeAppMetadata(base.TestCase): yaml_file = os.path.join(os.path.dirname(__file__), "data", "metadata_app_reapply_not_cycle_1_non_managed.yaml") - with open(yaml_file, 'r') as f: + with io.open(yaml_file, 'r', encoding='utf-8') as f: metadata_collection = yaml.safe_load_all(f) for metadata in metadata_collection: @@ -211,7 +212,7 @@ class TestKubeAppMetadata(base.TestCase): yaml_file = os.path.join(os.path.dirname(__file__), "data", "metadata_app_reapply_not_cycle_2.yaml") - with open(yaml_file, 'r') as f: + with io.open(yaml_file, 'r', encoding='utf-8') as f: metadata_collection = yaml.safe_load_all(f) for metadata in metadata_collection: @@ -265,7 +266,7 @@ class TestKubeAppMetadata(base.TestCase): yaml_file = os.path.join(os.path.dirname(__file__), "data", "metadata_app_reapply_cycle_1.yaml") - with open(yaml_file, 'r') as f: + with io.open(yaml_file, 'r', encoding='utf-8') as f: metadata_collection = yaml.safe_load_all(f) for metadata in metadata_collection: @@ -283,7 +284,7 @@ class TestKubeAppMetadata(base.TestCase): yaml_file = os.path.join(os.path.dirname(__file__), "data", "metadata_app_reapply_cycle_2.yaml") - with open(yaml_file, 'r') as f: + with io.open(yaml_file, 'r', encoding='utf-8') as f: metadata_collection = yaml.safe_load_all(f) for metadata in metadata_collection: @@ -301,7 +302,7 @@ class TestKubeAppMetadata(base.TestCase): yaml_file = os.path.join(os.path.dirname(__file__), "data", "metadata_app_reapply_cycle_3.yaml") - with open(yaml_file, 'r') as f: + with io.open(yaml_file, 'r', encoding='utf-8') as f: metadata_collection = yaml.safe_load_all(f) for metadata in metadata_collection: