From 1518543bf7a504ec09a7d88770b602e57e7674ea Mon Sep 17 00:00:00 2001 From: Dan Voiculeasa Date: Thu, 8 Jul 2021 00:26:26 +0300 Subject: [PATCH] py3: Fix base64 usage b64encode and b64decode returns a str in Python2 and a bytes in Python3. This is a problem when using http/url/rest libraries. Runtime errors are raised. Mixing str and bytes when formatting a text might introduce an unwanted "b" for bytes, which can lead to potential issues when sent over the network. To keep compatibility use oslo_serialization to force the return type to str. There is one place where base64 urlsafe version is specifically used to send and receive(Rest API for application upload). One of the tests is that platform-integ-app applies which exercises part of the changes. Cert-mon and DC part will be exercied when DC is available on f/centos8 branch. Story: 2006796 Task: 42797 Signed-off-by: Dan Voiculeasa Change-Id: I48b1c6c80363458945c6bc1a9cf7e16c743a7bd6 (cherry picked from commit 8a7c4b15c7203e4410f100a1dc01f165ddaf0f18) --- .../sysinv/cert_mon/certificate_mon_manager.py | 8 ++++---- sysinv/sysinv/sysinv/sysinv/cert_mon/utils.py | 6 +++--- .../sysinv/sysinv/sysinv/cert_mon/watcher.py | 8 ++++---- sysinv/sysinv/sysinv/sysinv/common/utils.py | 18 +++++++++--------- .../sysinv/sysinv/conductor/docker_registry.py | 4 ++-- .../sysinv/sysinv/sysinv/conductor/kube_app.py | 14 +++++++------- .../sysinv/sysinv/sysinv/conductor/manager.py | 8 ++++---- sysinv/sysinv/sysinv/sysinv/puppet/platform.py | 3 ++- .../sysinv/tests/conductor/test_manager.py | 4 ++-- 9 files changed, 37 insertions(+), 36 deletions(-) diff --git a/sysinv/sysinv/sysinv/sysinv/cert_mon/certificate_mon_manager.py b/sysinv/sysinv/sysinv/sysinv/cert_mon/certificate_mon_manager.py index 19451abc01..9305b9d8a6 100644 --- a/sysinv/sysinv/sysinv/sysinv/cert_mon/certificate_mon_manager.py +++ b/sysinv/sysinv/sysinv/sysinv/cert_mon/certificate_mon_manager.py @@ -17,11 +17,11 @@ # of this software may be licensed only pursuant to the terms # of an applicable Wind River license agreement. # -import base64 from eventlet import greenthread import greenlet from oslo_config import cfg from oslo_log import log +from oslo_serialization import base64 from oslo_service import periodic_task import time @@ -137,9 +137,9 @@ class CertificateMonManager(periodic_task.PeriodicTasks): raise Exception('%s certificate data missing: %s' % (subcloud_name, item)) - txt_ssl_cert = base64.b64decode(secret.data['tls.crt']) - txt_ssl_key = base64.b64decode(secret.data['tls.key']) - txt_ca_cert = base64.b64decode(secret.data['ca.crt']) + txt_ssl_cert = base64.decode_as_text(secret.data['tls.crt']) + txt_ssl_key = base64.decode_as_text(secret.data['tls.key']) + txt_ca_cert = base64.decode_as_text(secret.data['ca.crt']) except Exception as e: LOG.error('Cannot audit ssl certificate on %s' % subcloud_name) LOG.exception(e) diff --git a/sysinv/sysinv/sysinv/sysinv/cert_mon/utils.py b/sysinv/sysinv/sysinv/sysinv/cert_mon/utils.py index a97684e805..7fe2b1dd2b 100644 --- a/sysinv/sysinv/sysinv/sysinv/cert_mon/utils.py +++ b/sysinv/sysinv/sysinv/sysinv/cert_mon/utils.py @@ -17,7 +17,6 @@ # of this software may be licensed only pursuant to the terms # of an applicable Wind River license agreement. # -import base64 import json import os import re @@ -32,6 +31,7 @@ from keystoneclient.auth.identity import v3 from oslo_config import cfg from oslo_log import log from oslo_utils import encodeutils +from oslo_serialization import base64 from six.moves.urllib.request import Request from six.moves.urllib.error import HTTPError from six.moves.urllib.error import URLError @@ -109,8 +109,8 @@ def verify_adminep_cert_chain(): raise Exception('%s tls.crt data missing' % (constants.SC_ADMIN_ENDPOINT_SECRET_NAME)) - txt_ca_crt = base64.b64decode(secret_ica.data['tls.crt']) - txt_tls_crt = base64.b64decode(secret_adminep.data['tls.crt']) + txt_ca_crt = base64.decode_as_text(secret_ica.data['tls.crt']) + txt_tls_crt = base64.decode_as_text(secret_adminep.data['tls.crt']) with tempfile.NamedTemporaryFile() as ca_tmpfile: ca_tmpfile.write(txt_ca_crt) diff --git a/sysinv/sysinv/sysinv/sysinv/cert_mon/watcher.py b/sysinv/sysinv/sysinv/sysinv/cert_mon/watcher.py index 37dd4095ab..b0764c0e96 100644 --- a/sysinv/sysinv/sysinv/sysinv/cert_mon/watcher.py +++ b/sysinv/sysinv/sysinv/sysinv/cert_mon/watcher.py @@ -17,7 +17,6 @@ # of this software may be licensed only pursuant to the terms # of an applicable Wind River license agreement. # -import base64 import re import hashlib from datetime import datetime @@ -29,6 +28,7 @@ from kubernetes import config import os from oslo_config import cfg from oslo_log import log +from oslo_serialization import base64 from six.moves.urllib.error import URLError from sysinv.cert_mon import utils @@ -93,11 +93,11 @@ class CertUpdateEventData(object): self.tls_crt = None self.tls_key = None try: - self.ca_crt = base64.b64decode(data['ca.crt']).strip() \ + self.ca_crt = base64.decode_as_text(data['ca.crt']).strip() \ if 'ca.crt' in data else '' - self.tls_crt = base64.b64decode(data['tls.crt']).strip() \ + self.tls_crt = base64.decode_as_text(data['tls.crt']).strip() \ if 'tls.crt' in data else '' - self.tls_key = base64.b64decode(data['tls.key']).strip() \ + self.tls_key = base64.decode_as_text(data['tls.key']).strip() \ if 'tls.key' in data else '' except TypeError: LOG.error('Invalid secret data.') diff --git a/sysinv/sysinv/sysinv/sysinv/common/utils.py b/sysinv/sysinv/sysinv/sysinv/common/utils.py index 02cbdfc4b9..2fc58ceb3b 100644 --- a/sysinv/sysinv/sysinv/sysinv/common/utils.py +++ b/sysinv/sysinv/sysinv/sysinv/common/utils.py @@ -25,7 +25,6 @@ """Utilities and helper functions.""" import ast -import base64 import boto3 from botocore.config import Config import collections @@ -70,6 +69,7 @@ import netaddr from oslo_concurrency import lockutils from oslo_config import cfg from oslo_log import log as logging +from oslo_serialization import base64 from fm_api import constants as fm_constants @@ -1759,7 +1759,7 @@ def read_filtered_directory_content(dirpath, *filters): # they can be transferred over RPC and stored in DB content.decode('utf-8') except UnicodeError: - content = content.encode('base64') + content = base64.encode_as_text(content) content_dict['base64_encoded_files'] = \ content_dict.get("base64_encoded_files", []) + [filename] @@ -2578,7 +2578,7 @@ def get_aws_ecr_registry_credentials(dbapi, registry, username, password): response = client.get_authorization_token() token = response['authorizationData'][0]['authorizationToken'] - username, password = token.decode('base64').split(':') + username, password = base64.decode_as_text(token).split(':') except Exception as e: raise exception.SysinvException(_( "Failed to get AWS ECR credentials: %s" % e)) @@ -2602,7 +2602,7 @@ def extract_ca_private_key_bytes_from_pem(pem_content): raise exception.InvalidKubernetesCA end_search += len(constants.END_PRIVATE_KEY_MARKER) - base64_key = base64.b64encode(pem_content[begin_search:end_search]) + base64_key = base64.encode_as_text(pem_content[begin_search:end_search]) return base64_key @@ -2621,7 +2621,7 @@ def extract_ca_crt_bytes_from_pem(pem_content): raise exception.InvalidKubernetesCA end_search += len(constants.END_CERTIFICATE_MARKER) - base64_crt = base64.b64encode(pem_content[begin_search:end_search]) + base64_crt = base64.encode_as_text(pem_content[begin_search:end_search]) return base64_crt @@ -2924,8 +2924,8 @@ def get_admin_ep_cert(dc_role): raise Exception("Invalid admin endpoint certificate data.") try: - tls_crt = base64.b64decode(data['tls.crt']) - tls_key = base64.b64decode(data['tls.key']) + tls_crt = base64.decode_as_text(data['tls.crt']) + tls_key = base64.decode_as_text(data['tls.key']) except TypeError: raise Exception('admin endpoint secret is invalid %s' % endpoint_cert_secret_name) @@ -2944,7 +2944,7 @@ def get_admin_ep_cert(dc_role): LOG.error('Cannot read DC root CA certificate %s' % e) elif dc_role == constants.DISTRIBUTED_CLOUD_ROLE_SYSTEMCONTROLLER: try: - ca_crt = base64.b64decode(data['ca.crt']) + ca_crt = base64.decode_as_text(data['ca.crt']) except TypeError: raise Exception('admin endpoint secret is invalid %s' % endpoint_cert_secret_name) @@ -3001,7 +3001,7 @@ def get_root_ca_cert(): data = secret.data try: - ca_crt = base64.b64decode(data['ca.crt']) + ca_crt = base64.decode_as_text(data['ca.crt']) except TypeError: raise Exception('Secret is invalid %s' % secret_name) diff --git a/sysinv/sysinv/sysinv/sysinv/conductor/docker_registry.py b/sysinv/sysinv/sysinv/sysinv/conductor/docker_registry.py index 84e6d233f5..686e1e8c33 100644 --- a/sysinv/sysinv/sysinv/sysinv/conductor/docker_registry.py +++ b/sysinv/sysinv/sysinv/sysinv/conductor/docker_registry.py @@ -4,10 +4,10 @@ # SPDX-License-Identifier: Apache-2.0 # -import base64 import keyring import requests +from oslo_serialization import base64 from sysinv.common import constants from sysinv.common import exception @@ -68,7 +68,7 @@ def docker_registry_authenticate(www_authenticate): # make a request to the token server # the credentials are passed as a header while the rest # are passed as params - auth_string = base64.b64encode("%s:%s" % (REGISTRY_USERNAME, get_registry_password())) + auth_string = base64.encode_as_text("%s:%s" % (REGISTRY_USERNAME, get_registry_password())) token_server_request_headers = {"authorization": "Basic %s" % auth_string} # we need try twice. # SYSTEM_CERT_PATH if the docker registry cert is signed by a trusted CA diff --git a/sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py b/sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py index 8d215bef4b..370f815427 100644 --- a/sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py +++ b/sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py @@ -9,7 +9,6 @@ """ System Inventory Kubernetes Application Operator.""" -import base64 import copy import docker from eventlet.green import subprocess @@ -38,6 +37,7 @@ from eventlet import Timeout from fm_api import constants as fm_constants from fm_api import fm_api from oslo_log import log as logging +from oslo_serialization import base64 from sysinv._i18n import _ from sysinv.api.controllers.v1 import kube_app from sysinv.common import constants @@ -983,12 +983,12 @@ class AppOperator(object): secret = self._kube.kube_get_secret("registry-local-secret", kubernetes.NAMESPACE_KUBE_SYSTEM) if secret is None: return - secret_auth_body = base64.b64decode(secret.data['.dockerconfigjson']) + secret_auth_body = base64.decode_as_text(secret.data['.dockerconfigjson']) secret_auth_info = (secret_auth_body.split('auth":')[1]).split('"')[1] registry_auth = cutils.get_local_docker_registry_auth() registry_auth_info = '{0}:{1}'.format(registry_auth['username'], registry_auth['password']) - if secret_auth_info == base64.b64encode(registry_auth_info): + if secret_auth_info == base64.encode_as_text(registry_auth_info): LOG.debug("Auth info is the same, no update is needed for k8s secret.") return except Exception as e: @@ -997,8 +997,8 @@ class AppOperator(object): try: # update secret with new auth info token = '{{\"auths\": {{\"{0}\": {{\"auth\": \"{1}\"}}}}}}'.format( - constants.DOCKER_REGISTRY_SERVER, base64.b64encode(registry_auth_info)) - secret.data['.dockerconfigjson'] = base64.b64encode(token) + constants.DOCKER_REGISTRY_SERVER, base64.encode_as_text(registry_auth_info)) + secret.data['.dockerconfigjson'] = base64.encode_as_text(token) self._kube.kube_patch_secret("registry-local-secret", kubernetes.NAMESPACE_KUBE_SYSTEM, secret) LOG.info("Secret registry-local-secret under Namespace kube-system is updated") except Exception as e: @@ -1015,9 +1015,9 @@ class AppOperator(object): continue try: - secret_auth_body = base64.b64decode(secret.data['.dockerconfigjson']) + secret_auth_body = base64.decode_as_text(secret.data['.dockerconfigjson']) if constants.DOCKER_REGISTRY_SERVER in secret_auth_body: - secret.data['.dockerconfigjson'] = base64.b64encode(token) + secret.data['.dockerconfigjson'] = base64.encode_as_text(token) self._kube.kube_patch_secret(AppOperator.DOCKER_REGISTRY_SECRET, ns, secret) LOG.info("Secret %s under Namespace %s is updated" % (AppOperator.DOCKER_REGISTRY_SECRET, ns)) diff --git a/sysinv/sysinv/sysinv/sysinv/conductor/manager.py b/sysinv/sysinv/sysinv/sysinv/conductor/manager.py index 213802bb9a..937f7d650a 100644 --- a/sysinv/sysinv/sysinv/sysinv/conductor/manager.py +++ b/sysinv/sysinv/sysinv/sysinv/conductor/manager.py @@ -29,7 +29,6 @@ collection of inventory data for each host. """ -import base64 import errno import filecmp import fnmatch @@ -70,6 +69,7 @@ from netaddr import IPAddress from netaddr import IPNetwork from oslo_config import cfg from oslo_log import log +from oslo_serialization import base64 from oslo_serialization import jsonutils from oslo_utils import excutils from oslo_utils import timeutils @@ -12410,8 +12410,8 @@ class ConductorManager(service.PeriodicService): sc_endpoint_cert_secret_ns, sc_intermediate_ca_secret_name )) - tls_key = base64.b64encode(sc_ca_key) - tls_crt = base64.b64encode(sc_ca_cert) + tls_key = base64.encode_as_text(sc_ca_key) + tls_crt = base64.encode_as_text(sc_ca_cert) if tls_key == secret.data['tls.key'] and tls_crt == secret.data['tls.crt']: LOG.info('Intermediate CA cert is not changed') return @@ -14358,7 +14358,7 @@ class ConductorManager(service.PeriodicService): return dict(success="", error=msg) data = secret.data - tls_crt = base64.b64decode(data['tls.crt']) + tls_crt = base64.decode_as_bytes(data['tls.crt']) cert = cutils.extract_certs_from_pem(tls_crt)[0] # extract information regarding the new rootca diff --git a/sysinv/sysinv/sysinv/sysinv/puppet/platform.py b/sysinv/sysinv/sysinv/sysinv/puppet/platform.py index b647b715a4..537926f111 100644 --- a/sysinv/sysinv/sysinv/sysinv/puppet/platform.py +++ b/sysinv/sysinv/sysinv/sysinv/puppet/platform.py @@ -6,6 +6,7 @@ import keyring import os +from oslo_serialization import base64 from sysinv.common import constants from sysinv.common import exception from sysinv.common import utils @@ -522,7 +523,7 @@ class PlatformPuppet(base.BasePuppet): # convert these back to their native encoding encoded_files = tpm_data.pop("base64_encoded_files", []) for binary in encoded_files: - tpm_data[binary] = tpm_data[binary].decode('base64') + tpm_data[binary] = base64.decode_as_text(tpm_data[binary]) config.update({ 'platform::tpm::tpm_data': tpm_data }) diff --git a/sysinv/sysinv/sysinv/sysinv/tests/conductor/test_manager.py b/sysinv/sysinv/sysinv/sysinv/tests/conductor/test_manager.py index 82f8a38446..e09d8d1677 100644 --- a/sysinv/sysinv/sysinv/sysinv/tests/conductor/test_manager.py +++ b/sysinv/sysinv/sysinv/sysinv/tests/conductor/test_manager.py @@ -22,7 +22,6 @@ """Test class for Sysinv ManagerService.""" -import base64 import mock import os.path import tsconfig.tsconfig as tsc @@ -31,6 +30,7 @@ import uuid from cryptography import x509 from cryptography.hazmat.backends import default_backend +from oslo_serialization import base64 from sysinv.agent import rpcapi as agent_rpcapi from sysinv.common import constants from sysinv.common import device as dconstants @@ -48,7 +48,7 @@ from sysinv.tests.db import utils class FakeSecret(object): def __init__(self, crt): - self.data = {'tls.crt': base64.b64encode(crt)} + self.data = {'tls.crt': base64.encode_as_text(crt)} class FakeCephOperator(object):