starlingx
/
config
Code Issues Proposed changes

py3: Fix base64 usage 

b64encode and b64decode returns a str in Python2 and a
bytes in Python3. This is a problem when using http/url/rest
libraries. Runtime errors are raised. Mixing str and bytes when
formatting a text might introduce an unwanted "b" for bytes, which can
lead to potential issues when sent over the network.

To keep compatibility use oslo_serialization to force the
return type to str.
There is one place where base64 urlsafe version is specifically used to
send and receive(Rest API for application upload).

One of the tests is that platform-integ-app applies which exercises part
of the changes.
Cert-mon and DC part will be exercied when DC is available on f/centos8
branch.

Story: 2006796
Task: 42797
Signed-off-by: Dan Voiculeasa <dan.voiculeasa@windriver.com>
Change-Id: I48b1c6c80363458945c6bc1a9cf7e16c743a7bd6
(cherry picked from commit 8a7c4b15c7)
This commit is contained in:
Dan Voiculeasa 2021-07-08 00:26:26 +03:00 committed by Charles Short
parent 3bc4153c17
commit 1518543bf7
9 changed files with 37 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
sysinv/sysinv/sysinv/sysinv/cert_mon/certificate_mon_manager.py
View File

@ -17,11 +17,11 @@
# of this software may be licensed only pursuant to the terms # of this software may be licensed only pursuant to the terms
# of an applicable Wind River license agreement. # of an applicable Wind River license agreement.
# #
import base64
from eventlet import greenthread from eventlet import greenthread
import greenlet import greenlet
from oslo_config import cfg from oslo_config import cfg
from oslo_log import log from oslo_log import log
from oslo_serialization import base64
from oslo_service import periodic_task from oslo_service import periodic_task
import time import time
@ -137,9 +137,9 @@ class CertificateMonManager(periodic_task.PeriodicTasks):
raise Exception('%s certificate data missing: %s' raise Exception('%s certificate data missing: %s'
% (subcloud_name, item)) % (subcloud_name, item))
txt_ssl_cert = base64.b64decode(secret.data['tls.crt']) txt_ssl_cert = base64.decode_as_text(secret.data['tls.crt'])
txt_ssl_key = base64.b64decode(secret.data['tls.key']) txt_ssl_key = base64.decode_as_text(secret.data['tls.key'])
txt_ca_cert = base64.b64decode(secret.data['ca.crt']) txt_ca_cert = base64.decode_as_text(secret.data['ca.crt'])
except Exception as e: except Exception as e:
LOG.error('Cannot audit ssl certificate on %s' % subcloud_name) LOG.error('Cannot audit ssl certificate on %s' % subcloud_name)
LOG.exception(e) LOG.exception(e)

6
sysinv/sysinv/sysinv/sysinv/cert_mon/utils.py
View File

@ -17,7 +17,6 @@
# of this software may be licensed only pursuant to the terms # of this software may be licensed only pursuant to the terms
# of an applicable Wind River license agreement. # of an applicable Wind River license agreement.
# #
import base64
import json import json
import os import os
import re import re
@ -32,6 +31,7 @@ from keystoneclient.auth.identity import v3
from oslo_config import cfg from oslo_config import cfg
from oslo_log import log from oslo_log import log
from oslo_utils import encodeutils from oslo_utils import encodeutils
from oslo_serialization import base64
from six.moves.urllib.request import Request from six.moves.urllib.request import Request
from six.moves.urllib.error import HTTPError from six.moves.urllib.error import HTTPError
from six.moves.urllib.error import URLError from six.moves.urllib.error import URLError
@ -109,8 +109,8 @@ def verify_adminep_cert_chain():
raise Exception('%s tls.crt data missing' raise Exception('%s tls.crt data missing'
% (constants.SC_ADMIN_ENDPOINT_SECRET_NAME)) % (constants.SC_ADMIN_ENDPOINT_SECRET_NAME))
txt_ca_crt = base64.b64decode(secret_ica.data['tls.crt']) txt_ca_crt = base64.decode_as_text(secret_ica.data['tls.crt'])
txt_tls_crt = base64.b64decode(secret_adminep.data['tls.crt']) txt_tls_crt = base64.decode_as_text(secret_adminep.data['tls.crt'])
with tempfile.NamedTemporaryFile() as ca_tmpfile: with tempfile.NamedTemporaryFile() as ca_tmpfile:
ca_tmpfile.write(txt_ca_crt) ca_tmpfile.write(txt_ca_crt)

8
sysinv/sysinv/sysinv/sysinv/cert_mon/watcher.py
View File

@ -17,7 +17,6 @@
# of this software may be licensed only pursuant to the terms # of this software may be licensed only pursuant to the terms
# of an applicable Wind River license agreement. # of an applicable Wind River license agreement.
# #
import base64
import re import re
import hashlib import hashlib
from datetime import datetime from datetime import datetime
@ -29,6 +28,7 @@ from kubernetes import config
import os import os
from oslo_config import cfg from oslo_config import cfg
from oslo_log import log from oslo_log import log
from oslo_serialization import base64
from six.moves.urllib.error import URLError from six.moves.urllib.error import URLError
from sysinv.cert_mon import utils from sysinv.cert_mon import utils
@ -93,11 +93,11 @@ class CertUpdateEventData(object):
self.tls_crt = None self.tls_crt = None
self.tls_key = None self.tls_key = None
try: try:
self.ca_crt = base64.b64decode(data['ca.crt']).strip() \ self.ca_crt = base64.decode_as_text(data['ca.crt']).strip() \
if 'ca.crt' in data else '' if 'ca.crt' in data else ''
self.tls_crt = base64.b64decode(data['tls.crt']).strip() \ self.tls_crt = base64.decode_as_text(data['tls.crt']).strip() \
if 'tls.crt' in data else '' if 'tls.crt' in data else ''
self.tls_key = base64.b64decode(data['tls.key']).strip() \ self.tls_key = base64.decode_as_text(data['tls.key']).strip() \
if 'tls.key' in data else '' if 'tls.key' in data else ''
except TypeError: except TypeError:
LOG.error('Invalid secret data.') LOG.error('Invalid secret data.')

18
sysinv/sysinv/sysinv/sysinv/common/utils.py
View File

@ -25,7 +25,6 @@
"""Utilities and helper functions.""" """Utilities and helper functions."""
import ast import ast
import base64
import boto3 import boto3
from botocore.config import Config from botocore.config import Config
import collections import collections
@ -70,6 +69,7 @@ import netaddr
from oslo_concurrency import lockutils from oslo_concurrency import lockutils
from oslo_config import cfg from oslo_config import cfg
from oslo_log import log as logging from oslo_log import log as logging
from oslo_serialization import base64
from fm_api import constants as fm_constants from fm_api import constants as fm_constants
@ -1759,7 +1759,7 @@ def read_filtered_directory_content(dirpath, *filters):
# they can be transferred over RPC and stored in DB # they can be transferred over RPC and stored in DB
content.decode('utf-8') content.decode('utf-8')
except UnicodeError: except UnicodeError:
content = content.encode('base64') content = base64.encode_as_text(content)
content_dict['base64_encoded_files'] = \ content_dict['base64_encoded_files'] = \
content_dict.get("base64_encoded_files", []) + [filename] content_dict.get("base64_encoded_files", []) + [filename]
@ -2578,7 +2578,7 @@ def get_aws_ecr_registry_credentials(dbapi, registry, username, password):
response = client.get_authorization_token() response = client.get_authorization_token()
token = response['authorizationData'][0]['authorizationToken'] token = response['authorizationData'][0]['authorizationToken']
username, password = token.decode('base64').split(':') username, password = base64.decode_as_text(token).split(':')
except Exception as e: except Exception as e:
raise exception.SysinvException(_( raise exception.SysinvException(_(
"Failed to get AWS ECR credentials: %s" % e)) "Failed to get AWS ECR credentials: %s" % e))
@ -2602,7 +2602,7 @@ def extract_ca_private_key_bytes_from_pem(pem_content):
raise exception.InvalidKubernetesCA raise exception.InvalidKubernetesCA
end_search += len(constants.END_PRIVATE_KEY_MARKER) end_search += len(constants.END_PRIVATE_KEY_MARKER)
base64_key = base64.b64encode(pem_content[begin_search:end_search]) base64_key = base64.encode_as_text(pem_content[begin_search:end_search])
return base64_key return base64_key
@ -2621,7 +2621,7 @@ def extract_ca_crt_bytes_from_pem(pem_content):
raise exception.InvalidKubernetesCA raise exception.InvalidKubernetesCA
end_search += len(constants.END_CERTIFICATE_MARKER) end_search += len(constants.END_CERTIFICATE_MARKER)
base64_crt = base64.b64encode(pem_content[begin_search:end_search]) base64_crt = base64.encode_as_text(pem_content[begin_search:end_search])
return base64_crt return base64_crt
@ -2924,8 +2924,8 @@ def get_admin_ep_cert(dc_role):
raise Exception("Invalid admin endpoint certificate data.") raise Exception("Invalid admin endpoint certificate data.")
try: try:
tls_crt = base64.b64decode(data['tls.crt']) tls_crt = base64.decode_as_text(data['tls.crt'])
tls_key = base64.b64decode(data['tls.key']) tls_key = base64.decode_as_text(data['tls.key'])
except TypeError: except TypeError:
raise Exception('admin endpoint secret is invalid %s' % raise Exception('admin endpoint secret is invalid %s' %
endpoint_cert_secret_name) endpoint_cert_secret_name)
@ -2944,7 +2944,7 @@ def get_admin_ep_cert(dc_role):
LOG.error('Cannot read DC root CA certificate %s' % e) LOG.error('Cannot read DC root CA certificate %s' % e)
elif dc_role == constants.DISTRIBUTED_CLOUD_ROLE_SYSTEMCONTROLLER: elif dc_role == constants.DISTRIBUTED_CLOUD_ROLE_SYSTEMCONTROLLER:
try: try:
ca_crt = base64.b64decode(data['ca.crt']) ca_crt = base64.decode_as_text(data['ca.crt'])
except TypeError: except TypeError:
raise Exception('admin endpoint secret is invalid %s' % raise Exception('admin endpoint secret is invalid %s' %
endpoint_cert_secret_name) endpoint_cert_secret_name)
@ -3001,7 +3001,7 @@ def get_root_ca_cert():
data = secret.data data = secret.data
try: try:
ca_crt = base64.b64decode(data['ca.crt']) ca_crt = base64.decode_as_text(data['ca.crt'])
except TypeError: except TypeError:
raise Exception('Secret is invalid %s' % secret_name) raise Exception('Secret is invalid %s' % secret_name)

4
sysinv/sysinv/sysinv/sysinv/conductor/docker_registry.py
View File

@ -4,10 +4,10 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# #
import base64
import keyring import keyring
import requests import requests
from oslo_serialization import base64
from sysinv.common import constants from sysinv.common import constants
from sysinv.common import exception from sysinv.common import exception
@ -68,7 +68,7 @@ def docker_registry_authenticate(www_authenticate):
# make a request to the token server # make a request to the token server
# the credentials are passed as a header while the rest # the credentials are passed as a header while the rest
# are passed as params # are passed as params
auth_string = base64.b64encode("%s:%s" % (REGISTRY_USERNAME, get_registry_password())) auth_string = base64.encode_as_text("%s:%s" % (REGISTRY_USERNAME, get_registry_password()))
token_server_request_headers = {"authorization": "Basic %s" % auth_string} token_server_request_headers = {"authorization": "Basic %s" % auth_string}
# we need try twice. # we need try twice.
# SYSTEM_CERT_PATH if the docker registry cert is signed by a trusted CA # SYSTEM_CERT_PATH if the docker registry cert is signed by a trusted CA

14
sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py
View File

@ -9,7 +9,6 @@
""" System Inventory Kubernetes Application Operator.""" """ System Inventory Kubernetes Application Operator."""
import base64
import copy import copy
import docker import docker
from eventlet.green import subprocess from eventlet.green import subprocess
@ -38,6 +37,7 @@ from eventlet import Timeout
from fm_api import constants as fm_constants from fm_api import constants as fm_constants
from fm_api import fm_api from fm_api import fm_api
from oslo_log import log as logging from oslo_log import log as logging
from oslo_serialization import base64
from sysinv._i18n import _ from sysinv._i18n import _
from sysinv.api.controllers.v1 import kube_app from sysinv.api.controllers.v1 import kube_app
from sysinv.common import constants from sysinv.common import constants
@ -983,12 +983,12 @@ class AppOperator(object):
secret = self._kube.kube_get_secret("registry-local-secret", kubernetes.NAMESPACE_KUBE_SYSTEM) secret = self._kube.kube_get_secret("registry-local-secret", kubernetes.NAMESPACE_KUBE_SYSTEM)
if secret is None: if secret is None:
return return
secret_auth_body = base64.b64decode(secret.data['.dockerconfigjson']) secret_auth_body = base64.decode_as_text(secret.data['.dockerconfigjson'])
secret_auth_info = (secret_auth_body.split('auth":')[1]).split('"')[1] secret_auth_info = (secret_auth_body.split('auth":')[1]).split('"')[1]
registry_auth = cutils.get_local_docker_registry_auth() registry_auth = cutils.get_local_docker_registry_auth()
registry_auth_info = '{0}:{1}'.format(registry_auth['username'], registry_auth_info = '{0}:{1}'.format(registry_auth['username'],
registry_auth['password']) registry_auth['password'])
if secret_auth_info == base64.b64encode(registry_auth_info): if secret_auth_info == base64.encode_as_text(registry_auth_info):
LOG.debug("Auth info is the same, no update is needed for k8s secret.") LOG.debug("Auth info is the same, no update is needed for k8s secret.")
return return
except Exception as e: except Exception as e:
@ -997,8 +997,8 @@ class AppOperator(object):
try: try:
# update secret with new auth info # update secret with new auth info
token = '{{\"auths\": {{\"{0}\": {{\"auth\": \"{1}\"}}}}}}'.format( token = '{{\"auths\": {{\"{0}\": {{\"auth\": \"{1}\"}}}}}}'.format(
constants.DOCKER_REGISTRY_SERVER, base64.b64encode(registry_auth_info)) constants.DOCKER_REGISTRY_SERVER, base64.encode_as_text(registry_auth_info))
secret.data['.dockerconfigjson'] = base64.b64encode(token) secret.data['.dockerconfigjson'] = base64.encode_as_text(token)
self._kube.kube_patch_secret("registry-local-secret", kubernetes.NAMESPACE_KUBE_SYSTEM, secret) self._kube.kube_patch_secret("registry-local-secret", kubernetes.NAMESPACE_KUBE_SYSTEM, secret)
LOG.info("Secret registry-local-secret under Namespace kube-system is updated") LOG.info("Secret registry-local-secret under Namespace kube-system is updated")
except Exception as e: except Exception as e:
@ -1015,9 +1015,9 @@ class AppOperator(object):
continue continue
try: try:
secret_auth_body = base64.b64decode(secret.data['.dockerconfigjson']) secret_auth_body = base64.decode_as_text(secret.data['.dockerconfigjson'])
if constants.DOCKER_REGISTRY_SERVER in secret_auth_body: if constants.DOCKER_REGISTRY_SERVER in secret_auth_body:
secret.data['.dockerconfigjson'] = base64.b64encode(token) secret.data['.dockerconfigjson'] = base64.encode_as_text(token)
self._kube.kube_patch_secret(AppOperator.DOCKER_REGISTRY_SECRET, ns, secret) self._kube.kube_patch_secret(AppOperator.DOCKER_REGISTRY_SECRET, ns, secret)
LOG.info("Secret %s under Namespace %s is updated" LOG.info("Secret %s under Namespace %s is updated"
% (AppOperator.DOCKER_REGISTRY_SECRET, ns)) % (AppOperator.DOCKER_REGISTRY_SECRET, ns))

8
sysinv/sysinv/sysinv/sysinv/conductor/manager.py
View File

@ -29,7 +29,6 @@ collection of inventory data for each host.
""" """
import base64
import errno import errno
import filecmp import filecmp
import fnmatch import fnmatch
@ -70,6 +69,7 @@ from netaddr import IPAddress
from netaddr import IPNetwork from netaddr import IPNetwork
from oslo_config import cfg from oslo_config import cfg
from oslo_log import log from oslo_log import log
from oslo_serialization import base64
from oslo_serialization import jsonutils from oslo_serialization import jsonutils
from oslo_utils import excutils from oslo_utils import excutils
from oslo_utils import timeutils from oslo_utils import timeutils
@ -12410,8 +12410,8 @@ class ConductorManager(service.PeriodicService):
sc_endpoint_cert_secret_ns, sc_intermediate_ca_secret_name sc_endpoint_cert_secret_ns, sc_intermediate_ca_secret_name
)) ))
tls_key = base64.b64encode(sc_ca_key) tls_key = base64.encode_as_text(sc_ca_key)
tls_crt = base64.b64encode(sc_ca_cert) tls_crt = base64.encode_as_text(sc_ca_cert)
if tls_key == secret.data['tls.key'] and tls_crt == secret.data['tls.crt']: if tls_key == secret.data['tls.key'] and tls_crt == secret.data['tls.crt']:
LOG.info('Intermediate CA cert is not changed') LOG.info('Intermediate CA cert is not changed')
return return
@ -14358,7 +14358,7 @@ class ConductorManager(service.PeriodicService):
return dict(success="", error=msg) return dict(success="", error=msg)
data = secret.data data = secret.data
tls_crt = base64.b64decode(data['tls.crt']) tls_crt = base64.decode_as_bytes(data['tls.crt'])
cert = cutils.extract_certs_from_pem(tls_crt)[0] cert = cutils.extract_certs_from_pem(tls_crt)[0]
# extract information regarding the new rootca # extract information regarding the new rootca

3
sysinv/sysinv/sysinv/sysinv/puppet/platform.py
View File

@ -6,6 +6,7 @@
import keyring import keyring
import os import os
from oslo_serialization import base64
from sysinv.common import constants from sysinv.common import constants
from sysinv.common import exception from sysinv.common import exception
from sysinv.common import utils from sysinv.common import utils
@ -522,7 +523,7 @@ class PlatformPuppet(base.BasePuppet):
# convert these back to their native encoding # convert these back to their native encoding
encoded_files = tpm_data.pop("base64_encoded_files", []) encoded_files = tpm_data.pop("base64_encoded_files", [])
for binary in encoded_files: for binary in encoded_files:
tpm_data[binary] = tpm_data[binary].decode('base64') tpm_data[binary] = base64.decode_as_text(tpm_data[binary])
config.update({ config.update({
'platform::tpm::tpm_data': tpm_data 'platform::tpm::tpm_data': tpm_data
}) })

4
sysinv/sysinv/sysinv/sysinv/tests/conductor/test_manager.py
View File

@ -22,7 +22,6 @@
"""Test class for Sysinv ManagerService.""" """Test class for Sysinv ManagerService."""
import base64
import mock import mock
import os.path import os.path
import tsconfig.tsconfig as tsc import tsconfig.tsconfig as tsc
@ -31,6 +30,7 @@ import uuid
from cryptography import x509 from cryptography import x509
from cryptography.hazmat.backends import default_backend from cryptography.hazmat.backends import default_backend
from oslo_serialization import base64
from sysinv.agent import rpcapi as agent_rpcapi from sysinv.agent import rpcapi as agent_rpcapi
from sysinv.common import constants from sysinv.common import constants
from sysinv.common import device as dconstants from sysinv.common import device as dconstants
@ -48,7 +48,7 @@ from sysinv.tests.db import utils
class FakeSecret(object): class FakeSecret(object):
def __init__(self, crt): def __init__(self, crt):
self.data = {'tls.crt': base64.b64encode(crt)} self.data = {'tls.crt': base64.encode_as_text(crt)}
class FakeCephOperator(object): class FakeCephOperator(object):