From 81db20a48d4b7123f281842f6900c80f29d0237f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20Jord=C3=A3o=20Jardim?= Date: Tue, 6 Apr 2021 07:44:06 -0400 Subject: [PATCH] Python 2 to Python 3 compatibility The changes of https://review.opendev.org/c/starlingx/config/+/782575 was used to test the cgts-client, so it is need to be merged first. Removing python-neutronclient because this dependency is unnecessary, it was removed by copying a few very small utility functions from python-neutronclient into the cgtsclient. Development: When I was trying to find things to modify I followed the approach of build the client, get the tar file, I set up 2 environments one based on python2 and another python3, I installed the tar client in both environments and i exported the env vars that the client expect to get to request the controller, and doing that I could switch between the two python and indentifying what I should modify. Test: After all the modification I built an ISO and I installed that to run some commands and check if my changes got any side effects. After that followed the procedure to update the remote CLI docker image and insert the updated client there and I test this new image in the remote CLI. Story: 2007106 Task: 42268 Depends-On: I5086832605752bdb00a40a24596494c8fd987692 Signed-off-by: Rafael Jardim Change-Id: Ibf919260693f1cbe99993d1de01ecf785d604839 (cherry picked from commit bc34618d83a5e38d765cbde0a91ebf384b8857b8) --- sysinv/cgts-client/centos/cgts-client.spec | 3 +- .../cgts-client/cgtsclient/common/http.py | 44 +++++++++++++++---- .../cgts-client/cgts-client/requirements.txt | 2 +- sysinv/cgts-client/debian/control | 4 +- sysinv/cgts-client/opensuse/cgts-client.spec | 2 +- 5 files changed, 41 insertions(+), 14 deletions(-) diff --git a/sysinv/cgts-client/centos/cgts-client.spec b/sysinv/cgts-client/centos/cgts-client.spec index 011793c7ef..ededc56e74 100644 --- a/sysinv/cgts-client/centos/cgts-client.spec +++ b/sysinv/cgts-client/centos/cgts-client.spec @@ -16,12 +16,13 @@ BuildRequires: python3-wheel Requires: python3-httplib2 Requires: python3-prettytable Requires: bash-completion -Requires: python3-neutronclient +Requires: python3-dateutil Requires: python3-keystoneclient Requires: python3-oslo-i18n Requires: python3-oslo-serialization Requires: python3-oslo-utils Requires: python3-requests-toolbelt + # Needed for python2 and python3 compatible Requires: python3-six diff --git a/sysinv/cgts-client/cgts-client/cgtsclient/common/http.py b/sysinv/cgts-client/cgts-client/cgtsclient/common/http.py index 6f9a855273..c6902055b6 100644 --- a/sysinv/cgts-client/cgts-client/cgtsclient/common/http.py +++ b/sysinv/cgts-client/cgts-client/cgtsclient/common/http.py @@ -1,4 +1,4 @@ -# Copyright 2013, 2017 Wind River, Inc. +# Copyright 2013-2021 Wind River, Inc. # Copyright 2012 Openstack Foundation # All Rights Reserved. # @@ -15,9 +15,11 @@ # under the License. # +import hashlib import httplib2 import logging import os +from oslo_utils import encodeutils import requests from requests_toolbelt import MultipartEncoder import socket @@ -37,11 +39,11 @@ except ImportError: import simplejson as json from cgtsclient import exc as exceptions -from neutronclient.common import utils _logger = logging.getLogger(__name__) CHUNKSIZE = 1024 * 64 # 64kB +SENSITIVE_HEADERS = ('X-Auth-Token',) # httplib2 retries requests on socket.timeout which # is not idempotent and can lead to orhan objects. @@ -156,6 +158,32 @@ class HTTPClient(httplib2.Http): 'headers': resp_headers, 'body': body}) + @staticmethod + def http_log_req(_logger, args, kwargs): + if not _logger.isEnabledFor(logging.DEBUG): + return + + string_parts = ['curl -i'] + for element in args: + if element in ('GET', 'POST', 'DELETE', 'PUT'): + string_parts.append(' -X %s' % element) + else: + string_parts.append(' %s' % element) + + for (key, value) in kwargs['headers'].items(): + if key in SENSITIVE_HEADERS: + v = value.encode('utf-8') + h = hashlib.sha256(v) + d = h.hexdigest() + value = "{SHA256}%s" % d + header = ' -H "%s: %s"' % (key, value) + string_parts.append(header) + + if 'body' in kwargs and kwargs['body']: + string_parts.append(" -d '%s'" % (kwargs['body'])) + req = encodeutils.safe_encode("".join(string_parts)) + _logger.debug("REQ: %s", req) + def _cs_request(self, *args, **kwargs): kargs = {} kargs.setdefault('headers', kwargs.get('headers', {})) @@ -172,24 +200,22 @@ class HTTPClient(httplib2.Http): if 'body' in kwargs: kargs['body'] = kwargs['body'] - args = utils.safe_encode_list(args) - kargs = utils.safe_encode_dict(kargs) if self.log_credentials: log_kargs = kargs else: log_kargs = self._strip_credentials(kargs) - utils.http_log_req(_logger, args, log_kargs) + self.http_log_req(_logger, args, log_kargs) try: resp, body = self.request(*args, **kargs) - except httplib2.SSLHandshakeError as e: - raise exceptions.SslCertificateValidationError(reason=e) + except requests.exceptions.SSLError as e: + raise exceptions.SslCertificateValidationError(reason=str(e)) except Exception as e: # Wrap the low-level connection error (socket timeout, redirect # limit, decompression error, etc) into our custom high-level # connection exception (it is excepted in the upper layers of code) _logger.debug("throwing ConnectionFailed : %s", e) - raise exceptions.CommunicationError(e) + raise exceptions.CommunicationError(str(e)) finally: # Temporary Fix for gate failures. RPC calls and HTTP requests # seem to be stepping on each other resulting in bogus fd's being @@ -199,7 +225,7 @@ class HTTPClient(httplib2.Http): # Read body into string if it isn't obviously image data body_str = None if 'content-type' in resp and resp['content-type'] != 'application/octet-stream': - body_str = ''.join([chunk for chunk in body]) + body_str = ''.join([chunk for chunk in body.decode('utf8')]) self.http_log_resp(_logger, resp, body_str) body = body_str else: diff --git a/sysinv/cgts-client/cgts-client/requirements.txt b/sysinv/cgts-client/cgts-client/requirements.txt index b4859f301e..107d5d1d90 100644 --- a/sysinv/cgts-client/cgts-client/requirements.txt +++ b/sysinv/cgts-client/cgts-client/requirements.txt @@ -1,6 +1,6 @@ -python-neutronclient keyring oslo.i18n # Apache-2.0 oslo.serialization>=1.10.0,!=2.19.1 # Apache-2.0 oslo.utils>=3.5.0 # Apache-2.0 requests-toolbelt +python-dateutil diff --git a/sysinv/cgts-client/debian/control b/sysinv/cgts-client/debian/control index f0b0b44914..297bb392d0 100644 --- a/sysinv/cgts-client/debian/control +++ b/sysinv/cgts-client/debian/control @@ -8,9 +8,9 @@ Build-Depends: python-setuptools, python-pbr, python-keystoneclient, python-fixtures, + python-dateutil, python-httplib2, python-dateutil, - python-neutronclient, python-six, python-mock Standards-Version: 3.9.6 @@ -23,7 +23,7 @@ Depends: ${misc:Depends}, python-httplib2, python-prettytable, bash-completion, - python-neutronclient, python-keystoneclient, + python-dateutil, python-six Description: This package contains the cgts-client project. diff --git a/sysinv/cgts-client/opensuse/cgts-client.spec b/sysinv/cgts-client/opensuse/cgts-client.spec index 5a9146c241..f6d6d13fad 100644 --- a/sysinv/cgts-client/opensuse/cgts-client.spec +++ b/sysinv/cgts-client/opensuse/cgts-client.spec @@ -14,8 +14,8 @@ BuildRequires: fdupes Requires: python-httplib2 Requires: python-prettytable Requires: bash-completion -Requires: python-neutronclient Requires: python-keystoneclient +Requires: python-dateutil # Needed for python2 and python3 compatible Requires: python-six