From c876311111e1606350bccaba99f13e794da194dc Mon Sep 17 00:00:00 2001 From: Luz Cazares Date: Wed, 21 Jun 2017 00:46:31 +0000 Subject: [PATCH] Add python35 support To be aligned with community goals Add python35 support to refstack-client Change-Id: I42ccc6128fc6a83ddee0e4014cbe2c8793b84012 --- refstack_client/list_parser.py | 2 +- refstack_client/refstack_client.py | 52 ++++++++++-------- refstack_client/scripts/prep_cloud.py | 1 + refstack_client/tests/smoke/test_distros.py | 1 + refstack_client/tests/unit/test_client.py | 54 ++++++++++--------- .../tests/unit/test_list_parser.py | 3 +- setup.cfg | 3 +- setup_env | 27 ++++++---- tox.ini | 3 +- 9 files changed, 83 insertions(+), 63 deletions(-) diff --git a/refstack_client/list_parser.py b/refstack_client/list_parser.py index 12e20bc..5106a2a 100644 --- a/refstack_client/list_parser.py +++ b/refstack_client/list_parser.py @@ -150,7 +150,7 @@ class TestListParser(object): """ temp = tempfile.NamedTemporaryFile(delete=False) for test_id in test_ids: - temp.write("%s\n" % test_id) + temp.write(("%s\n" % test_id).encode('utf-8')) temp.flush() # Register the created file for cleanup. diff --git a/refstack_client/refstack_client.py b/refstack_client/refstack_client.py index 69b6bd9..927a4ab 100755 --- a/refstack_client/refstack_client.py +++ b/refstack_client/refstack_client.py @@ -24,9 +24,10 @@ Tempest configuration file. """ +from __future__ import absolute_import + import argparse import binascii -import ConfigParser import hashlib import itertools import json @@ -43,10 +44,10 @@ from cryptography.hazmat.primitives.asymmetric import padding import requests import requests.exceptions -import six.moves +from six import moves from six.moves.urllib import parse -from subunit_processor import SubunitProcessor -from list_parser import TestListParser +from refstack_client.subunit_processor import SubunitProcessor +from refstack_client.list_parser import TestListParser import yaml @@ -54,7 +55,7 @@ def get_input(): """ Wrapper for raw_input. Necessary for testing. """ - return raw_input().lower() # pragma: no cover + return moves.input().lower() # pragma: no cover def read_accounts_yaml(path): @@ -83,7 +84,7 @@ class RefstackClient: # set default log level to INFO. if self.args.silent: self.logger.setLevel(logging.WARNING) - elif self.args.verbose > 0: + elif self.args.verbose: self.logger.setLevel(logging.DEBUG) else: self.logger.setLevel(logging.INFO) @@ -113,7 +114,9 @@ class RefstackClient: exit(1) self.conf_file = self.args.conf_file - self.conf = ConfigParser.SafeConfigParser() + # Note: SafeConfigParser deprecated on Python 3.2 + # Use ConfigParser directly + self.conf = moves.configparser.ConfigParser() self.conf.read(self.args.conf_file) def _prep_upload(self): @@ -150,10 +153,10 @@ class RefstackClient: # Prefer Keystone V3 API if it is enabled auth_version = ( 'v3' if (conf_file.has_option('identity-feature-enabled', - 'api_v3') - and conf_file.getboolean('identity-feature-enabled', - 'api_v3') - and conf_file.has_option('identity', 'uri_v3')) + 'api_v3') and + conf_file.getboolean('identity-feature-enabled', + 'api_v3') and + conf_file.has_option('identity', 'uri_v3')) else 'v2') if auth_version == 'v2': auth_url = '%s/tokens' % (conf_file.get('identity', 'uri') @@ -217,7 +220,7 @@ class RefstackClient: 'configuration guide (http://docs.openstack.' 'org/developer/tempest/configuration.html).') exit(1) - except ConfigParser.Error as e: + except moves.configparser.Error as e: # Most likely a missing section or option in the config file. self.logger.error("Invalid Config File: %s" % e) exit(1) @@ -246,12 +249,15 @@ class RefstackClient: } return auth_version, auth_url, data elif auth_version == 'v3': - identity = {'methods': ['password'], 'password': - {'user': {'name': auth_config['username'], - 'domain': { - 'name': auth_config['domain_name'] - }, - 'password': auth_config['password']}}} + identity = { + 'methods': ['password'], + 'password': { + 'user': { + 'name': auth_config['username'], + 'domain': {'name': auth_config['domain_name']}, + 'password': auth_config['password'] + }}} + data = { 'auth': { 'identity': identity, @@ -297,7 +303,7 @@ class RefstackClient: message = ('Invalid request with error ' 'code: %s. Error message: %s' '' % (rsp['error']['code'], - rsp['error']['message'])) + rsp['error']['message'])) raise requests.exceptions.HTTPError(message) # If a Key or Index Error was raised, one of the expected keys or # indices for retrieving the identity service ID was not found. @@ -321,7 +327,7 @@ class RefstackClient: raise ValueError('Invalid Keystone endpoint format. Make sure ' 'the endpoint (%s) includes the URL scheme ' '(i.e. http/https).' % endpoint) - return hashlib.md5(url_parts.hostname).hexdigest() + return hashlib.md5(url_parts.hostname.encode('utf-8')).hexdigest() def _form_result_content(self, cpid, duration, results): '''This method will create the content for the request. The spec at @@ -345,7 +351,7 @@ class RefstackClient: if self.args.quiet: return True try: - inp = six.moves.input(q + ' (yes/y): ') + inp = moves.input(q + ' (yes/y): ') except KeyboardInterrupt: return else: @@ -403,7 +409,7 @@ class RefstackClient: if response.status_code == 201: resp = response.json() - print 'Test results uploaded!\nURL: %s' % resp.get('url', '') + print('Test results uploaded!\nURL: %s' % resp.get('url', '')) def test(self): '''Execute Tempest test against the cloud.''' @@ -566,7 +572,7 @@ class RefstackClient: for r in page_of_results: print('%s - %s' % (r['created_at'], r['url'])) try: - six.moves.input('Press Enter to go to next page...') + moves.input('Press Enter to go to next page...') except KeyboardInterrupt: return diff --git a/refstack_client/scripts/prep_cloud.py b/refstack_client/scripts/prep_cloud.py index 618b03c..0811f1a 100644 --- a/refstack_client/scripts/prep_cloud.py +++ b/refstack_client/scripts/prep_cloud.py @@ -392,5 +392,6 @@ def main(): # TODO(tkammer): add network implementation + if __name__ == "__main__": main() diff --git a/refstack_client/tests/smoke/test_distros.py b/refstack_client/tests/smoke/test_distros.py index 942e63b..e2f31b3 100644 --- a/refstack_client/tests/smoke/test_distros.py +++ b/refstack_client/tests/smoke/test_distros.py @@ -56,5 +56,6 @@ class TestSequenceFunctions(unittest.TestCase): distro_image = 'opensuse/13.2' self.run_test(distro_image) + if __name__ == '__main__': unittest.main() diff --git a/refstack_client/tests/unit/test_client.py b/refstack_client/tests/unit/test_client.py index aa43667..f01c68d 100755 --- a/refstack_client/tests/unit/test_client.py +++ b/refstack_client/tests/unit/test_client.py @@ -25,7 +25,7 @@ import mock from mock import MagicMock import unittest -import refstack_client.refstack_client as rc +from refstack_client import refstack_client as rc class TestRefstackClient(unittest.TestCase): @@ -356,11 +356,13 @@ class TestRefstackClient(unittest.TestCase): client._get_cpid_from_keystone(auth_version, auth_url, content) client._generate_cpid_from_endpoint.assert_called_with(auth_url) - #Test when catalog has other non-identity services. - ks3_other_services = {'token': {'catalog': [{'type': 'compute', - 'id': 'test-id1'}, - {'type': 'identity', - 'id': 'test-id2'}]}} + # Test when catalog has other non-identity services. + ks3_other_services = {'token': { + 'catalog': [{'type': 'compute', + 'id': 'test-id1'}, + {'type': 'identity', + 'id': 'test-id2'}] + }} client._generate_cpid_from_endpoint = MagicMock() @httmock.all_requests @@ -413,7 +415,7 @@ class TestRefstackClient(unittest.TestCase): args = rc.parse_cli_args(self.mock_argv()) client = rc.RefstackClient(args) cpid = client._generate_cpid_from_endpoint('http://some.url:5000/v2') - expected = hashlib.md5('some.url').hexdigest() + expected = hashlib.md5('some.url'.encode('utf-8')).hexdigest() self.assertEqual(expected, cpid) with self.assertRaises(ValueError): @@ -539,11 +541,11 @@ class TestRefstackClient(unittest.TestCase): return expected_response with httmock.HTTMock(refstack_api_mock): - client.post_results("http://127.0.0.1", content, - sign_with=self.test_path + '/rsa_key') + rsapath = os.path.join(self.test_path, 'rsa_key') + client.post_results("http://127.0.0.1", content, sign_with=rsapath) client.logger.info.assert_called_with( - 'http://127.0.0.1/v1/results/ Response: ' - '%s' % expected_response) + 'http://127.0.0.1/v1/results/ Response: %s' % + expected_response) def test_run_tempest(self): """ @@ -766,8 +768,8 @@ class TestRefstackClient(unittest.TestCase): """ upload_file_path = self.test_path + "/.testrepository/0.json" args = rc.parse_cli_args( - self.mock_argv(command='upload', priv_key='rsa_key') - + [upload_file_path]) + self.mock_argv(command='upload', priv_key='rsa_key') + + [upload_file_path]) client = rc.RefstackClient(args) client.post_results = MagicMock() @@ -791,15 +793,15 @@ class TestRefstackClient(unittest.TestCase): """ upload_file_path = self.test_path + "/.testrepository/0" args = rc.parse_cli_args( - self.mock_argv(command='upload-subunit', priv_key='rsa_key') - + ['--keystone-endpoint', 'http://0.0.0.0:5000/v2.0'] - + [upload_file_path]) + self.mock_argv(command='upload-subunit', priv_key='rsa_key') + + ['--keystone-endpoint', 'http://0.0.0.0:5000/v2.0'] + + [upload_file_path]) client = rc.RefstackClient(args) client.post_results = MagicMock() client.upload_subunit() expected_json = { 'duration_seconds': 0, - 'cpid': hashlib.md5('0.0.0.0').hexdigest(), + 'cpid': hashlib.md5('0.0.0.0'.encode('utf-8')).hexdigest(), 'results': [ {'name': 'tempest.passed.test'}, {'name': 'tempest.tagged_passed.test', @@ -860,13 +862,13 @@ class TestRefstackClient(unittest.TestCase): args = rc.parse_cli_args(self.mock_argv(command='list')) client = rc.RefstackClient(args) results = [[{"cpid": "42", - "created_at": "2015-04-28 13:57:05", - "test_id": "1", - "url": "http://127.0.0.1:8000/output.html?test_id=1"}, - {"cpid": "42", - "created_at": "2015-04-28 13:57:05", - "test_id": "2", - "url": "http://127.0.0.1:8000/output.html?test_id=2"}]] + "created_at": "2015-04-28 13:57:05", + "test_id": "1", + "url": "http://127.0.0.1:8000/output.html?test_id=1"}, + {"cpid": "42", + "created_at": "2015-04-28 13:57:05", + "test_id": "2", + "url": "http://127.0.0.1:8000/output.html?test_id=2"}]] mock_results = MagicMock() mock_results.__iter__.return_value = results client.yield_results = MagicMock(return_value=mock_results) @@ -882,8 +884,8 @@ class TestRefstackClient(unittest.TestCase): os.path.join(self.test_path, 'rsa_key')]) client = rc.RefstackClient(args) pubkey, signature = client._sign_pubkey() - self.assertTrue(pubkey.startswith('ssh-rsa AAAA')) - self.assertTrue(signature.startswith('413cb954')) + self.assertTrue(pubkey.decode('utf8').startswith('ssh-rsa AAAA')) + self.assertTrue(signature.decode('utf8').startswith('413cb954')) def test_set_env_params(self): """ diff --git a/refstack_client/tests/unit/test_list_parser.py b/refstack_client/tests/unit/test_list_parser.py index e7204d2..8ba0a7a 100644 --- a/refstack_client/tests/unit/test_list_parser.py +++ b/refstack_client/tests/unit/test_list_parser.py @@ -165,7 +165,8 @@ class TestTestListParser(unittest.TestCase): # ID list. with open(test_file, 'rb') as f: file_contents = f.read() - testcase_list = filter(None, file_contents.split('\n')) + testcase_list = list(filter(None, + file_contents.decode('utf-8').split('\n'))) self.assertEqual(test_ids, testcase_list) diff --git a/setup.cfg b/setup.cfg index 706298f..8d2e5cc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,7 +16,8 @@ classifier = Programming Language :: Python Programming Language :: Python :: 2 Programming Language :: Python :: 2.7 - Programming Language :: Python :: 3.3 + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.5 [files] packages = diff --git a/setup_env b/setup_env index 476ef09..cbc10d4 100755 --- a/setup_env +++ b/setup_env @@ -2,6 +2,7 @@ #Default Tempest commit: SHA 8f98c4b60bf06a8c15e8c054848d2440c46077d0 (February 17, 2017 tags 15.0.0) CHECKOUT_POINT=8f98c4b60bf06a8c15e8c054848d2440c46077d0 +PY_VERSION="2.7.8" # Prints help function usage { @@ -13,6 +14,7 @@ function usage { echo " -h Print this usage message" echo " -c Tempest test runner commit. You can specify SHA or branch here" echo " If no commit or tag is specified, tempest will be install from commit" + echo " -p [ 2 | 3 ] - Uses either python 2.7 or 3.5. Default to python 2.7" echo " -q Run quietly. If .tempest folder exists, refstack-client is considered as installed" echo " -t Tempest test runner tag. You can specify tag here" echo " ${CHECKOUT_POINT}" @@ -30,11 +32,16 @@ function check_tag { # By default tempest uses commit ${CHECKOUT_POINT} -while getopts c:t:qh FLAG; do +while getopts c:p:t:qh FLAG; do case ${FLAG} in c) CHECKOUT_POINT=${OPTARG} ;; + p) + if [ ${OPTARG} == '3' ]; then + PY_VERSION="3.5.2" + fi + ;; t) CHECKOUT_POINT="-q ${OPTARG}" ;; @@ -110,10 +117,10 @@ cd ${WORKDIR} # Setup binary requirements if [ -n "$(command -v apt-get)" ]; then # For apt-get-based Linux distributions (Ubuntu, Debian) - sudo apt-get -y install curl wget tar unzip python-dev build-essential libssl-dev libxslt-dev libsasl2-dev libffi-dev libbz2-dev libyaml-dev + sudo apt-get -y install curl wget tar unzip python-dev build-essential libssl-dev libxslt-dev libsasl2-dev libffi-dev libbz2-dev libyaml-dev python3-dev elif [ -n "$(command -v yum)" ]; then # For yum-based distributions (RHEL, Centos) - sudo yum -y install curl wget tar unzip make python-devel.x86_64 gcc gcc-c++ libffi-devel libxml2-devel bzip2-devel libxslt-devel openssl-devel libyaml-devel + sudo yum -y install curl wget tar unzip make python-devel.x86_64 gcc gcc-c++ libffi-devel libxml2-devel bzip2-devel libxslt-devel openssl-devel libyaml-devel python3-devel elif [ -n "$(command) -v zypper" ]; then # For zypper-based distributions (openSuSe, SELS) sudo zypper --non-interactive install curl wget tar unzip make python-devel.x86_64 gcc gcc-c++ libffi-devel libxml2-devel zlib-devel libxslt-devel libopenssl-devel python-xml libyaml-devel @@ -123,9 +130,9 @@ else fi # Build local python interpreter if needed -if [ ! -n "$(command -v python2.7)" ]; then - PY_VERSION="2.7.8" - echo "python2.7 not found. Building python ${PY_VERSION}..." +sub_pystr="python$(echo $PY_VERSION | cut -c 1-3)" +if [ ! -n "$(command -v $sub_pystr)" ]; then + echo "$sub_pystr not found. Building python ${PY_VERSION}..." mkdir ${WORKDIR}/.localpython mkdir ${WORKDIR}/.python_src cd ${WORKDIR}/.python_src @@ -133,14 +140,14 @@ if [ ! -n "$(command -v python2.7)" ]; then tar zxvf Python-${PY_VERSION}.tgz cd Python-${PY_VERSION} - ./configure --prefix=${WORKDIR}/.localpython + ./configure --prefix=${WORKDIR}/.localpython --without-pymalloc make && make install cd ${WORKDIR} rm -rf ${WORKDIR}/.python_src - PYPATH="${WORKDIR}/.localpython/bin/python" + PYPATH="${WORKDIR}/.localpython/bin/$sub_pystr" else -echo "python2.7 found!" -PYPATH="python2.7" + echo "$sub_pystr found!" + PYPATH="$sub_pystr" fi # Setup virtual environments for refstack-client and tempest diff --git a/tox.ini b/tox.ini index e26b368..b07c13b 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27,pep8 +envlist = pep8,py35,py27 minversion = 1.6 skipsdist = True @@ -17,6 +17,7 @@ commands = /bin/rm -f .testrepository/times.dbm distribute = false [testenv:pep8] +deps = flake8 commands = flake8 distribute = false