openinfra/refstack-client
Code Issues Proposed changes

Merge "Add python35 support"

Browse Source
This commit is contained in:
Jenkins 2017-08-15 19:02:20 +00:00 committed by Gerrit Code Review
commit a61137f767
9 changed files with 83 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
refstack_client/list_parser.py
View File

@ -150,7 +150,7 @@ class TestListParser(object):
""" """
temp = tempfile.NamedTemporaryFile(delete=False) temp = tempfile.NamedTemporaryFile(delete=False)
for test_id in test_ids: for test_id in test_ids:
temp.write("%s\n" % test_id) temp.write(("%s\n" % test_id).encode('utf-8'))
temp.flush() temp.flush()
# Register the created file for cleanup. # Register the created file for cleanup.

52
refstack_client/refstack_client.py
View File

@ -24,9 +24,10 @@ Tempest configuration file.
""" """
from __future__ import absolute_import
import argparse import argparse
import binascii import binascii
import ConfigParser
import hashlib import hashlib
import itertools import itertools
import json import json
@ -43,10 +44,10 @@ from cryptography.hazmat.primitives.asymmetric import padding
import requests import requests
import requests.exceptions import requests.exceptions
import six.moves from six import moves
from six.moves.urllib import parse from six.moves.urllib import parse
from subunit_processor import SubunitProcessor from refstack_client.subunit_processor import SubunitProcessor
from list_parser import TestListParser from refstack_client.list_parser import TestListParser
import yaml import yaml
@ -54,7 +55,7 @@ def get_input():
""" """
Wrapper for raw_input. Necessary for testing. Wrapper for raw_input. Necessary for testing.
""" """
return raw_input().lower() # pragma: no cover return moves.input().lower() # pragma: no cover
def read_accounts_yaml(path): def read_accounts_yaml(path):
@ -83,7 +84,7 @@ class RefstackClient:
# set default log level to INFO. # set default log level to INFO.
if self.args.silent: if self.args.silent:
self.logger.setLevel(logging.WARNING) self.logger.setLevel(logging.WARNING)
elif self.args.verbose > 0: elif self.args.verbose:
self.logger.setLevel(logging.DEBUG) self.logger.setLevel(logging.DEBUG)
else: else:
self.logger.setLevel(logging.INFO) self.logger.setLevel(logging.INFO)
@ -113,7 +114,9 @@ class RefstackClient:
exit(1) exit(1)
self.conf_file = self.args.conf_file self.conf_file = self.args.conf_file
self.conf = ConfigParser.SafeConfigParser() # Note: SafeConfigParser deprecated on Python 3.2
# Use ConfigParser directly
self.conf = moves.configparser.ConfigParser()
self.conf.read(self.args.conf_file) self.conf.read(self.args.conf_file)
def _prep_upload(self): def _prep_upload(self):
@ -150,10 +153,10 @@ class RefstackClient:
# Prefer Keystone V3 API if it is enabled # Prefer Keystone V3 API if it is enabled
auth_version = ( auth_version = (
'v3' if (conf_file.has_option('identity-feature-enabled', 'v3' if (conf_file.has_option('identity-feature-enabled',
'api_v3') 'api_v3') and
and conf_file.getboolean('identity-feature-enabled', conf_file.getboolean('identity-feature-enabled',
'api_v3') 'api_v3') and
and conf_file.has_option('identity', 'uri_v3')) conf_file.has_option('identity', 'uri_v3'))
else 'v2') else 'v2')
if auth_version == 'v2': if auth_version == 'v2':
auth_url = '%s/tokens' % (conf_file.get('identity', 'uri') auth_url = '%s/tokens' % (conf_file.get('identity', 'uri')
@ -217,7 +220,7 @@ class RefstackClient:
'configuration guide (http://docs.openstack.' 'configuration guide (http://docs.openstack.'
'org/developer/tempest/configuration.html).') 'org/developer/tempest/configuration.html).')
exit(1) exit(1)
except ConfigParser.Error as e: except moves.configparser.Error as e:
# Most likely a missing section or option in the config file. # Most likely a missing section or option in the config file.
self.logger.error("Invalid Config File: %s" % e) self.logger.error("Invalid Config File: %s" % e)
exit(1) exit(1)
@ -246,12 +249,15 @@ class RefstackClient:
} }
return auth_version, auth_url, data return auth_version, auth_url, data
elif auth_version == 'v3': elif auth_version == 'v3':
identity = {'methods': ['password'], 'password': identity = {
{'user': {'name': auth_config['username'], 'methods': ['password'],
'domain': { 'password': {
'name': auth_config['domain_name'] 'user': {
}, 'name': auth_config['username'],
'password': auth_config['password']}}} 'domain': {'name': auth_config['domain_name']},
'password': auth_config['password']
}}}
data = { data = {
'auth': { 'auth': {
'identity': identity, 'identity': identity,
@ -297,7 +303,7 @@ class RefstackClient:
message = ('Invalid request with error ' message = ('Invalid request with error '
'code: %s. Error message: %s' 'code: %s. Error message: %s'
'' % (rsp['error']['code'], '' % (rsp['error']['code'],
rsp['error']['message'])) rsp['error']['message']))
raise requests.exceptions.HTTPError(message) raise requests.exceptions.HTTPError(message)
# If a Key or Index Error was raised, one of the expected keys or # If a Key or Index Error was raised, one of the expected keys or
# indices for retrieving the identity service ID was not found. # indices for retrieving the identity service ID was not found.
@ -321,7 +327,7 @@ class RefstackClient:
raise ValueError('Invalid Keystone endpoint format. Make sure ' raise ValueError('Invalid Keystone endpoint format. Make sure '
'the endpoint (%s) includes the URL scheme ' 'the endpoint (%s) includes the URL scheme '
'(i.e. http/https).' % endpoint) '(i.e. http/https).' % endpoint)
return hashlib.md5(url_parts.hostname).hexdigest() return hashlib.md5(url_parts.hostname.encode('utf-8')).hexdigest()
def _form_result_content(self, cpid, duration, results): def _form_result_content(self, cpid, duration, results):
'''This method will create the content for the request. The spec at '''This method will create the content for the request. The spec at
@ -345,7 +351,7 @@ class RefstackClient:
if self.args.quiet: if self.args.quiet:
return True return True
try: try:
inp = six.moves.input(q + ' (yes/y): ') inp = moves.input(q + ' (yes/y): ')
except KeyboardInterrupt: except KeyboardInterrupt:
return return
else: else:
@ -403,7 +409,7 @@ class RefstackClient:
if response.status_code == 201: if response.status_code == 201:
resp = response.json() resp = response.json()
print 'Test results uploaded!\nURL: %s' % resp.get('url', '') print('Test results uploaded!\nURL: %s' % resp.get('url', ''))
def test(self): def test(self):
'''Execute Tempest test against the cloud.''' '''Execute Tempest test against the cloud.'''
@ -566,7 +572,7 @@ class RefstackClient:
for r in page_of_results: for r in page_of_results:
print('%s - %s' % (r['created_at'], r['url'])) print('%s - %s' % (r['created_at'], r['url']))
try: try:
six.moves.input('Press Enter to go to next page...') moves.input('Press Enter to go to next page...')
except KeyboardInterrupt: except KeyboardInterrupt:
return return

1
refstack_client/scripts/prep_cloud.py
View File

@ -392,5 +392,6 @@ def main():
# TODO(tkammer): add network implementation # TODO(tkammer): add network implementation
if __name__ == "__main__": if __name__ == "__main__":
main() main()

1
refstack_client/tests/smoke/test_distros.py
View File

@ -56,5 +56,6 @@ class TestSequenceFunctions(unittest.TestCase):
distro_image = 'opensuse/13.2' distro_image = 'opensuse/13.2'
self.run_test(distro_image) self.run_test(distro_image)
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()

54
refstack_client/tests/unit/test_client.py
View File

@ -25,7 +25,7 @@ import mock
from mock import MagicMock from mock import MagicMock
import unittest import unittest
import refstack_client.refstack_client as rc from refstack_client import refstack_client as rc
class TestRefstackClient(unittest.TestCase): class TestRefstackClient(unittest.TestCase):
@ -356,11 +356,13 @@ class TestRefstackClient(unittest.TestCase):
client._get_cpid_from_keystone(auth_version, auth_url, content) client._get_cpid_from_keystone(auth_version, auth_url, content)
client._generate_cpid_from_endpoint.assert_called_with(auth_url) client._generate_cpid_from_endpoint.assert_called_with(auth_url)
#Test when catalog has other non-identity services. # Test when catalog has other non-identity services.
ks3_other_services = {'token': {'catalog': [{'type': 'compute', ks3_other_services = {'token': {
'id': 'test-id1'}, 'catalog': [{'type': 'compute',
{'type': 'identity', 'id': 'test-id1'},
'id': 'test-id2'}]}} {'type': 'identity',
'id': 'test-id2'}]
}}
client._generate_cpid_from_endpoint = MagicMock() client._generate_cpid_from_endpoint = MagicMock()
@httmock.all_requests @httmock.all_requests
@ -413,7 +415,7 @@ class TestRefstackClient(unittest.TestCase):
args = rc.parse_cli_args(self.mock_argv()) args = rc.parse_cli_args(self.mock_argv())
client = rc.RefstackClient(args) client = rc.RefstackClient(args)
cpid = client._generate_cpid_from_endpoint('http://some.url:5000/v2') cpid = client._generate_cpid_from_endpoint('http://some.url:5000/v2')
expected = hashlib.md5('some.url').hexdigest() expected = hashlib.md5('some.url'.encode('utf-8')).hexdigest()
self.assertEqual(expected, cpid) self.assertEqual(expected, cpid)
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
@ -539,11 +541,11 @@ class TestRefstackClient(unittest.TestCase):
return expected_response return expected_response
with httmock.HTTMock(refstack_api_mock): with httmock.HTTMock(refstack_api_mock):
client.post_results("http://127.0.0.1", content, rsapath = os.path.join(self.test_path, 'rsa_key')
sign_with=self.test_path + '/rsa_key') client.post_results("http://127.0.0.1", content, sign_with=rsapath)
client.logger.info.assert_called_with( client.logger.info.assert_called_with(
'http://127.0.0.1/v1/results/ Response: ' 'http://127.0.0.1/v1/results/ Response: %s' %
'%s' % expected_response) expected_response)
def test_run_tempest(self): def test_run_tempest(self):
""" """
@ -766,8 +768,8 @@ class TestRefstackClient(unittest.TestCase):
""" """
upload_file_path = self.test_path + "/.testrepository/0.json" upload_file_path = self.test_path + "/.testrepository/0.json"
args = rc.parse_cli_args( args = rc.parse_cli_args(
self.mock_argv(command='upload', priv_key='rsa_key') self.mock_argv(command='upload', priv_key='rsa_key') +
+ [upload_file_path]) [upload_file_path])
client = rc.RefstackClient(args) client = rc.RefstackClient(args)
client.post_results = MagicMock() client.post_results = MagicMock()
@ -791,15 +793,15 @@ class TestRefstackClient(unittest.TestCase):
""" """
upload_file_path = self.test_path + "/.testrepository/0" upload_file_path = self.test_path + "/.testrepository/0"
args = rc.parse_cli_args( args = rc.parse_cli_args(
self.mock_argv(command='upload-subunit', priv_key='rsa_key') self.mock_argv(command='upload-subunit', priv_key='rsa_key') +
+ ['--keystone-endpoint', 'http://0.0.0.0:5000/v2.0'] ['--keystone-endpoint', 'http://0.0.0.0:5000/v2.0'] +
+ [upload_file_path]) [upload_file_path])
client = rc.RefstackClient(args) client = rc.RefstackClient(args)
client.post_results = MagicMock() client.post_results = MagicMock()
client.upload_subunit() client.upload_subunit()
expected_json = { expected_json = {
'duration_seconds': 0, 'duration_seconds': 0,
'cpid': hashlib.md5('0.0.0.0').hexdigest(), 'cpid': hashlib.md5('0.0.0.0'.encode('utf-8')).hexdigest(),
'results': [ 'results': [
{'name': 'tempest.passed.test'}, {'name': 'tempest.passed.test'},
{'name': 'tempest.tagged_passed.test', {'name': 'tempest.tagged_passed.test',
@ -860,13 +862,13 @@ class TestRefstackClient(unittest.TestCase):
args = rc.parse_cli_args(self.mock_argv(command='list')) args = rc.parse_cli_args(self.mock_argv(command='list'))
client = rc.RefstackClient(args) client = rc.RefstackClient(args)
results = [[{"cpid": "42", results = [[{"cpid": "42",
"created_at": "2015-04-28 13:57:05", "created_at": "2015-04-28 13:57:05",
"test_id": "1", "test_id": "1",
"url": "http://127.0.0.1:8000/output.html?test_id=1"}, "url": "http://127.0.0.1:8000/output.html?test_id=1"},
{"cpid": "42", {"cpid": "42",
"created_at": "2015-04-28 13:57:05", "created_at": "2015-04-28 13:57:05",
"test_id": "2", "test_id": "2",
"url": "http://127.0.0.1:8000/output.html?test_id=2"}]] "url": "http://127.0.0.1:8000/output.html?test_id=2"}]]
mock_results = MagicMock() mock_results = MagicMock()
mock_results.__iter__.return_value = results mock_results.__iter__.return_value = results
client.yield_results = MagicMock(return_value=mock_results) client.yield_results = MagicMock(return_value=mock_results)
@ -882,8 +884,8 @@ class TestRefstackClient(unittest.TestCase):
os.path.join(self.test_path, 'rsa_key')]) os.path.join(self.test_path, 'rsa_key')])
client = rc.RefstackClient(args) client = rc.RefstackClient(args)
pubkey, signature = client._sign_pubkey() pubkey, signature = client._sign_pubkey()
self.assertTrue(pubkey.startswith('ssh-rsa AAAA')) self.assertTrue(pubkey.decode('utf8').startswith('ssh-rsa AAAA'))
self.assertTrue(signature.startswith('413cb954')) self.assertTrue(signature.decode('utf8').startswith('413cb954'))
def test_set_env_params(self): def test_set_env_params(self):
""" """

3
refstack_client/tests/unit/test_list_parser.py
View File

@ -165,7 +165,8 @@ class TestTestListParser(unittest.TestCase):
# ID list. # ID list.
with open(test_file, 'rb') as f: with open(test_file, 'rb') as f:
file_contents = f.read() file_contents = f.read()
testcase_list = filter(None, file_contents.split('\n')) testcase_list = list(filter(None,
file_contents.decode('utf-8').split('\n')))
self.assertEqual(test_ids, testcase_list) self.assertEqual(test_ids, testcase_list)

3
setup.cfg
View File

@ -16,7 +16,8 @@ classifier =
Programming Language :: Python Programming Language :: Python
Programming Language :: Python :: 2 Programming Language :: Python :: 2
Programming Language :: Python :: 2.7 Programming Language :: Python :: 2.7
Programming Language :: Python :: 3.3 Programming Language :: Python :: 3
Programming Language :: Python :: 3.5
[files] [files]
packages = packages =

27
setup_env
View File

@ -2,6 +2,7 @@
#Default Tempest commit: SHA 8f98c4b60bf06a8c15e8c054848d2440c46077d0 (February 17, 2017 tags 15.0.0) #Default Tempest commit: SHA 8f98c4b60bf06a8c15e8c054848d2440c46077d0 (February 17, 2017 tags 15.0.0)
CHECKOUT_POINT=8f98c4b60bf06a8c15e8c054848d2440c46077d0 CHECKOUT_POINT=8f98c4b60bf06a8c15e8c054848d2440c46077d0
PY_VERSION="2.7.8"
# Prints help # Prints help
function usage { function usage {
@ -13,6 +14,7 @@ function usage {
echo " -h Print this usage message" echo " -h Print this usage message"
echo " -c Tempest test runner commit. You can specify SHA or branch here" echo " -c Tempest test runner commit. You can specify SHA or branch here"
echo " If no commit or tag is specified, tempest will be install from commit" echo " If no commit or tag is specified, tempest will be install from commit"
echo " -p [ 2 | 3 ] - Uses either python 2.7 or 3.5. Default to python 2.7"
echo " -q Run quietly. If .tempest folder exists, refstack-client is considered as installed" echo " -q Run quietly. If .tempest folder exists, refstack-client is considered as installed"
echo " -t Tempest test runner tag. You can specify tag here" echo " -t Tempest test runner tag. You can specify tag here"
echo " ${CHECKOUT_POINT}" echo " ${CHECKOUT_POINT}"
@ -30,11 +32,16 @@ function check_tag {
# By default tempest uses commit ${CHECKOUT_POINT} # By default tempest uses commit ${CHECKOUT_POINT}
while getopts c:t:qh FLAG; do while getopts c:p:t:qh FLAG; do
case ${FLAG} in case ${FLAG} in
c) c)
CHECKOUT_POINT=${OPTARG} CHECKOUT_POINT=${OPTARG}
;; ;;
p)
if [ ${OPTARG} == '3' ]; then
PY_VERSION="3.5.2"
fi
;;
t) t)
CHECKOUT_POINT="-q ${OPTARG}" CHECKOUT_POINT="-q ${OPTARG}"
;; ;;
@ -110,10 +117,10 @@ cd ${WORKDIR}
# Setup binary requirements # Setup binary requirements
if [ -n "$(command -v apt-get)" ]; then if [ -n "$(command -v apt-get)" ]; then
# For apt-get-based Linux distributions (Ubuntu, Debian) # For apt-get-based Linux distributions (Ubuntu, Debian)
sudo apt-get -y install curl wget tar unzip python-dev build-essential libssl-dev libxslt-dev libsasl2-dev libffi-dev libbz2-dev libyaml-dev sudo apt-get -y install curl wget tar unzip python-dev build-essential libssl-dev libxslt-dev libsasl2-dev libffi-dev libbz2-dev libyaml-dev python3-dev
elif [ -n "$(command -v yum)" ]; then elif [ -n "$(command -v yum)" ]; then
# For yum-based distributions (RHEL, Centos) # For yum-based distributions (RHEL, Centos)
sudo yum -y install curl wget tar unzip make python-devel.x86_64 gcc gcc-c++ libffi-devel libxml2-devel bzip2-devel libxslt-devel openssl-devel libyaml-devel sudo yum -y install curl wget tar unzip make python-devel.x86_64 gcc gcc-c++ libffi-devel libxml2-devel bzip2-devel libxslt-devel openssl-devel libyaml-devel python3-devel
elif [ -n "$(command) -v zypper" ]; then elif [ -n "$(command) -v zypper" ]; then
# For zypper-based distributions (openSuSe, SELS) # For zypper-based distributions (openSuSe, SELS)
sudo zypper --non-interactive install curl wget tar unzip make python-devel.x86_64 gcc gcc-c++ libffi-devel libxml2-devel zlib-devel libxslt-devel libopenssl-devel python-xml libyaml-devel sudo zypper --non-interactive install curl wget tar unzip make python-devel.x86_64 gcc gcc-c++ libffi-devel libxml2-devel zlib-devel libxslt-devel libopenssl-devel python-xml libyaml-devel
@ -123,9 +130,9 @@ else
fi fi
# Build local python interpreter if needed # Build local python interpreter if needed
if [ ! -n "$(command -v python2.7)" ]; then sub_pystr="python$(echo $PY_VERSION | cut -c 1-3)"
PY_VERSION="2.7.8" if [ ! -n "$(command -v $sub_pystr)" ]; then
echo "python2.7 not found. Building python ${PY_VERSION}..." echo "$sub_pystr not found. Building python ${PY_VERSION}..."
mkdir ${WORKDIR}/.localpython mkdir ${WORKDIR}/.localpython
mkdir ${WORKDIR}/.python_src mkdir ${WORKDIR}/.python_src
cd ${WORKDIR}/.python_src cd ${WORKDIR}/.python_src
@ -133,14 +140,14 @@ if [ ! -n "$(command -v python2.7)" ]; then
tar zxvf Python-${PY_VERSION}.tgz tar zxvf Python-${PY_VERSION}.tgz
cd Python-${PY_VERSION} cd Python-${PY_VERSION}
./configure --prefix=${WORKDIR}/.localpython ./configure --prefix=${WORKDIR}/.localpython --without-pymalloc
make && make install make && make install
cd ${WORKDIR} cd ${WORKDIR}
rm -rf ${WORKDIR}/.python_src rm -rf ${WORKDIR}/.python_src
PYPATH="${WORKDIR}/.localpython/bin/python" PYPATH="${WORKDIR}/.localpython/bin/$sub_pystr"
else else
echo "python2.7 found!" echo "$sub_pystr found!"
PYPATH="python2.7" PYPATH="$sub_pystr"
fi fi
# Setup virtual environments for refstack-client and tempest # Setup virtual environments for refstack-client and tempest

3
tox.ini
View File

@ -1,5 +1,5 @@
[tox] [tox]
envlist = py27,pep8 envlist = pep8,py35,py27
minversion = 1.6 minversion = 1.6
skipsdist = True skipsdist = True
@ -17,6 +17,7 @@ commands = /bin/rm -f .testrepository/times.dbm
distribute = false distribute = false
[testenv:pep8] [testenv:pep8]
deps = flake8
commands = flake8 commands = flake8
distribute = false distribute = false