From b0c10ad39aad62effcb55965aad6c2f27467aa4e Mon Sep 17 00:00:00 2001 From: Nolan Brubaker Date: Thu, 18 May 2017 11:00:26 -0700 Subject: [PATCH] Move keystone and glance to a single file This single file will become a CLI tool that can launch any test. This is preliminary work at merging the previous two files together and getting a running result. The loop was consolidated into one function, with each test also being a function. Some utility workers were also provided. Later work will add a command line interface for choosing a test. For now, it simply executes the glance test since it's more comprehensive. Change-Id: I68aa10a6f7126b7a74e474e74b7f0948c6dc20f8 --- bowling_ball/rolling_tests.py | 181 ++++++++++++++++++++++++++++ bowling_ball/tests/glance_api.py | 111 ----------------- bowling_ball/tests/glance_upload.py | 131 -------------------- bowling_ball/tests/keystone.py | 86 ------------- 4 files changed, 181 insertions(+), 328 deletions(-) create mode 100644 bowling_ball/rolling_tests.py delete mode 100755 bowling_ball/tests/glance_api.py delete mode 100644 bowling_ball/tests/glance_upload.py delete mode 100644 bowling_ball/tests/keystone.py diff --git a/bowling_ball/rolling_tests.py b/bowling_ball/rolling_tests.py new file mode 100644 index 00000000..5193c7a8 --- /dev/null +++ b/bowling_ball/rolling_tests.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python +# Copyright 2017, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# (c) 2017, Nolan Brubaker + +import datetime +from keystoneauth1.identity import v3 +from keystoneauth1 import session +from keystoneauth1.exceptions.connection import ConnectFailure +from keystoneauth1.exceptions.http import BadGateway +from keystoneauth1.exceptions.http import InternalServerError +from keystoneclient.v3 import client as key_client +import logging +import os +import sys +import time +from glanceclient import Client +from glanceclient import exc +import tempfile + +logger = logging.getLogger(__name__) + + +def configure_logging(service): + """Configure a stream and file log for a given service + + :param: service - name of service for log file. + generates `/var/log/{service_name}_query.log` + """ + logger.setLevel(logging.INFO) + console = logging.StreamHandler() + logfile = logging.FileHandler('/var/log/keystone_query.log', 'a') + + console.setLevel(logging.INFO) + logfile.setLevel(logging.INFO) + + formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') + # Make sure we're using UTC for everything. + formatter.converter = time.gmtime + + console.setFormatter(formatter) + logfile.setFormatter(formatter) + + logger.addHandler(console) + logger.addHandler(logfile) + + +def keystone_test(logger): + configure_logging('keystone') + + auth_url = os.environ['OS_AUTH_URL'] + password = os.environ['OS_PASSWORD'] + + auth = v3.Password(auth_url=auth_url, username="admin", + password=password, project_name="admin", + user_domain_id="default", project_domain_id="default") + sess = session.Session(auth=auth) + keystone = key_client.Client(session=sess) + test_list = keystone.projects.list() + if test_list: + msg = "New project list." + else: + msg = "Failed to get project list" + return msg + + +def test_loop(test_function): + """Main loop to execute tests + + Executes and times interactions with OpenStack services to gather timing + data. + :param: test_function - function object that performs some action + against an OpenStack service API. + """ + disconnected = None + # Has to be a tuple for python syntax reasons. + # This is currently the set needed for glance; should probably + # provide some way of letting a test say which exceptions should + # be caught for a service. + exc_list = (ConnectFailure, InternalServerError, BadGateway, + exc.CommunicationError, exc.HTTPInternalServerError) + try: + while True: + try: + # Pause for a bit so we're not generating more data than we + # can handle + time.sleep(1) + start_time = datetime.datetime.now() + + # Let the test function report it's own errors + msg = test_function(logger) + + end_time = datetime.datetime.now() + + if disconnected: + dis_delta = end_time - disconnected + disconnected = None + logger.info("Reconnect {}s".format( + dis_delta.total_seconds())) + + delta = end_time - start_time + + logger.info("{}s {}s.".format(msg, delta.total_seconds())) + except (exc_list): + if not disconnected: + disconnected = datetime.datetime.now() + except KeyboardInterrupt: + sys.exit() + + +def get_session(): + auth_url = os.environ['OS_AUTH_URL'] + password = os.environ['OS_PASSWORD'] + auth = v3.Password(auth_url=auth_url, username="admin", + password=password, project_name="admin", + user_domain_id="default", + project_domain_id="default") + sess = session.Session(auth=auth) + return sess + + +def get_keystone_client(session): + return key_client.Client(session=session) + + +def get_glance_endpoint(keystone): + """Get the glance admin endpoint + + Because we don't want to set up SSL handling, use the plain HTTP + endpoints. + """ + service_id = keystone.services.find(name='glance') + glance_endpoint = keystone.endpoints.list(service=service_id, + interface='admin')[0] + # The glance client wants the URL, not the keystone object + return glance_endpoint.url + + +def glance_test(logger): + configure_logging('glance') + # make a bogus file to give to glance. + + sess = get_session() + keystone = get_keystone_client(sess) + endpoint = get_glance_endpoint(keystone) + + temp_file = tempfile.TemporaryFile() + temp_file.write(os.urandom(1024 * 1024)) + temp_file.seek(0) + + glance = Client(version='2', endpoint=endpoint, session=sess) + image = glance.images.create(name="Rolling test", + disk_format="raw", + container_format="bare") + try: + glance.images.upload(image.id, temp_file) + except exc.HTTPInternalServerError: + # TODO: set msg and error type instead. + logger.error("Failed to upload") + return + finally: + glance.images.delete(image.id) + temp_file.close() + + msg = "Image created and deleted." + return msg + +if __name__ == "__main__": + test_loop(glance_test) diff --git a/bowling_ball/tests/glance_api.py b/bowling_ball/tests/glance_api.py deleted file mode 100755 index 3db0f85d..00000000 --- a/bowling_ball/tests/glance_api.py +++ /dev/null @@ -1,111 +0,0 @@ -#!/usr/bin/env python -# Copyright 2017, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# (c) 2017, Nolan Brubaker - -import datetime -from glanceclient import Client -from keystoneauth1.identity import v3 -from keystoneauth1 import session -from keystoneauth1.exceptions.connection import ConnectFailure -from keystoneauth1.exceptions.http import BadGateway -from keystoneauth1.exceptions.http import InternalServerError -from keystoneclient.v3 import client as key_client -import logging -import os -import sys -import time - -logger = logging.getLogger(__name__) - - -def configure_logging(): - logger.setLevel(logging.INFO) - console = logging.StreamHandler() - logfile = logging.FileHandler('/var/log/glance_query.log', 'a') - - console.setLevel(logging.INFO) - logfile.setLevel(logging.INFO) - - formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') - # Make sure we're using UTC for everything. - formatter.converter = time.gmtime - - console.setFormatter(formatter) - logfile.setFormatter(formatter) - - logger.addHandler(console) - logger.addHandler(logfile) - - -configure_logging() - - -def get_session(): - auth_url = os.environ['OS_AUTH_URL'] - password = os.environ['OS_PASSWORD'] - auth = v3.Password(auth_url=auth_url, username="admin", - password=password, project_name="admin", - user_domain_id="default", - project_domain_id="default") - sess = session.Session(auth=auth) - return sess - - -def get_keystone_client(session): - return key_client.Client(session=session) - - -def get_glance_endpoint(keystone): - """Get the glance admin endpoint - - Because we don't want to set up SSL handling, use the plain HTTP - endpoints. - """ - service_id = keystone.services.find(name='glance') - glance_endpoint = keystone.endpoints.list(service=service_id, - interface='admin')[0] - # The glance client wants the URL, not the keystone object - return glance_endpoint.url - -disconnected = None -try: - while True: - try: - time.sleep(1) - start_time = datetime.datetime.now() - - sess = get_session() - keystone = get_keystone_client(sess) - endpoint = get_glance_endpoint(keystone) - glance = Client(version='2', endpoint=endpoint, session=sess) - # The image.list method returns a generator, but we just care about - # response time - image_list = glance.images.list() - - end_time = datetime.datetime.now() - - if disconnected: - dis_delta = end_time - disconnected - disconnected = None - logger.info("Reconnect {}s".format(dis_delta.total_seconds())) - - delta = end_time - start_time - logger.info("New list {}s".format(delta.total_seconds())) - except (ConnectFailure, InternalServerError, BadGateway): - if not disconnected: - disconnected = datetime.datetime.now() -except KeyboardInterrupt: - sys.exit() diff --git a/bowling_ball/tests/glance_upload.py b/bowling_ball/tests/glance_upload.py deleted file mode 100644 index 1757f39f..00000000 --- a/bowling_ball/tests/glance_upload.py +++ /dev/null @@ -1,131 +0,0 @@ -#!/usr/bin/env python -# Copyright 2017, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# (c) 2017, Nolan Brubaker - -import datetime -from glanceclient import Client -from glanceclient import exc -from keystoneauth1.identity import v3 -from keystoneauth1 import session -from keystoneauth1.exceptions.connection import ConnectFailure -from keystoneauth1.exceptions.http import BadGateway -from keystoneauth1.exceptions.http import InternalServerError -from keystoneclient.v3 import client as key_client -import logging -import os -import sys -import tempfile -import time - -logger = logging.getLogger(__name__) - - -def configure_logging(): - logger.setLevel(logging.INFO) - console = logging.StreamHandler() - logfile = logging.FileHandler('/var/log/glance_query.log', 'a') - - console.setLevel(logging.INFO) - logfile.setLevel(logging.INFO) - - formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') - # Make sure we're using UTC for everything. - formatter.converter = time.gmtime - - console.setFormatter(formatter) - logfile.setFormatter(formatter) - - logger.addHandler(console) - logger.addHandler(logfile) - - -configure_logging() - - -def get_session(): - auth_url = os.environ['OS_AUTH_URL'] - password = os.environ['OS_PASSWORD'] - auth = v3.Password(auth_url=auth_url, username="admin", - password=password, project_name="admin", - user_domain_id="default", - project_domain_id="default") - sess = session.Session(auth=auth) - return sess - - -def get_keystone_client(session): - return key_client.Client(session=session) - - -def get_glance_endpoint(keystone): - """Get the glance admin endpoint - - Because we don't want to set up SSL handling, use the plain HTTP - endpoints. - """ - service_id = keystone.services.find(name='glance') - glance_endpoint = keystone.endpoints.list(service=service_id, - interface='admin')[0] - # The glance client wants the URL, not the keystone object - return glance_endpoint.url - -sess = get_session() -keystone = get_keystone_client(sess) -endpoint = get_glance_endpoint(keystone) - -# Has to be a tuple for python syntax reasons. -exc_list = (ConnectFailure, InternalServerError, BadGateway, - exc.CommunicationError) - -disconnected = None -try: - while True: - try: - time.sleep(1) - - # make a bogus file to give to glance. - temp_file = tempfile.TemporaryFile() - temp_file.write(os.urandom(1024 * 1024)) - temp_file.seek(0) - - start_time = datetime.datetime.now() - - glance = Client(version='2', endpoint=endpoint, session=sess) - image = glance.images.create(name="Rolling test", - disk_format="raw", - container_format="bare") - try: - glance.images.upload(image.id, temp_file) - except exc.HTTPInternalServerError: - logger.error("Failed to upload") - finally: - glance.images.delete(image.id) - temp_file.close() - - end_time = datetime.datetime.now() - - if disconnected: - dis_delta = end_time - disconnected - disconnected = None - logger.info("Reconnect {}s".format(dis_delta.total_seconds())) - - delta = end_time - start_time - logger.info("Image made/deleted {}s".format(delta.total_seconds())) - except exc_list: - if not disconnected: - disconnected = datetime.datetime.now() -except KeyboardInterrupt: - sys.exit() diff --git a/bowling_ball/tests/keystone.py b/bowling_ball/tests/keystone.py deleted file mode 100644 index 42581d01..00000000 --- a/bowling_ball/tests/keystone.py +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env python -# Copyright 2017, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# (c) 2017, Nolan Brubaker - -import datetime -from keystoneauth1.identity import v3 -from keystoneauth1 import session -from keystoneauth1.exceptions.connection import ConnectFailure -from keystoneauth1.exceptions.http import InternalServerError -from keystoneclient.v3 import client -import logging -import os -import sys -import time - -logger = logging.getLogger(__name__) - - -def configure_logging(): - logger.setLevel(logging.INFO) - console = logging.StreamHandler() - logfile = logging.FileHandler('/var/log/keystone_query.log', 'a') - - console.setLevel(logging.INFO) - logfile.setLevel(logging.INFO) - - formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') - # Make sure we're using UTC for everything. - formatter.converter = time.gmtime - - console.setFormatter(formatter) - logfile.setFormatter(formatter) - - logger.addHandler(console) - logger.addHandler(logfile) - -configure_logging() - -auth_url = os.environ['OS_AUTH_URL'] -password = os.environ['OS_PASSWORD'] - -auth = v3.Password(auth_url=auth_url, username="admin", - password=password, project_name="admin", - user_domain_id="default", project_domain_id="default") - -disconnected = None -try: - while True: - try: - # Pause for a bit so we're not generating more data than we - # can handle - time.sleep(1) - start_time = datetime.datetime.now() - - sess = session.Session(auth=auth) - keystone = client.Client(session=sess) - keystone.projects.list() - - end_time = datetime.datetime.now() - - if disconnected: - dis_delta = end_time - disconnected - disconnected = None - logger.info("Reconnect {}s".format(dis_delta.total_seconds())) - - delta = end_time - start_time - - logger.info("New list: {}s.".format(delta.total_seconds())) - except (ConnectFailure, InternalServerError): - if not disconnected: - disconnected = datetime.datetime.now() -except KeyboardInterrupt: - sys.exit()