x
/
kloudbuster
Code Issues Proposed changes

Migrate code to python3/Ubuntu 20.04 

Change-Id: I18a21e04d009afdee3afc2723afdbade24bfdf71
This commit is contained in:
ahothan 2020-07-24 01:44:41 -07:00
parent de38fad996
commit 1316bd443d
50 changed files with 817 additions and 475 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -68,3 +68,5 @@ scale/dib/kloudbuster.d/
# kb_web
!kb_server/public/ui/components/*/*.css
!kb_server/public/ui/components/*/*.js
.pytest_cache/

41
Dockerfile
View File

@ -1,30 +1,33 @@
# docker file for creating a container that has kloudbuster installed and ready to use
# this will build from uptreams master latest
FROM ubuntu:16.04
MAINTAINER kloudbuster-core <kloudbuster-core@lists.launchpad.net>
FROM ubuntu:20.04
# Simpler would be to clone direct from upstream (latest)
# but the content might differ from the curent repo
# So we'd rather copy the current kloudbuster directory
# along with the pre-built qcow2 image
COPY ./ /kloudbuster/
# The name of the kloudbuster wheel package
# must be placed under ./dist directory before calling docker build
# example: ./dist/kloudbuster-8.0.0-py3-none-any.whl
ARG WHEEL_PKG
# The name of the kloudbuster VM qcow2 image
# must be placed in the current directory
# example: ./kloudbuster-8.0.0.qcow2
ARG VM_IMAGE
# copy the wheel package so it can be installed inside the container
COPY ./dist/$WHEEL_PKG /
# copy the VM image under /
COPY $VM_IMAGE /
# copy the VM Image
# Install KloudBuster script and dependencies
# Note the dot_git directory must be renamed to .git
# in order for pip install -e . to work properly
RUN apt-get update && apt-get install -y \
git \
libyaml-dev \
python \
python-dev \
python-pip \
&& pip install -U -q pip \
&& hash -r pip \
&& pip install -U -q setuptools \
&& cd /kloudbuster \
&& pip install -q -e . \
&& rm -rf .git \
&& rm -rf /var/lib/apt/lists/* \
&& apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/*
RUN apt-get update \
&& apt-get install -y python3 python3-pip python-is-python3 \
&& pip3 install /$WHEEL_PKG \
&& rm -f /$WHEEL_PKG

2
README.rst
View File

@ -1,5 +1,5 @@
=====================
KloudBuster version 7
KloudBuster version 8
=====================
How good is your OpenStack **data plane** or **storage service** under real

13
doc/source/readme.rst
View File

@ -1,5 +1,5 @@
=====================
KloudBuster version 7
KloudBuster version 8
=====================
How good is your OpenStack **data plane** or **storage service** under real
@ -89,8 +89,6 @@ Feature List
* Aggregated results provide an easy to understand way to assess the scale of
the cloud under test
* KloudBuster VM image pre-built and available from the OpenStack Community App
Catalog (https://apps.openstack.org/)
**Diagrams** describing how the scale test resources are staged and how the
traffic flows are available in :ref:`arch`.
@ -100,6 +98,15 @@ graphical charts generated straight off the tool.
**Examples of results** are available in :ref:`gallery`.
New in Release 8
----------------
* Kloudbuster is now fully python 3 compatible, python 2.7 is no longer supported.
* Validated againts OpenStack Train release
New in Release 7
----------------

87
kb_build.sh
View File

@ -10,12 +10,6 @@ export DIB_DEV_USER_PWDLESS_SUDO=Y
# Set the data sources to have ConfigDrive only
export DIB_CLOUD_INIT_DATASOURCES="ConfigDrive"
# Check we are in a virtual environment
function check_in_venv {
IN_VENV=$(python -c 'import sys; print hasattr(sys, "real_prefix")')
echo $IN_VENV
}
function cleanup_qcow2 {
echo
echo "Error: found unrelated qcow2 files that would make the container image too large."
@ -34,17 +28,12 @@ function build_vm {
fi
echo "Building $kb_image_name.qcow2..."
pip install "diskimage-builder>=2.15"
pip3 install "diskimage-builder>=2.15"
cd ./kb_dib
# Add the kloudbuster elements directory to the DIB elements path
export ELEMENTS_PATH=./elements
# canned user/password for direct login
export DIB_DEV_USER_USERNAME=kloudbuster
export DIB_DEV_USER_PASSWORD=kloudbuster
export DIB_DEV_USER_PWDLESS_SUDO=Y
# Install Ubuntu 18.04
export DIB_RELEASE=bionic
@ -64,10 +53,21 @@ function build_vm {
# Build container
function build_container {
echo "docker build --tag=berrypatch/kloudbuster:$KB_TAG ."
sudo docker build --tag=berrypatch/kloudbuster:$KB_TAG .
echo "sudo docker build --tag=berrypatch/kloudbuster:latest ."
sudo docker build --tag=berrypatch/kloudbuster:latest .
# Create a wheel package
# ./dist/kloudbuster-$KB_TAG-py3-none-any.whl
python setup.py build bdist_wheel || { echo "Error building package"; exit 5; }
wheel_pkg="kloudbuster-$KB_TAG-py3-none-any.whl"
if [ -f ./dist/$wheel_pkg ]; then
echo "Created package: ./dist/$wheel_pkg"
else
echo "Error: Cannot find created package: ./dist/$wheel_pkg"
exit 4
fi
build_args="--build-arg WHEEL_PKG=$wheel_pkg --build-arg VM_IMAGE=$kb_image_name.qcow2"
echo "docker build $build_args --tag=berrypatch/kloudbuster:$KB_TAG ."
sudo docker build $build_args --tag=berrypatch/kloudbuster:$KB_TAG .
echo "sudo docker build $build_args --tag=berrypatch/kloudbuster:latest ."
sudo docker build $build_args --tag=berrypatch/kloudbuster:latest .
}
function help {
@ -78,7 +78,7 @@ function help {
echo "Builds the KloudBuster VM and Docker container images"
echo "The Docker container image will include the VM image for easier upload"
echo
echo "Must run in a virtual environment and must be called from the root of the repository"
echo "Kloudbuster must be installed for this script to run (typically would run from a virtual environment)"
exit 1
}
@ -96,27 +96,50 @@ while [[ $# -gt 0 ]]; do
# Shift after checking all the cases to get the next option
shift
done
in_venv=$(check_in_venv)
if [ $in_venv != "True" ]; then
echo "Error: Must be in a virtual environment to run!"
exit 2
# check that we have python3/pip3 enabled
python -c 'print 0' >/dev/null 2>/dev/null
if [ $? -eq 0 ]; then
echo "Error: python 3 is required as default python version"
exit 3
fi
# check that we are in a virtual environment
INVENV=$(python -c 'import sys;print(hasattr(sys, "real_prefix") or (hasattr(sys, "base_prefix") and sys.base_prefix != sys.prefix))')
if [ $INVENV != "True" ]; then
echo "Error: must run inside a venv as many packages will be installed"
exit 4
fi
# check that kloudbuster binary is installed
# Get the kloudbuster version (must be retrieved from stderr)
KB_TAG=$(kloudbuster --version 2>&1)
if [ $? != 0 ]; then
echo "Installing kloudbuster..."
# Install kloudbuster in the virtual env in editable mode
pip3 install -q -e .
KB_TAG=$(kloudbuster --version 2>&1)
if [ $? != 0 ]; then
echo "Error: cannot retrieve version from kloudbuster..."
echo
kloudbuster --version
exit 2
fi
fi
# check that docker is installed
if [ $build_vm_only = 0 ]; then
docker --version >/dev/null 2>/dev/null
if [ $? -ne 0 ]; then
echo "Error: docker is not installed"
exit 4
fi
fi
# check we're at the root of the kloudbuster repo
if [ ! -d kloudbuster -o ! -f Dockerfile ]; then
echo "Error: Must be called from the root of the kloudbuster repository to run!"
exit 2
fi
# Install kloudbuster in the virtual env
pip install -q -U setuptools
pip install -q -e .
# Get the kloudbuster version (must be retrieved from stderr)
KB_TAG=$(kloudbuster --version 2>&1)
if [ $? != 0 ]; then
echo "Error retrieving kloudbuster version:"
echo
kloudbuster --version
exit 2
fi
echo
echo "Building KloudBuster with tag $KB_TAG"

6
kb_dib/elements/kloudbuster/README.rst
View File

@ -10,7 +10,7 @@ The same image can run using one of the following roles (Assigned from the user-
- Client VM for a given traffic type (e.g. http client or tcp/udp client)
- Redis server (only 1 instance in the client cloud)
The default login on the VM is
VMs are launched using cloud config and can be access with ssh:
- username: kb
- password: kb
- username: cloud-user
- no password, use key pairs to create the VM

4
kb_dib/elements/kloudbuster/package-installs.yaml
View File

@ -6,8 +6,8 @@ libssl-dev:
libyaml-dev:
nginx:
ntpdate:
python-pip:
python-dev:
python3-pip:
python3-dev:
redis-server:
xfsprogs:
zlib1g-dev:

4
kb_dib/elements/kloudbuster/post-install.d/01-pip-package
View File

@ -1,5 +1,3 @@
#!/bin/sh
pip install --upgrade pip
hash -r pip
pip install setuptools wheel
pip3 install setuptools wheel

14
kb_dib/elements/kloudbuster/post-install.d/02-kb-script
View File

@ -56,7 +56,7 @@ echo 'mkdir -p /mnt/config' >> /etc/rc.local
echo 'mount /dev/disk/by-label/config-2 /mnt/config' >> /etc/rc.local
echo 'cp /mnt/config/openstack/latest/user_data /kb_test/' >> /etc/rc.local
echo 'cd /kb_test' >> /etc/rc.local
echo 'python kb_vm_agent.py &' >> /etc/rc.local
echo 'python3 kb_vm_agent.py &' >> /etc/rc.local
chmod +x /etc/rc.local
# =================
@ -65,24 +65,24 @@ chmod +x /etc/rc.local
cd /kb_test
git clone https://opendev.org/x/kloudbuster.git
cd kloudbuster
pip install -r requirements.txt
pip3 install -r requirements.txt
# ======
# Client
# ======
# python redis client, HdrHistorgram_py
pip install redis hdrhistogram
pip3 install redis hdrhistogram
# Install HdrHistorgram_c
cd /tmp
git clone git://github.com/HdrHistogram/HdrHistogram_c.git
git clone https://github.com/HdrHistogram/HdrHistogram_c.git
cd HdrHistogram_c
cmake .
make install
# Install the http traffic generator
cd /tmp
git clone git://github.com/yicwang/wrk2.git
git clone https://github.com/yicwang/wrk2.git
cd wrk2
make
mv wrk /usr/local/bin/wrk2
@ -113,7 +113,7 @@ rm -rf /tmp/wrk2
rm -rf /tmp/fio
# Uninstall unneeded packages
apt-get -y --purge remove libyaml-dev libssl-dev zlib1g-dev libaio-dev python-pip python-dev build-essential cmake
apt-get -y --purge remove libyaml-dev libssl-dev zlib1g-dev libaio-dev python3-pip python3-dev build-essential cmake
apt-get -y --purge autoremove
apt-get -y install python
## apt-get -y install python
apt-get -y autoclean

7
kb_dib/elements/kloudbuster/post-install.d/99-cloudcfg-edit
View File

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
import yaml
cloudcfg = "/etc/cloud/cloud.cfg"
@ -7,11 +7,12 @@ user = "cloud-user"
with open(cloudcfg) as f:
cfg = yaml.safe_load(f)
synver = "1"
try:
if cfg['system_info']['default_user']['name']:
synver = "2"
except KeyError:
synver = "1"
pass
if synver == "1":
if cfg['user'] == user:
@ -27,7 +28,7 @@ elif synver == "2":
# Change the user to cloud-user
cfg['system_info']['default_user']['name'] = user
cfg['system_info']['default_user']['gecos'] = "Cloud User"
print cfg['system_info']['default_user']['name']
print(cfg['system_info']['default_user']['name'])
with open(cloudcfg, "w") as f:
yaml.dump(cfg, f, default_flow_style=False)

81
kb_dib/elements/kloudbuster/static/kb_test/kb_vm_agent.py
View File

@ -13,10 +13,8 @@
# under the License.
#
from hdrh.histogram import HdrHistogram
import json
import multiprocessing
import redis
import socket
import struct
import subprocess
@ -27,6 +25,9 @@ import threading
import time
import traceback
from hdrh.histogram import HdrHistogram
import redis
# Define the version of the KloudBuster agent and VM image
#
# When VM is up running, the agent will send the READY message to the
@ -36,11 +37,11 @@ import traceback
# and can be left constant moving forward.
__version__ = '7'
# TODO(Logging on Agent)
# to add later logging on Agent
def exec_command(cmd, cwd=None):
p = subprocess.Popen(cmd, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
(stdout, stderr) = p.communicate()
(_, stderr) = p.communicate()
if p.returncode:
syslog.syslog("Command failed: " + ' '.join(cmd))
if stderr:
@ -54,7 +55,7 @@ def refresh_clock(clocks, force_sync=False):
command = "sudo ntpdate" + step + clocks
exec_command(command.split(" "))
class KB_Instance(object):
class KB_Instance():
# Check whether the HTTP Service is up running
@staticmethod
@ -73,7 +74,7 @@ class KB_Instance(object):
if if_name:
debug_msg += " and %s" % if_name
cmd += " dev %s" % if_name
print debug_msg
print(debug_msg)
return cmd
@staticmethod
@ -105,7 +106,7 @@ class KB_Instance(object):
else:
debug_msg = "with next hop %s" % if_name
cmd += " dev %s" % if_name
print debug_msg
print(debug_msg)
return cmd
# Run the HTTP benchmarking tool
@ -167,7 +168,7 @@ class KB_Instance(object):
cmd = '%s %s %s %s' % (dest_path, fixed_opt, required_opt, optional_opt)
return cmd
class KBA_Client(object):
class KBA_Client():
def __init__(self, user_data):
host = user_data['redis_server']
@ -185,10 +186,10 @@ class KBA_Client(object):
def setup_channels(self):
# Check for connections to redis server
while (True):
while True:
try:
self.redis_obj.get("test")
except (redis.exceptions.ConnectionError):
except redis.exceptions.ConnectionError:
time.sleep(1)
continue
break
@ -230,6 +231,8 @@ class KBA_Client(object):
self.last_process = p
lines_iterator = iter(p.stdout.readline, b"")
for line in lines_iterator:
# line is bytes, so need to make it a str
line = line.decode('utf-8')
# One exception, if this is the very last report, we will send it
# through "DONE" command, not "REPORT". So what's happening here
# is to determine whether this is the last report.
@ -267,23 +270,25 @@ class KBA_Client(object):
# When 'ACK' is received, means the master node
# acknowledged the current VM. So stopped sending more
# "hello" packet to the master node.
# Unfortunately, there is no thread.stop() in Python 2.x
self.stop_hello.set()
elif message['cmd'] == 'EXEC':
self.last_cmd = ""
arange = message['data']['active_range']
my_id = int(self.vm_name[self.vm_name.rindex('I') + 1:])
if (not arange) or (my_id >= arange[0] and my_id <= arange[1]):
if (not arange) or (arange[0] <= my_id <= arange[1]):
try:
par = message['data'].get('parameter', '')
str_par = 'par' if par else ''
cmd_res_tuple = eval('self.exec_%s(%s)' % (message['data']['cmd'], str_par))
cmd = message['data']['cmd']
if isinstance(cmd, bytes):
cmd = cmd.decode('utf-8')
cmd_res_tuple = eval('self.exec_%s(%s)' % (cmd, str_par))
cmd_res_dict = dict(zip(("status", "stdout", "stderr"), cmd_res_tuple))
except Exception as exc:
except Exception:
cmd_res_dict = {
"status": 1,
"stdout": self.last_cmd,
"stderr": str(exc)
"stderr": traceback.format_exc() + '\nmessage: ' + str(message['data'])
}
if self.__class__.__name__ == "KBA_Multicast_Client":
self.report('DONE_MC', message['client-type'], cmd_res_dict)
@ -291,14 +296,14 @@ class KBA_Client(object):
self.report('DONE', message['client-type'], cmd_res_dict)
else:
# Unexpected
print 'ERROR: Unexpected command received!'
print('ERROR: Unexpected command received!')
class KBA_HTTP_Client(KBA_Client):
def exec_setup_static_route(self):
self.last_cmd = KB_Instance.get_static_route(self.user_data['target_subnet_ip'])
result = self.exec_command(self.last_cmd)
if (self.user_data['target_subnet_ip'] not in result[1]):
if self.user_data['target_subnet_ip'] not in result[1]:
self.last_cmd = KB_Instance.add_static_route(
self.user_data['target_subnet_ip'],
self.user_data['target_shared_interface_ip'])
@ -323,7 +328,7 @@ class KBA_Multicast_Client(KBA_Client):
self.last_cmd = KB_Instance.get_static_route(self.user_data['target_subnet_ip'])
result = self.exec_command(self.last_cmd)
if (self.user_data['target_subnet_ip'] not in result[1]):
if self.user_data['target_subnet_ip'] not in result[1]:
self.last_cmd = KB_Instance.add_static_route(
self.user_data['target_subnet_ip'],
self.user_data['target_shared_interface_ip'])
@ -340,10 +345,10 @@ class KBA_Multicast_Client(KBA_Client):
'megabytes': 'megabytes', 'rate_Mbps': 'mbps', 'msmaxjitter': 'jitter',
'msavgOWD': 'latency'} # Format/Include Keys
try:
return {kmap[k]: abs(float(v))
for (k, v) in [c.split("=")
for c in p_out.split(" ")]
if k in kmap}
return {
kmap[k]: abs(float(v)) for (k, v) in [c.split("=")
for c in p_out.split(" ")] if k in kmap
}
except Exception:
return {'error': '0'}
@ -365,12 +370,12 @@ class KBA_Multicast_Client(KBA_Client):
queue.put([cmds[cmd][0], out])
# End Function #
for cmd in cmds:
for _ in cmds:
multiprocessing.Process(target=spawn, args=(cmd_index, queue)).start()
cmd_index += 1
p_err = ""
try:
while(j < len(cmds)):
while j < len(cmds):
out = queue.get(True, timeout)
key = out[0]
j += 1
@ -500,7 +505,7 @@ class KBA_Storage_Client(KBA_Client):
grp_msb_bits = clat['FIO_IO_U_PLAT_BITS']
buckets_per_grp = clat['FIO_IO_U_PLAT_VAL']
for bucket in xrange(total_buckets):
for bucket in range(total_buckets):
if clat[str(bucket)]:
grp = bucket / buckets_per_grp
subbucket = bucket % buckets_per_grp
@ -511,7 +516,8 @@ class KBA_Storage_Client(KBA_Client):
val = int(base + (base / buckets_per_grp) * (subbucket - 0.5))
histogram.record_value(val, clat[str(bucket)])
p_output['jobs'][0][test]['clat']['hist'] = histogram.encode()
# histogram.encode() returns a base64 compressed histogram as bytes
p_output['jobs'][0][test]['clat']['hist'] = histogram.encode().decode('utf-8')
p_output['jobs'][0][test]['clat'].pop('bins')
p_output['jobs'][0][test]['clat'].pop('percentile')
@ -534,7 +540,7 @@ class KBA_Storage_Client(KBA_Client):
return self.encode_bins(p_out)
class KBA_Server(object):
class KBA_Server():
def __init__(self, user_data):
self.user_data = user_data
@ -544,14 +550,14 @@ class KBA_Server(object):
html_size = self.user_data['http_server_configs']['html_size']
cmd_str = 'dd if=/dev/zero of=/data/www/index.html bs=%s count=1' % html_size
cmd = cmd_str.split()
return False if exec_command(cmd) else True
return not bool(exec_command(cmd))
def start_nginx_server(self):
cmd = ['sudo', 'service', 'nginx', 'start']
return exec_command(cmd)
def start_nuttcp_server(self):
cmd = ['/usr/local/bin/nuttcp', '-S' '-P5000']
cmd = ['/usr/local/bin/nuttcp', '-S', '-P5000']
return exec_command(cmd)
def start_multicast_listener(self, mc_addrs, multicast_ports, start_address="231.0.0.128"):
@ -574,7 +580,7 @@ class KBA_Server(object):
s.bind((m_addr, port))
s.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq)
while True:
d, e = s.recvfrom(10240)
s.recvfrom(10240)
# End Function #
@ -587,7 +593,7 @@ class KBA_Server(object):
while True:
continue
class KBA_Proxy(object):
class KBA_Proxy():
def start_redis_server(self):
cmd = ['sudo', 'service', 'redis-server', 'start']
return exec_command(cmd)
@ -600,18 +606,19 @@ if __name__ == "__main__":
except Exception as e:
# KloudBuster starts without user-data
cwd = 'kloudbuster/kb_server'
cmd = ['python', 'setup.py', 'develop']
cmd = ['python3', 'setup.py', 'develop']
rc = exec_command(cmd, cwd=cwd)
if not rc:
syslog.syslog("Starting kloudbuster HTTP server")
cmd = ['/usr/local/bin/pecan', 'serve', 'config.py']
sys.exit(exec_command(cmd, cwd=cwd))
if user_data.get('role') == 'KB-PROXY':
role = user_data.get('role')
if role == 'KB-PROXY':
agent = KBA_Proxy()
syslog.syslog("Starting kloudbuster proxy server")
sys.exit(agent.start_redis_server())
if user_data.get('role').endswith('Server'):
if role.endswith('Server'):
agent = KBA_Server(user_data)
if user_data['role'].startswith('Multicast'):
KB_Instance.add_multicast_route()
@ -631,11 +638,11 @@ if __name__ == "__main__":
sys.exit(agent.start_nginx_server())
else:
sys.exit(1)
elif user_data.get('role').endswith('Client'):
if user_data['role'].startswith('HTTP'):
elif role.endswith('Client'):
if role.startswith('HTTP'):
syslog.syslog("Starting kloudbuster HTTP client")
agent = KBA_HTTP_Client(user_data)
elif user_data['role'].startswith('Multicast'):
elif role.startswith('Multicast'):
KB_Instance.add_multicast_route()
refresh_clock(user_data.get('ntp_clocks'), force_sync=True)
agent = KBA_Multicast_Client(user_data)

4
kb_server/kb_server/controllers/api_cfg.py
View File

@ -29,7 +29,7 @@ from pecan import response
LOG = logging.getLogger("kloudbuster")
class ConfigController(object):
class ConfigController():
# Decorator to check for missing or invalid session ID
def check_session_id(func):
@ -198,7 +198,7 @@ class ConfigController(object):
allowed_status = ['READY']
except Exception as e:
response.status = 400
response.text = u"Invalid JSON: \n%s" % (e.message)
response.text = u"Invalid JSON: \n%s" % str(e)
return response.text
# http_tool_configs and storage_tool_config for client VMs is allowed to be

2
kb_server/kb_server/controllers/api_kb.py
View File

@ -26,7 +26,7 @@ from pecan import response
LOG = logging.getLogger("kloudbuster")
class KBController(object):
class KBController():
def __init__(self):
self.kb_thread = None

4
kb_server/kb_server/controllers/kb_session.py
View File

@ -17,7 +17,7 @@ import threading
KB_SESSIONS = {}
KB_SESSIONS_LOCK = threading.Lock()
class KBSessionManager(object):
class KBSessionManager():
@staticmethod
def has(session_id):
@ -46,7 +46,7 @@ class KBSessionManager(object):
KB_SESSIONS_LOCK.release()
class KBSession(object):
class KBSession():
def __init__(self):
self.kb_status = 'READY'
self.first_run = True

4
kb_server/kb_server/controllers/root.py
View File

@ -19,7 +19,7 @@ from pecan import expose
from pecan import response
class APIController(object):
class APIController():
@expose()
def _lookup(self, primary_key, *remainder):
if primary_key == "config":
@ -30,7 +30,7 @@ class APIController(object):
abort(404)
class RootController(object):
class RootController():
@expose()
def index(self):
response.status = 301

20
kloudbuster/base_compute.py
View File

@ -15,7 +15,7 @@
import os
import time
import log as logging
import kloudbuster.log as logging
from novaclient.exceptions import BadRequest
LOG = logging.getLogger(__name__)
@ -24,7 +24,7 @@ class KBVolAttachException(Exception):
pass
class BaseCompute(object):
class BaseCompute():
"""
The Base class for nova compute resources
1. Creates virtual machines with specific configs
@ -46,13 +46,12 @@ class BaseCompute(object):
self.shared_interface_ip = None
self.vol = None
# Create a server instance with associated
# security group, keypair with a provided public key
def create_server(self, image_name, flavor_type, keyname,
nic, sec_group, avail_zone=None, user_data=None,
config_drive=True, retry_count=100):
"""
Create a server instance with associated security group, keypair with a provided public key.
Create a VM instance given following parameters
1. VM Name
2. Image Name
@ -93,6 +92,7 @@ class BaseCompute(object):
LOG.error('Instance creation error:' + instance.fault['message'])
return None
time.sleep(2)
return None
def attach_vol(self):
if self.vol.status != 'available':
@ -117,7 +117,7 @@ class BaseCompute(object):
def detach_vol(self):
if self.instance and self.vol:
attached_vols = self.novaclient.volumes.get_server_volumes(self.instance.id)
if len(attached_vols):
if attached_vols:
try:
self.novaclient.volumes.delete_server_volume(self.instance.id, self.vol.id)
except BadRequest:
@ -133,7 +133,7 @@ class BaseCompute(object):
return flavor
class SecGroup(object):
class SecGroup():
def __init__(self, novaclient, neutronclient):
self.secgroup = None
@ -238,7 +238,7 @@ class SecGroup(object):
LOG.error('Failed while deleting security group %s.' % self.secgroup['id'])
return False
class KeyPair(object):
class KeyPair():
def __init__(self, novaclient):
self.keypair = None
@ -268,7 +268,7 @@ class KeyPair(object):
if self.keypair:
self.novaclient.keypairs.delete(self.keypair)
class Flavor(object):
class Flavor():
def __init__(self, novaclient):
self.novaclient = novaclient
@ -304,7 +304,7 @@ class Flavor(object):
except Exception:
pass
class NovaQuota(object):
class NovaQuota():
def __init__(self, novaclient, tenant_id):
self.novaclient = novaclient

26
kloudbuster/base_network.py
View File

@ -14,11 +14,11 @@
import time
from perf_instance import PerfInstance
from kloudbuster.perf_instance import PerfInstance
import base_compute
import base_storage
import log as logging
import kloudbuster.base_compute as base_compute
import kloudbuster.base_storage as base_storage
import kloudbuster.log as logging
import netaddr
from neutronclient.common.exceptions import NetworkInUseClient
@ -101,7 +101,7 @@ def find_provider_network(neutron_client, name):
networks = neutron_client.list_networks()['networks']
for network in networks:
if network['provider:physical_network']:
if name == "" or name == network['name']:
if name in ("", network['name']):
return network
if name != "":
LOG.error("The provider network: " + name + " was not found.")
@ -116,11 +116,11 @@ def find_first_network(neutron_client):
If no external network is found return None
"""
networks = neutron_client.list_networks()['networks']
if (len(networks) > 0):
if networks:
return networks[0]
return None
class BaseNetwork(object):
class BaseNetwork():
"""
The Base class for neutron network operations
1. Creates networks with 1 subnet inside each network
@ -177,7 +177,7 @@ class BaseNetwork(object):
vol_size = 0
# Schedule to create the required number of VMs
for instance_count in xrange(vm_total):
for instance_count in range(vm_total):
vm_name = network_prefix + "-I" + str(instance_count)
perf_instance = PerfInstance(vm_name, self, config_scale)
self.instance_list.append(perf_instance)
@ -197,7 +197,8 @@ class BaseNetwork(object):
if config_scale['use_floatingip']:
# Create the floating ip for the instance
# store it and the ip address in perf_instance object
perf_instance.fip = create_floating_ip(self.neutron_client, external_network)
port_id = perf_instance.instance.interface_list()[0].id
perf_instance.fip = create_floating_ip(self.neutron_client, external_network, port_id)
perf_instance.fip_ip = perf_instance.fip['floatingip']['floating_ip_address']
self.res_logger.log('floating_ips',
perf_instance.fip['floatingip']['floating_ip_address'],
@ -270,7 +271,7 @@ class BaseNetwork(object):
if len(self.network['subnets']) > 0:
subnet = self.neutron_client.show_subnet(self.network['subnets'][0])['subnet']
self.network['subnet_ip'] = subnet['cidr']
self.network['is_ipv6'] = True if subnet['ipv6_address_mode'] else False
self.network['is_ipv6'] = bool(subnet['ipv6_address_mode'])
def get_cidr_from_subnet_id(self, subnetID):
sub = self.neutron_client.show_subnet(subnetID)
@ -281,6 +282,7 @@ class BaseNetwork(object):
"""Generate next CIDR for network or subnet, without IP overlapping.
"""
global cidr
# pylint: disable=not-callable
cidr = str(netaddr.IPNetwork(cidr).next())
return cidr
@ -304,7 +306,7 @@ class BaseNetwork(object):
def get_all_instances(self):
return self.instance_list
class Router(object):
class Router():
"""
Router class to create a new routers
Supports addition and deletion
@ -496,7 +498,7 @@ class Router(object):
class NeutronQuota(object):
class NeutronQuota():
def __init__(self, neutronclient, tenant_id):
self.neutronclient = neutronclient

6
kloudbuster/base_storage.py
View File

@ -14,14 +14,14 @@
import time
import log as logging
import kloudbuster.log as logging
LOG = logging.getLogger(__name__)
class KBVolCreationException(Exception):
pass
class BaseStorage(object):
class BaseStorage():
"""
The Base class for cinder storage resources
"""
@ -69,7 +69,7 @@ class BaseStorage(object):
# self.cinderclient.volumes.detach(volume)
class CinderQuota(object):
class CinderQuota():
def __init__(self, cinderclient, tenant_id):
self.cinderclient = cinderclient

3
kloudbuster/cfg.scale.yaml
View File

@ -32,8 +32,9 @@ image_name:
# vm_image_file: /kloudbuster/kloudbuster-7.0.0.qcow2
# If empty, KloudBuster will attempt to locate that file (with the default name)
# under the following directories:
# - root of the kloudbuster package
# - current directory
# - home directory
# - top directory ("/")
vm_image_file:
# Keystone admin role name (default should work in most deployments)

4
kloudbuster/credentials.py
View File

@ -21,11 +21,11 @@ from keystoneauth1 import session
import os
import re
import log as logging
import kloudbuster.log as logging
LOG = logging.getLogger(__name__)
class Credentials(object):
class Credentials():
def get_session(self):
dct = {

10
kloudbuster/fio_tool.py
View File

@ -15,7 +15,7 @@
import json
from perf_tool import PerfTool
from kloudbuster.perf_tool import PerfTool
from hdrh.histogram import HdrHistogram
@ -99,7 +99,7 @@ class FioTool(PerfTool):
histogram.decode_and_add(item['results'][clat])
latency_dict = histogram.get_percentile_to_value_dict(perc_list)
for key, value in latency_dict.iteritems():
for key, value in latency_dict.items():
all_res[clat].append([key, value])
all_res[clat].sort()
@ -108,10 +108,10 @@ class FioTool(PerfTool):
@staticmethod
def consolidate_samples(results, vm_count):
all_res = FioTool.consolidate_results(results)
total_count = float(len(results)) / vm_count
total_count = len(results) // vm_count
if not total_count:
return all_res
all_res['read_iops'] = int(all_res['read_iops'] / total_count)
all_res['write_iops'] = int(all_res['write_iops'] / total_count)
all_res['read_iops'] = all_res['read_iops'] // total_count
all_res['write_iops'] = all_res['write_iops'] // total_count
return all_res

172
kloudbuster/force_cleanup.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
# Copyright 2016 Cisco Systems, Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
@ -26,7 +26,7 @@
# #
# It is safe to use the script with the resource list generated by #
# KloudBuster, usage: #
# $ python force_cleanup.py --file kb_20150807_183001_svr.log #
# $ python3 force_cleanup.py --file kb_20150807_183001_svr.log #
# #
# Note: If running under single-tenant or tenant/user reusing mode, you have #
# to cleanup the server resources first, then client resources. #
@ -57,20 +57,25 @@ import traceback
# openstack python clients
import cinderclient
from keystoneclient import client as keystoneclient
from cinderclient.client import Client as CinderClient
import keystoneclient
from keystoneclient.client import Client as KeystoneClient
import neutronclient
from neutronclient.neutron.client import Client as NeutronClient
from novaclient.client import Client as NovaClient
from novaclient.exceptions import NotFound
from tabulate import tabulate
# kloudbuster base code
import credentials
import kloudbuster.credentials as credentials
resource_name_re = None
def prompt_to_run():
print "Warning: You didn't specify a resource list file as the input. "\
"The script will delete all resources shown above."
answer = raw_input("Are you sure? (y/n) ")
print("Warning: You didn't specify a resource list file as the input. "
"The script will delete all resources shown above.")
answer = input("Are you sure? (y/n) ")
if answer.lower() != 'y':
sys.exit(0)
@ -83,7 +88,7 @@ def fetch_resources(fetcher, options=None):
except Exception as e:
res_list = []
traceback.print_exc()
print "Warning exception while listing resources:" + str(e)
print('Warning exception while listing resources:', str(e))
resources = {}
for res in res_list:
# some objects provide direct access some
@ -98,16 +103,15 @@ def fetch_resources(fetcher, options=None):
resources[resid] = resname
return resources
class AbstractCleaner(object):
__metaclass__ = ABCMeta
class AbstractCleaner(metaclass=ABCMeta):
def __init__(self, res_category, res_desc, resources, dryrun):
self.dryrun = dryrun
self.category = res_category
self.resources = {}
if not resources:
print 'Discovering %s resources...' % (res_category)
for rtype, fetch_args in res_desc.iteritems():
print('Discovering %s resources...' % (res_category))
for rtype, fetch_args in res_desc.items():
if resources:
if rtype in resources:
self.resources[rtype] = resources[rtype]
@ -116,20 +120,20 @@ class AbstractCleaner(object):
def report_deletion(self, rtype, name):
if self.dryrun:
print ' + ' + rtype + ' ' + name + ' should be deleted (but is not deleted: dry run)'
print(' + ' + rtype + ' ' + name + ' should be deleted (but is not deleted: dry run)')
else:
print ' + ' + rtype + ' ' + name + ' is successfully deleted'
print(' + ' + rtype + ' ' + name + ' is successfully deleted')
def report_not_found(self, rtype, name):
print ' ? ' + rtype + ' ' + name + ' not found (already deleted?)'
print(' ? ' + rtype + ' ' + name + ' not found (already deleted?)')
def report_error(self, rtype, name, reason):
print ' - ' + rtype + ' ' + name + ' ERROR:' + reason
print(' - ' + rtype + ' ' + name + ' ERROR:' + reason)
def get_resource_list(self):
result = []
for rtype, rdict in self.resources.iteritems():
for resid, resname in rdict.iteritems():
for rtype, rdict in self.resources.items():
for resid, resname in rdict.items():
result.append([rtype, resname, resid])
return result
@ -139,21 +143,20 @@ class AbstractCleaner(object):
class StorageCleaner(AbstractCleaner):
def __init__(self, sess, resources, dryrun):
from cinderclient import client as cclient
from novaclient import client as nclient
self.nova = nclient.Client('2', endpoint_type='publicURL', session=sess)
self.cinder = cclient.Client('2', endpoint_type='publicURL', session=sess)
self.nova = NovaClient('2', endpoint_type='publicURL', session=sess)
self.cinder = CinderClient('2', endpoint_type='publicURL', session=sess)
res_desc = {'volumes': [self.cinder.volumes.list, {"all_tenants": 1}]}
super(StorageCleaner, self).__init__('Storage', res_desc, resources, dryrun)
def clean(self):
print '*** STORAGE cleanup'
print('*** STORAGE cleanup')
try:
kb_volumes = []
kb_detaching_volumes = []
for id, name in self.resources['volumes'].iteritems():
for id, name in self.resources['volumes'].items():
try:
vol = self.cinder.volumes.get(id)
if vol.attachments:
@ -162,15 +165,15 @@ class StorageCleaner(AbstractCleaner):
if not self.dryrun:
ins_id = vol.attachments[0]['server_id']
self.nova.volumes.delete_server_volume(ins_id, id)
print ' . VOLUME ' + vol.name + ' detaching...'
print(' . VOLUME ' + vol.name + ' detaching...')
else:
print ' . VOLUME ' + vol.name + ' to be detached...'
print(' . VOLUME ' + vol.name + ' to be detached...')
kb_detaching_volumes.append(vol)
except NotFound:
print 'WARNING: Volume %s attached to an instance that no longer '\
'exists (will require manual cleanup of the database)' % (id)
print('WARNING: Volume %s attached to an instance that no longer '
'exists (will require manual cleanup of the database)' % id)
except Exception as e:
print str(e)
print(str(e))
else:
# no attachments
kb_volumes.append(vol)
@ -180,8 +183,8 @@ class StorageCleaner(AbstractCleaner):
# check that the volumes are no longer attached
if kb_detaching_volumes:
if not self.dryrun:
print ' . Waiting for %d volumes to be fully detached...' % \
(len(kb_detaching_volumes))
print(' . Waiting for %d volumes to be fully detached...' %
(len(kb_detaching_volumes)))
retry_count = 5 + len(kb_detaching_volumes)
while True:
retry_count -= 1
@ -190,19 +193,19 @@ class StorageCleaner(AbstractCleaner):
latest_vol = self.cinder.volumes.get(kb_detaching_volumes[0].id)
if self.dryrun or not latest_vol.attachments:
if not self.dryrun:
print ' + VOLUME ' + vol.name + ' detach complete'
print(' + VOLUME ' + vol.name + ' detach complete')
kb_detaching_volumes.remove(vol)
kb_volumes.append(vol)
if kb_detaching_volumes and not self.dryrun:
if retry_count:
print ' . VOLUME %d left to be detached, retries left=%d...' % \
(len(kb_detaching_volumes), retry_count)
print(' . VOLUME %d left to be detached, retries left=%d...' %
len(kb_detaching_volumes), retry_count)
time.sleep(2)
else:
print ' - VOLUME detach timeout, %d volumes left:' % \
(len(kb_detaching_volumes))
print(' - VOLUME detach timeout, %d volumes left:' %
len(kb_detaching_volumes))
for vol in kb_detaching_volumes:
print ' ', vol.name, vol.status, vol.id, vol.attachments
print(' ', vol.name, vol.status, vol.id, vol.attachments)
break
else:
break
@ -213,17 +216,15 @@ class StorageCleaner(AbstractCleaner):
try:
vol.force_delete()
except cinderclient.exceptions.BadRequest as exc:
print str(exc)
print(str(exc))
self.report_deletion('VOLUME', vol.name)
except KeyError:
pass
class ComputeCleaner(AbstractCleaner):
def __init__(self, sess, resources, dryrun):
from neutronclient.neutron import client as nclient
from novaclient import client as novaclient
self.neutron_client = nclient.Client('2.0', endpoint_type='publicURL', session=sess)
self.nova_client = novaclient.Client('2', endpoint_type='publicURL', session=sess)
self.neutron_client = NeutronClient('2.0', endpoint_type='publicURL', session=sess)
self.nova_client = NovaClient('2', endpoint_type='publicURL', session=sess)
res_desc = {
'instances': [self.nova_client.servers.list, {"all_tenants": 1}],
'flavors': [self.nova_client.flavors.list],
@ -232,15 +233,16 @@ class ComputeCleaner(AbstractCleaner):
super(ComputeCleaner, self).__init__('Compute', res_desc, resources, dryrun)
def clean(self):
print '*** COMPUTE cleanup'
print('*** COMPUTE cleanup')
try:
# Get a list of floating IPs
fip_lst = self.neutron_client.list_floatingips()['floatingips']
deleting_instances = self.resources['instances']
for id, name in self.resources['instances'].iteritems():
for id, name in self.resources['instances'].items():
try:
if self.nova_client.servers.get(id).addresses.values():
ins_addr = self.nova_client.servers.get(id).addresses.values()[0]
addrs = list(self.nova_client.servers.get(id).addresses.values())
if addrs:
ins_addr = addrs[0]
fips = [x['addr'] for x in ins_addr if x['OS-EXT-IPS:type'] == 'floating']
else:
fips = []
@ -259,33 +261,36 @@ class ComputeCleaner(AbstractCleaner):
deleting_instances.remove(id)
self.report_not_found('INSTANCE', name)
if not self.dryrun and len(deleting_instances):
print ' . Waiting for %d instances to be fully deleted...' % \
(len(deleting_instances))
if not self.dryrun and deleting_instances:
print(' . Waiting for %d instances to be fully deleted...' %
len(deleting_instances))
retry_count = 5 + len(deleting_instances)
while True:
retry_count -= 1
for ins_id in deleting_instances.keys():
# get a copy of the initial list content
instances_list = list(deleting_instances)
for ins_id in instances_list:
try:
self.nova_client.servers.get(ins_id)
except NotFound:
self.report_deletion('INSTANCE', deleting_instances[ins_id])
deleting_instances.pop(ins_id)
if not len(deleting_instances):
if not deleting_instances:
# all deleted
break
if retry_count:
print ' . INSTANCE %d left to be deleted, retries left=%d...' % \
(len(deleting_instances), retry_count)
print(' . INSTANCE %d left to be deleted, retries left=%d...' %
(len(deleting_instances), retry_count))
time.sleep(2)
else:
print ' - INSTANCE deletion timeout, %d instances left:' % \
(len(deleting_instances))
print(' - INSTANCE deletion timeout, %d instances left:' %
len(deleting_instances))
for ins_id in deleting_instances.keys():
try:
ins = self.nova_client.servers.get(ins_id)
print ' ', ins.name, ins.status, ins.id
print(' ', ins.name, ins.status, ins.id)
except NotFound:
print(' ', deleting_instances[ins_id],
'(just deleted)', ins_id)
@ -294,7 +299,7 @@ class ComputeCleaner(AbstractCleaner):
pass
try:
for id, name in self.resources['flavors'].iteritems():
for id, name in self.resources['flavors'].items():
try:
flavor = self.nova_client.flavors.find(name=name)
if not self.dryrun:
@ -306,7 +311,7 @@ class ComputeCleaner(AbstractCleaner):
pass
try:
for id, name in self.resources['keypairs'].iteritems():
for id, name in self.resources['keypairs'].items():
try:
if self.dryrun:
self.nova_client.keypairs.get(name)
@ -321,8 +326,7 @@ class ComputeCleaner(AbstractCleaner):
class NetworkCleaner(AbstractCleaner):
def __init__(self, sess, resources, dryrun):
from neutronclient.neutron import client as nclient
self.neutron = nclient.Client('2.0', endpoint_type='publicURL', session=sess)
self.neutron = NeutronClient('2.0', endpoint_type='publicURL', session=sess)
# because the response has an extra level of indirection
# we need to extract it to present the list of network or router objects
@ -357,10 +361,10 @@ class NetworkCleaner(AbstractCleaner):
pass
def clean(self):
print '*** NETWORK cleanup'
print('*** NETWORK cleanup')
try:
for id, name in self.resources['sec_groups'].iteritems():
for id, name in self.resources['sec_groups'].items():
try:
if self.dryrun:
self.neutron.show_security_group(id)
@ -373,7 +377,7 @@ class NetworkCleaner(AbstractCleaner):
pass
try:
for id, name in self.resources['floating_ips'].iteritems():
for id, name in self.resources['floating_ips'].items():
try:
if self.dryrun:
self.neutron.show_floatingip(id)
@ -386,7 +390,7 @@ class NetworkCleaner(AbstractCleaner):
pass
try:
for id, name in self.resources['routers'].iteritems():
for id, name in self.resources['routers'].items():
try:
if self.dryrun:
self.neutron.show_router(id)
@ -412,7 +416,7 @@ class NetworkCleaner(AbstractCleaner):
except KeyError:
pass
try:
for id, name in self.resources['networks'].iteritems():
for id, name in self.resources['networks'].items():
try:
if self.dryrun:
self.neutron.show_network(id)
@ -429,7 +433,7 @@ class NetworkCleaner(AbstractCleaner):
class KeystoneCleaner(AbstractCleaner):
def __init__(self, sess, resources, dryrun):
self.keystone = keystoneclient.Client(endpoint_type='publicURL', session=sess)
self.keystone = KeystoneClient(endpoint_type='publicURL', session=sess)
self.tenant_api = self.keystone.tenants \
if self.keystone.version == 'v2.0' else self.keystone.projects
res_desc = {
@ -439,9 +443,9 @@ class KeystoneCleaner(AbstractCleaner):
super(KeystoneCleaner, self).__init__('Keystone', res_desc, resources, dryrun)
def clean(self):
print '*** KEYSTONE cleanup'
print('*** KEYSTONE cleanup')
try:
for id, name in self.resources['users'].iteritems():
for id, name in self.resources['users'].items():
try:
if self.dryrun:
self.keystone.users.get(id)
@ -454,7 +458,7 @@ class KeystoneCleaner(AbstractCleaner):
pass
try:
for id, name in self.resources['tenants'].iteritems():
for id, name in self.resources['tenants'].items():
try:
if self.dryrun:
self.tenant_api.get(id)
@ -466,7 +470,7 @@ class KeystoneCleaner(AbstractCleaner):
except KeyError:
pass
class KbCleaners(object):
class KbCleaners():
def __init__(self, creds_obj, resources, dryrun):
self.cleaners = []
@ -479,13 +483,13 @@ class KbCleaners(object):
for cleaner in self.cleaners:
table.extend(cleaner.get_resource_list())
count = len(table) - 1
print
print()
if count:
print 'SELECTED RESOURCES:'
print tabulate(table, headers="firstrow", tablefmt="psql")
print('SELECTED RESOURCES:')
print(tabulate(table, headers="firstrow", tablefmt="psql"))
else:
print 'There are no resources to delete.'
print
print('There are no resources to delete.')
print()
return count
def clean(self):
@ -511,7 +515,7 @@ def get_resources_from_cleanup_log(logfile):
if not resid:
# normally only the keypairs have no ID
if restype != "keypairs":
print 'Error: resource type %s has no ID - ignored!!!' % (restype)
print('Error: resource type %s has no ID - ignored!!!' % (restype))
else:
resid = '0'
if restype not in resources:
@ -556,9 +560,9 @@ def main():
try:
resource_name_re = re.compile(opts.filter)
except Exception as exc:
print 'Provided filter is not a valid python regular expression: ' + opts.filter
print str(exc)
sys.exit(1)
print('Provided filter is not a valid python regular expression: ' + opts.filter)
print(str(exc))
return 1
else:
resource_name_re = re.compile('KB')
@ -566,19 +570,21 @@ def main():
cleaners = KbCleaners(cred, resources, opts.dryrun)
if opts.dryrun:
print
print()
print('!!! DRY RUN - RESOURCES WILL BE CHECKED BUT WILL NOT BE DELETED !!!')
print
print()
# Display resources to be deleted
count = cleaners.show_resources()
if not count:
sys.exit(0)
return 0
if not opts.file and not opts.dryrun:
prompt_to_run()
cleaners.clean()
return 0
if __name__ == '__main__':
main()
sys.exit(main())

25
kloudbuster/kb_config.py
View File

@ -15,14 +15,15 @@
import os
import sys
import yaml
from pathlib import Path
from __init__ import __version__
from attrdict import AttrDict
import log as logging
from oslo_config import cfg
from pkg_resources import resource_string
import credentials
import kloudbuster.credentials as credentials
from kloudbuster.__init__ import __version__
import kloudbuster.log as logging
CONF = cfg.CONF
LOG = logging.getLogger(__name__)
@ -68,7 +69,7 @@ def get_absolute_path_for_file(file_name):
return abs_file_path
class KBConfig(object):
class KBConfig():
def __init__(self):
# The default configuration file for KloudBuster
default_cfg = resource_string(__name__, "cfg.scale.yaml")
@ -127,17 +128,13 @@ class KBConfig(object):
# Check if the default image is located at the default locations
# if vm_image_file is empty
if not self.config_scale['vm_image_file']:
# check current directory
default_image_file = default_image_name + '.qcow2'
if os.path.isfile(default_image_file):
self.config_scale['vm_image_file'] = default_image_file
else:
# check at the root of the package
# root is up one level where this module resides
pkg_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
default_image_file = pkg_root + '/' + default_image_file
img_path_list = [os.getcwd(), str(Path.home()), '/']
for img_path in img_path_list:
default_image_file = os.path.join(img_path, default_image_name + '.qcow2')
if os.path.isfile(default_image_file):
self.config_scale['vm_image_file'] = default_image_file
LOG.info('Found VM image: %s', default_image_file)
break
# A bit of config dict surgery, extract out the client and server side
# and transplant the remaining (common part) into the client and server dict
@ -243,4 +240,4 @@ class KBConfig(object):
self.config_scale['number_tenants'] = 1
except Exception as e:
LOG.error('Cannot parse the count of tenant/user from the config file.')
raise KBConfigParseException(e.message)
raise KBConfigParseException(str(e))

4
kloudbuster/kb_res_logger.py
View File

@ -12,7 +12,7 @@
# License for the specific language governing permissions and limitations
# under the License.
import log as logging
import kloudbuster.log as logging
from time import gmtime
from time import strftime
@ -21,7 +21,7 @@ LOG = logging.getLogger(__name__)
class KBResTypeInvalid(Exception):
pass
class KBResLogger(object):
class KBResLogger():
def __init__(self):
self.resource_list = {}

58
kloudbuster/kb_runner_base.py
View File

@ -12,21 +12,31 @@
# License for the specific language governing permissions and limitations
# under the License.
from __future__ import division
import abc
from collections import deque
import json
import log as logging
import redis
import sys
import threading
import time
import kloudbuster.log as logging
# A set of warned VM version mismatches
vm_version_mismatches = set()
LOG = logging.getLogger(__name__)
def cmp(x, y):
"""
Replacement for built-in function cmp that was removed in Python 3
Compare the two objects x and y and return an integer according to
the outcome. The return value is negative if x < y, zero if x == y
and strictly positive if x > y.
"""
return (x > y) - (x < y)
class KBException(Exception):
pass
@ -36,7 +46,7 @@ class KBVMUpException(KBException):
class KBProxyConnectionException(KBException):
pass
class KBRunner(object):
class KBRunner():
"""
Control the testing VMs on the testing cloud
"""
@ -51,6 +61,7 @@ class KBRunner(object):
self.tool_result = {}
self.agent_version = None
self.report = None
self.msg_thread = None
# Redis
self.redis_obj = None
@ -61,13 +72,13 @@ class KBRunner(object):
def msg_handler(self):
for message in self.pubsub.listen():
if message['data'] == "STOP":
if message['data'] == b"STOP":
break
LOG.kbdebug(message)
self.message_queue.append(message)
def setup_redis(self, redis_server, redis_server_port=6379, timeout=120):
LOG.info("Setting up the redis connections...")
LOG.info("Connecting to redis server in proxy VM %s:%d...", redis_server, redis_server_port)
connection_pool = redis.ConnectionPool(
host=redis_server, port=redis_server_port, db=0)
@ -77,14 +88,11 @@ class KBRunner(object):
success = False
retry_count = max(timeout // self.config.polling_interval, 1)
# Check for connections to redis server
for retry in xrange(retry_count):
for retry in range(retry_count):
try:
self.redis_obj.get("test")
success = True
except (redis.exceptions.ConnectionError):
# clear active exception to avoid the exception summary
# appended to LOG.info by oslo log
sys.exc_clear()
except redis.exceptions.ConnectionError:
LOG.info("Connecting to redis server... Retry #%d/%d", retry, retry_count)
time.sleep(self.config.polling_interval)
continue
@ -125,9 +133,9 @@ class KBRunner(object):
retry = cnt_succ = cnt_failed = 0
clist = self.client_dict.copy()
samples = []
perf_tool = self.client_dict.values()[0].perf_tool
perf_tool = list(self.client_dict.values())[0].perf_tool
while (retry < retry_count and len(clist)):
while (retry < retry_count and clist):
time.sleep(polling_interval)
sample_count = 0
while True:
@ -135,10 +143,9 @@ class KBRunner(object):
msg = self.message_queue.popleft()
except IndexError:
# No new message, commands are in executing
# clear active exc to prevent LOG pollution
sys.exc_clear()
break
# pylint: disable=eval-used
payload = eval(msg['data'])
vm_name = payload['sender-id']
cmd = payload['cmd']
@ -149,11 +156,10 @@ class KBRunner(object):
instance = self.full_client_dict[vm_name]
if instance.up_flag:
continue
else:
clist[vm_name].up_flag = True
clist.pop(vm_name)
cnt_succ = cnt_succ + 1
self.agent_version = payload['data']
clist[vm_name].up_flag = True
clist.pop(vm_name)
cnt_succ = cnt_succ + 1
self.agent_version = payload['data']
elif cmd == 'REPORT':
sample_count = sample_count + 1
# Parse the results from HTTP Tools
@ -183,8 +189,8 @@ class KBRunner(object):
else:
LOG.error('[%s] received invalid command: %s' + (vm_name, cmd))
log_msg = "VMs: %d Ready, %d Failed, %d Pending... Retry #%d" %\
(cnt_succ, cnt_failed, len(clist), retry)
log_msg = "VMs: %d Ready, %d Failed, %d Pending... Retry #%d/%d" %\
(cnt_succ, cnt_failed, len(clist), retry, retry_count)
if sample_count != 0:
log_msg += " (%d sample(s) received)" % sample_count
LOG.info(log_msg)
@ -202,6 +208,7 @@ class KBRunner(object):
LOG.info("Waiting for agents on VMs to come up...")
cnt_succ = self.polling_vms(timeout)[0]
if cnt_succ != len(self.client_dict):
print('Exception %d != %d' % (cnt_succ, len(self.client_dict)))
raise KBVMUpException("Some VMs failed to start.")
self.send_cmd('ACK', None, None)
@ -213,7 +220,7 @@ class KBRunner(object):
self.host_stats[phy_host] = []
self.host_stats[phy_host].append(self.result[vm])
perf_tool = self.client_dict.values()[0].perf_tool
perf_tool = list(self.client_dict.values())[0].perf_tool
for phy_host in self.host_stats:
self.host_stats[phy_host] = perf_tool.consolidate_results(self.host_stats[phy_host])
@ -224,3 +231,8 @@ class KBRunner(object):
def stop(self):
self.send_cmd('ABORT', None, None)
def get_sorted_vm_list(self):
vm_list = self.full_client_dict.keys()
vm_list.sort(cmp=lambda x, y: cmp(int(x[x.rfind('I') + 1:]), int(y[y.rfind('I') + 1:])))
return vm_list

25
kloudbuster/kb_runner_http.py
View File

@ -12,11 +12,9 @@
# License for the specific language governing permissions and limitations
# under the License.
from __future__ import division
from kb_runner_base import KBException
from kb_runner_base import KBRunner
import log as logging
from kloudbuster.kb_runner_base import KBException
from kloudbuster.kb_runner_base import KBRunner
import kloudbuster.log as logging
LOG = logging.getLogger(__name__)
@ -84,8 +82,7 @@ class KBRunner_HTTP(KBRunner):
self.check_http_service(active_range)
if self.config.prompt_before_run:
print "Press enter to start running benchmarking tools..."
raw_input()
_ = input("Press enter to start running benchmarking tools...")
LOG.info("Running HTTP Benchmarking...")
self.report = {'seq': 0, 'report': None}
@ -93,9 +90,10 @@ class KBRunner_HTTP(KBRunner):
self.run_http_test(active_range)
# Call the method in corresponding tools to consolidate results
perf_tool = self.client_dict.values()[0].perf_tool
LOG.kbdebug(self.result.values())
self.tool_result = perf_tool.consolidate_results(self.result.values())
perf_tool = list(self.client_dict.values())[0].perf_tool
results = list(self.result.values())
LOG.kbdebug(results)
self.tool_result = perf_tool.consolidate_results(results)
self.tool_result['http_rate_limit'] =\
len(self.client_dict) * self.config.http_tool_configs.rate_limit
self.tool_result['total_connections'] =\
@ -120,8 +118,7 @@ class KBRunner_HTTP(KBRunner):
multiple = self.config.progression.vm_multiple
limit = self.config.progression.http_stop_limit
timeout = self.config.http_tool_configs.timeout
vm_list = self.full_client_dict.keys()
vm_list.sort(cmp=lambda x, y: cmp(int(x[x.rfind('I') + 1:]), int(y[y.rfind('I') + 1:])))
vm_list = self.get_sorted_vm_list()
self.client_dict = {}
cur_stage = 1
@ -137,7 +134,7 @@ class KBRunner_HTTP(KBRunner):
if self.tool_result and 'latency_stats' in self.tool_result:
err = self.tool_result['http_sock_err'] + self.tool_result['http_sock_timeout']
pert_dict = dict(self.tool_result['latency_stats'])
if limit[1] in pert_dict.keys():
if limit[1] in pert_dict:
timeout_at_percentile = pert_dict[limit[1]] // 1000000
elif limit[1] != 0:
LOG.warning('Percentile %s%% is not a standard statistic point.' % limit[1])
@ -146,7 +143,7 @@ class KBRunner_HTTP(KBRunner):
'reaches the stop limit.')
break
for idx in xrange(cur_vm_count, target_vm_count):
for idx in range(cur_vm_count, target_vm_count):
self.client_dict[vm_list[idx]] = self.full_client_dict[vm_list[idx]]
description = "-- %s --" % self.header_formatter(cur_stage, len(self.client_dict))
LOG.info(description)

24
kloudbuster/kb_runner_multicast.py
View File

@ -12,11 +12,9 @@
# License for the specific language governing permissions and limitations
# under the License.
from __future__ import division
from kb_runner_base import KBException
from kb_runner_base import KBRunner
import log as logging
from kloudbuster.kb_runner_base import KBException
from kloudbuster.kb_runner_base import KBRunner
import kloudbuster.log as logging
LOG = logging.getLogger(__name__)
class KBMulticastServerUpException(KBException):
@ -41,12 +39,12 @@ class KBRunner_Multicast(KBRunner):
def setup_static_route(self, active_range, timeout=30):
func = {'cmd': 'setup_static_route', 'active_range': active_range}
self.send_cmd('EXEC', 'multicast', func)
self.polling_vms(timeout)[0]
_ = self.polling_vms(timeout)[0]
def check_multicast_service(self, active_range, timeout=30):
func = {'cmd': 'check_multicast_service', 'active_range': active_range}
self.send_cmd('EXEC', 'multicast', func)
self.polling_vms(timeout)[0]
_ = self.polling_vms(timeout)[0]
def run_multicast_test(self, active_range, opts, timeout):
func = {'cmd': 'run_multicast_test', 'active_range': active_range,
@ -61,10 +59,10 @@ class KBRunner_Multicast(KBRunner):
@staticmethod
def json_to_csv(jsn):
csv = "Test,receivers,addresses,ports,bitrate,pkt_size,"
firstKey = [x for x in jsn.keys()][0]
firstKey = list(jsn)[0]
keys = jsn[firstKey].keys()
csv += ",".join(keys) + "\r\n"
for obj_k in jsn.keys():
for obj_k in jsn:
obj = jsn[obj_k]
obj_vals = map(str, obj.values())
csv += '"' + obj_k + '"' + "," + obj_k + "," + ",".join(obj_vals) + "\r\n"
@ -81,8 +79,7 @@ class KBRunner_Multicast(KBRunner):
self.check_multicast_service(active_range)
if self.config.prompt_before_run:
print "Press enter to start running benchmarking tools..."
raw_input()
_ = input("Press enter to start running benchmarking tools...")
LOG.info("Running Multicast Benchmarking...")
self.report = {'seq': 0, 'report': None}
@ -101,8 +98,7 @@ class KBRunner_Multicast(KBRunner):
def run(self, test_only=False, run_label=None):
self.tool_result = {}
vm_list = self.full_client_dict.keys()
vm_list.sort(cmp=lambda x, y: cmp(int(x[x.rfind('I') + 1:]), int(y[y.rfind('I') + 1:])))
vm_list = self.get_sorted_vm_list()
self.client_dict = {}
cur_stage = 1
@ -125,7 +121,7 @@ class KBRunner_Multicast(KBRunner):
server_port = 5000
for nReceiver in receivers:
for idx in range(0, nReceiver):
for _ in range(0, nReceiver):
self.client_dict[vm_list[0]] = self.full_client_dict[vm_list[0]]
if nReceiver > 1:

32
kloudbuster/kb_runner_storage.py
View File

@ -12,11 +12,9 @@
# License for the specific language governing permissions and limitations
# under the License.
from __future__ import division
from kb_runner_base import KBException
from kb_runner_base import KBRunner
import log as logging
from kloudbuster.kb_runner_base import KBException
from kloudbuster.kb_runner_base import KBRunner
import kloudbuster.log as logging
LOG = logging.getLogger(__name__)
@ -74,8 +72,7 @@ class KBRunner_Storage(KBRunner):
# timeout is calculated as 30s/GB/client VM
timeout = 60 * self.config.storage_stage_configs.io_file_size * len(self.client_dict)
parameter = {'size': str(self.config.storage_stage_configs.io_file_size) + 'GiB'}
parameter['mkfs'] = True \
if self.config.storage_stage_configs.target == 'volume' else False
parameter['mkfs'] = bool(self.config.storage_stage_configs.target == 'volume')
func = {'cmd': 'init_volume', 'active_range': active_range,
'parameter': parameter}
@ -114,11 +111,10 @@ class KBRunner_Storage(KBRunner):
self.init_volume(active_range)
if self.config.prompt_before_run:
print "Press enter to start running benchmarking tools..."
raw_input()
_ = input("Press enter to start running benchmarking tools...")
test_count = len(self.config.storage_tool_configs)
perf_tool = self.client_dict.values()[0].perf_tool
perf_tool = list(self.client_dict.values())[0].perf_tool
self.tool_result = []
vm_count = active_range[1] - active_range[0] + 1\
if active_range else len(self.full_client_dict)
@ -130,9 +126,10 @@ class KBRunner_Storage(KBRunner):
timeout_vms = self.run_storage_test(active_range, dict(cur_config))
# Call the method in corresponding tools to consolidate results
LOG.kbdebug(self.result.values())
results = list(self.result.values())
LOG.kbdebug(results)
tc_result = perf_tool.consolidate_results(self.result.values())
tc_result = perf_tool.consolidate_results(results)
tc_result['description'] = cur_config['description']
tc_result['mode'] = cur_config['mode']
tc_result['block_size'] = cur_config['block_size']
@ -168,8 +165,8 @@ class KBRunner_Storage(KBRunner):
start = self.config.progression.vm_start
multiple = self.config.progression.vm_multiple
limit = self.config.progression.storage_stop_limit
vm_list = self.full_client_dict.keys()
vm_list.sort(cmp=lambda x, y: cmp(int(x[x.rfind('I') + 1:]), int(y[y.rfind('I') + 1:])))
vm_list = self.get_sorted_vm_list()
self.client_dict = {}
cur_stage = 1
@ -183,7 +180,7 @@ class KBRunner_Storage(KBRunner):
if target_vm_count > len(self.full_client_dict):
break
for idx in xrange(cur_vm_count, target_vm_count):
for idx in range(cur_vm_count, target_vm_count):
self.client_dict[vm_list[idx]] = self.full_client_dict[vm_list[idx]]
description = "-- %s --" % self.header_formatter(cur_stage, len(self.client_dict))
@ -210,9 +207,8 @@ class KBRunner_Storage(KBRunner):
if req_iops or req_rate:
degrade_iops = (req_iops - cur_iops) * 100 / req_iops if req_iops else 0
degrade_rate = (req_rate - cur_rate) * 100 / req_rate if req_rate else 0
if ((cur_tc['mode'] in ['randread', 'randwrite'] and
degrade_iops > limit)
or (cur_tc['mode'] in ['read', 'write'] and degrade_rate > limit)):
if (cur_tc['mode'] in ['randread', 'randwrite'] and degrade_iops > limit) or \
(cur_tc['mode'] in ['read', 'write'] and degrade_rate > limit):
LOG.warning('KloudBuster is stopping the iteration '
'because the result reaches the stop limit.')
tc_flag = False

4
kloudbuster/kb_scheduler.py
View File

@ -12,7 +12,7 @@
# License for the specific language governing permissions and limitations
# under the License.
import log as logging
import kloudbuster.log as logging
LOG = logging.getLogger(__name__)
@ -22,7 +22,7 @@ class KBVMMappingAlgoNotSup(Exception):
class KBVMPlacementAlgoNotSup(Exception):
pass
class KBScheduler(object):
class KBScheduler():
"""
1. VM Placements
2. Mapping client VMs to target servers

1
kloudbuster/kb_vm_agent.py
View File

@ -1 +0,0 @@
../kb_dib/elements/kloudbuster/static/kb_test/kb_vm_agent.py

148
kloudbuster/kloudbuster.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
# Copyright 2016 Cisco Systems, Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
@ -13,7 +13,7 @@
# License for the specific language governing permissions and limitations
# under the License.
from __init__ import __version__
from kloudbuster.__init__ import __version__
from concurrent.futures import ThreadPoolExecutor
import datetime
@ -26,30 +26,32 @@ import time
import traceback
import webbrowser
import base_compute
import base_network
from cinderclient import client as cinderclient
from cinderclient.client import Client as CinderClient
from glanceclient import exc as glance_exception
from glanceclient.v2 import client as glanceclient
from kb_config import KBConfig
from kb_res_logger import KBResLogger
from kb_runner_base import KBException
from kb_runner_http import KBRunner_HTTP
from kb_runner_multicast import KBRunner_Multicast
from kb_runner_storage import KBRunner_Storage
from kb_scheduler import KBScheduler
from glanceclient.v2.client import Client as GlanceClient
import keystoneauth1
from keystoneclient import client as keystoneclient
import log as logging
from neutronclient.neutron import client as neutronclient
from novaclient import client as novaclient
from neutronclient.neutron.client import Client as NeutronClient
from novaclient.client import Client as NovaClient
from oslo_config import cfg
from pkg_resources import resource_filename
from pkg_resources import resource_string
from tabulate import tabulate
import tenant
import kloudbuster.base_compute as base_compute
import kloudbuster.base_network as base_network
from kloudbuster.kb_config import KBConfig
from kloudbuster.kb_res_logger import KBResLogger
from kloudbuster.kb_runner_base import KBException
from kloudbuster.kb_runner_http import KBRunner_HTTP
from kloudbuster.kb_runner_multicast import KBRunner_Multicast
from kloudbuster.kb_runner_storage import KBRunner_Storage
from kloudbuster.kb_scheduler import KBScheduler
import kloudbuster.log as logging
import kloudbuster.tenant as tenant
CONF = cfg.CONF
LOG = logging.getLogger(__name__)
@ -69,7 +71,7 @@ FLAVOR_KB_CLIENT = 'KB.client'
FLAVOR_KB_SERVER = 'KB.server'
class Kloud(object):
class Kloud():
def __init__(self, scale_cfg, cred, reusing_tenants, vm_img,
testing_side=False, storage_mode=False, multicast_mode=False):
self.tenant_list = []
@ -96,12 +98,12 @@ class Kloud(object):
# these client handles use the kloudbuster credentials (usually admin)
# to do tenant creation, tenant nova+cinder quota allocation and the like
self.keystone = keystoneclient.Client(session=self.osclient_session)
self.neutron_client = neutronclient.Client('2.0', endpoint_type='publicURL',
session=self.osclient_session)
self.nova_client = novaclient.Client('2', endpoint_type='publicURL',
session=self.osclient_session)
self.cinder_client = cinderclient.Client('2', endpoint_type='publicURL',
session=self.osclient_session)
self.neutron_client = NeutronClient('2.0', endpoint_type='publicURL',
session=self.osclient_session)
self.nova_client = NovaClient('2', endpoint_type='publicURL',
session=self.osclient_session)
self.cinder_client = CinderClient('2', endpoint_type='publicURL',
session=self.osclient_session)
LOG.info("Creating kloud: " + self.prefix)
if self.placement_az:
LOG.info('%s Availability Zone: %s' % (self.name, self.placement_az))
@ -113,6 +115,7 @@ class Kloud(object):
flavor_manager = base_compute.Flavor(self.nova_client)
fcand = {'vcpus': sys.maxint, 'ram': sys.maxint, 'disk': sys.maxint}
# find the smallest flavor that is at least 1vcpu, 1024MB ram and 10MB disk
find_flag = False
for flavor in flavor_manager.list():
flavor = vars(flavor)
if flavor['vcpus'] < 1 or flavor['ram'] < 1024 or flavor['disk'] < 10:
@ -150,7 +153,7 @@ class Kloud(object):
reusing_users=user_list)
self.tenant_list.append(tenant_instance)
else:
for tenant_count in xrange(self.scale_cfg['number_tenants']):
for tenant_count in range(self.scale_cfg['number_tenants']):
tenant_name = self.prefix + "-T" + str(tenant_count)
tenant_instance = tenant.Tenant(tenant_name, self, tenant_quota)
self.res_logger.log('tenants', tenant_instance.tenant_name,
@ -192,7 +195,7 @@ class Kloud(object):
def delete_resources(self):
if not self.reusing_tenants:
for fn, flavor in self.flavors.iteritems():
for fn, flavor in self.flavors.items():
LOG.info('Deleting flavor %s', fn)
try:
flavor.delete()
@ -249,7 +252,10 @@ class Kloud(object):
if instance.vol:
instance.attach_vol()
instance.fixed_ip = instance.instance.networks.values()[0][0]
# example:
# instance.instance.networks = OrderedDict([('KBc-T0-U-R0-N0', ['10.1.0.194'])])
# there should be only 1 item in the ordered dict
instance.fixed_ip = list(instance.instance.networks.values())[0][0]
u_fip = instance.config['use_floatingip']
if self.scale_cfg['provider_network']:
instance.fip = None
@ -273,13 +279,13 @@ class Kloud(object):
def create_vms(self, vm_creation_concurrency):
try:
with ThreadPoolExecutor(max_workers=vm_creation_concurrency) as executor:
for feature in executor.map(self.create_vm, self.get_all_instances()):
for _ in executor.map(self.create_vm, self.get_all_instances()):
self.vm_up_count += 1
except Exception:
self.exc_info = sys.exc_info()
except Exception as exc:
self.exc_info = exc
class KloudBuster(object):
class KloudBuster():
"""
Creates resources on the cloud for loading up the cloud
1. Tenants
@ -317,8 +323,8 @@ class KloudBuster(object):
LOG.warning("REUSING MODE: The flavor configs will be ignored.")
else:
self.tenants_list = {'server': None, 'client': None}
# TODO(check on same auth_url instead)
self.single_cloud = False if client_cred else True
# !TODO(check on same auth_url instead)
self.single_cloud = bool(not client_cred)
if not client_cred:
self.client_cred = server_cred
# Automatically enable the floating IP for server cloud under dual-cloud mode
@ -340,7 +346,7 @@ class KloudBuster(object):
def get_hypervisor_list(self, cred):
ret_list = []
sess = cred.get_session()
nova_client = novaclient('2', endpoint_type='publicURL',
nova_client = NovaClient('2', endpoint_type='publicURL',
http_log_debug=True, session=sess)
for hypervisor in nova_client.hypervisors.list():
if vars(hypervisor)['status'] == 'enabled':
@ -351,7 +357,7 @@ class KloudBuster(object):
def get_az_list(self, cred):
ret_list = []
sess = cred.get_session()
nova_client = novaclient('2', endpoint_type='publicURL',
nova_client = NovaClient('2', endpoint_type='publicURL',
http_log_debug=True, session=sess)
for az in nova_client.availability_zones.list():
zoneName = vars(az)['zoneName']
@ -364,14 +370,11 @@ class KloudBuster(object):
def check_and_upload_image(self, kloud_name, image_name, image_url, sess, retry_count):
'''Check a VM image and upload it if not found
'''
glance_client = glanceclient.Client('2', session=sess)
try:
# Search for the image
img = glance_client.images.list(filters={'name': image_name}).next()
# image found
return img
except StopIteration:
sys.exc_clear()
glance_client = GlanceClient('2', session=sess)
# Search for the image
images = list(glance_client.images.list(filters={'name': image_name}))
if images:
return images[0]
# Trying to upload image
LOG.info("KloudBuster VM Image is not found in %s, trying to upload it..." % kloud_name)
@ -381,7 +384,7 @@ class KloudBuster(object):
retry = 0
try:
LOG.info("Uploading VM Image from %s..." % image_url)
with open(image_url) as f_image:
with open(image_url, "rb") as f_image:
img = glance_client.images.create(name=image_name,
disk_format="qcow2",
container_format="bare",
@ -411,7 +414,7 @@ class KloudBuster(object):
return None
except Exception:
LOG.error(traceback.format_exc())
LOG.error("Failed while uploading the image: %s", str(exc))
LOG.exception("Failed while uploading the image")
return None
return img
@ -448,7 +451,7 @@ class KloudBuster(object):
row = [instance.vm_name, instance.host, instance.fixed_ip,
instance.fip_ip, instance.subnet_ip, instance.shared_interface_ip]
table.append(row)
LOG.info('Provision Details (Tested Kloud)\n' +
LOG.info('Provision Details (Tested Kloud)\n%s',
tabulate(table, headers="firstrow", tablefmt="psql"))
table = [["VM Name", "Host", "Internal IP", "Floating IP", "Subnet"]]
@ -457,7 +460,7 @@ class KloudBuster(object):
row = [instance.vm_name, instance.host, instance.fixed_ip,
instance.fip_ip, instance.subnet_ip]
table.append(row)
LOG.info('Provision Details (Testing Kloud)\n' +
LOG.info('Provision Details (Testing Kloud)\n%s',
tabulate(table, headers="firstrow", tablefmt="psql"))
def gen_server_user_data(self, test_mode):
@ -544,7 +547,7 @@ class KloudBuster(object):
self.stage()
self.run_test()
except KBException as e:
LOG.error(e.message)
LOG.error(str(e))
except base_network.KBGetProvNetException:
pass
except Exception:
@ -619,7 +622,7 @@ class KloudBuster(object):
cur_vm_count = 1 if start else multiple
# Minus 1 for KB-Proxy
total_vm = self.get_tenant_vm_count(self.client_cfg) - 1
while (cur_vm_count <= total_vm):
while cur_vm_count <= total_vm:
log_info += "\n" + self.kb_runner.header_formatter(stage, cur_vm_count)
cur_vm_count = (stage + 1 - start) * multiple
stage += 1
@ -653,10 +656,10 @@ class KloudBuster(object):
self.client_vm_create_thread.join()
if self.testing_kloud and self.testing_kloud.exc_info:
raise self.testing_kloud.exc_info[1], None, self.testing_kloud.exc_info[2]
raise self.testing_kloud.exc_info[1].with_traceback(self.testing_kloud.exc_info[2])
if self.kloud and self.kloud.exc_info:
raise self.kloud.exc_info[1], None, self.kloud.exc_info[2]
raise self.kloud.exc_info[1].with_traceback(self.kloud.exc_info[2])
# Function that print all the provisioning info
self.print_provision_info()
@ -673,11 +676,11 @@ class KloudBuster(object):
while 1:
if self.interactive:
print()
runlabel = raw_input('>> KB ready, enter label for next run or "q" to quit: ')
runlabel = input('>> KB ready, enter label for next run or "q" to quit: ')
if runlabel.lower() == "q":
break
for run_result in self.kb_runner.run(test_only, runlabel):
for _ in self.kb_runner.run(test_only, runlabel):
if not self.multicast_mode or len(self.final_result['kb_result']) == 0:
self.final_result['kb_result'].append(self.kb_runner.tool_result)
if self.tsdb_connector:
@ -740,8 +743,7 @@ class KloudBuster(object):
def get_tenant_vm_count(self, config):
# this does not apply for storage mode!
return (config['routers_per_tenant'] * config['networks_per_router'] *
config['vms_per_network'])
return config['routers_per_tenant'] * config['networks_per_router'] * config['vms_per_network']
def calc_neutron_quota(self):
total_vm = self.get_tenant_vm_count(self.server_cfg)
@ -751,7 +753,7 @@ class KloudBuster(object):
self.server_cfg['networks_per_router']
server_quota['subnet'] = server_quota['network']
server_quota['router'] = self.server_cfg['routers_per_tenant']
if (self.server_cfg['use_floatingip']):
if self.server_cfg['use_floatingip']:
# (1) Each VM has one floating IP
# (2) Each Router has one external IP
server_quota['floatingip'] = total_vm + server_quota['router']
@ -775,7 +777,7 @@ class KloudBuster(object):
client_quota['network'] = 1
client_quota['subnet'] = 1
client_quota['router'] = 1
if (self.client_cfg['use_floatingip']):
if self.client_cfg['use_floatingip']:
# (1) Each VM has one floating IP
# (2) Each Router has one external IP, total of 1 router
# (3) KB-Proxy node has one floating IP
@ -882,20 +884,23 @@ def generate_charts(json_results, html_file_name, is_config):
'''Save results in HTML format file.'''
LOG.info('Saving results to HTML file: ' + html_file_name + '...')
try:
if json_results['test_mode'] == "storage":
test_mode = json_results['test_mode']
if test_mode == "storage":
template_path = resource_filename(__name__, 'template_storage.html')
elif json_results['test_mode'] == "http":
elif test_mode == "http":
template_path = resource_filename(__name__, 'template_http.html')
else:
raise
LOG.error('Invalid test mode, : %s', test_mode)
return 1
except Exception:
LOG.error('Invalid json file.')
sys.exit(1)
return 1
with open(html_file_name, 'w') as hfp, open(template_path, 'r') as template:
create_html(hfp,
template,
json.dumps(json_results, sort_keys=True),
is_config)
return 0
def main():
@ -994,26 +999,26 @@ def main():
if CONF.charts_from_json:
if not CONF.html:
LOG.error('Destination html filename must be specified using --html.')
sys.exit(1)
return 1
with open(CONF.charts_from_json, 'r') as jfp:
json_results = json.load(jfp)
generate_charts(json_results, CONF.html, None)
sys.exit(0)
return 0
if CONF.show_config:
print resource_string(__name__, "cfg.scale.yaml")
sys.exit(0)
print(resource_string(__name__, "cfg.scale.yaml").decode('utf-8'))
return 0
if CONF.multicast and CONF.storage:
LOG.error('--multicast and --storage can not both be chosen.')
sys.exit(1)
return 1
try:
kb_config = KBConfig()
kb_config.init_with_cli()
except TypeError:
LOG.exception('Error parsing the configuration file')
sys.exit(1)
return 1
# The KloudBuster class is just a wrapper class
# levarages tenant and user class for resource creations and deletion
@ -1030,13 +1035,13 @@ def main():
kloudbuster.run()
if CONF.json:
'''Save results in JSON format file.'''
# Save results in JSON format file
LOG.info('Saving results in json file: ' + CONF.json + "...")
with open(CONF.json, 'w') as jfp:
json.dump(kloudbuster.final_result, jfp, indent=4, sort_keys=True)
if CONF.multicast and CONF.csv and 'kb_result' in kloudbuster.final_result:
'''Save results in JSON format file.'''
# Save results in JSON format file
if len(kloudbuster.final_result['kb_result']) > 0:
LOG.info('Saving results in csv file: ' + CONF.csv + "...")
with open(CONF.csv, 'w') as jfp:
@ -1044,7 +1049,8 @@ def main():
if CONF.html:
generate_charts(kloudbuster.final_result, CONF.html, kb_config.config_scale)
return 0
if __name__ == '__main__':
main()
sys.exit(main())

3
kloudbuster/log.py
View File

@ -43,6 +43,8 @@ WARN = logging.WARN
WARNING = logging.WARNING
def setup(product_name, logfile=None):
# pylint: disable=protected-access
dbg_color = handlers.ColorHandler.LEVEL_COLORS[logging.DEBUG]
handlers.ColorHandler.LEVEL_COLORS[logging.KBDEBUG] = dbg_color
CONF.logging_default_format_string = '%(asctime)s %(levelname)s %(message)s'
@ -62,6 +64,7 @@ def setup(product_name, logfile=None):
project=product_name).logger.setLevel(logging.KBDEBUG)
def getLogger(name="unknown", version="unknown"):
# pylint: disable=protected-access
if name not in oslogging._loggers:
oslogging._loggers[name] = KloudBusterContextAdapter(
logging.getLogger(name), {"project": "kloudbuster",

2
kloudbuster/nuttcp_tool.py
View File

@ -15,7 +15,7 @@
import json
from perf_tool import PerfTool
from kloudbuster.perf_tool import PerfTool
class NuttcpTool(PerfTool):

8
kloudbuster/perf_instance.py
View File

@ -13,10 +13,10 @@
# under the License.
#
from base_compute import BaseCompute
from fio_tool import FioTool
from nuttcp_tool import NuttcpTool
from wrk_tool import WrkTool
from kloudbuster.base_compute import BaseCompute
from kloudbuster.fio_tool import FioTool
from kloudbuster.nuttcp_tool import NuttcpTool
from kloudbuster.wrk_tool import WrkTool
# An openstack instance (can be a VM or a LXC)

5
kloudbuster/perf_tool.py
View File

@ -15,14 +15,13 @@
import abc
import log as logging
import kloudbuster.log as logging
LOG = logging.getLogger(__name__)
# A base class for all tools that can be associated to an instance
class PerfTool(object):
__metaclass__ = abc.ABCMeta
class PerfTool(metaclass=abc.ABCMeta):
def __init__(self, instance, tool_name):
self.instance = instance

4
kloudbuster/prometheus.py
View File

@ -17,7 +17,7 @@ import requests
import time
class Prometheus(object):
class Prometheus():
def __init__(self, config):
self.server_address = "http://{}:{}/api/v1/".format(config['server_ip'],
config['server_port'])
@ -38,5 +38,5 @@ class Prometheus(object):
self.step_size)).json()
except requests.exceptions.RequestException as e:
print e
print(e)
return None

3
kloudbuster/requirements.txt
View File

@ -6,7 +6,6 @@ pytz>=2016.4
pbr>=3.0.1
Babel>=2.3.4
futures>=3.1.1
python-cinderclient>=2.0.1
python-glanceclient>=2.6.0
python-openstackclient>=3.11.0
@ -27,5 +26,3 @@ tabulate>=0.7.7
pyyaml>=3.12
requests
# Workaround for pip install failed on RHEL/CentOS
functools32>=3.2.3

15
kloudbuster/start_server.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
# Copyright 2016 Cisco Systems, Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
@ -22,7 +22,7 @@ def exec_command(cmd, cwd=None, show_console=False):
p = subprocess.Popen(cmd, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
if show_console:
for line in iter(p.stdout.readline, b""):
print line,
print(line)
p.communicate()
return p.returncode
@ -40,10 +40,10 @@ def launch_kb(cwd):
except OSError:
continue
if os.uname()[0] == "Darwin":
print
print "To run the KloudBuster web server you need to install the coreutils package:"
print " brew install coreutils"
print
print()
print("To run the KloudBuster web server you need to install the coreutils package:")
print(" brew install coreutils")
print()
raise OSError('Cannot find stdbuf or gstdbuf command')
def main():
@ -52,8 +52,9 @@ def main():
try:
return launch_kb(cwd)
except KeyboardInterrupt:
print 'Terminating server...'
print('Terminating server...')
return 1
if __name__ == '__main__':
sys.exit(main())

15
kloudbuster/tenant.py
View File

@ -12,20 +12,19 @@
# License for the specific language governing permissions and limitations
# under the License.
import base_compute
import base_network
import base_storage
from keystoneclient import exceptions as keystone_exception
import log as logging
import users
import kloudbuster.base_compute as base_compute
import kloudbuster.base_network as base_network
import kloudbuster.base_storage as base_storage
import kloudbuster.log as logging
import kloudbuster.users as users
LOG = logging.getLogger(__name__)
class KBQuotaCheckException(Exception):
pass
class Tenant(object):
class Tenant():
"""
Holds the tenant resources
1. Provides ability to create users in a tenant
@ -109,7 +108,7 @@ class Tenant(object):
meet_quota = True
quota = quota_manager.get()
for key, value in self.tenant_quota[quota_type].iteritems():
for key, value in self.tenant_quota[quota_type].items():
if quota[key] < value:
meet_quota = False
break

4
kloudbuster/tsdb.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
# Copyright 2018 Cisco Systems, Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
@ -16,7 +16,7 @@
import time
class TSDB(object):
class TSDB():
def __init__(self, config):
pass

13
kloudbuster/users.py
View File

@ -12,17 +12,18 @@
# License for the specific language governing permissions and limitations
# under the License.
import base_compute
import base_network
import kloudbuster.base_compute as base_compute
import kloudbuster.base_network as base_network
import kloudbuster.log as logging
from cinderclient import client as cinderclient
from keystoneclient import exceptions as keystone_exception
import log as logging
from neutronclient.neutron import client as neutronclient
from novaclient import client as novaclient
LOG = logging.getLogger(__name__)
class User(object):
class User():
"""
User class that stores router list
Creates and deletes N routers based on num of routers
@ -143,8 +144,8 @@ class User(object):
self.key_pair.add_public_key(self.key_name, config_scale.public_key_file)
# Find the external network that routers need to attach to
if self.tenant.kloud.multicast_mode or (
self.tenant.kloud.storage_mode and config_scale.provider_network):
if self.tenant.kloud.multicast_mode or (self.tenant.kloud.storage_mode and
config_scale.provider_network):
router_instance = base_network.Router(
self, provider_network=config_scale.provider_network)
self.router_list.append(router_instance)

7
kloudbuster/wrk_tool.py
View File

@ -15,10 +15,9 @@
import json
from perf_tool import PerfTool
from hdrh.histogram import HdrHistogram
import log as logging
from kloudbuster.perf_tool import PerfTool
import kloudbuster.log as logging
LOG = logging.getLogger(__name__)
@ -119,7 +118,7 @@ class WrkTool(PerfTool):
err_flag = True
perc_list = [50, 75, 90, 99, 99.9, 99.99, 99.999]
latency_dict = histogram.get_percentile_to_value_dict(perc_list)
for key, value in latency_dict.iteritems():
for key, value in latency_dict.items():
all_res['latency_stats'].append([key, value])
all_res['latency_stats'].sort()

287
pylintrc Normal file
View File

@ -0,0 +1,287 @@
[MASTER]
extension-pkg-whitelist=netifaces,lxml
ignore=CVS
ignore-patterns=
jobs=1
limit-inference-results=100
load-plugins=
persistent=yes
suggestion-mode=yes
unsafe-load-any-extension=no
init-hook=import sys; sys.path.append('installer/')
[MESSAGES CONTROL]
confidence=
disable=missing-docstring,
invalid-name,
global-statement,
broad-except,
useless-object-inheritance,
useless-else-on-loop,
no-member,
arguments-differ,
redundant-keyword-arg,
cell-var-from-loop,
no-self-use,
consider-using-set-comprehension,
wrong-import-position,
wrong-import-order,
redefined-outer-name,
no-else-return,
assignment-from-no-return,
dangerous-default-value,
no-name-in-module,
function-redefined,
redefined-builtin,
unused-argument,
too-many-instance-attributes,
too-many-locals,
too-many-function-args,
too-many-branches,
too-many-arguments
enable=c-extension-no-member
[REPORTS]
evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
output-format=text
reports=no
score=yes
[REFACTORING]
max-nested-blocks=10
never-returning-functions=sys.exit
[LOGGING]
logging-format-style=old
logging-modules=logging
[SPELLING]
max-spelling-suggestions=4
spelling-dict=
spelling-ignore-words=
spelling-private-dict-file=
spelling-store-unknown-words=no
[MISCELLANEOUS]
notes=XXX,
TODO
[TYPECHECK]
contextmanager-decorators=contextlib.contextmanager
generated-members=
ignore-mixin-members=yes
ignore-none=yes
ignore-on-opaque-inference=yes
missing-member-hint=yes
missing-member-hint-distance=1
missing-member-max-choices=1
[VARIABLES]
additional-builtins=mibBuilder,OPENSTACK_NEUTRON_NETWORK
allow-global-unused-variables=yes
callbacks=cb_,
_cb
dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_
ignored-argument-names=_.*|^ignored_|^unused_
init-import=no
redefining-builtins-modules=builtins,io
[FORMAT]
expected-line-ending-format=
ignore-long-lines=^\s*(# )?<?https?://\S+>?$
indent-after-paren=4
indent-string=' '
max-line-length=150
max-module-lines=2500
no-space-check=trailing-comma,
dict-separator
single-line-class-stmt=no
single-line-if-stmt=no
[SIMILARITIES]
ignore-comments=yes
ignore-docstrings=yes
ignore-imports=no
min-similarity-lines=10
[BASIC]
argument-naming-style=snake_case
attr-naming-style=snake_case
bad-names=foo,
bar,
baz,
toto,
tutu,
tata
class-attribute-naming-style=any
class-naming-style=PascalCase
const-naming-style=UPPER_CASE
docstring-min-length=-1
function-naming-style=snake_case
good-names=i,
j,
k,
ex,
Run,
_
include-naming-hint=yes
inlinevar-naming-style=any
method-naming-style=snake_case
module-naming-style=snake_case
name-group=
no-docstring-rgx=^_
property-classes=abc.abstractproperty
variable-naming-style=snake_case
[STRING]
check-str-concat-over-line-jumps=no
[IMPORTS]
allow-wildcard-with-all=no
analyse-fallback-blocks=no
deprecated-modules=optparse,tkinter.tix
ext-import-graph=
import-graph=
int-import-graph=
known-standard-library=
known-third-party=enchant
[CLASSES]
defining-attr-methods=__init__,
__new__,
setUp
exclude-protected=_asdict,
_fields,
_replace,
_source,
_make
valid-classmethod-first-arg=cls
valid-metaclass-classmethod-first-arg=cls
[DESIGN]
max-args=15
max-attributes=32
max-bool-expr=10
max-branches=80
max-locals=40
max-parents=12
additional-builtins=OPENSTACK_NEUTRON_NETWORK
max-public-methods=100
max-returns=50
max-statements=300
min-public-methods=0
[EXCEPTIONS]
overgeneral-exceptions=BaseException,
Exception

6
requirements.txt
View File

@ -6,7 +6,6 @@ pytz>=2016.4
pbr>=3.0.1
Babel>=2.3.4
futures>=3.1.1
python-cinderclient>=2.0.1
python-glanceclient>=2.6.0
python-openstackclient>=3.11.0
@ -14,7 +13,7 @@ python-neutronclient>=6.2.0
python-novaclient>=9.0.0
python-keystoneclient>=3.10.0
attrdict>=2.0.0
hdrhistogram>=0.5.2
hdrhistogram>=0.8.0
# ipaddress is required to get TLS working
# otherwise certificates with numeric IP addresses in the ServerAltName field will fail
ipaddress>= 1.0.16
@ -24,6 +23,3 @@ pecan>=1.2.1
redis>=2.10.5
tabulate>=0.7.7
pyyaml>=3.12
# Workaround for pip install failed on RHEL/CentOS
functools32>=3.2.3

4
setup.cfg
View File

@ -16,8 +16,8 @@ classifier =
Operating System :: POSIX :: Linux
Operating System :: MacOS
Programming Language :: Python
Programming Language :: Python :: 2
Programming Language :: Python :: 2.7
Programming Language :: Python :: 3
Programming Language :: Python :: 3.6
[files]
packages =

5
setup.py
View File

@ -25,6 +25,7 @@ except ImportError:
pass
setuptools.setup(
setup_requires=['pbr'],
setup_requires=['pbr', 'wheel'],
scripts=['kloudbuster/kb_extract_img_from_docker.sh'],
pbr=True)
pbr=True,
python_requires='>=3.6')

11
test-requirements.txt
View File

@ -2,15 +2,6 @@
# of appearance. Changing the order has an impact on the overall integration
# process, which may cause wedges in the gate later.
hacking<0.11,>=0.10.0
coverage>=3.6
discover
python-subunit>=0.0.18
sphinx>=1.4.0,<2.0
sphinx_rtd_theme>=0.1.9
oslosphinx>=2.5.0 # Apache-2.0
oslotest>=1.10.0 # Apache-2.0
testrepository>=0.0.18
testscenarios>=0.4
testtools>=1.4.0
oslosphinx>=2.5.0

0
kloudbuster/tests/__init__.py → tests/__init__.py
View File

5
kloudbuster/tests/test_kloudbuster.py → tests/test_kloudbuster.py
View File

@ -18,3 +18,8 @@ test_kloudbuster
Tests for `kloudbuster` module.
"""
from kloudbuster.kb_config import KBConfig
def test_config():
cfg = KBConfig()
cfg.update_configs()

58
tox.ini
View File

@ -1,41 +1,51 @@
[tox]
minversion = 1.6
envlist = py27,pep8
envlist = py3,pylint,pep8
skipsdist = True
basepython = python3
[testenv]
usedevelop = True
install_command = pip install -U {opts} {packages}
setenv =
VIRTUAL_ENV={envdir}
deps = -r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
# commands = python setup.py test --slowest --testr-args='{posargs}'
[testenv:py3]
deps =
pytest>=5.4
pytest-cov>=2.8
mock>=4.0
-r{toxinidir}/requirements.txt
commands =
{posargs:pytest --cov=kloudbuster --cov-report=term-missing -vv tests}
[testenv:pep8]
commands = flake8
deps =
pep8>=1.5.7
flake8>=3.8.3
-r{toxinidir}/requirements.txt
whitelist_externals = flake8
commands = flake8 kloudbuster
[testenv:venv]
commands = {posargs}
[testenv:cover]
commands = python setup.py test --coverage --testr-args='{posargs}'
[testenv:pylint]
deps =
pylint>=2.4
pytest>=5.4
pytest-cov>=2.8
mock>=4.0
-r{toxinidir}/requirements.txt
commands = pylint --rcfile=pylintrc kloudbuster
[testenv:docs]
commands = python setup.py build_sphinx
[testenv:debug]
commands = oslo_debug_helper {posargs}
deps =
sphinx>=1.4.0
sphinx_rtd_theme>=0.1.9
oslosphinx>=2.5.0
commands = python3 setup.py build_sphinx
[flake8]
max-line-length = 100
max-line-length = 150
show-source = True
# H233: Python 3.x incompatible use of print operator
# H236: Python 3.x incompatible __metaclass__, use six.add_metaclass()
# E302: expected 2 blank linee
# E302: expected 2 blank lines
# E303: too many blank lines (2)
# H306: imports not in alphabetical order
# H404: multi line docstring should start without a leading new line
# H405: multi line docstring summary not separated with an empty line
ignore = H233,H236,E302,E303,H404,H405
# W504 line break after binary operator
ignore = E302,E303,H306,H404,H405,W504
builtins = _
exclude=.venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*egg,build