x
/
kloudbuster
Code Issues Proposed changes

Replace configure with AttrDict 

1. Replace configure with AttrDict;
2. Fix the issue while tring to update the configs using Rest API (PUT);
3. Adding timestamp to the logging when calling from Rest API;
4. Use decorator in Rest API server to avoid code duplication;

Change-Id: I3490bc5ca5b982c6ad424bbb485fcf3c18cdb586
This commit is contained in:
Yichen Wang 2015-09-24 13:57:29 -07:00
parent af8fd33451
commit 322dd1e7d1
6 changed files with 131 additions and 167 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

139
kb_server/kb_server/controllers/api_cfg.py
View File

@ -12,6 +12,7 @@
# License for the specific language governing permissions and limitations
# under the License.
import functools
import hashlib
import json
import os
@ -20,18 +21,34 @@ import traceback
kb_main_path = os.path.split(os.path.abspath(__file__))[0] + "/../../../kloudbuster"
sys.path.append(kb_main_path)
from attrdict import AttrDict
from credentials import Credentials
from kb_session import KBSession
from kb_session import KBSessionManager
import log as logging
from configure import Configuration
from kb_config import KBConfig
from pecan import expose
from pecan import response
class ConfigController(object):
# Decorator to check for missing or invalid session ID
def check_session_id(func):
@functools.wraps(func)
def wrapper(self, *args, **kwargs):
if not len(args):
response.status = 404
response.text = u"Please specify the session_id."
return response.text
if not KBSessionManager.has(args[0]):
response.status = 404
response.text = u"Session ID is not found or invalid."
return response.text
return func(self, *args, **kwargs)
return wrapper
def update_config(self, kb_config, user_config):
# Parsing server and client configs from application input
# Save the public key into a temporary file
@ -41,52 +58,31 @@ class ConfigController(object):
f.write(user_config['kb_cfg']['public_key_file'])
kb_config.config_scale['public_key_file'] = pubkey_filename
if 'prompt_before_run' in user_config['kb_cfg']:
kb_config.config_scale['prompt_before_run'] = False
kb_config.config_scale['prompt_before_run'] = False
if user_config['kb_cfg']:
alt_config = Configuration.from_dict(user_config['kb_cfg']).configure()
kb_config.config_scale = kb_config.config_scale.merge(alt_config)
# Parsing the KloudBuster/topology/tenants configs from application input
alt_config = AttrDict(user_config['kb_cfg']) if 'kb_cfg' in user_config else None
topo_cfg = AttrDict(user_config['topo_cfg']) if 'topo_cfg' in user_config else None
tenants_list = AttrDict(user_config['tenants_list'])\
if 'tenants_list' in user_config else None
# Parsing topology configs from application input
if 'topo_cfg' in user_config:
topo_cfg = Configuration.from_string(user_config['topo_cfg']).configure()
else:
topo_cfg = None
# Parsing tenants configs from application input
if 'tenants_list' in user_config:
tenants_list = Configuration.from_string(user_config['tenants_list']).configure()
else:
tenants_list = None
kb_config.update_with_rest_api(topo_cfg=topo_cfg, tenants_list=tenants_list)
key = ['alt_cfg', 'topo_cfg', 'tenants_list']
val = [alt_config, topo_cfg, tenants_list]
kwargs = dict([(k, v) for k, v in zip(key, val) if v])
kb_config.update_with_rest_api(**kwargs)
@expose(generic=True)
def default_config(self):
kb_config = KBConfig()
pdict = eval(str(kb_config.config_scale))
return json.dumps(pdict)
return json.dumps(dict(kb_config.config_scale))
@expose(generic=True)
@check_session_id
def running_config(self, *args):
if len(args):
session_id = args[0]
else:
response.status = 404
response.text = u"Please specify the session_id."
return response.text
if KBSessionManager.has(session_id):
kb_config_obj = KBSessionManager.get(session_id).kb_config
config_scale = kb_config_obj.config_scale
config_scale['server'] = kb_config_obj.server_cfg
config_scale['client'] = kb_config_obj.client_cfg
config_scale = eval(str(config_scale))
return json.dumps(config_scale)
else:
response.status = 404
response.text = u"Session ID is not found or invalid."
return response.text
session_id = args[0]
kb_config_obj = KBSessionManager.get(session_id).kb_config
config_scale = dict(kb_config_obj.config_scale)
return json.dumps(config_scale)
@running_config.when(method='POST')
def running_config_POST(self, arg):
@ -121,7 +117,7 @@ class ConfigController(object):
return response.text
except Exception:
response.status = 400
response.text = u"Error while parsing configurations: \n%s" % (traceback.format_exc)
response.text = u"Error while parsing configurations: \n%s" % (traceback.format_exc())
return response.text
logging.setup("kloudbuster", logfile="/tmp/kb_log_%s" % session_id)
@ -136,51 +132,32 @@ class ConfigController(object):
return str(session_id)
@running_config.when(method='PUT')
@check_session_id
def running_config_PUT(self, *args, **kwargs):
if len(args):
session_id = args[0]
session_id = args[0]
status = KBSessionManager.get(session_id).kb_status
if status == "READY":
# Expectation:
# {
# 'kb_cfg': {<USER_OVERRIDED_CONFIGS>},
# 'topo_cfg': {<TOPOLOGY_CONFIGS>}
# 'tenants_cfg': {<TENANT_AND_USER_LISTS_FOR_REUSING>}
# }
kb_config = KBSessionManager.get(session_id).kb_config
user_config = json.loads(kwargs['arg'])
self.update_config(kb_config, user_config)
return "OK!"
else:
response.status = 404
response.text = u"Please specify the session_id."
return response.text
if KBSessionManager.has(session_id):
status = KBSessionManager.get(session_id).kb_status
if status == "READY":
# Expectation:
# {
# 'kb_cfg': {<USER_OVERRIDED_CONFIGS>},
# 'topo_cfg': {<TOPOLOGY_CONFIGS>}
# 'tenants_cfg': {<TENANT_AND_USER_LISTS_FOR_REUSING>}
# }
kb_config = KBConfig()
KBSessionManager.get(session_id).kb_config = kb_config
user_config = json.loads(kwargs['arg'])
self.update_config(kb_config, user_config)
return "OK!"
else:
response.status = 403
response.text = u"Cannot update configuration if KloudBuster is not at READY"
return response.text
else:
response.status = 404
response.text = u"Session ID is not found or invalid."
response.status = 403
response.text = u"Cannot update configuration if KloudBuster is not at READY."
return response.text
@running_config.when(method='DELETE')
@check_session_id
def running_config_DELETE(self, *args):
if len(args):
session_id = args[0]
else:
response.status = 404
response.text = u"Please specify the session_id."
return response.text
if KBSessionManager.has(session_id):
kb_session = KBSessionManager.get(session_id)
if kb_session.kloudbuster:
kb_session.kloudbuster.dispose()
KBSessionManager.delete(session_id)
return "OK!"
else:
response.status = 404
response.text = u"Session ID is not found or invalid."
return response.text
session_id = args[0]
kb_session = KBSessionManager.get(session_id)
if kb_session.kloudbuster:
kb_session.kloudbuster.dispose()
KBSessionManager.delete(session_id)
return "OK!"

109
kb_server/kb_server/controllers/api_kb.py
View File

@ -12,6 +12,7 @@
# License for the specific language governing permissions and limitations
# under the License.
import functools
import json
import os
import sys
@ -31,6 +32,22 @@ class KBController(object):
def __init__(self):
self.kb_thread = None
# Decorator to check for missing or invalid session ID
def check_session_id(func):
@functools.wraps(func)
def wrapper(self, *args, **kwargs):
if not len(args):
response.status = 404
response.text = u"Please specify the session_id."
return response.text
if not KBSessionManager.has(args[0]):
response.status = 404
response.text = u"Session ID is not found or invalid."
return response.text
return func(self, *args, **kwargs)
return wrapper
def kb_thread_handler(self, session_id):
kb_session = KBSessionManager.get(session_id)
kb_session.kb_status = 'RUNNING'
@ -49,75 +66,41 @@ class KBController(object):
kb_session.kb_status = 'ERROR'
@expose(generic=True)
@check_session_id
def status(self, *args):
if len(args):
session_id = args[0]
else:
response.status = 400
response.text = u"Please specify the session_id."
return response.text
if KBSessionManager.has(session_id):
status = KBSessionManager.get(session_id).kb_status
return status
else:
response.status = 404
response.text = u"Session ID is not found or invalid."
return response.text
session_id = args[0]
status = KBSessionManager.get(session_id).kb_status
return status
@expose(generic=True)
@check_session_id
def log(self, *args, **kwargs):
if len(args):
session_id = args[0]
else:
session_id = args[0]
offset = kwargs.get('offset', 0)
try:
offset = int(offset)
except ValueError:
response.status = 400
response.text = u"Please specify the session_id."
response.text = u"Parameter 'offset' is invalid."
return response.text
offset = 0
if 'offset' in kwargs:
try:
offset = int(kwargs['offset'])
except ValueError:
response.status = 400
response.text = u"Parameter 'offset' is invalid."
return response.text
if KBSessionManager.has(session_id):
kb_session = KBSessionManager.get(session_id)
plog = kb_session.kloudbuster.dump_logs(offset=offset)\
if kb_session.kloudbuster else ""
return json.dumps(plog)
else:
response.status = 404
response.text = u"Session ID is not found or invalid."
return response.text
kb_session = KBSessionManager.get(session_id)
plog = kb_session.kloudbuster.dump_logs(offset=offset)\
if kb_session.kloudbuster else ""
return json.dumps(plog)
@expose(generic=True)
@check_session_id
def report(self, *args, **kwargs):
if len(args):
session_id = args[0]
else:
response.status = 400
response.text = u"Please specify the session_id."
return response.text
final = False
if 'final' in kwargs:
final = True if kwargs['final'].lower() == 'true' else False
session_id = args[0]
preport = None
if KBSessionManager.has(session_id):
kb_session = KBSessionManager.get(session_id)
if kb_session.kloudbuster and kb_session.kloudbuster.kb_runner:
preport = kb_session.kloudbuster.final_result\
if final else kb_session.kloudbuster.kb_runner.report
final = True if kwargs.get('final', '').lower() == 'true' else False
kb_session = KBSessionManager.get(session_id)
if kb_session.kloudbuster and kb_session.kloudbuster.kb_runner:
preport = kb_session.kloudbuster.final_result\
if final else kb_session.kloudbuster.kb_runner.report
return json.dumps(preport)
else:
response.status = 404
response.text = u"Session ID is not found or invalid."
return response.text
return json.dumps(preport)
@expose(generic=True)
def version(self):
@ -130,17 +113,9 @@ class KBController(object):
return response.text
@run.when(method='POST')
@check_session_id
def run_POST(self, *args):
if len(args):
session_id = args[0]
else:
response.status = 400
response.text = u"Please specify the session_id."
return response.text
if not KBSessionManager.has(session_id):
response.status = 404
response.text = u"Session ID is not found or invalid."
return response.text
session_id = args[0]
if KBSessionManager.get(session_id).kb_status == 'RUNNING':
response.status = 403
response.text = u"An instance of KloudBuster is already running."

44
kloudbuster/kb_config.py
View File

@ -14,8 +14,9 @@
import os
import sys
import yaml
import configure
from attrdict import AttrDict
import log as logging
from oslo_config import cfg
from pkg_resources import resource_string
@ -69,7 +70,8 @@ class KBConfig(object):
# The default configuration file for KloudBuster
default_cfg = resource_string(__name__, "cfg.scale.yaml")
# Read the configuration file
self.config_scale = configure.Configuration.from_string(default_cfg).configure()
self.config_scale = AttrDict(yaml.safe_load(default_cfg))
self.alt_config = None
self.cred_tested = None
self.cred_testing = None
self.server_cfg = None
@ -91,10 +93,18 @@ class KBConfig(object):
self.config_scale['public_key_file'] = pub_key
LOG.info('Using %s as public key for all VMs' % (pub_key))
if self.alt_cfg:
self.config_scale.update(self.alt_cfg)
# Use the default image name for Glance
# defaults to something like "kloudbuster_v3"
if not self.config_scale['image_name']:
self.config_scale['image_name'] = kb_vm_agent.get_image_name()
# A bit of config dict surgery, extract out the client and server side
# and transplant the remaining (common part) into the client and server dict
self.server_cfg = self.config_scale.pop('server')
self.client_cfg = self.config_scale.pop('client')
self.server_cfg = AttrDict(self.config_scale.pop('server'))
self.client_cfg = AttrDict(self.config_scale.pop('client'))
self.server_cfg.update(self.config_scale)
self.client_cfg.update(self.config_scale)
@ -107,12 +117,8 @@ class KBConfig(object):
self.client_cfg['vms_per_network'] =\
self.get_total_vm_count(self.server_cfg) + 1
# Use the default image name for Glance
# defaults to something like "kloudbuster_v3"
if not self.server_cfg['image_name']:
self.server_cfg['image_name'] = kb_vm_agent.get_image_name()
if not self.client_cfg['image_name']:
self.client_cfg['image_name'] = kb_vm_agent.get_image_name()
self.config_scale['server'] = self.server_cfg
self.config_scale['client'] = self.client_cfg
def init_with_cli(self):
self.get_credentials()
@ -126,8 +132,9 @@ class KBConfig(object):
self.cred_testing = kwargs['cred_testing']
def update_with_rest_api(self, **kwargs):
self.topo_cfg = kwargs['topo_cfg']
self.tenants_list = kwargs['tenants_list']
self.alt_cfg = kwargs.get('alt_cfg', None)
self.topo_cfg = kwargs.get('topo_cfg', self.topo_cfg)
self.tenants_list = kwargs.get('tenants_list', self.tenants_list)
self.update_configs()
def get_total_vm_count(self, config):
@ -150,19 +157,22 @@ class KBConfig(object):
def get_configs(self):
if CONF.config:
try:
alt_config = configure.Configuration.from_file(CONF.config).configure()
self.config_scale = self.config_scale.merge(alt_config)
except configure.ConfigurationError:
with open(CONF.config) as f:
alt_config = AttrDict(yaml.safe_load(f))
self.config_scale = self.config_scale + alt_config
except TypeError:
# file can be empty
pass
def get_topo_cfg(self):
if CONF.topology:
self.topo_cfg = configure.Configuration.from_file(CONF.topology).configure()
with open(CONF.topology) as f:
self.topo_cfg = AttrDict(yaml.safe_load(f))
def get_tenants_list(self):
if CONF.tenants_list:
self.tenants_list = configure.Configuration.from_file(CONF.tenants_list).configure()
with open(CONF.tenants_list) as f:
self.tenants_list = AttrDict(yaml.safe_load(f))
try:
self.config_scale['number_tenants'] = 1
except Exception as e:

2
kloudbuster/kb_runner.py
View File

@ -220,7 +220,7 @@ class KBRunner(object):
def run_http_test(self, active_range):
func = {'cmd': 'run_http_test', 'active_range': active_range,
'parameter': self.config.http_tool_configs}
'parameter': dict(self.config.http_tool_configs)}
self.send_cmd('EXEC', 'http', func)
# Give additional 30 seconds for everybody to report results
timeout = self.config.http_tool_configs.duration + 30

2
kloudbuster/log.py
View File

@ -53,7 +53,9 @@ def setup(product_name, logfile=None):
if os.path.exists(logfile):
os.remove(logfile)
CONF.log_file = logfile
fmt = logging.Formatter(fmt=CONF.logging_default_format_string)
hdlr = logging.FileHandler(logfile)
hdlr.setFormatter(fmt)
for name in oslogging._loggers:
if name:
oslogging.getLogger(name).logger.addHandler(hdlr)

2
requirements.txt
View File

@ -5,7 +5,7 @@
pbr>=1.3
Babel>=1.3
configure>=0.5
attrdict>=2.0.0
hdrhistogram>=0.3.1
oslo.log>=1.0.0
pecan>=0.9.0