From dbd455bc7d49c15e8e60f3bee1caf901e4501916 Mon Sep 17 00:00:00 2001 From: Yichen Wang Date: Tue, 13 Oct 2015 01:27:07 -0700 Subject: [PATCH] Add support to stop testing via Rest API 1. Add support to stop testing via Rest API; 2. Automatically enable floating IP if under dual-cloud mode; 3. Enhance to support running KloudBuster without keypair; 4. Disable the percentile_of_packet_not_timeout check for progression runs; Change-Id: I4904d7322719b4784f5cd40250529dfa7578bd0a --- .../kloudbuster/static/kb_test/kb_vm_agent.py | 37 +++++++++++-------- kb_server/kb_server/controllers/api_kb.py | 35 +++++++++++++++++- kb_server/kloudbuster-swagger.yaml | 23 ++++++++++++ kloudbuster/cfg.scale.yaml | 3 +- kloudbuster/kb_config.py | 3 ++ kloudbuster/kb_runner.py | 7 +++- kloudbuster/kloudbuster.py | 14 +++++-- 7 files changed, 99 insertions(+), 23 deletions(-) diff --git a/kb_dib/elements/kloudbuster/static/kb_test/kb_vm_agent.py b/kb_dib/elements/kloudbuster/static/kb_test/kb_vm_agent.py index 8faff92..175e787 100644 --- a/kb_dib/elements/kloudbuster/static/kb_test/kb_vm_agent.py +++ b/kb_dib/elements/kloudbuster/static/kb_test/kb_vm_agent.py @@ -31,6 +31,8 @@ import redis # are added to the agent VM __version__ = '4' +# TODO(Logging on Agent) + def get_image_name(): '''Return the versioned VM image name that corresponds to this agent code. This string must match the way DIB names the kloudbuster image. @@ -61,7 +63,6 @@ class KB_Instance(object): if if_name: debug_msg += " and %s" % if_name cmd += " dev %s" % if_name - # TODO(Logging on Agent) print debug_msg return cmd @@ -89,7 +90,6 @@ class KB_Instance(object): else: debug_msg = "with next hop %s" % if_name cmd += " dev %s" % if_name - # TODO(Logging on Agent) print debug_msg return cmd @@ -121,6 +121,7 @@ class KB_VM_Agent(object): self.orches_chan_name = "kloudbuster_orches" self.report_chan_name = "kloudbuster_report" self.last_cmd = None + self.last_process = None def setup_channels(self): # Check for connections to redis server @@ -151,6 +152,7 @@ class KB_VM_Agent(object): cmds = ['bash', '-c'] cmds.append(cmd) p = subprocess.Popen(cmds, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self.last_process = p (stdout, stderr) = p.communicate() return (p.returncode, stdout, stderr) @@ -162,6 +164,7 @@ class KB_VM_Agent(object): cmds.append(cmd) p_output = '' p = subprocess.Popen(cmds, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self.last_process = p lines_iterator = iter(p.stdout.readline, b"") for line in lines_iterator: @@ -181,6 +184,23 @@ class KB_VM_Agent(object): stderr = p.communicate()[1] return (p.returncode, p_output, stderr) + + def work(self): + for item in self.pubsub.listen(): + if item['type'] != 'message': + continue + # Convert the string representation of dict to real dict obj + message = eval(item['data']) + if message['cmd'] == 'ABORT': + try: + self.last_process.kill() + except Exception: + pass + else: + work_thread = threading.Thread(target=agent.process_cmd, args=[message]) + work_thread.daemon = True + work_thread.start() + def process_cmd(self, message): if message['cmd'] == 'ACK': # When 'ACK' is received, means the master node @@ -205,22 +225,10 @@ class KB_VM_Agent(object): "stderr": str(exc) } self.report('DONE', message['client-type'], cmd_res_dict) - elif message['cmd'] == 'ABORT': - # TODO(Add support to abort a session) - pass else: # Unexpected - # TODO(Logging on Agent) print 'ERROR: Unexpected command received!' - def work(self): - for item in self.pubsub.listen(): - if item['type'] != 'message': - continue - # Convert the string representation of dict to real dict obj - message = eval(item['data']) - self.process_cmd(message) - def exec_setup_static_route(self): self.last_cmd = KB_Instance.get_static_route(self.user_data['target_subnet_ip']) result = self.exec_command(self.last_cmd) @@ -273,7 +281,6 @@ if __name__ == "__main__": with open('user-data', 'r') as f: user_data = eval(f.read()) except Exception as e: - # TODO(Logging on Agent) print e.message sys.exit(1) diff --git a/kb_server/kb_server/controllers/api_kb.py b/kb_server/kb_server/controllers/api_kb.py index 59cd72d..e80440e 100644 --- a/kb_server/kb_server/controllers/api_kb.py +++ b/kb_server/kb_server/controllers/api_kb.py @@ -84,6 +84,17 @@ class KBController(object): LOG.warn(traceback.format_exc()) kb_session.kb_status = 'ERROR' + def kb_stop_test_thread_handler(self, session_id): + kb_session = KBSessionManager.get(session_id) + kb_session.kb_status = 'STOPPING' + kloudbuster = kb_session.kloudbuster + try: + kloudbuster.stop_test() + kb_session.kb_status = 'STAGED' + except Exception: + LOG.warn(traceback.format_exc()) + kb_session.kb_status = 'ERROR' + def kb_cleanup_thread_handler(self, session_id): kb_session = KBSessionManager.get(session_id) kb_session.kb_status = 'CLEANING' @@ -176,7 +187,7 @@ class KBController(object): session_id = args[0] if KBSessionManager.get(session_id).kb_status != 'STAGED': response.status = 403 - response.text = u"Unable to start the tests when status is not STAGED." + response.text = u"Unable to start the tests when status is not at STAGED." return response.text self.kb_thread = threading.Thread(target=self.kb_run_test_thread_handler, args=[session_id]) @@ -185,6 +196,28 @@ class KBController(object): return "OK!" + @expose(generic=True) + def stop_test(self, *args): + response.status = 400 + response.text = u"Please POST to this resource." + return response.text + + @stop_test.when(method='POST') + @check_session_id + def stop_test_POST(self, *args): + session_id = args[0] + if KBSessionManager.get(session_id).kb_status != 'RUNNING': + response.status = 403 + response.text = u"Unable to stop the tests when status is not at RUNNING." + return response.text + + self.kb_thread = threading.Thread(target=self.kb_stop_test_thread_handler, + args=[session_id]) + self.kb_thread.daemon = True + self.kb_thread.start() + + return "OK!" + @expose(generic=True) def cleanup(self, *args): response.status = 400 diff --git a/kb_server/kloudbuster-swagger.yaml b/kb_server/kloudbuster-swagger.yaml index 358866b..9d9d2dd 100644 --- a/kb_server/kloudbuster-swagger.yaml +++ b/kb_server/kloudbuster-swagger.yaml @@ -274,6 +274,29 @@ paths: 404: description: The session_id is not found or invalid + /kloudbuster/stop_test/{session_id}: + post: + description: | + Stop the KloudBuster tests for a given session + parameters: + - name: session_id + type: string + format: md5sum + in: path + description: | + The session to be stopped + required: true + tags: + - kloudbuster + responses: + 200: + description: Scheduled to stop the tests for the given session + 403: + description: | + Unable to stop the tests when status is not at RUNNING + 404: + description: The session_id is not found or invalid + /kloudbuster/cleanup/{session_id}: post: description: | diff --git a/kloudbuster/cfg.scale.yaml b/kloudbuster/cfg.scale.yaml index 70b1239..b3985c3 100644 --- a/kloudbuster/cfg.scale.yaml +++ b/kloudbuster/cfg.scale.yaml @@ -121,7 +121,8 @@ client: # (1) The timeout value is defined in the client:http_tool_config section; # (2) The percentile of packets must be in the below list: # [50, 75, 90, 99, 99.9, 99.99, 99.999] - stop_limit: [50, 99.99] + # (3) Sets percentile to 0 to disable timeout checks; + stop_limit: [50, 0] # Assign floating IP for every client side test VM # Default: no floating IP (only assign internal fixed IP) diff --git a/kloudbuster/kb_config.py b/kloudbuster/kb_config.py index a339dcd..934db7d 100644 --- a/kloudbuster/kb_config.py +++ b/kloudbuster/kb_config.py @@ -92,6 +92,9 @@ class KBConfig(object): if os.path.isfile(pub_key): self.config_scale['public_key_file'] = pub_key LOG.info('Using %s as public key for all VMs' % (pub_key)) + else: + LOG.warn('No public key is found or specified to instantiate VMs. ' + 'You will not be able to access the VMs spawned by KloudBuster.') if self.alt_cfg: self.config_scale = self.config_scale + AttrDict(self.alt_cfg) diff --git a/kloudbuster/kb_runner.py b/kloudbuster/kb_runner.py index a6b828b..37f7200 100644 --- a/kloudbuster/kb_runner.py +++ b/kloudbuster/kb_runner.py @@ -319,6 +319,7 @@ class KBRunner(object): while True: cur_vm_count = len(self.client_dict) target_vm_count = start + (cur_stage - 1) * step + timeout_at_percentile = 0 if target_vm_count > len(self.full_client_dict): break if self.tool_result and 'latency_stats' in self.tool_result: @@ -326,8 +327,7 @@ class KBRunner(object): pert_dict = dict(self.tool_result['latency_stats']) if limit[1] in pert_dict.keys(): timeout_at_percentile = pert_dict[limit[1]] // 1000000 - else: - timeout_at_percentile = 0 + elif limit[1] != 0: LOG.warn('Percentile %s%% is not a standard statistic point.' % limit[1]) if err > limit[0] or timeout_at_percentile > timeout: LOG.warn('KloudBuster is stopping the iteration because the result ' @@ -347,3 +347,6 @@ class KBRunner(object): else: self.single_run(http_test_only=http_test_only) yield self.tool_result + + def stop(self): + self.send_cmd('ABORT', 'http', None) diff --git a/kloudbuster/kloudbuster.py b/kloudbuster/kloudbuster.py index a311b10..e690e33 100755 --- a/kloudbuster/kloudbuster.py +++ b/kloudbuster/kloudbuster.py @@ -226,10 +226,12 @@ class KloudBuster(object): else: self.tenants_list = {'server': None, 'client': None} # TODO(check on same auth_url instead) - if server_cred == client_cred: - self.single_cloud = True - else: - self.single_cloud = False + self.single_cloud = True if server_cred == client_cred else False + # Automatically enable the floating IP for server cloud under dual-cloud mode + if not self.single_cloud and not self.server_cfg['use_floatingip']: + self.server_cfg['use_floatingip'] = True + LOG.info('Automatically setting "use_floatingip" to True for server cloud...') + self.kloud = Kloud(server_cfg, server_cred, self.tenants_list['server']) self.testing_kloud = Kloud(client_cfg, client_cred, self.tenants_list['client'], @@ -438,6 +440,10 @@ class KloudBuster(object): self.final_result.append(self.kb_runner.tool_result) LOG.info('SUMMARY: %s' % self.final_result) + def stop_test(self): + self.kb_runner.stop() + LOG.info('Testing is stopped by request.') + def cleanup(self): # Cleanup: start with tested side first # then testing side last (order is important because of the shared network)