Feature commit #3 to support running fio

Change-Id: I7f6205b0381d06e2ddb3eee0ed7f2129a1cc6f98

kloudbuster/cfg.scale.yaml

@@ -195,9 +195,9 @@ client:
     #     ['randread', 'randwrite', 'read', 'write']
     # runtime: (Required)
     #     Test duration in seconds
-    # block_size: (Optional, default=4k)
+    # block_size: (Required, default=4k)
     #     Block size for I/O units
-    # iodepth: (Optional, default=1)
+    # iodepth: (Required, default=1)
     #     Number of I/O units to keep in flight against the file
     # rate_iops: (Optional, default=unlimited)
     #     Cap the bandwidth to this number of IOPS

kloudbuster/fio_tool.py

@@ -33,26 +33,51 @@ class FioTool(PerfTool):
             return self.parse_error(stderr)
 
         # Sample Output:
-        # {
-        # }
+        # Refer to kloudbuster/fio_example.json for a sample output
 
         try:
             result = json.loads(stdout)
-            result = result
+            read_iops = result['jobs'][0]['read']['iops']
+            read_bw = result['jobs'][0]['read']['bw']
+            write_iops = result['jobs'][0]['write']['iops']
+            write_bw = result['jobs'][0]['write']['bw']
         except Exception:
             return self.parse_error('Could not parse: "%s"' % (stdout))
 
         parsed_output = {'tool': self.name}
+        if read_iops:
+            parsed_output['read_iops'] = read_iops
+        if read_bw:
+            parsed_output['read_bw'] = read_bw
+        if write_iops:
+            parsed_output['write_iops'] = write_iops
+        if write_bw:
+            parsed_output['write_bw'] = write_bw
 
-        # TODO()
         return parsed_output
 
     @staticmethod
     def consolidate_results(results):
-        # TODO()
-        return {'Test': 'Test'}
+        all_res = {'tool': 'fio'}
+        total_count = len(results)
+        if not total_count:
+            return all_res
+
+        for key in ['read_iops', 'read_bw', 'write_iops', 'write_bw']:
+            all_res[key] = 0
+            for item in results:
+                all_res[key] += item['results'].get(key, 0)
+            all_res[key] = int(all_res[key])
+
+        return all_res
 
     @staticmethod
     def consolidate_samples(results, vm_count):
-        # TODO()
-        return {'Test': 'Test'}
+        all_res = FioTool.consolidate_results(results)
+        total_count = float(len(results)) / vm_count
+        if not total_count:
+            return all_res
+
+        all_res['read_iops'] = int(all_res['read_iops'] / total_count)
+        all_res['write_iops'] = int(all_res['write_iops'] / total_count)
+        return all_res

kloudbuster/kb_runner_storage.py

@@ -66,6 +66,9 @@ class KBRunner_Storage(KBRunner):
 
             test_count = len(self.config.storage_tool_configs)
             perf_tool = self.client_dict.values()[0].perf_tool
+            self.tool_result = []
+            vm_count = active_range[1] - active_range[0] + 1\
+                if active_range else len(self.full_client_dict)
             for idx, cur_config in enumerate(self.config.storage_tool_configs):
                 LOG.info("Runing test case %d of %d..." % (idx + 1, test_count))
                 self.report = {'seq': 0, 'report': None}
@@ -73,11 +76,17 @@ class KBRunner_Storage(KBRunner):
                 self.run_storage_test(active_range, dict(cur_config))
                 # Call the method in corresponding tools to consolidate results
                 LOG.kbdebug(self.result.values())
-                self.tool_result = perf_tool.consolidate_results(self.result.values())
-                vm_count = active_range[1] - active_range[0] + 1\
-                    if active_range else len(self.full_client_dict)
-                self.tool_result['total_client_vms'] = vm_count
-                self.tool_result['total_server_vms'] = self.tool_result['total_client_vms']
+                tc_result = perf_tool.consolidate_results(self.result.values())
+                tc_result['mode'] = cur_config['mode']
+                tc_result['block_size'] = cur_config['block_size']
+                tc_result['iodepth'] = cur_config['iodepth']
+                if 'rate_iops' in cur_config:
+                    tc_result['rate_iops'] = cur_config['rate_iops']
+                if 'rate' in cur_config:
+                    tc_result['rate'] = cur_config['rate']
+                tc_result['total_client_vms'] = vm_count
+                tc_result['total_server_vms'] = tc_result['total_client_vms']
+                self.tool_result.append(tc_result)
         except KBInitVolumeException:
             raise KBException("Could not initilize the volume.")
 

kloudbuster/wrk_tool.py

@@ -104,8 +104,7 @@ class WrkTool(PerfTool):
                     'http_sock_timeout', 'http_throughput_kbytes']:
             all_res[key] = 0
             for item in results:
-                if (key in item['results']):
-                    all_res[key] += item['results'][key]
+                all_res[key] += item['results'].get(key, 0)
             all_res[key] = int(all_res[key])
 
         if 'latency_stats' in results[0]['results']: