Dynamic workloads : Support for multiple OCP clusters

This patch adds support for multiple Openshift clusters in
kube-burner dynamic workloads.

Change-Id: I15a629c43fe359e40a28a75493fe27483e31d1d2

ansible/install/roles/e2e-benchmarking/tasks/main.yml

@@ -9,6 +9,8 @@
 
 - name: Deploy benchmark operator and make changes to scripts
   shell: |
+    export KUBECONFIG={{item}}
     ./install_e2e_benchmarking.sh
+  loop: "{{ lookup('file', '{{ browbeat_path }}/ansible/kubeconfig_paths').splitlines() }}"
   args:
     chdir: "{{ browbeat_path }}/ansible"

ansible/kubeconfig_paths

@@ -0,0 +1 @@
+/home/stack/.kube/config

browbeat-config.yaml

@@ -589,6 +589,8 @@ workloads:
         # shift_on_stack_workload can be poddensity, clusterdensity, maxnamespaces,
         # or maxservices
         shift_on_stack_workload: poddensity
+        shift_on_stack_kubeconfig_paths:
+          - /home/stack/.kube/config
         # workloads can be 'all', a single workload(Eg. : create_delete_servers),
         # or a comma separated string(Eg. : create_delete_servers,migrate_servers).
         # Currently supported workloads : create_delete_servers, migrate_servers

doc/source/installation.rst

@@ -55,13 +55,16 @@ On the Undercloud
    internet. Some useful documentation can be found at:
    https://access.redhat.com/documentation/en/red-hat-openstack-platform/11/single/networking-guide/
 
-(Optional) Clone e2e-benchmarking repository
+(Optional) Clone e2e-benchmarking repository and deploy benchmark-operator
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 e2e-benchmarking is a repository that is used to run workloads to stress an Openshift
 cluster, and is needed to run shift-on-stack workloads in Browbeat.
 
-To enable the e2e-benchmarking repository to be cloned, set install_e2e_benchmarking: true
+To enable the e2e-benchmarking repository to be cloned and benchmark-operator to be deployed,
-in ansible/install/group_vars/all.yml before running the command mentioned below.
+set install_e2e_benchmarking: true in ansible/install/group_vars/all.yml.
+After that, add the kubeconfig paths of all your Openshift clusters in the ansible/kubeconfig_paths
+file. Move the default kubeconfig file(/home/stack/.kube/config) to another location so that it isn't
+used for all Openshift clusters. After that, run the command mentioned below.
 
   [stack@undercloud ansible]$ ansible-playbook -i hosts.yml install/browbeat.yml
 

rally/rally-plugins/dynamic-workloads/dynamic_workload.py

@@ -10,6 +10,8 @@
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
 
+import os
+
 from rally_openstack import consts
 from rally.task import scenario
 from rally.task import types
@@ -55,7 +57,8 @@ class DynamicWorkload(vm.VMDynamicScenario, trunk.TrunkDynamicScenario,
         num_pools, num_clients, delete_num_lbs, delete_num_members, num_create_vms, num_delete_vms,
         provider_phys_net, iface_name, iface_mac, num_vms_provider_net,
         shift_on_stack_job_iterations, shift_on_stack_qps, shift_on_stack_burst,
-        shift_on_stack_workload, workloads="all", router_create_args=None, network_create_args=None,
+        shift_on_stack_workload, shift_on_stack_kubeconfig_paths, workloads="all",
+        router_create_args=None, network_create_args=None,
         subnet_create_args=None, **kwargs):
 
         workloads_list = workloads.split(",")
@@ -83,6 +86,11 @@ class DynamicWorkload(vm.VMDynamicScenario, trunk.TrunkDynamicScenario,
         self.ext_net_name = self.clients("neutron").show_network(ext_net_id)["network"][
             "name"]
 
+        try:
+            self.browbeat_dir = DynamicWorkload.browbeat_dir
+        except AttributeError:
+            DynamicWorkload.browbeat_dir = os.getcwd()
+
         if workloads == "all" or "create_delete_servers" in workloads_list:
             self.boot_servers(smallest_image, smallest_flavor, num_create_vms,
                               subnet_create_args=subnet_create_args)
@@ -141,6 +149,10 @@ class DynamicWorkload(vm.VMDynamicScenario, trunk.TrunkDynamicScenario,
             self.provider_net_nova_delete(provider_phys_net)
 
         if "shift_on_stack" in workloads_list:
+            num_openshift_clusters = len(shift_on_stack_kubeconfig_paths)
             self.run_kube_burner_workload(shift_on_stack_workload,
                                           shift_on_stack_job_iterations,
-                                          shift_on_stack_qps, shift_on_stack_burst)
+                                          shift_on_stack_qps, shift_on_stack_burst,
+                                          shift_on_stack_kubeconfig_paths[
+                                              ((self.context["iteration"] - 1)
+                                               % num_openshift_clusters)])

rally/rally-plugins/dynamic-workloads/dynamic_workload.yml

@@ -27,6 +27,7 @@
 {% set shift_on_stack_qps = shift_on_stack_qps or 20 %}
 {% set shift_on_stack_burst = shift_on_stack_burst or 20 %}
 {% set shift_on_stack_workload = shift_on_stack_workload or 'poddensity' %}
+{% set shift_on_stack_kubeconfig_paths = shift_on_stack_kubeconfig_paths or ['/home/stack/.kube/config'] %}
 {% set router_external = router_external or True %}
 {% set sla_max_avg_duration = sla_max_avg_duration or 60 %}
 {% set sla_max_failure = sla_max_failure or 0 %}
@@ -73,6 +74,7 @@ BrowbeatPlugin.dynamic_workload:
       shift_on_stack_qps: {{shift_on_stack_qps}}
       shift_on_stack_burst: {{shift_on_stack_burst}}
       shift_on_stack_workload: '{{shift_on_stack_workload}}'
+      shift_on_stack_kubeconfig_paths: {{shift_on_stack_kubeconfig_paths}}
       provider_phys_net: '{{ provider_phys_net }}'
       iface_name: '{{ iface_name }}'
       iface_mac: '{{ iface_mac }}'

rally/rally-plugins/dynamic-workloads/shift_on_stack.py

@@ -19,22 +19,23 @@ import dynamic_utils
 class ShiftStackDynamicScenario(dynamic_utils.NovaUtils,
                                 dynamic_utils.NeutronUtils,
                                 dynamic_utils.LockUtils):
-    def run_kube_burner_workload(self, workload, job_iterations, qps, burst):
+    def run_kube_burner_workload(self, workload, job_iterations, qps, burst, kubeconfig):
         """Run kube-burner workloads through e2e-benchmarking
         :param workload: str, kube-burner workload to run
         :param job_iterations: int, number of job iterations
         :param qps: int, queries per second
         :param burst: int, burst value to throttle
+        :param kubeconfig: str, path to kubeconfig file
         """
-        browbeat_dir = os.getcwd()
         os.chdir(
-            browbeat_dir + "/ansible/gather/e2e-benchmarking/workloads/kube-burner"
+            self.browbeat_dir + "/ansible/gather/e2e-benchmarking/workloads/kube-burner"
         )
         e2e_benchmarking_dir = os.getcwd()
 
         script_file_name = "run_" + workload + "_test_fromgit.sh"
         script_file_path = e2e_benchmarking_dir + "/" + script_file_name
         script_file = open(script_file_path, "r")
+
         updated_file_content = ""
 
         if workload == "poddensity":
@@ -47,7 +48,10 @@ class ShiftStackDynamicScenario(dynamic_utils.NovaUtils,
             job_iters_param = "SERVICE_COUNT"
 
         for line in script_file:
-            if "TEST_JOB_ITERATIONS" in line:
+            if "/usr/bin/bash" in line:
+                updated_file_content += line
+                updated_file_content += "export KUBECONFIG=${KUBECONFIG:-"+kubeconfig+"}\n"
+            elif "TEST_JOB_ITERATIONS" in line:
                 first_part_of_line = line.split("TEST")[0]
                 updated_file_content += (
                     first_part_of_line + "TEST_JOB_ITERATIONS=${" + job_iters_param +
@@ -56,7 +60,8 @@ class ShiftStackDynamicScenario(dynamic_utils.NovaUtils,
                 updated_file_content += "export QPS=" + str(qps) + "\n"
                 updated_file_content += "export BURST=" + str(burst) + "\n"
                 updated_file_content += "export CLEANUP_WHEN_FINISH=true\n"
-            else:
+            elif ("export KUBECONFIG" not in line and "export QPS" not in line and
+                  "export BURST" not in line and "export CLEANUP_WHEN_FINISH" not in line):
                 updated_file_content += line
 
         with open(script_file_path, "w") as script_file_writer:
@@ -65,4 +70,4 @@ class ShiftStackDynamicScenario(dynamic_utils.NovaUtils,
         subprocess.run("./" + script_file_name + " 2>&1 | tee -a log.txt && exit ${PIPESTATUS}",
                        shell=True, check=True, executable="/bin/bash")
 
-        os.chdir(browbeat_dir)
+        os.chdir(self.browbeat_dir)