Merge "Converging browbeat and shaker installers. Adding shaker network for automatic network configuration."

README.md

@@ -20,17 +20,15 @@ Table of Contents
       * [Run performance stress tests through browbeat:](#run-performance-stress-tests-through-browbeat)
 
 # Browbeat
-This started as a project to help determine the number of database connections a given OpenStack deployment uses. It has since
+This started as a project to help determine the number of database connections a given OpenStack deployment uses via stress tests. It has since grown into a set of Ansible playbooks to help check deployments for known issues, install tools and change parameters of the overcloud.
-grown into a set of Ansible playbooks to help check deployments for known issues, install tools and change parameters of the
-overcloud.
 
 # Before running browbeat
 * Execute the ansible/gen_hostfile.sh script (builds the ssh config)
-* Install Tools (rally , shaker, connmon, etc)
+* Install Tools (connmon, collectd, graphite, grafana)
-* Configure browbeat.cfg to match your tests
+* Configure browbeat-config.yaml to match your tests
 
 # How to run Browbeat?
-On the Red Hat OpenStack Director host, as the Stack user jump into a venv w/ Rally and you simply run:
+On the Red Hat OpenStack Director host, as the Stack user jump into the browbeat venv and you simply run:
 
     ./browbeat.py --help
 
@@ -57,11 +55,14 @@ $ cd browbeat/ansible
 $ ./gen_hostfile.sh <undercloud-ip> ~/.ssh/config
 $ vi install/group_vars/all # Make sure to edit the dns_server to the correct ip address
 $ ansible-playbook -i hosts install/browbeat.yml
+$ vi install/group_vars/all # Edit shaker subnet/start/end/gw settings
+$ ansible-playbook -i hosts install/shaker_network.yml
+$ ansible-playbook -i hosts install/shaker_build.yml
 ```
 
-### (Optional) Install shaker:
+### (Optional) Install collectd:
 ```
-$ ansible-playbook -i hosts install/shaker.yml
+$ ansible-playbook -i hosts install/collectd.yml
 ```
 
 ### (Optional) Install connmon:
@@ -99,11 +100,14 @@ $ ssh undercloud-root
 [stack@ospd ansible]$ sudo pip install ansible
 [stack@ospd ansible]$ vi install/group_vars/all # Make sure to edit the dns_server to the correct ip address
 [stack@ospd ansible]$ ansible-playbook -i hosts install/browbeat.yml
+[stack@ospd ansible]$ vi install/group_vars/all # Edit shaker subnet/start/end/gw settings
+[stack@ospd ansible]$ ansible-playbook -i hosts install/shaker_network.yml
+[stack@ospd ansible]$ ansible-playbook -i hosts install/shaker_build.yml
 ```
 
-### (Optional) Install shaker:
+### (Optional) Install collectd:
 ```
-[stack@ospd ansible]$ ansible-playbook -i hosts install/shaker.yml
+[stack@ospd ansible]$ ansible-playbook -i hosts install/collectd.yml
 ```
 
 ### (Optional) Install connmon:

ansible/install/browbeat.yml

@@ -1,12 +1,17 @@
 ---
 #
-# Playbook to install browbeat on undercloud
+# Playbook to install Browbeat (Rally + Shaker) on undercloud
 #
 
+- hosts: controller
+  remote_user: heat-admin
+  roles:
+    - shaker-check
+
 - hosts: undercloud
   remote_user: stack
   vars:
     results_in_httpd: true
   roles:
-  - common
+    - common
-  - browbeat
+    - browbeat

ansible/install/build_image.yml

@@ -1,16 +0,0 @@
----
-#
-# Playbook to build shaker-image
-#
-- hosts: undercloud
-  remote_user: stack
-  tasks:
-  - name: build shaker image
-    become: true
-    shell: >
-           source /home/stack/overcloudrc; source /home/stack/shaker-venv/bin/activate;
-           shaker-image-builder --flavor-name {{ shaker_flavor }} --image-builder-template
-           /home/stack/shaker-venv/lib/python2.7/site-packages/shaker/resources/image_builder_templates/centos.yaml
-           --os-region-name regionOne
-    register: image_result
-    failed_when: image_result.rc != 0

ansible/install/group_vars/all

@@ -38,3 +38,13 @@ shaker_port: 5555
 
 # Should choose m1.small or larger
 shaker_flavor: m1.small
+
+# Automates creating a public network that shaker can build its image off of
+shaker_public_subnet: 1.1.1.1/22
+shaker_pool_start: 1.1.1.1
+shaker_pool_end: 1.1.1.1
+shaker_pool_gw: 1.1.1.1
+
+# Defaults here should not require changing
+shaker_network_name: shaker_public
+shaker_router_name: shaker_router

ansible/install/roles/browbeat/tasks/main.yml

@@ -3,6 +3,10 @@
 # Browbeat Install
 #
 
+- name: Install python development tools
+  yum: name=python-devel state=present
+  become: true
+
 - name: Install browbeat matplotlib dependencies
   yum: name={{ item }} state=present
   become: true
@@ -11,12 +15,12 @@
     - libpng-devel
 
 - name: Install pip
-  become: true
   easy_install: name=pip
+  become: true
 
 - name: Install virtualenv
-  become: true
   pip: name=virtualenv
+  become: true
 
 - name: Create browbeat virtualenv
   command: virtualenv /home/stack/browbeat-venv creates="/home/stack/browbeat-venv"
@@ -24,6 +28,9 @@
 - name: Create rally virtualenv
   command: virtualenv /home/stack/rally-venv creates="/home/stack/rally-venv"
 
+- name: Create shaker virtualenv
+  command: virtualenv /home/stack/shaker-venv creates="/home/stack/shaker-venv"
+
 - name: Clone browbeat on undercloud
   git: repo=https://github.com/jtaleric/browbeat.git dest=/home/stack/browbeat
   when: "'localhost' not in '{{ inventory_hostname }}'"
@@ -37,12 +44,12 @@
   when: "'localhost' not in '{{ inventory_hostname }}'"
 
 - name: Install requirements.txt into browbeat-venv
-  become: true
   pip: requirements=/home/stack/browbeat/requirements.txt virtualenv=/home/stack/browbeat-venv
+  become: true
 
 - name: Install rally into rally-venv
-  become: true
   pip: name=rally virtualenv=/home/stack/rally-venv
+  become: true
 
 - name: Setup rally database
   shell: . /home/stack/rally-venv/bin/activate; rally-manage db recreate
@@ -50,6 +57,21 @@
 - name: Setup rally deployment
   shell: . /home/stack/rally-venv/bin/activate; . /home/stack/overcloudrc; rally deployment create --fromenv --name overcloud
 
+- name: Install shaker
+  pip: name=pyshaker version=0.0.10 virtualenv=/home/stack/shaker-venv
+
+- name: Check for shaker port in iptables
+  shell: iptables -nvL | grep -q "dpt:"{{ shaker_port }}""
+  become: true
+  changed_when: false
+  register: shaker_iptables
+  ignore_errors: true
+
+- name: Open up shaker port in iptables
+  shell: /usr/sbin/iptables -I INPUT 1 -p tcp --dport {{ shaker_port }} -j ACCEPT
+  become: true
+  when: shaker_iptables.rc == 1
+
 #
 # Serve results out of httpd if results_in_httpd is set to true
 #

ansible/install/roles/shaker-check/tasks/main.yml

@@ -1,6 +1,6 @@
 ---
 #
-# Tasks to check requirements for isntalling shaker
+# Tasks to check requirements for installing shaker
 #
 
 - name: Check for heat

ansible/install/roles/shaker-network/tasks/main.yml

@@ -0,0 +1,18 @@
+---
+#
+# Setup up network for shaker
+#
+
+- name: Create shaker public network
+  shell: ". /home/stack/overcloudrc; neutron net-create {{shaker_network_name}} --router:external | grep -E ' id ' | awk '{print $4}'"
+  register: public_net_id
+
+- name: Create shaker public subnet
+  shell: ". /home/stack/overcloudrc; neutron subnet-create {{public_net_id.stdout}} {{shaker_public_subnet}} --allocation-pool start={{shaker_pool_start}},end={{shaker_pool_end}} --gateway={{shaker_pool_gw}} --disable-dhcp | grep -E ' id ' | awk '{print $4}'"
+  register: subnet_id
+
+- name: Create shaker router
+  shell: ". /home/stack/overcloudrc; neutron router-create {{shaker_router_name}} | grep -E ' id ' | awk '{print $4}'"
+
+- name: Set shaker router gateway
+  shell: ". /home/stack/overcloudrc; neutron router-gateway-set {{shaker_router_name}} {{shaker_network_name}}"

ansible/install/roles/shaker/tasks/main.yml

@@ -1,34 +0,0 @@
----
-#
-# Tasks to install shaker for data plane testing
-#
-
-- name: Install python development tools
-  yum: name=python-devel state=present
-  become: true
-
-- name: Install pip
-  become: true
-  easy_install: name=pip
-
-- name: Install virtualenv
-  become: true
-  pip: name=virtualenv
-
-- name: Create virtualenv
-  command: virtualenv /home/stack/shaker-venv creates="/home/stack/shaker-venv"
-
-- name: Install shaker
-  pip: name=pyshaker version=0.0.10 virtualenv=/home/stack/shaker-venv
-
-- name: Check for shaker port in iptables
-  shell: iptables -nvL | grep -q "dpt:"{{ shaker_port }}""
-  become: true
-  changed_when: false
-  register: shaker_iptables
-  ignore_errors: true
-
-- name: Open up shaker port in iptables
-  shell: /usr/sbin/iptables -I INPUT 1 -p tcp --dport {{ shaker_port }} -j ACCEPT
-  become: true
-  when: shaker_iptables.rc == 1

ansible/install/shaker.yml

@@ -1,15 +0,0 @@
----
-#
-# Playbook to install shaker on undercloud/overcloud
-#
-
-- hosts: controller
-  remote_user: heat-admin
-  roles:
-  - shaker-check
-
-- hosts: undercloud
-  remote_user: stack
-  roles:
-  - common
-  - shaker

ansible/install/shaker_build.yml

@@ -0,0 +1,17 @@
+---
+#
+# Playbook to build shaker-image
+#
+
+- hosts: undercloud
+  remote_user: stack
+  tasks:
+    - name: build shaker image
+      shell: >
+             source /home/stack/overcloudrc; source /home/stack/shaker-venv/bin/activate;
+             shaker-image-builder --flavor-name {{ shaker_flavor }} --image-builder-template
+             /home/stack/shaker-venv/lib/python2.7/site-packages/shaker/resources/image_builder_templates/centos.yaml
+             --os-region-name regionOne
+      become: true
+      register: image_result
+      failed_when: image_result.rc != 0

ansible/install/shaker_network.yml

@@ -0,0 +1,9 @@
+---
+#
+# Playbook for shaker-network
+#
+
+- hosts: undercloud
+  remote_user: stack
+  roles:
+    - shaker-network

browbeat-complete.yaml

@@ -16,6 +16,7 @@ ansible:
     nova_db: ansible/browbeat/adjustment-db.yml
     workers: ansible/browbeat/adjustment-workers.yml
   grafana_snapshot: ansible/browbeat/snapshot-general-performance-dashboard.yml
+  shaker_build: ansible/install/shaker_build.yml
 grafana:
   enabled: true
   cloud_name: openstack

browbeat-config.yaml

@@ -9,7 +9,6 @@ ansible:
   install:
     connmon: ansible/install/connmon.yml
     browbeat: ansible/install/browbeat.yml
-    shaker: ansible/install/shaker.yml
   check: ansible/check/site.yml
   adjust:
     keystone_token: ansible/browbeat/adjustment-keystone-token.yml
@@ -17,7 +16,7 @@ ansible:
     nova_db: ansible/browbeat/adjustment-db.yml
     workers: ansible/browbeat/adjustment-workers.yml
   grafana_snapshot: ansible/browbeat/snapshot-general-performance-dashboard.yml
-  shaker_build: ansible/install/build_image.yml
+  shaker_build: ansible/install/shaker_build.yml
 grafana:
   enabled: true
   cloud_name: openstack

browbeat.py

@@ -32,7 +32,7 @@ except ImportError :
     exit(1)
 
 # Browbeat specific options
-_install_opts=['connmon','browbeat', 'shaker']
+_install_opts = ['connmon', 'browbeat', 'shaker-build']
 _config_file = 'browbeat-config.yaml'
 _config = None
 
@@ -70,8 +70,8 @@ if __name__ == '__main__':
             help='Run the Browbeat Overcloud Checks')
     _cli.add_argument('-w','--workloads',action='store_true',
             help='Run the Browbeat workloads')
-    _cli.add_argument('-i','--install',nargs=1,choices=_install_opts,dest='install',
+    _cli.add_argument('-i', '--install', nargs=1, choices=_install_opts, dest='install',
-            help='Install Browbeat Tools')
+            help='Install/Setup Browbeat Tools/Workloads')
     _cli.add_argument('--debug',action='store_true',
             help='Enable Debug messages')
     _cli_args = _cli.parse_args()
@@ -93,15 +93,13 @@ if __name__ == '__main__':
             hosts_path=_cli_args.hosts
         if _cli_args.install[0] == 'all' :
             for tool in _install_opts:
-                if tool == "shaker":
+                if tool == 'shaker-build':
-                    _run_playbook(_config['ansible']['install']['shaker'],hosts_path)
                     _run_playbook(_config['ansible']['shaker_build'],hosts_path)
                 else:
                     _run_playbook(_config['ansible']['install'][tool],hosts_path)
 
         elif _cli_args.install[0] in _install_opts :
-            if _cli_args.install[0] == "shaker":
+            if _cli_args.install[0] == 'shaker-build':
-                _run_playbook(_config['ansible']['install']['shaker'],hosts_path)
                 _run_playbook(_config['ansible']['shaker_build'],hosts_path)
             else:
                 _run_playbook(_config['ansible']['install'][_cli_args.install[0]],hosts_path)