Update tox.ini with Ansible linting and Browbeat config linting

This commit enanbles Ansible linting and does some
minor refactoring to make existing Ansible roles
compatible with the new rules.

Several Ansible linting rules have been excluded to keep the number
of changes from being too onerous.

Also a new script in ci-scripts is used to check very config file
included in the Browbeat repo for validity using the template
Browbeat uses when it runs.

Here's a list of the new linting rules

* Ansible tasks must have names
* When you use shell you must use become not sudo
* Using become_user without using become is not allowed
* If a repo is pulled it must be a pinned version of commit, not latest
* Always_run is deprecated don't use it
* Variables without {{}} and not in when statements are deprecated don't use them
* No Trailing whitepaces
* YAML checking, catches big syntax errors but not less obvious ones

Change-Id: Ic531c91c408996d4e7d8899afe8b21d364998680

ansible/browbeat/adjustment-workers.yml

@@ -34,5 +34,5 @@
   roles:
     - { role: keystone-workers, when: service == "keystone" }
     - { role: nova-workers, when: service == "nova" }
-    - { role: neutron-workers, when: service == "neutron" } 
+    - { role: neutron-workers, when: service == "neutron" }
     - { role: cinder-workers, when: service == "cinder" }

ansible/check/site.yml

@@ -57,10 +57,15 @@
     ansible_python_interpreter: "/usr/bin/python"
   name: Generating bug report
   tasks:
-    - file: "path={{result_dir}} state=directory"
-    - template: src=templates/bug_report.j2 dest={{result_dir}}/bug_report.log
+    - name: Create results dir
+      file: "path={{result_dir}} state=directory"
+    - name: Template Bug report
+      template: src=templates/bug_report.j2 dest={{result_dir}}/bug_report.log
       become: false
-    - template: src=templates/mysql_report.j2 dest={{result_dir}}/mysql_report.log
+    - name: Template MySQL report
+      template: src=templates/mysql_report.j2 dest={{result_dir}}/mysql_report.log
       become: false
-    - replace: dest={{result_dir}}/mysql_report.log regexp='\[([^\s+]+)' replace=''
-    - replace: dest={{result_dir}}/mysql_report.log regexp='\r' replace=''
+    - name: Run MySQL report regex
+      replace: dest={{result_dir}}/mysql_report.log regexp='\[([^\s+]+)' replace=''
+    - name: Run MySQL report regex
+      replace: dest={{result_dir}}/mysql_report.log regexp='\r' replace=''

ansible/install/roles/grafana_docker/handlers/main.yml

@@ -0,0 +1,3 @@
+- name: restart systemd-grafana
+  shell: /usr/bin/systemctl daemon-reload && /usr/bin/systemctl enable grafana-server && /usr/bin/systemctl restart grafana-server
+  become: true

ansible/install/roles/grafana_docker/tasks/main.yml

@@ -65,10 +65,6 @@
     group=root
     mode=0644
   become: true
-  register: systemd_grafana_needs_restart
+  notify: restart systemd-grafana
 
-- name: bounce systemd and grafana-server container
-  shell: /usr/bin/systemctl daemon-reload && /usr/bin/systemctl enable grafana-server && /usr/bin/systemctl restart grafana-server
-  become: true
-  when: systemd_grafana_needs_restart.changed
 

ansible/install/roles/graphite/handlers/main.yml

@@ -0,0 +1,6 @@
+- name: restart apache
+  service: name=httpd state=restarted enabled=true
+  become: true
+- name: restart carbon-cache
+  service: name=carbon-cache state=restarted enabled=true
+  become: true

ansible/install/roles/graphite/tasks/main.yml

@@ -29,7 +29,8 @@
   shell: /root/setup-graphite-db.exp {{ graphite_username }} {{ graphite_password }} && chown apache:apache /var/lib/graphite-web/graphite.db
   become: true
   when: graphite_db_installed.rc != 0
-  register: apache_needs_restart
+  notify:
+     - restart apache
 
 - name: Setup httpd graphite-web config
   template:
@@ -39,7 +40,8 @@
     group=root
     mode=0644
   become: true
-  register: apache_needs_restart
+  notify:
+     - restart apache
 
 ### begin firewall ###
 # we need TCP/80 open
@@ -110,12 +112,8 @@
 - name: Remove httpd welcome config
   become: true
   file: path=/etc/httpd/conf.d/welcome.conf state=absent
-  register: apache_needs_restart
-
-- name: Bounce Apache
-  service: name=httpd state=restarted enabled=true
-  become: true
-  when: apache_needs_restart.changed
+  notify:
+     - restart apache
 
 #
 # setup the python-carbon service
@@ -133,7 +131,8 @@
     group=root
     mode=0644
   become: true
-  register: carbon_cache_needs_restart
+  notify:
+     - restart carbon-cache
 
 - name: copy carbon storage aggregation config
   copy:
@@ -143,7 +142,8 @@
     group=root
     mode=0644
   become: true
-  register: carbon_cache_needs_restart
+  notify:
+     - restart carbon-cache
 
 - name: copy carbon config
   copy:
@@ -153,10 +153,7 @@
     group=root
     mode=0644
   become: true
-  register: carbon_cache_needs_restart
+  notify:
+     - restart carbon-cache
 
-- name: bounce carbon cache
-  service: name=carbon-cache state=restarted enabled=true
-  become: true
-  when: carbon_cache_needs_restart.changed
 

ansible/install/roles/graphite_docker/handlers/main.yml

@@ -0,0 +1,3 @@
+- name: restart systemd-graphite
+  shell: /usr/bin/systemctl daemon-reload && /usr/bin/systemctl enable graphite-web && /usr/bin/systemctl restart graphite-web
+  become: true

ansible/install/roles/graphite_docker/tasks/main.yml

@@ -87,12 +87,8 @@
     group=root
     mode=0644
   become: true
-  register: systemd_graphite_needs_restart
+  notify: restart systemd-graphite
 
-- name: bounce systemd and graphite-web container
-  shell: /usr/bin/systemctl daemon-reload && /usr/bin/systemctl enable graphite-web && /usr/bin/systemctl restart graphite-web
-  become: true
-  when: systemd_graphite_needs_restart.changed
 
 - name: Setup carbon-cache systemd config
   template:
@@ -102,10 +98,6 @@
     group=root
     mode=0644
   become: true
-  register: systemd_carbon_needs_restart
+  notify: restart systemd-graphite
 
-- name: bounce systemd and carbon-cache container
-  shell: /usr/bin/systemctl daemon-reload && /usr/bin/systemctl enable carbon-cache && /usr/bin/systemctl restart carbon-cache
-  become: true
-  when: systemd_carbon_needs_restart.changed
 

ansible/install/roles/kibana-visualization/vars/shaker_cloud_comparison_visualizations.yml

@@ -9,7 +9,7 @@ shaker_cloud_comparison_visualizations:
     markdown: "# Shaker-L2-Flent\\\\n ## Throughput in Mbps\\\\nTo filter based on concurrency please query for key record.concurrency and to filter based based on number of compute nodes, please query for key acommodation.compute_nodes"
   - title: "Browbeat-Shaker-Cloud-Comprison-L2-Throughput"
     template: "shaker-cloud-comparison.json.j2"
-    query: "shaker_test_info.deployment.template: l2.hot AND (result.result_type: tcp_download OR result.result_type: tcp_upload)" 
+    query: "shaker_test_info.deployment.template: l2.hot AND (result.result_type: tcp_download OR result.result_type: tcp_upload)"
   - title: "Shaker-Cloud-Comparison-L2-Latency"
     template: "markdown.json.j2"
     markdown: "# Shaker-L2-Flent\\\\n ## Latency in ms\\\\nTo filter based on concurrency please query for key record.concurrency and to filter based based on number of compute nodes, please query for accommodation.compute_nodes"
@@ -21,7 +21,7 @@ shaker_cloud_comparison_visualizations:
     markdown: "# Shaker-L3-East-West-Flent\\\\n## Throughput in Mbps\\\\nTo filter based on concurrency please query for key record.concurrency and to filter based based on number of compute nodes, please query for key accommodation.compute_nodes"
   - title: "Browbeat-Shaker-Cloud-Comparison-L3-East-West-Throughput"
     template: "shaker-cloud-comparison.json.j2"
-    query: "shaker_test_info.deployment.template: l3_east_west.hot AND (result.result_type: tcp_download OR result.result_type: tcp_upload)" 
+    query: "shaker_test_info.deployment.template: l3_east_west.hot AND (result.result_type: tcp_download OR result.result_type: tcp_upload)"
   - title: "Shaker-Cloud-Comparison-L3-East-West-Latency"
     template: "markdown.json.j2"
     markdown: "# Shaker-L3-East-West-Flent\\\\n## Latency in ms\\\\nTo filter based on concurrency please query for key record.concurrency and to filter based based on number of compute nodes, please query for key accommodation.compute_nodes"

browbeat-complete.yaml

@@ -5,6 +5,7 @@ browbeat:
   cloud_name: openstack
 elasticsearch:
   enabled: false
+  regather: false
   host: 1.1.1.1
   port: 9200
   metadata_files:

ci-scripts/linters/lint-browbeat-config.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+import yaml
+import sys
+from pykwalify import core as pykwalify_core
+from pykwalify import errors as pykwalify_errors
+stream = open(sys.argv[1], 'r')
+schema = yaml.load(stream)
+check = pykwalify_core.Core(sys.argv[2], schema_data=schema)
+try:
+    check.validate(raise_exception=True)
+    print ("Validation successful")
+    exit(0)
+except pykwalify_errors.SchemaError as e:
+    print ("Config " + sys.argv[2] + " is not valid!")
+    raise Exception('File does not conform to schema: {}'.format(e))

conf/browbeat-glance-complete.yaml

@@ -5,6 +5,7 @@ browbeat:
   cloud_name: openstack
 elasticsearch:
   enabled: false
+  regather: false
   host: 1.1.1.1
   port: 9200
   metadata_files:

conf/browbeat-keystone-complete.yaml

@@ -6,6 +6,7 @@ browbeat:
   cloud_name: openstack
 elasticsearch:
   enabled: false
+  regather: false
   host: 1.1.1.1
   port: 9200
   regather: true

conf/browbeat-keystone-minimal.yaml

@@ -6,6 +6,7 @@ browbeat:
   cloud_name: openstack
 elasticsearch:
   enabled: false
+  regather: false
   host: 1.1.1.1
   port: 9200
   regather: true

conf/ceilometer-minimal.yaml

@@ -5,6 +5,7 @@ browbeat:
   cloud_name: openstack
 elasticsearch:
   enabled: false
+  regather: false
   host: 1.1.1.1
   port: 9200
   metadata_files:

conf/ceilometer-persist-response-timing.yaml

@@ -8,6 +8,7 @@ browbeat:
   cloud_name: openstack
 elasticsearch:
   enabled: false
+  regather: false
   host: 1.1.1.1
   port: 9200
   metadata_files:

conf/perfkit-benchmarks.yaml

@@ -10,6 +10,7 @@ browbeat:
   cloud_name: openstack
 elasticsearch:
   enabled: false
+  regather: false
   host: 1.1.1.1
   port: 9200
   metadata_files:

conf/telemetry-nova-persist-instances.yaml

@@ -9,6 +9,7 @@ browbeat:
   cloud_name: openstack
 elasticsearch:
   enabled: false
+  regather: false
   host: 1.1.1.1
   port: 9200
   regather: true

tox.ini

@@ -1,7 +1,6 @@
 [tox]
 minversion = 2.0
-#envlist = py34-constraints,py27-constraints,pypy-constraints,pep8-constraints
-envlist = py27-constraints,pypy-constraints,pep8-constraints
+envlist = py27-constraints,pep8-constraints,linters
 skipsdist = True
 
 [testenv]
@@ -13,17 +12,23 @@ setenv =
    VIRTUAL_ENV={envdir}
 deps = -r{toxinidir}/test-requirements.txt
 commands = python setup.py test
-#commands = python setup.py test --slowest --testr-args='{posargs}'
 
 [testenv:common-constraints]
 install_command = pip install -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} {opts} {packages}
 
+
 [testenv:linters]
 whitelist_externals = bash
 commands =
+  pip install pykwalify
   flake8
-#  bash -c "cd ansible; find . -type f -regex '.*.y[a]?ml' -print0 | xargs -t -n1 -0 \
-#    ansible-lint"
+  bash -c "cd ansible; find . -type f -regex '.*.y[a]?ml' -print0 | xargs -t -n1 -0 \
+    ansible-lint \
+    -x ANSIBLE0013,ANSIBLE0012,ANSIBLE0006,ANSIBLE0007,ANSIBLE0010,ANSIBLE0016"
+  python ci-scripts/linters/lint-browbeat-config.py lib/validate.yaml browbeat-config.yaml
+  python ci-scripts/linters/lint-browbeat-config.py lib/validate.yaml browbeat-complete.yaml
+  bash -c "for config in $(ls conf/); do \
+    python ci-scripts/linters/lint-browbeat-config.py lib/validate.yaml conf/$config; done"
 
 [testenv:pep8]
 commands = flake8 {posargs} --exclude=ansible