From c60b55ec8380c6f0d3747ff524579f627b02b607 Mon Sep 17 00:00:00 2001 From: jkilpatr Date: Wed, 25 Jan 2017 10:31:55 -0500 Subject: [PATCH] Update tox.ini with Ansible linting and Browbeat config linting This commit enanbles Ansible linting and does some minor refactoring to make existing Ansible roles compatible with the new rules. Several Ansible linting rules have been excluded to keep the number of changes from being too onerous. Also a new script in ci-scripts is used to check very config file included in the Browbeat repo for validity using the template Browbeat uses when it runs. Here's a list of the new linting rules * Ansible tasks must have names * When you use shell you must use become not sudo * Using become_user without using become is not allowed * If a repo is pulled it must be a pinned version of commit, not latest * Always_run is deprecated don't use it * Variables without {{}} and not in when statements are deprecated don't use them * No Trailing whitepaces * YAML checking, catches big syntax errors but not less obvious ones Change-Id: Ic531c91c408996d4e7d8899afe8b21d364998680 --- ansible/browbeat/adjustment-workers.yml | 2 +- ansible/check/site.yml | 15 +++++++---- .../roles/grafana_docker/handlers/main.yml | 3 +++ .../roles/grafana_docker/tasks/main.yml | 6 +---- .../install/roles/graphite/handlers/main.yml | 6 +++++ ansible/install/roles/graphite/tasks/main.yml | 27 +++++++++---------- .../roles/graphite_docker/handlers/main.yml | 3 +++ .../roles/graphite_docker/tasks/main.yml | 12 ++------- ...shaker_cloud_comparison_visualizations.yml | 4 +-- browbeat-complete.yaml | 1 + ci-scripts/linters/lint-browbeat-config.py | 27 +++++++++++++++++++ conf/browbeat-glance-complete.yaml | 1 + conf/browbeat-keystone-complete.yaml | 1 + conf/browbeat-keystone-minimal.yaml | 1 + conf/ceilometer-minimal.yaml | 1 + conf/ceilometer-persist-response-timing.yaml | 1 + conf/perfkit-benchmarks.yaml | 1 + conf/telemetry-nova-persist-instances.yaml | 1 + tox.ini | 15 +++++++---- 19 files changed, 85 insertions(+), 43 deletions(-) create mode 100644 ansible/install/roles/grafana_docker/handlers/main.yml create mode 100644 ansible/install/roles/graphite/handlers/main.yml create mode 100644 ansible/install/roles/graphite_docker/handlers/main.yml create mode 100644 ci-scripts/linters/lint-browbeat-config.py diff --git a/ansible/browbeat/adjustment-workers.yml b/ansible/browbeat/adjustment-workers.yml index 9a933e4c1..a39c39046 100644 --- a/ansible/browbeat/adjustment-workers.yml +++ b/ansible/browbeat/adjustment-workers.yml @@ -34,5 +34,5 @@ roles: - { role: keystone-workers, when: service == "keystone" } - { role: nova-workers, when: service == "nova" } - - { role: neutron-workers, when: service == "neutron" } + - { role: neutron-workers, when: service == "neutron" } - { role: cinder-workers, when: service == "cinder" } diff --git a/ansible/check/site.yml b/ansible/check/site.yml index 62a6508fa..e7dd14d01 100644 --- a/ansible/check/site.yml +++ b/ansible/check/site.yml @@ -57,10 +57,15 @@ ansible_python_interpreter: "/usr/bin/python" name: Generating bug report tasks: - - file: "path={{result_dir}} state=directory" - - template: src=templates/bug_report.j2 dest={{result_dir}}/bug_report.log + - name: Create results dir + file: "path={{result_dir}} state=directory" + - name: Template Bug report + template: src=templates/bug_report.j2 dest={{result_dir}}/bug_report.log become: false - - template: src=templates/mysql_report.j2 dest={{result_dir}}/mysql_report.log + - name: Template MySQL report + template: src=templates/mysql_report.j2 dest={{result_dir}}/mysql_report.log become: false - - replace: dest={{result_dir}}/mysql_report.log regexp='\[([^\s+]+)' replace='' - - replace: dest={{result_dir}}/mysql_report.log regexp='\r' replace='' + - name: Run MySQL report regex + replace: dest={{result_dir}}/mysql_report.log regexp='\[([^\s+]+)' replace='' + - name: Run MySQL report regex + replace: dest={{result_dir}}/mysql_report.log regexp='\r' replace='' diff --git a/ansible/install/roles/grafana_docker/handlers/main.yml b/ansible/install/roles/grafana_docker/handlers/main.yml new file mode 100644 index 000000000..826781eb9 --- /dev/null +++ b/ansible/install/roles/grafana_docker/handlers/main.yml @@ -0,0 +1,3 @@ +- name: restart systemd-grafana + shell: /usr/bin/systemctl daemon-reload && /usr/bin/systemctl enable grafana-server && /usr/bin/systemctl restart grafana-server + become: true diff --git a/ansible/install/roles/grafana_docker/tasks/main.yml b/ansible/install/roles/grafana_docker/tasks/main.yml index 95b0de90e..22fc11704 100644 --- a/ansible/install/roles/grafana_docker/tasks/main.yml +++ b/ansible/install/roles/grafana_docker/tasks/main.yml @@ -65,10 +65,6 @@ group=root mode=0644 become: true - register: systemd_grafana_needs_restart + notify: restart systemd-grafana -- name: bounce systemd and grafana-server container - shell: /usr/bin/systemctl daemon-reload && /usr/bin/systemctl enable grafana-server && /usr/bin/systemctl restart grafana-server - become: true - when: systemd_grafana_needs_restart.changed diff --git a/ansible/install/roles/graphite/handlers/main.yml b/ansible/install/roles/graphite/handlers/main.yml new file mode 100644 index 000000000..3987061b4 --- /dev/null +++ b/ansible/install/roles/graphite/handlers/main.yml @@ -0,0 +1,6 @@ +- name: restart apache + service: name=httpd state=restarted enabled=true + become: true +- name: restart carbon-cache + service: name=carbon-cache state=restarted enabled=true + become: true diff --git a/ansible/install/roles/graphite/tasks/main.yml b/ansible/install/roles/graphite/tasks/main.yml index 5ba52cb2d..105f85135 100644 --- a/ansible/install/roles/graphite/tasks/main.yml +++ b/ansible/install/roles/graphite/tasks/main.yml @@ -29,7 +29,8 @@ shell: /root/setup-graphite-db.exp {{ graphite_username }} {{ graphite_password }} && chown apache:apache /var/lib/graphite-web/graphite.db become: true when: graphite_db_installed.rc != 0 - register: apache_needs_restart + notify: + - restart apache - name: Setup httpd graphite-web config template: @@ -39,7 +40,8 @@ group=root mode=0644 become: true - register: apache_needs_restart + notify: + - restart apache ### begin firewall ### # we need TCP/80 open @@ -110,12 +112,8 @@ - name: Remove httpd welcome config become: true file: path=/etc/httpd/conf.d/welcome.conf state=absent - register: apache_needs_restart - -- name: Bounce Apache - service: name=httpd state=restarted enabled=true - become: true - when: apache_needs_restart.changed + notify: + - restart apache # # setup the python-carbon service @@ -133,7 +131,8 @@ group=root mode=0644 become: true - register: carbon_cache_needs_restart + notify: + - restart carbon-cache - name: copy carbon storage aggregation config copy: @@ -143,7 +142,8 @@ group=root mode=0644 become: true - register: carbon_cache_needs_restart + notify: + - restart carbon-cache - name: copy carbon config copy: @@ -153,10 +153,7 @@ group=root mode=0644 become: true - register: carbon_cache_needs_restart + notify: + - restart carbon-cache -- name: bounce carbon cache - service: name=carbon-cache state=restarted enabled=true - become: true - when: carbon_cache_needs_restart.changed diff --git a/ansible/install/roles/graphite_docker/handlers/main.yml b/ansible/install/roles/graphite_docker/handlers/main.yml new file mode 100644 index 000000000..50049b3dd --- /dev/null +++ b/ansible/install/roles/graphite_docker/handlers/main.yml @@ -0,0 +1,3 @@ +- name: restart systemd-graphite + shell: /usr/bin/systemctl daemon-reload && /usr/bin/systemctl enable graphite-web && /usr/bin/systemctl restart graphite-web + become: true diff --git a/ansible/install/roles/graphite_docker/tasks/main.yml b/ansible/install/roles/graphite_docker/tasks/main.yml index 5e58bf758..cd7f3294d 100644 --- a/ansible/install/roles/graphite_docker/tasks/main.yml +++ b/ansible/install/roles/graphite_docker/tasks/main.yml @@ -87,12 +87,8 @@ group=root mode=0644 become: true - register: systemd_graphite_needs_restart + notify: restart systemd-graphite -- name: bounce systemd and graphite-web container - shell: /usr/bin/systemctl daemon-reload && /usr/bin/systemctl enable graphite-web && /usr/bin/systemctl restart graphite-web - become: true - when: systemd_graphite_needs_restart.changed - name: Setup carbon-cache systemd config template: @@ -102,10 +98,6 @@ group=root mode=0644 become: true - register: systemd_carbon_needs_restart + notify: restart systemd-graphite -- name: bounce systemd and carbon-cache container - shell: /usr/bin/systemctl daemon-reload && /usr/bin/systemctl enable carbon-cache && /usr/bin/systemctl restart carbon-cache - become: true - when: systemd_carbon_needs_restart.changed diff --git a/ansible/install/roles/kibana-visualization/vars/shaker_cloud_comparison_visualizations.yml b/ansible/install/roles/kibana-visualization/vars/shaker_cloud_comparison_visualizations.yml index 75a43c2ef..24a0badfa 100644 --- a/ansible/install/roles/kibana-visualization/vars/shaker_cloud_comparison_visualizations.yml +++ b/ansible/install/roles/kibana-visualization/vars/shaker_cloud_comparison_visualizations.yml @@ -9,7 +9,7 @@ shaker_cloud_comparison_visualizations: markdown: "# Shaker-L2-Flent\\\\n ## Throughput in Mbps\\\\nTo filter based on concurrency please query for key record.concurrency and to filter based based on number of compute nodes, please query for key acommodation.compute_nodes" - title: "Browbeat-Shaker-Cloud-Comprison-L2-Throughput" template: "shaker-cloud-comparison.json.j2" - query: "shaker_test_info.deployment.template: l2.hot AND (result.result_type: tcp_download OR result.result_type: tcp_upload)" + query: "shaker_test_info.deployment.template: l2.hot AND (result.result_type: tcp_download OR result.result_type: tcp_upload)" - title: "Shaker-Cloud-Comparison-L2-Latency" template: "markdown.json.j2" markdown: "# Shaker-L2-Flent\\\\n ## Latency in ms\\\\nTo filter based on concurrency please query for key record.concurrency and to filter based based on number of compute nodes, please query for accommodation.compute_nodes" @@ -21,7 +21,7 @@ shaker_cloud_comparison_visualizations: markdown: "# Shaker-L3-East-West-Flent\\\\n## Throughput in Mbps\\\\nTo filter based on concurrency please query for key record.concurrency and to filter based based on number of compute nodes, please query for key accommodation.compute_nodes" - title: "Browbeat-Shaker-Cloud-Comparison-L3-East-West-Throughput" template: "shaker-cloud-comparison.json.j2" - query: "shaker_test_info.deployment.template: l3_east_west.hot AND (result.result_type: tcp_download OR result.result_type: tcp_upload)" + query: "shaker_test_info.deployment.template: l3_east_west.hot AND (result.result_type: tcp_download OR result.result_type: tcp_upload)" - title: "Shaker-Cloud-Comparison-L3-East-West-Latency" template: "markdown.json.j2" markdown: "# Shaker-L3-East-West-Flent\\\\n## Latency in ms\\\\nTo filter based on concurrency please query for key record.concurrency and to filter based based on number of compute nodes, please query for key accommodation.compute_nodes" diff --git a/browbeat-complete.yaml b/browbeat-complete.yaml index 9acac75d1..cd03d49af 100644 --- a/browbeat-complete.yaml +++ b/browbeat-complete.yaml @@ -5,6 +5,7 @@ browbeat: cloud_name: openstack elasticsearch: enabled: false + regather: false host: 1.1.1.1 port: 9200 metadata_files: diff --git a/ci-scripts/linters/lint-browbeat-config.py b/ci-scripts/linters/lint-browbeat-config.py new file mode 100644 index 000000000..95138bdbb --- /dev/null +++ b/ci-scripts/linters/lint-browbeat-config.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import yaml +import sys +from pykwalify import core as pykwalify_core +from pykwalify import errors as pykwalify_errors +stream = open(sys.argv[1], 'r') +schema = yaml.load(stream) +check = pykwalify_core.Core(sys.argv[2], schema_data=schema) +try: + check.validate(raise_exception=True) + print ("Validation successful") + exit(0) +except pykwalify_errors.SchemaError as e: + print ("Config " + sys.argv[2] + " is not valid!") + raise Exception('File does not conform to schema: {}'.format(e)) diff --git a/conf/browbeat-glance-complete.yaml b/conf/browbeat-glance-complete.yaml index 3065f5953..cfebfa58a 100644 --- a/conf/browbeat-glance-complete.yaml +++ b/conf/browbeat-glance-complete.yaml @@ -5,6 +5,7 @@ browbeat: cloud_name: openstack elasticsearch: enabled: false + regather: false host: 1.1.1.1 port: 9200 metadata_files: diff --git a/conf/browbeat-keystone-complete.yaml b/conf/browbeat-keystone-complete.yaml index beebe3f1e..34b229536 100644 --- a/conf/browbeat-keystone-complete.yaml +++ b/conf/browbeat-keystone-complete.yaml @@ -6,6 +6,7 @@ browbeat: cloud_name: openstack elasticsearch: enabled: false + regather: false host: 1.1.1.1 port: 9200 regather: true diff --git a/conf/browbeat-keystone-minimal.yaml b/conf/browbeat-keystone-minimal.yaml index ba6335945..9b0a926ac 100644 --- a/conf/browbeat-keystone-minimal.yaml +++ b/conf/browbeat-keystone-minimal.yaml @@ -6,6 +6,7 @@ browbeat: cloud_name: openstack elasticsearch: enabled: false + regather: false host: 1.1.1.1 port: 9200 regather: true diff --git a/conf/ceilometer-minimal.yaml b/conf/ceilometer-minimal.yaml index 3a7bbf2e3..c2e793c78 100644 --- a/conf/ceilometer-minimal.yaml +++ b/conf/ceilometer-minimal.yaml @@ -5,6 +5,7 @@ browbeat: cloud_name: openstack elasticsearch: enabled: false + regather: false host: 1.1.1.1 port: 9200 metadata_files: diff --git a/conf/ceilometer-persist-response-timing.yaml b/conf/ceilometer-persist-response-timing.yaml index df6bcab65..b4ab758d0 100644 --- a/conf/ceilometer-persist-response-timing.yaml +++ b/conf/ceilometer-persist-response-timing.yaml @@ -8,6 +8,7 @@ browbeat: cloud_name: openstack elasticsearch: enabled: false + regather: false host: 1.1.1.1 port: 9200 metadata_files: diff --git a/conf/perfkit-benchmarks.yaml b/conf/perfkit-benchmarks.yaml index 6bf32f138..516bb774c 100644 --- a/conf/perfkit-benchmarks.yaml +++ b/conf/perfkit-benchmarks.yaml @@ -10,6 +10,7 @@ browbeat: cloud_name: openstack elasticsearch: enabled: false + regather: false host: 1.1.1.1 port: 9200 metadata_files: diff --git a/conf/telemetry-nova-persist-instances.yaml b/conf/telemetry-nova-persist-instances.yaml index a03479017..66766bea3 100644 --- a/conf/telemetry-nova-persist-instances.yaml +++ b/conf/telemetry-nova-persist-instances.yaml @@ -9,6 +9,7 @@ browbeat: cloud_name: openstack elasticsearch: enabled: false + regather: false host: 1.1.1.1 port: 9200 regather: true diff --git a/tox.ini b/tox.ini index de60a7289..008545e9c 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,6 @@ [tox] minversion = 2.0 -#envlist = py34-constraints,py27-constraints,pypy-constraints,pep8-constraints -envlist = py27-constraints,pypy-constraints,pep8-constraints +envlist = py27-constraints,pep8-constraints,linters skipsdist = True [testenv] @@ -13,17 +12,23 @@ setenv = VIRTUAL_ENV={envdir} deps = -r{toxinidir}/test-requirements.txt commands = python setup.py test -#commands = python setup.py test --slowest --testr-args='{posargs}' [testenv:common-constraints] install_command = pip install -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} {opts} {packages} + [testenv:linters] whitelist_externals = bash commands = + pip install pykwalify flake8 -# bash -c "cd ansible; find . -type f -regex '.*.y[a]?ml' -print0 | xargs -t -n1 -0 \ -# ansible-lint" + bash -c "cd ansible; find . -type f -regex '.*.y[a]?ml' -print0 | xargs -t -n1 -0 \ + ansible-lint \ + -x ANSIBLE0013,ANSIBLE0012,ANSIBLE0006,ANSIBLE0007,ANSIBLE0010,ANSIBLE0016" + python ci-scripts/linters/lint-browbeat-config.py lib/validate.yaml browbeat-config.yaml + python ci-scripts/linters/lint-browbeat-config.py lib/validate.yaml browbeat-complete.yaml + bash -c "for config in $(ls conf/); do \ + python ci-scripts/linters/lint-browbeat-config.py lib/validate.yaml conf/$config; done" [testenv:pep8] commands = flake8 {posargs} --exclude=ansible