openstack/openstack-ansible-ops
Code Issues Proposed changes

Fixes for tests/gates

Browse Source 
Bump ansible version to 2.10.5
Prefer python3 over python2
Fix ansible str vs int comparisons
Print a message if setting defaultIndex in kibana 6 fails
Use saved_objects API for setting defaultIndex in kibana 7
Update version in dashboard path for 7x
ILM doesn't like bodies from GET requests
Stop using forked galera_server role, now that 633321 is merged
Force osquery inventory to python3
Update zuul jobs

Change-Id: Ibfc20b1605245927ad4de4a54e751a13defb1ee0
This commit is contained in:
Georgina Shippey 2020-07-20 18:48:12 +01:00 committed by Erik Berg
parent 5ea9543b23
commit b05893ae39
12 changed files with 98 additions and 121 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
bootstrap-embedded-ansible/bootstrap-embedded-ansible.sh
View File

@ -20,7 +20,7 @@ export OPTS=()
export CLONE_DIR="$(dirname $(readlink -f ${BASH_SOURCE[0]}))"
OPTS+=('CLONE_DIR')
export ANSIBLE_VERSION="${ANSIBLE_VERSION:-2.7.5.0}"
export ANSIBLE_VERSION="${ANSIBLE_VERSION:-2.10.5}"
OPTS+=('ANSIBLE_VERSION')
export ANSIBLE_EMBED_HOME="${HOME}/ansible_venv"
@ -38,20 +38,21 @@ export ID="$(echo ${ID} | awk -F'-' '{print $1}')"
if [[ ! -e "${ANSIBLE_EMBED_HOME}/bin/ansible" ]]; then
if [ ${ID} = "ubuntu" ]; then
apt-get update
apt-get -y install python-virtualenv
apt-get -y install virtualenv
elif [ ${ID} = "opensuse" ] || [ ${ID} = "suse" ]; then
zypper install -y insserv
zypper install -y python-virtualenv
elif [ ${ID} = "centos" ] || [ ${ID} = "redhat" ] || [ ${ID} = "rhel" ]; then
yum install -y python-virtualenv
yum install -y python3-virtualenv
else
echo "Unknown operating system"
exit 99
fi
echo "done installing python-virtualenv"
if [[ -f "/usr/bin/python2" ]]; then
virtualenv --system-site-packages --python="/usr/bin/python2" "${ANSIBLE_EMBED_HOME}"
elif [[ -f "/usr/bin/python3" ]]; then
if [[ -f "/usr/bin/python3" ]]; then
virtualenv --system-site-packages --python="/usr/bin/python3" "${ANSIBLE_EMBED_HOME}"
elif [[ -f "/usr/bin/python2" ]]; then
virtualenv --system-site-packages --python="/usr/bin/python2" "${ANSIBLE_EMBED_HOME}"
else
virtualenv "${ANSIBLE_EMBED_HOME}"
fi

46
elk_metrics_6x/createElasticIndexes.yml
View File

@ -17,7 +17,7 @@
vars:
_elastic_refresh_interval: "{{ (elasticsearch_number_of_replicas | int) * 5 }}"
elastic_refresh_interval: "{{ (_elastic_refresh_interval > 0) | ternary(30, _elastic_refresh_interval) }}"
elastic_refresh_interval: "{{ (_elastic_refresh_interval | int > 0) | ternary(30, _elastic_refresh_interval) }}"
vars_files:
- vars/variables.yml
@ -186,7 +186,7 @@
- role: elastic_retention
post_tasks:
- name: Create kibana indexe patterns
- name: Create kibana index patterns
uri:
url: "http://127.0.0.1:5601/api/saved_objects/index-pattern/{{ item.name }}"
method: POST
@ -227,24 +227,30 @@
delay: 30
run_once: true
- name: Create basic indexes
uri:
url: "http://127.0.0.1:5601/api/kibana/settings/defaultIndex"
method: POST
body: "{{ item.index_options | to_json }}"
status_code: 200
body_format: json
headers:
Content-Type: "application/json"
kbn-xsrf: "{{ inventory_hostname | to_uuid }}"
with_items:
- name: "default"
index_options:
value: "default"
register: kibana_indexes
until: kibana_indexes is success
retries: 6
delay: 30
- block:
- name: Set default index
uri:
url: "http://127.0.0.1:5601/api/kibana/settings/defaultIndex"
method: POST
body: "{{ item.index_options | to_json }}"
status_code: 200
body_format: json
headers:
Content-Type: "application/json"
kbn-xsrf: "{{ inventory_hostname | to_uuid }}"
with_items:
- name: "default"
index_options:
value: "default"
register: _kibana_default_index
rescue:
- debug:
msg: "{{ message.split('\n') }}"
vars:
message: |
Failed to set the defaultIndex, most likely due to a timeout.
It's not a big deal, and you'll find that it usually gets set
at some point whenever elastic gets around to it.
run_once: true
tags:

2
elk_metrics_6x/roles/elastic_curator/tasks/main.yml
View File

@ -47,7 +47,7 @@
- name: Ensure curator is installed
pip:
name: "elasticsearch-curator<6"
name: "elasticsearch-curator<=5.7.0"
state: "{{ elk_package_state | default('present') }}"
extra_args: --isolated
virtualenv: /opt/elasticsearch-curator

1
elk_metrics_6x/roles/elastic_curator/vars/ubuntu.yml
View File

@ -16,3 +16,4 @@
curator_distro_packages:
- python-virtualenv
- virtualenv
- libyaml-dev

56
elk_metrics_7x/createElasticIndexes.yml
View File

@ -20,7 +20,7 @@
vars:
_elastic_refresh_interval: "{{ (elasticsearch_beat_settings.number_of_replicas | int) * 5 }}"
elastic_refresh_interval: "{{ (_elastic_refresh_interval > 0) | ternary(30, _elastic_refresh_interval) }}"
elastic_refresh_interval: "{{ (_elastic_refresh_interval | int > 0) | ternary(30, _elastic_refresh_interval) }}"
vars_files:
- vars/variables.yml
@ -227,25 +227,41 @@
delay: 30
run_once: true
- name: Create basic indexes
uri:
url: "http://127.0.0.1:5601/api/kibana/settings/defaultIndex"
method: POST
body: "{{ item.index_options | to_json }}"
status_code: 200
body_format: json
headers:
Content-Type: "application/json"
kbn-xsrf: "{{ inventory_hostname | to_uuid }}"
with_items:
- name: "default"
index_options:
value: "default"
register: kibana_indexes
until: kibana_indexes is success
retries: 6
delay: 30
run_once: true
- block:
- name: find objects of type config
uri:
url: "http://127.0.0.1:5601/api/saved_objects/_find?type=config"
headers:
Content-Type: "application/json"
kbn-xsrf: "{{ inventory_hostname | to_uuid }}"
return_content: true
register: _objects
- name: fail if we're not getting exactly 1 config object back
assert:
that: _objects.json.saved_objects | length == 1
- name: set fact for config object
set_fact:
config: "{{ _objects.json.saved_objects[0] }}"
- name: Set default index
uri:
url: "http://127.0.0.1:5601/api/saved_objects/config/{{ config.id }}"
method: PUT
body:
attributes:
defaultIndex: "default"
headers:
Content-Type: "application/json"
kbn-xsrf: "{{ inventory_hostname | to_uuid }}"
register: _kibana_default_index
until: _kibana_default_index is success
retries: 6
delay: 30
when:
- config.attributes.defaultIndex is defined
- config.attributes.defaultIndex != "default"
run_once: true
tags:
- server-install

4
elk_metrics_7x/roles/elastic_beat_setup/tasks/main.yml
View File

@ -37,8 +37,8 @@
- name: Load templates
shell: >-
{% if item == '--dashboards' %}
sed -i 's@\\\"index\\\": \\\"{{ elastic_beat_name }}-\*\\\"@\\\"index\\\": \\\"{{ elastic_beat_name }}\\\"@g' /usr/share/{{ elastic_beat_name }}/kibana/6/dashboard/*.json
sed -i 's@"id": "{{ elastic_beat_name }}\-\*",@"id": "{{ elastic_beat_name }}",@g' /usr/share/{{ elastic_beat_name }}/kibana/6/index-pattern/*.json
sed -i 's@\\\"index\\\": \\\"{{ elastic_beat_name }}-\*\\\"@\\\"index\\\": \\\"{{ elastic_beat_name }}\\\"@g' /usr/share/{{ elastic_beat_name }}/kibana/7/dashboard/*.json
sed -i 's@"id": "{{ elastic_beat_name }}\-\*",@"id": "{{ elastic_beat_name }}",@g' /usr/share/{{ elastic_beat_name }}/kibana/7/index-pattern/*.json
{% endif %}
{{ elastic_beat_name }} setup
{% if elastic_beat_name == "heartbeat" and item == "--index-management" -%}

2
elk_metrics_7x/roles/elastic_ilm/tasks/elastic_ilm_update_policy.yml
View File

@ -4,7 +4,6 @@
url: "http://{{ elasticsearch_data_node_details[0] }}/_ilm/policy/{{ ilm_policy_name }}"
method: GET
status_code: 200,404
body_format: json
register: check_policy
when: ilm_policy_name is defined and ilm_policy is defined
@ -34,7 +33,6 @@
url: "http://{{ elasticsearch_data_node_details[0] }}/_template/{{ ilm_policy_template }}/"
method: GET
status_code: 200,404
body_format: json
register: template
when: ilm_policy_template is defined and ilm_policy_name is defined

5
osquery/ansible-role-requirements.yml
View File

@ -15,11 +15,10 @@
scm: git
src: https://opendev.org/openstack/openstack-ansible-galera_client
version: master
# Galera server is using a forked role until https://review.openstack.org/#/c/633321/ is merged
- name: galera_server
scm: git
src: https://github.com/os-cloud/openstack-ansible-galera_server
version: xinetd-check
src: https://opendev.org/openstack/openstack-ansible-galera_server
version: master
- name: apt_package_pinning
scm: git
src: https://opendev.org/openstack/openstack-ansible-apt_package_pinning

2
osquery/installDB.yml
View File

@ -34,6 +34,8 @@
roles:
- role: galera_server
galera_install_client: True
galera_install_server: True
galera_cluster_members: "{{ groups['kolide-db_all'] }}"
galera_wsrep_node_name: "{{ inventory_hostname }}"
galera_cluster_name: telemetry_galera_cluster

1
osquery/tests/inventory/test-metal-inventory.yml
View File

@ -6,6 +6,7 @@ all:
ansible_connection: local
ansible_host: 127.0.0.1
ansible_user: root
ansible_python_interpreter: /usr/bin/python3
osquery_all:

79
zuul.d/jobs.yaml
View File

@ -14,11 +14,12 @@
# limitations under the License.
- job:
name: "openstack-ansible-ops:elk_metrics_6x-ubuntu-xenial"
name: "openstack-ansible-ops:elk_metrics_6x-ubuntu-bionic"
parent: base
description: "Runs a gate test on the elk_metrics_6x project."
run: "elk_metrics_6x/tests/test.yml"
post-run: "elk_metrics_6x/tests/post-run.yml"
nodeset: ubuntu-bionic
files:
- ^elk_metrics_6x/.*
- ^bootstrap-embedded-ansible/.*
@ -40,53 +41,30 @@
osa_test_repo: "openstack/openstack-ansible-ops"
test_clustered_elk: false
- job:
name: "openstack-ansible-ops:elk_metrics_7x-ubuntu-focal"
parent: "openstack-ansible-ops:elk_metrics_7x-ubuntu-bionic"
nodeset: ubuntu-focal
voting: false
- job:
name: "openstack-ansible-ops:elk_metrics_6x-centos-7"
parent: "openstack-ansible-ops:elk_metrics_6x-ubuntu-xenial"
parent: "openstack-ansible-ops:elk_metrics_6x-ubuntu-bionic"
nodeset: centos-7
voting: true
voting: false
- job:
name: "openstack-ansible-ops:elk_metrics_6x-centos-7-clustered"
parent: "openstack-ansible-ops:elk_metrics_6x-ubuntu-xenial"
parent: "openstack-ansible-ops:elk_metrics_6x-ubuntu-bionic"
nodeset: centos-7
voting: false
vars:
osa_test_repo: "openstack/openstack-ansible-ops"
test_clustered_elk: true
- job:
name: "openstack-ansible-ops:elk_metrics_6x-opensuse-15"
parent: "openstack-ansible-ops:elk_metrics_6x-ubuntu-xenial"
nodeset: opensuse-15
voting: false
- job:
name: "openstack-ansible-ops:elk_metrics_6x-opensuse-15-clustered"
parent: "openstack-ansible-ops:elk_metrics_6x-ubuntu-xenial"
nodeset: opensuse-15
voting: false
vars:
osa_test_repo: "openstack/openstack-ansible-ops"
test_clustered_elk: true
- job:
name: "openstack-ansible-ops:elk_metrics_6x-ubuntu-bionic"
parent: "openstack-ansible-ops:elk_metrics_6x-ubuntu-xenial"
nodeset: ubuntu-bionic
voting: true
- job:
name: "openstack-ansible-ops:elk_metrics_6x-ubuntu-xenial-clustered"
parent: "openstack-ansible-ops:elk_metrics_6x-ubuntu-xenial"
voting: false
vars:
osa_test_repo: "openstack/openstack-ansible-ops"
test_clustered_elk: true
- job:
name: "openstack-ansible-ops:elk_metrics_6x-ubuntu-bionic-clustered"
parent: "openstack-ansible-ops:elk_metrics_6x-ubuntu-xenial"
parent: "openstack-ansible-ops:elk_metrics_6x-ubuntu-bionic"
nodeset: ubuntu-bionic
voting: false
vars:
@ -94,8 +72,9 @@
test_clustered_elk: true
- job:
name: "openstack-ansible-ops:osquery-ubuntu-xenial"
name: "openstack-ansible-ops:osquery-ubuntu-bionic"
parent: base
nodeset: ubuntu-bionic
description: "Runs a gate test on the osquery project."
run: "osquery/tests/test.yml"
post-run: "osquery/tests/post-run.yml"
@ -104,30 +83,18 @@
- ^osquery/.*
- ^bootstrap-embedded-ansible/.*
- job:
name: "openstack-ansible-ops:osquery-ubuntu-xenial-clustered"
parent: "openstack-ansible-ops:osquery-ubuntu-xenial"
nodeset: ubuntu-bionic
vars:
osa_test_repo: "openstack/openstack-ansible-ops"
test_clustered_kolide: true
- job:
name: "openstack-ansible-ops:osquery-ubuntu-bionic"
parent: "openstack-ansible-ops:osquery-ubuntu-xenial"
nodeset: ubuntu-bionic
- job:
name: "openstack-ansible-ops:osquery-ubuntu-bionic-clustered"
parent: "openstack-ansible-ops:osquery-ubuntu-xenial"
parent: "openstack-ansible-ops:osquery-ubuntu-bionic"
nodeset: ubuntu-bionic
vars:
osa_test_repo: "openstack/openstack-ansible-ops"
test_clustered_kolide: true
- job:
name: "openstack-ansible-ops:skydive-ubuntu-xenial"
name: "openstack-ansible-ops:skydive-ubuntu-bionic"
parent: base
nodeset: ubuntu-bionic
description: "Runs a gate test on the skydive project."
run: "skydive/tests/test.yml"
post-run: "skydive/tests/post-run.yml"
@ -136,17 +103,7 @@
- ^skydive/.*
- ^bootstrap-embedded-ansible/.*
- job:
name: "openstack-ansible-ops:skydive-ubuntu-bionic"
parent: "openstack-ansible-ops:skydive-ubuntu-xenial"
nodeset: ubuntu-bionic
- job:
name: "openstack-ansible-ops:skydive-centos-7"
parent: "openstack-ansible-ops:skydive-ubuntu-xenial"
parent: "openstack-ansible-ops:skydive-ubuntu-bionic"
nodeset: centos-7
- job:
name: "openstack-ansible-ops:skydive-opensuse-15"
parent: "openstack-ansible-ops:skydive-ubuntu-xenial"
nodeset: opensuse-15

8
zuul.d/project.yaml
View File

@ -22,24 +22,20 @@
- openstack-ansible-linters
- openstack-ansible-ops:elk_metrics_6x-centos-7
# - openstack-ansible-ops:elk_metrics_6x-centos-7-clustered
- openstack-ansible-ops:elk_metrics_6x-opensuse-15
# - openstack-ansible-ops:elk_metrics_6x-opensuse-15-clustered
- openstack-ansible-ops:elk_metrics_6x-ubuntu-xenial
- openstack-ansible-ops:elk_metrics_6x-ubuntu-bionic
- openstack-ansible-ops:elk_metrics_7x-ubuntu-bionic
- openstack-ansible-ops:elk_metrics_7x-ubuntu-focal
# - openstack-ansible-ops:elk_metrics_6x-ubuntu-xenial-clustered
# - openstack-ansible-ops:elk_metrics_6x-ubuntu-bionic-clustered
- openstack-ansible-ops:osquery-ubuntu-xenial
# - openstack-ansible-ops:osquery-ubuntu-xenial-clustered
- openstack-ansible-ops:osquery-ubuntu-bionic
# - openstack-ansible-ops:osquery-ubuntu-bionic-clustered
- openstack-ansible-ops:skydive-ubuntu-xenial
- openstack-ansible-ops:skydive-ubuntu-bionic
- openstack-ansible-ops:skydive-centos-7
- openstack-ansible-ops:skydive-opensuse-15
gate:
jobs:
- openstack-ansible-linters
- openstack-ansible-ops:elk_metrics_6x-ubuntu-bionic
- openstack-ansible-ops:elk_metrics_7x-ubuntu-bionic
- openstack-ansible-ops:osquery-ubuntu-bionic
- openstack-ansible-ops:skydive-ubuntu-bionic