Convert template setup to a role

This change reduces code throughout the playbooks thereby speeding up
the task execution.
  * A new role named `elastic_beat_setup` was created to
    facilitate template setup as needed.
  * Beats retention policies are now defined on the elastic-logstash
    nodes instead of on all target hosts. This method will speed-up
    deployments on massive installations while streamlining all deployments.
  * Kibana variable assumptions have been fixed. This will allow for
    deployments without Kibana to be accomplished.

Change-Id: I36343264042e81dfcb68bad0f6c3a503e525eceb
Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>

elk_metrics_6x/installAPMserver.yml

@@ -29,3 +29,21 @@
 
   tags:
     - apm-server
+
+- name: Setup apm-server rollup
+  hosts: elastic-logstash[0]
+  become: true
+  vars:
+    haproxy_ssl: false
+
+  vars_files:
+    - vars/variables.yml
+
+  environment: "{{ deployment_environment_variables | default({}) }}"
+
+  roles:
+    - role: elastic_rollup
+      index_name: apm
+
+  tags:
+    - apm-server

elk_metrics_6x/installAuditbeat.yml

@@ -24,8 +24,28 @@
 
   roles:
     - role: elastic_auditbeat
+
+  tags:
+    - beat-install
+
+- name: Setup auditbeat rollup
+  hosts: elastic-logstash[0]
+  become: true
+  vars:
+    haproxy_ssl: false
+
+  vars_files:
+    - vars/variables.yml
+
+  environment: "{{ deployment_environment_variables | default({}) }}"
+
+  roles:
     - role: elastic_rollup
       index_name: auditbeat
 
   tags:
-    - beat-install
+    - auditbeat
+
+- import_playbook: fieldRefresh.yml
+  vars:
+    index_pattern: auditbeat-*

elk_metrics_6x/installFilebeat.yml

@@ -24,8 +24,28 @@
 
   roles:
     - role: elastic_filebeat
+
+  tags:
+    - beat-install
+
+- name: Setup filebeat rollup
+  hosts: elastic-logstash[0]
+  become: true
+  vars:
+    haproxy_ssl: false
+
+  vars_files:
+    - vars/variables.yml
+
+  environment: "{{ deployment_environment_variables | default({}) }}"
+
+  roles:
     - role: elastic_rollup
       index_name: filebeat
 
   tags:
-    - beat-install
+    - filebeat
+
+- import_playbook: fieldRefresh.yml
+  vars:
+    index_pattern: filebeat-*

elk_metrics_6x/installHeartbeat.yml

@@ -38,8 +38,28 @@
 
   roles:
     - role: elastic_heartbeat
+
+  tags:
+    - beat-install
+
+- name: Setup heartbeat rollup
+  hosts: elastic-logstash[0]
+  become: true
+  vars:
+    haproxy_ssl: false
+
+  vars_files:
+    - vars/variables.yml
+
+  environment: "{{ deployment_environment_variables | default({}) }}"
+
+  roles:
     - role: elastic_rollup
       index_name: heartbeat
 
   tags:
-    - beat-install
+    - heartbeat
+
+- import_playbook: fieldRefresh.yml
+  vars:
+    index_pattern: heartbeat-*

elk_metrics_6x/installJournalbeat.yml

@@ -62,6 +62,21 @@
         - journal_dir.stat.exists | bool
         - ansible_service_mgr == 'systemd'
 
+  tags:
+    - beat-install
+
+- name: Setup journalbeat rollup
+  hosts: elastic-logstash[0]
+  become: true
+  vars:
+    haproxy_ssl: false
+
+  vars_files:
+    - vars/variables.yml
+
+  environment: "{{ deployment_environment_variables | default({}) }}"
+
+  roles:
     - role: elastic_rollup
       when:
         - journal_dir.stat.exists | bool
@@ -69,4 +84,4 @@
       index_name: journalbeat
 
   tags:
-    - beat-install
+    - journalbeat

elk_metrics_6x/installMetricbeat.yml

@@ -24,8 +24,28 @@
 
   roles:
     - role: elastic_metricbeat
+
+  tags:
+    - beat-install
+
+- name: Setup metricbeat rollup
+  hosts: elastic-logstash[0]
+  become: true
+  vars:
+    haproxy_ssl: false
+
+  vars_files:
+    - vars/variables.yml
+
+  environment: "{{ deployment_environment_variables | default({}) }}"
+
+  roles:
     - role: elastic_rollup
       index_name: metricbeat
 
   tags:
-    - beat-install
+    - metricbeat
+
+- import_playbook: fieldRefresh.yml
+  vars:
+    index_pattern: metricbeat-*

elk_metrics_6x/installPacketbeat.yml

@@ -24,8 +24,28 @@
 
   roles:
     - role: elastic_packetbeat
+
+  tags:
+    - beat-install
+
+- name: Setup packetbeat rollup
+  hosts: elastic-logstash[0]
+  become: true
+  vars:
+    haproxy_ssl: false
+
+  vars_files:
+    - vars/variables.yml
+
+  environment: "{{ deployment_environment_variables | default({}) }}"
+
+  roles:
     - role: elastic_rollup
       index_name: packetbeat
 
   tags:
-    - beat-install
+    - packetbeat
+
+- import_playbook: fieldRefresh.yml
+  vars:
+    index_pattern: packetbeat-*

elk_metrics_6x/roles/elastic_apm_server/tasks/main.yml

@@ -69,9 +69,14 @@
   notify:
     - Enable and restart apm server
 
-- include_tasks: apm_setup.yml
+- name: Run the beat setup role
-  tags:
+  include_role:
-    - setup
+    name: elastic_beat_setup
+    private: true
+  when:
+    - (groups['kibana'] | length) > 0
+  vars:
+    elastic_beat_name: "apm-server"
 
 - name: Force beat handlers
   meta: flush_handlers

elk_metrics_6x/roles/elastic_apm_server/templates/apm-server.yml.j2

@@ -137,7 +137,9 @@ apm-server:
 {{ elk_macros.setup_template('apm', inventory_hostname, data_nodes, elasticsearch_number_of_replicas) }}
 
 #============================== Kibana =====================================
-{{ elk_macros.setup_kibana(hostvars[groups['kibana'][0]]['ansible_host'] + ':' + kibana_port|string) }}
+{% if (groups['kibana'] | length) > 0 %}
+{{ elk_macros.setup_kibana(hostvars[groups['kibana'][0]]['ansible_host'] ~ ':' ~ kibana_port) }}
+{% endif %}
 
 #================================ Logging ======================================
 {{ elk_macros.beat_logging('apm-server') }}

elk_metrics_6x/roles/elastic_auditbeat/tasks/auditbeat_setup.yml

@@ -1,60 +0,0 @@
----
-# Copyright 2018, Rackspace US, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-- name: Refresh local facts
-  setup:
-    filter: ansible_local
-    gather_subset: "!all"
-  tags:
-    - always
-
-- name: Load templates
-  shell: >-
-    auditbeat setup
-    {{ item }}
-    -E 'output.logstash.enabled=false'
-    -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}'
-    -E 'setup.template.enabled=true'
-    -E 'setup.template.overwrite=true'
-    -e -v
-  with_items:
-    - "--template"
-    - "--pipelines"
-    - "--machine-learning"
-    - "--dashboards"
-  register: templates
-  environment:
-    no_proxy: "{{ hostvars[groups['kibana'][0]]['ansible_host'] }}"
-  until: templates is success
-  retries: 5
-  delay: 5
-  run_once: true
-  when:
-    - ((ansible_local['elastic']['setup']['auditbeat_loaded_templates'] is undefined) or
-       (not (ansible_local['elastic']['setup']['auditbeat_loaded_templates'] | bool))) or
-      (elk_package_state | default('present')) == "latest"
-  tags:
-    - setup
-
-- name: Set template fact
-  ini_file:
-    dest: "/etc/ansible/facts.d/elastic.fact"
-    section: "setup"
-    option: auditbeat_loaded_templates
-    value: true
-  when:
-    - templates is changed
-  tags:
-    - setup

elk_metrics_6x/roles/elastic_auditbeat/tasks/main.yml

@@ -82,9 +82,14 @@
   notify:
     - Enable and restart auditbeat
 
-- include_tasks: auditbeat_setup.yml
+- name: Run the beat setup role
-  tags:
+  include_role:
-    - setup
+    name: elastic_beat_setup
+    private: true
+  when:
+    - (groups['kibana'] | length) > 0
+  vars:
+    elastic_beat_name: "auditbeat"
 
 - name: Force beat handlers
   meta: flush_handlers

elk_metrics_6x/roles/elastic_auditbeat/templates/auditbeat.yml.j2

@@ -666,7 +666,9 @@ processors:
 {{ elk_macros.setup_template('auditbeat', inventory_hostname, data_nodes, elasticsearch_number_of_replicas) }}
 
 #============================== Kibana =====================================
-{{ elk_macros.setup_kibana(hostvars[groups['kibana'][0]]['ansible_host'] + ':' + kibana_port|string) }}
+{% if (groups['kibana'] | length) > 0 %}
+{{ elk_macros.setup_kibana(hostvars[groups['kibana'][0]]['ansible_host'] ~ ':' ~ kibana_port) }}
+{% endif %}
 
 #================================ Logging ======================================
 {{ elk_macros.beat_logging('auditbeat') }}

elk_metrics_6x/roles/elastic_beat_setup/defaults/main.yml

@@ -0,0 +1,31 @@
+---
+# Copyright 2018, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Each setup flag is run one at a time.
+elastic_setup_flags:
+  - "--template"
+  - "--pipelines"
+  - "--machine-learning"
+  - "--dashboards"
+
+# Setup options are cast as a string with, one option per line.
+elastic_beat_setup_options: >-
+  -E 'output.logstash.enabled=false'
+  -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}'
+  -E 'setup.template.enabled=true'
+  -E 'setup.template.overwrite=true'
+
+# The node defined here will be used with the environment variable, "no_proxy".
+elastic_beat_kibana_host: "{{ hostvars[groups['kibana'][0]]['ansible_host'] }}"

elk_metrics_6x/roles/elastic_beat_setup/meta/main.yml

@@ -0,0 +1,35 @@
+---
+# Copyright 2018, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+galaxy_info:
+  author: OpenStack
+  description: Elastic v6.x repositories role
+  company: Rackspace
+  license: Apache2
+  min_ansible_version: 2.5
+  platforms:
+    - name: Ubuntu
+      versions:
+        - trusty
+        - xenial
+        - bionic
+  categories:
+    - cloud
+    - development
+    - beats
+    - elastic-beats
+    - elasticsearch
+    - elastic-stack
+dependencies: []

elk_metrics_6x/roles/elastic_beat_setup/tasks/main.yml

@@ -13,6 +13,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+- name: Check for coordination_nodes var
+  fail:
+    msg: >-
+      To use this role the variable `coordination_nodes` must be defined.
+  when:
+    - coordination_nodes is undefined
+
+- name: Check for elastic_beat_name var
+  fail:
+    msg: >-
+      To use this role the variable `elastic_beat_name` must be defined.
+  when:
+    - elastic_beat_name is undefined
+
 - name: Refresh local facts
   setup:
     filter: ansible_local
@@ -22,28 +36,21 @@
 
 - name: Load templates
   shell: >-
-    apm-server setup
+    {{ elastic_beat_name }} setup
     {{ item }}
-    -E 'apm-server.host=localhost:8200'
+    {{ elastic_beat_setup_options }}
-    -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}'
-    -E 'setup.template.enabled=true'
-    -E 'setup.template.overwrite=true'
     -e -v
-  with_items:
+  with_items: "{{ elastic_setup_flags }}"
-    - "--template"
-    - "--pipelines"
-    - "--machine-learning"
-    - "--dashboards"
   register: templates
   environment:
-    no_proxy: "{{ hostvars[groups['kibana'][0]]['ansible_host'] }}"
+    no_proxy: "{{ elastic_beat_kibana_host }}"
   until: templates is success
   retries: 5
   delay: 5
   run_once: true
   when:
-    - ((ansible_local['elastic']['setup']['apm_loaded_templates'] is undefined) or
+    - ((ansible_local['elastic']['setup'][elastic_beat_name + '_loaded_templates'] is undefined) or
-       (not (ansible_local['elastic']['setup']['apm_loaded_templates'] | bool))) or
+       (not (ansible_local['elastic']['setup'][elastic_beat_name + '_loaded_templates'] | bool))) or
       (elk_package_state | default('present')) == "latest"
   tags:
     - setup
@@ -52,7 +59,7 @@
   ini_file:
     dest: "/etc/ansible/facts.d/elastic.fact"
     section: "setup"
-    option: apm_loaded_templates
+    option: "{{ elastic_beat_name + '_loaded_templates' }}"
     value: true
   when:
     - templates is changed

elk_metrics_6x/roles/elastic_data_hosts/vars/data-node-variables.yml

@@ -121,9 +121,14 @@ master_nodes: |-
   {{ nodes }}
 master_node_count: "{{ master_nodes | length }}"
 
-coordination_nodes: >-
+coordination_nodes: |-
+  {% if (groups['kibana'] | length) > 0 %}
+  {%   set c_nodes = groups['kibana'] %}
+  {% else %}
+  {%   set c_nodes = groups['elastic-logstash'] %}
+  {% endif %}
   {{
-    (groups['kibana'] | map('extract', hostvars, 'ansible_host') | list)
+    (c_nodes | map('extract', hostvars, 'ansible_host') | list)
       | map('regex_replace', '(.*)' ,'\1:' ~ elastic_port)
       | list
   }}

elk_metrics_6x/roles/elastic_filebeat/tasks/filebeat_setup.yml

@@ -1,60 +0,0 @@
----
-# Copyright 2018, Rackspace US, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-- name: Refresh local facts
-  setup:
-    filter: ansible_local
-    gather_subset: "!all"
-  tags:
-    - always
-
-- name: Load templates
-  shell: >-
-    filebeat setup
-    {{ item }}
-    -E 'output.logstash.enabled=false'
-    -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}'
-    -E 'setup.template.enabled=true'
-    -E 'setup.template.overwrite=true'
-    -e -v
-  with_items:
-    - "--template"
-    - "--pipelines"
-    - "--machine-learning"
-    - "--dashboards"
-  register: templates
-  environment:
-    no_proxy: "{{ hostvars[groups['kibana'][0]]['ansible_host'] }}"
-  until: templates is success
-  retries: 5
-  delay: 5
-  run_once: true
-  when:
-    - ((ansible_local['elastic']['setup']['filebeat_loaded_templates'] is undefined) or
-       (not (ansible_local['elastic']['setup']['filebeat_loaded_templates'] | bool))) or
-      (elk_package_state | default('present')) == "latest"
-  tags:
-    - setup
-
-- name: Set template fact
-  ini_file:
-    dest: "/etc/ansible/facts.d/elastic.fact"
-    section: "setup"
-    option: filebeat_loaded_templates
-    value: true
-  when:
-    - templates is changed
-  tags:
-    - setup

elk_metrics_6x/roles/elastic_filebeat/tasks/main.yml

@@ -82,9 +82,14 @@
   notify:
     - Enable and restart filebeat
 
-- include_tasks: filebeat_setup.yml
+- name: Run the beat setup role
-  tags:
+  include_role:
-    - setup
+    name: elastic_beat_setup
+    private: true
+  when:
+    - (groups['kibana'] | length) > 0
+  vars:
+    elastic_beat_name: "filebeat"
 
 - name: Force beat handlers
   meta: flush_handlers

elk_metrics_6x/roles/elastic_filebeat/templates/filebeat.yml.j2

@@ -912,7 +912,9 @@ processors:
 {{ elk_macros.setup_template('filebeat', inventory_hostname, data_nodes, elasticsearch_number_of_replicas) }}
 
 #============================== Kibana =====================================
-{{ elk_macros.setup_kibana(hostvars[groups['kibana'][0]]['ansible_host'] + ':' + kibana_port|string) }}
+{% if (groups['kibana'] | length) > 0 %}
+{{ elk_macros.setup_kibana(hostvars[groups['kibana'][0]]['ansible_host'] ~ ':' ~ kibana_port) }}
+{% endif %}
 
 #================================ Logging ======================================
 {{ elk_macros.beat_logging('filebeat') }}

elk_metrics_6x/roles/elastic_heartbeat/tasks/heartbeat_setup.yml

@@ -1,60 +0,0 @@
----
-# Copyright 2018, Rackspace US, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-- name: Refresh local facts
-  setup:
-    filter: ansible_local
-    gather_subset: "!all"
-  tags:
-    - always
-
-- name: Load templates
-  shell: >-
-    heartbeat setup
-    {{ item }}
-    -E 'output.logstash.enabled=false'
-    -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}'
-    -E 'setup.template.enabled=true'
-    -E 'setup.template.overwrite=true'
-    -e -v
-  with_items:
-    - "--template"
-    - "--pipelines"
-    - "--machine-learning"
-    - "--dashboards"
-  register: templates
-  environment:
-    no_proxy: "{{ hostvars[groups['kibana'][0]]['ansible_host'] }}"
-  until: templates is success
-  retries: 5
-  delay: 5
-  run_once: true
-  when:
-    - ((ansible_local['elastic']['setup']['heartbeat_loaded_templates'] is undefined) or
-       (not (ansible_local['elastic']['setup']['heartbeat_loaded_templates'] | bool))) or
-      (elk_package_state | default('present')) == "latest"
-  tags:
-    - setup
-
-- name: Set template fact
-  ini_file:
-    dest: "/etc/ansible/facts.d/elastic.fact"
-    section: "setup"
-    option: heartbeat_loaded_templates
-    value: true
-  when:
-    - templates is changed
-  tags:
-    - setup

elk_metrics_6x/roles/elastic_heartbeat/tasks/main.yml

@@ -88,9 +88,14 @@
   notify:
     - Enable and restart heartbeat
 
-- include_tasks: heartbeat_setup.yml
+- name: Run the beat setup role
-  tags:
+  include_role:
-    - setup
+    name: elastic_beat_setup
+    private: true
+  when:
+    - (groups['kibana'] | length) > 0
+  vars:
+    elastic_beat_name: "heartbeat"
 
 - name: Force beat handlers
   meta: flush_handlers

elk_metrics_6x/roles/elastic_heartbeat/templates/heartbeat.yml.j2

@@ -780,7 +780,9 @@ processors:
 {{ elk_macros.setup_template('heartbeat', inventory_hostname, data_nodes, elasticsearch_number_of_replicas) }}
 
 #============================== Kibana =====================================
-{{ elk_macros.setup_kibana(hostvars[groups['kibana'][0]]['ansible_host'] + ':' + kibana_port|string) }}
+{% if (groups['kibana'] | length) > 0 %}
+{{ elk_macros.setup_kibana(hostvars[groups['kibana'][0]]['ansible_host'] ~ ':' ~ kibana_port) }}
+{% endif %}
 
 #================================ Logging ======================================
 {{ elk_macros.beat_logging('heartbeat') }}

elk_metrics_6x/roles/elastic_journalbeat/tasks/journalbeat_setup.yml

@@ -1,60 +0,0 @@
----
-# Copyright 2018, Rackspace US, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-- name: Refresh local facts
-  setup:
-    filter: ansible_local
-    gather_subset: "!all"
-  tags:
-    - always
-
-- name: Load templates
-  shell: >-
-    journalbeat setup
-    {{ item }}
-    -E 'output.logstash.enabled=false'
-    -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}'
-    -E 'setup.template.enabled=true'
-    -E 'setup.template.overwrite=true'
-    -e -v
-  with_items:
-    - "--template"
-    - "--pipelines"
-    - "--machine-learning"
-    - "--dashboards"
-  register: templates
-  environment:
-    no_proxy: "{{ hostvars[groups['kibana'][0]]['ansible_host'] }}"
-  until: templates is success
-  retries: 5
-  delay: 5
-  run_once: true
-  when:
-    - ((ansible_local['elastic']['setup']['journalbeat_loaded_templates'] is undefined) or
-       (not (ansible_local['elastic']['setup']['journalbeat_loaded_templates'] | bool))) or
-      (elk_package_state | default('present')) == "latest"
-  tags:
-    - setup
-
-- name: Set template fact
-  ini_file:
-    dest: "/etc/ansible/facts.d/elastic.fact"
-    section: "setup"
-    option: journalbeat_loaded_templates
-    value: true
-  when:
-    - templates is changed
-  tags:
-    - setup

elk_metrics_6x/roles/elastic_journalbeat/tasks/main.yml

@@ -88,9 +88,14 @@
   notify:
     - Enable and restart journalbeat
 
-- include_tasks: journalbeat_setup.yml
+- name: Run the beat setup role
-  tags:
+  include_role:
-    - setup
+    name: elastic_beat_setup
+    private: true
+  when:
+    - (groups['kibana'] | length) > 0
+  vars:
+    elastic_beat_name: "journalbeat"
 
 - name: Force beat handlers
   meta: flush_handlers

elk_metrics_6x/roles/elastic_journalbeat/templates/journalbeat.yml.j2

@@ -579,7 +579,9 @@ tags:
 {{ elk_macros.setup_template('journalbeat', inventory_hostname, data_nodes, elasticsearch_number_of_replicas) }}
 
 #============================== Kibana =====================================
-{{ elk_macros.setup_kibana(hostvars[groups['kibana'][0]]['ansible_host'] + ':' + kibana_port|string) }}
+{% if (groups['kibana'] | length) > 0 %}
+{{ elk_macros.setup_kibana(hostvars[groups['kibana'][0]]['ansible_host'] ~ ':' ~ kibana_port) }}
+{% endif %}
 
 #================================ Logging ======================================
 {{ elk_macros.beat_logging('journalbeat') }}

elk_metrics_6x/roles/elastic_metricbeat/tasks/main.yml

@@ -305,9 +305,14 @@
   notify:
     - Enable and restart metricbeat
 
-- include_tasks: metricbeat_setup.yml
+- name: Run the beat setup role
-  tags:
+  include_role:
-    - setup
+    name: elastic_beat_setup
+    private: true
+  when:
+    - (groups['kibana'] | length) > 0
+  vars:
+    elastic_beat_name: "metricbeat"
 
 - name: Force beat handlers
   meta: flush_handlers

elk_metrics_6x/roles/elastic_metricbeat/tasks/metricbeat_setup.yml

@@ -1,60 +0,0 @@
----
-# Copyright 2018, Rackspace US, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-- name: Refresh local facts
-  setup:
-    filter: ansible_local
-    gather_subset: "!all"
-  tags:
-    - always
-
-- name: Load templates
-  shell: >-
-    metricbeat setup
-    {{ item }}
-    -E 'output.logstash.enabled=false'
-    -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}'
-    -E 'setup.template.enabled=true'
-    -E 'setup.template.overwrite=true'
-    -e -v
-  with_items:
-    - "--template"
-    - "--pipelines"
-    - "--machine-learning"
-    - "--dashboards"
-  register: templates
-  environment:
-    no_proxy: "{{ hostvars[groups['kibana'][0]]['ansible_host'] }}"
-  until: templates is success
-  retries: 5
-  delay: 5
-  run_once: true
-  when:
-    - ((ansible_local['elastic']['setup']['metricbeat_loaded_templates'] is undefined) or
-       (not (ansible_local['elastic']['setup']['metricbeat_loaded_templates'] | bool))) or
-      (elk_package_state | default('present')) == "latest"
-  tags:
-    - setup
-
-- name: Set template fact
-  ini_file:
-    dest: "/etc/ansible/facts.d/elastic.fact"
-    section: "setup"
-    option: metricbeat_loaded_templates
-    value: true
-  when:
-    - templates is changed
-  tags:
-    - setup

elk_metrics_6x/roles/elastic_metricbeat/templates/metricbeat.yml.j2

@@ -1102,7 +1102,9 @@ processors:
 {{ elk_macros.setup_template('metricbeat', inventory_hostname, data_nodes, elasticsearch_number_of_replicas) }}
 
 #================================ Kibana =======================================
-{{ elk_macros.setup_kibana(hostvars[groups['kibana'][0]]['ansible_host'] + ':' + kibana_port|string) }}
+{% if (groups['kibana'] | length) > 0 %}
+{{ elk_macros.setup_kibana(hostvars[groups['kibana'][0]]['ansible_host'] ~ ':' ~ kibana_port) }}
+{% endif %}
 
 #================================ Logging ======================================
 {{ elk_macros.beat_logging('metricbeat') }}

elk_metrics_6x/roles/elastic_packetbeat/tasks/main.yml

@@ -73,9 +73,14 @@
   notify:
     - Enable and restart packetbeat
 
-- include_tasks: packetbeat_setup.yml
+- name: Run the beat setup role
-  tags:
+  include_role:
-    - setup
+    name: elastic_beat_setup
+    private: true
+  when:
+    - (groups['kibana'] | length) > 0
+  vars:
+    elastic_beat_name: "packetbeat"
 
 - name: Force beat handlers
   meta: flush_handlers

elk_metrics_6x/roles/elastic_packetbeat/tasks/packetbeat_setup.yml

@@ -1,60 +0,0 @@
----
-# Copyright 2018, Rackspace US, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-- name: Refresh local facts
-  setup:
-    filter: ansible_local
-    gather_subset: "!all"
-  tags:
-    - always
-
-- name: Load templates
-  shell: >-
-    packetbeat setup
-    {{ item }}
-    -E 'output.logstash.enabled=false'
-    -E 'output.elasticsearch.hosts={{ coordination_nodes | to_json }}'
-    -E 'setup.template.enabled=true'
-    -E 'setup.template.overwrite=true'
-    -e -v
-  with_items:
-    - "--template"
-    - "--pipelines"
-    - "--machine-learning"
-    - "--dashboards"
-  register: templates
-  environment:
-    no_proxy: "{{ hostvars[groups['kibana'][0]]['ansible_host'] }}"
-  until: templates is success
-  retries: 5
-  delay: 5
-  run_once: true
-  when:
-    - ((ansible_local['elastic']['setup']['packetbeat_loaded_templates'] is undefined) or
-       (not (ansible_local['elastic']['setup']['packetbeat_loaded_templates'] | bool))) or
-      (elk_package_state | default('present')) == "latest"
-  tags:
-    - setup
-
-- name: Set template fact
-  ini_file:
-    dest: "/etc/ansible/facts.d/elastic.fact"
-    section: "setup"
-    option: packetbeat_loaded_templates
-    value: true
-  when:
-    - templates is changed
-  tags:
-    - setup

elk_metrics_6x/roles/elastic_packetbeat/templates/packetbeat.yml.j2

@@ -1056,7 +1056,9 @@ processors:
 {{ elk_macros.setup_template('packetbeat', inventory_hostname, data_nodes, elasticsearch_number_of_replicas) }}
 
 #================================ Kibana =======================================
-{{ elk_macros.setup_kibana(hostvars[groups['kibana'][0]]['ansible_host'] + ':' + kibana_port|string) }}
+{% if (groups['kibana'] | length) > 0 %}
+{{ elk_macros.setup_kibana(hostvars[groups['kibana'][0]]['ansible_host'] ~ ':' ~ kibana_port) }}
+{% endif %}
 
 #================================ Logging ======================================
 {{ elk_macros.beat_logging('packetbeat') }}

elk_metrics_6x/site-beats-core.yml

@@ -12,28 +12,13 @@
 # limitations under the License.
 
 - import_playbook: installMetricbeat.yml
-- import_playbook: fieldRefresh.yml
-  vars:
-    index_pattern: metricbeat-*
 
 - import_playbook: installPacketbeat.yml
-- import_playbook: fieldRefresh.yml
-  vars:
-    index_pattern: packetbeat-*
 
 - import_playbook: installAuditbeat.yml
-- import_playbook: fieldRefresh.yml
-  vars:
-    index_pattern: auditbeat-*
 
 - import_playbook: installHeartbeat.yml
-- import_playbook: fieldRefresh.yml
-  vars:
-    index_pattern: heartbeat-*
 
 - import_playbook: installFilebeat.yml
-- import_playbook: fieldRefresh.yml
-  vars:
-    index_pattern: filebeat-*
 
 - import_playbook: installJournalbeat.yml