Added telegraf role

Added ansible role to deploy telegraf

Roll out telegraf to all nodes

Introduce parameter to toggle deployment of performance monitoring

Co-Authored-By: zhubingbing <zhubingbing10@gmail.com>

Change-Id: Ia09b20ce65bf557c1a1030eda99df5cc88debd01
Partially-Implements: Blueprint performance-monitoring

ansible/group_vars/all.yml

@@ -263,6 +263,7 @@ enable_rally: "no"
 enable_sahara: "no"
 enable_senlin: "no"
 enable_swift: "no"
+enable_telegraf: "no"
 enable_tempest: "no"
 enable_watcher: "no"
 

ansible/inventory/all-in-one

@@ -23,6 +23,9 @@ compute
 [kibana:children]
 control
 
+[telegraf:children]
+monitoring
+
 [elasticsearch:children]
 control
 

ansible/inventory/multinode

@@ -44,6 +44,9 @@ monitoring
 [kibana:children]
 control
 
+[telegraf:children]
+monitoring
+
 [elasticsearch:children]
 control
 

ansible/roles/telegraf/defaults/main.yml

@@ -0,0 +1,18 @@
+---
+project_name: "telegraf"
+
+####################
+# Docker
+####################
+telegraf_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-telegraf"
+telegraf_tag: "{{ openstack_release }}"
+telegraf_image_full: "{{ telegraf_image }}:{{ telegraf_tag }}"
+
+
+####################
+# Protocols
+####################
+elasticsearch_proto: "http"
+haproxy_proto: "http"
+influxdb_proto: "http"
+rabbitmq_proto: "http"

ansible/roles/telegraf/meta/main.yml

@@ -0,0 +1,3 @@
+---
+dependencies:
+  - { role: common }

ansible/roles/telegraf/tasks/config.yml

@@ -0,0 +1,26 @@
+---
+- name: Ensuring config directories exist
+  file:
+    path: "{{ node_config_directory }}/{{ item }}"
+    state: "directory"
+    recurse: yes
+  with_items:
+    - "telegraf"
+    - "telegraf/config"
+
+- name: Copying over default config.json files
+  template:
+    src: "telegraf.json.j2"
+    dest: "{{ node_config_directory }}/telegraf/config.json"
+
+- name: Copying over telegraf config file
+  template:
+    src: "telegraf.conf.j2"
+    dest: "{{ node_config_directory }}/telegraf/telegraf.conf"
+
+- name: Copying over telegraf plugin files
+  copy:
+    src: "{{ item }}"
+    dest: "{{ node_config_directory }}/telegraf/config"
+  with_fileglob:
+    - "{{ role_path }}/templates/config/*.conf"

ansible/roles/telegraf/tasks/deploy.yml

@@ -0,0 +1,4 @@
+---
+- include: config.yml
+
+- include: start.yml

ansible/roles/telegraf/tasks/do_reconfigure.yml

@@ -0,0 +1,42 @@
+---
+- name: Ensuring the containers up
+  kolla_docker:
+    name: "telegraf"
+    action: "get_container_state"
+  register: container_state
+  failed_when: container_state.Running == false
+
+- include: config.yml
+
+- name: Check the configs
+  command: docker exec telegraf /usr/local/bin/kolla_set_configs --check
+  changed_when: false
+  failed_when: false
+  register: check_results
+
+# NOTE(jeffrey4l): when config_strategy == 'COPY_ALWAYS'
+# and container env['KOLLA_CONFIG_STRATEGY'] == 'COPY_ONCE',
+# just remove the container and start again
+- name: Containers config strategy
+  kolla_docker:
+    name: "telegraf"
+    action: "get_container_env"
+  register: container_envs
+
+- name: Remove the containers
+  kolla_docker:
+    name: "telegraf"
+    action: "remove_container"
+  register: remove_containers
+  when:
+    - config_strategy == "COPY_ONCE"
+
+- include: start.yml
+  when: remove_containers.changed
+
+- name: Restart containers
+  kolla_docker:
+    name: "telegraf"
+    action: "restart_container"
+  when:
+    - config_strategy == 'COPY_ALWAYS'

ansible/roles/telegraf/tasks/main.yml

@@ -0,0 +1,2 @@
+---
+- include: "{{ action }}.yml"

ansible/roles/telegraf/tasks/pull.yml

@@ -0,0 +1,6 @@
+---
+- name: Pulling telegraf image
+  kolla_docker:
+    action: "pull_image"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ telegraf_image_full }}"

ansible/roles/telegraf/tasks/reconfigure.yml

@@ -0,0 +1,2 @@
+---
+- include: do_reconfigure.yml

ansible/roles/telegraf/tasks/start.yml

@@ -0,0 +1,19 @@
+---
+- name: Starting telegraf container
+  kolla_docker:
+    action: "start_container"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ telegraf_image_full }}"
+    name: "telegraf"
+    environment:
+      HOST_PROC: "/rootfs/proc"
+      HOST_SYS: "/rootfs/sys"
+    volumes:
+      - "{{ node_config_directory }}/telegraf/:{{ container_config_directory }}/:ro"
+      - "/etc/localtime:/etc/localtime:ro"
+      - "kolla_logs:/var/log/kolla/"
+      - "/sys:/rootfs/sys:ro"
+      - "/proc:/rootfs/proc:ro"
+      - "/var/run/docker.sock:/var/run/docker.sock:ro"
+      - "haproxy_socket:/var/lib/kolla/haproxy/:rw"
+    pid_mode: "host"

ansible/roles/telegraf/tasks/upgrade.yml

@@ -0,0 +1,4 @@
+---
+- include: config.yml
+
+- include: start.yml

ansible/roles/telegraf/templates/telegraf.conf.j2

@@ -0,0 +1,55 @@
+[global_tags]
+[agent]
+  interval = "10s"
+  round_interval = true
+  metric_batch_size = 1000
+  metric_buffer_limit = 10000
+  collection_jitter = "0s"
+  flush_interval = "10s"
+  flush_jitter = "0s"
+  debug = false
+  quiet = false
+  hostname = ""
+  omit_hostname = false
+[[outputs.influxdb]]
+  urls = [{% for host in groups['influxdb'] %}"{{ influxdb_proto }}://{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address']}}:{{ influxdb_http_port }}"{% if not loop.last %},{% endif %}{% endfor %}]
+  database = "telegraf" # required
+  retention_policy = "default"
+  write_consistency = "any"
+  timeout = "5s"
+[[inputs.filestat]]
+files = ["/var/log/kolla/**.log"]
+[[inputs.cpu]]
+  percpu = true
+  totalcpu = true
+  fielddrop = ["time_*"]
+[[inputs.disk]]
+  ignore_fs = ["tmpfs", "devtmpfs"]
+[[inputs.diskio]]
+[[inputs.kernel]]
+[[inputs.mem]]
+[[inputs.processes]]
+[[inputs.swap]]
+[[inputs.system]]
+[[inputs.net]]
+  interfaces = []
+{% if enable_haproxy | bool %}
+[[inputs.haproxy]]
+  servers = ["{{ haproxy_proto }}://{{ haproxy_user }}:{{ haproxy_password }}@{{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}:{{ haproxy_stats_port }}"]
+{% endif %}
+{% if enable_memcached | bool %}
+[[inputs.memcached]]
+  servers = ["{{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}:{{ memcached_port }}"]
+{% endif %}
+{% if enable_elasticsearch | bool %}
+[[inputs.elasticsearch]]
+  servers = ["{{ elasticsearch_proto }}://{{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}:{{ elasticsearch_port }}"]
+  local = true
+  cluster_health = true
+{% endif %}
+{% if inventory_hostname in groups['rabbitmq'] %}
+[[inputs.rabbitmq]]
+  url = "{{ rabbitmq_proto }}://{{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}:{{ rabbitmq_management_port }}"
+  username = "{{ rabbitmq_user }}"
+  password = "{{ rabbitmq_password }}"
+{% endif %}

ansible/roles/telegraf/templates/telegraf.json.j2

@@ -0,0 +1,17 @@
+{
+    "command": "telegraf -config /etc/telegraf/telegraf.conf -config-directory /etc/telegraf/telegraf.d/",
+    "config_files": [
+        {
+            "source": "{{ container_config_directory }}/telegraf.conf",
+            "dest": "/etc/telegraf/telegraf.conf",
+            "owner": "telegraf",
+            "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/config/*",
+            "dest": "/etc/telegraf/telegraf.d/",
+            "owner": "telegraf",
+            "perm": "0600"
+        }
+    ]
+}

ansible/site.yml

@@ -28,6 +28,14 @@
         tags: influxdb,
         when: enable_influxdb | bool }
 
+- hosts:
+    - telegraf
+  serial: '{{ "30%" if action == "upgrade" else "0" }}'
+  roles:
+    - { role: telegraf,
+        tags: telegraf,
+        when: enable_telegraf | bool }
+
 - hosts: haproxy
   serial: '{{ "30%" if action == "upgrade" else "0" }}'
   roles:

etc/kolla/globals.yml

@@ -137,6 +137,7 @@ kolla_internal_vip_address: "10.10.10.254"
 #enable_rally: "no"
 #enable_senlin: "no"
 #enable_swift: "no"
+#enable_telegraf: "no"
 #enable_tempest: "no"
 #enable_watcher: "no"