Add grafana

This change introduces grafana into the stack which gives us a great
way to visualize the data. The grafana role from cloudalchemy is being
used for the bulk of the deployment.

Because the grafana deployment playbook is now standalone the mentions
of grafana in the other ops directories have been removed.

Change-Id: I23e1c96cd1fda7ece9b86a69f9f0326913de714d
Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>

cluster_metrics/playbook-grafana.yml

@@ -1,73 +0,0 @@
----
-# Copyright 2016, Rackspace US, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-- name: Deploy grafana
-  hosts: "cluster-metrics"
-  gather_facts: true
-  user: root
-  pre_tasks:
-    - name: Create DB for service
-      mysql_db:
-        login_user: "{{ galera_root_user }}"
-        login_password: "{{ galera_root_password }}"
-        login_host: "{{ galera_address|default('127.0.0.1') }}"
-        name: "{{ grafana_db_name }}"
-        state: "present"
-      delegate_to: "{{ groups['galera_all'][0] }}"
-      no_log: true
-    - name: Grant access to the DB for the service
-      mysql_user:
-        login_user: "{{ galera_root_user }}"
-        login_password: "{{ galera_root_password }}"
-        login_host: "{{ galera_address|default('127.0.0.1') }}"
-        name: "{{ grafana_db_user }}"
-        password: "{{ grafana_db_password }}"
-        host: "{{ item }}"
-        state: "present"
-        priv: "{{ grafana_db_name }}.*:ALL"
-      delegate_to: "{{ groups['galera_all'][0] }}"
-      no_log: true
-      with_items:
-        - "localhost"
-        - "%"
-  tasks:
-    - name: Ensure https repos function
-      apt:
-        pkg: "apt-transport-https"
-        state: "latest"
-    - name: Add grafana apt-keys
-      apt_key:
-        url: "https://packagecloud.io/gpg.key"
-        state: "present"
-    - name: Add grafana repo
-      apt_repository:
-        repo: "deb https://packagecloud.io/grafana/stable/debian/ wheezy main"
-        state: "present"
-        update_cache: yes
-    - name: Install grafana
-      apt:
-        pkg: "grafana"
-        state: "latest"
-    - name: Drop grafana config file
-      template:
-        src: templates/grafana.ini.j2
-        dest: /etc/grafana/grafana.ini
-    - name: Enable and start grafana
-      service:
-        name: "grafana-server"
-        enabled: true
-        state: restarted
-  vars_files:
-    - vars.yml

cluster_metrics/playbook-metrics-lb.yml

@@ -70,12 +70,3 @@
               read_queries:
                  rule: "path_sub -i query"
                  backend_name: "influxdb"
-        - service:
-            haproxy_service_name: grafana
-            haproxy_backend_nodes: "{{ groups['cluster-metrics'] | default([]) }}"
-            haproxy_ssl: "{{ haproxy_ssl }}"
-            haproxy_port: 8089
-            haproxy_balance_type: tcp
-            haproxy_backend_options:
-              - tcp-check
-

cluster_metrics/readme.rst

@@ -8,7 +8,7 @@ Gather and visualize cluster wide metrics
 About this repository
 ---------------------
 
-This set of playbooks will deploy InfluxDB, Telegraf, Grafana and Kapacitor for the purpose of collecting
+This set of playbooks will deploy InfluxDB, Telegraf, and Kapacitor for the purpose of collecting
 metrics on an OpenStack cluster.
 
 Process
@@ -34,7 +34,7 @@ Add the export to update the inventory file location
 
     export ANSIBLE_INVENTORY=/opt/openstack-ansible/playbooks/inventory/dynamic_inventory.py
 
-If you are running the HA Proxy you should run the following playbook as well to enable the grafana port 8089
+If you are running the HA Proxy you should run the following playbook as well.
 
 .. code-block:: bash
 
@@ -67,19 +67,6 @@ variable in the ``user_variables.yml`` file as a list containing all targets tha
 
     openstack-ansible playbook-influx-telegraf.yml --forks 100
 
-Install grafana
-
-If you're proxy'ing grafana you will need to provide the full ``root_path`` when you run the playbook add the following
-``-e grafana_root_url='https://cloud.something:8443/grafana/'``
-
-.. code-block:: bash
-
-    openstack-ansible playbook-grafana.yml -e galera_root_user=root -e galera_address='127.0.0.1'
-
-Once that last playbook is completed you will have a functioning InfluxDB, Telegraf, and Grafana metric collection system
-active and collecting metrics. Grafana will need some setup, however functional dashboards have been provided in the
-``grafana-dashboards`` directory.
-
 Install Kapacitor
 
 .. code-block:: bash

cluster_metrics/templates/grafana.ini.j2

@@ -1,66 +0,0 @@
-# {{ ansible_managed }}
-[paths]
-
-[server]
-http_port = {{ grafana_port }}
-{% if grafana_root_url is defined %}
-root_url = {{ grafana_root_url }}
-{% endif %}
-
-[database]
-type = mysql
-host = {{ galera_address }}:3306
-name = {{ grafana_db_name }}
-user = {{ grafana_db_user }}
-password = {{ grafana_db_password }}
-
-[session]
-
-[analytics]
-check_for_updates = true
-
-[security]
-admin_user = admin
-admin_password = {{ grafana_admin_password }}
-
-[snapshots]
-
-[users]
-allow_sign_up = false
-allow_org_create = false
-
-[auth.anonymous]
-enabled = true
-org_name = OpenStack
-org_role = Viewer
-
-[auth.github]
-
-[auth.google]
-
-[auth.proxy]
-
-[auth.basic]
-
-[auth.ldap]
-
-[smtp]
-
-[emails]
-
-[log]
-
-[log.console]
-
-[log.file]
-
-[log.syslog]
-
-[event_publisher]
-
-[dashboards.json]
-
-[metrics]
-
-[grafana_net]
-url = https://grafana.net

cluster_metrics/vars.yml

@@ -13,14 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
-# Grafana vars
-grafana_port: 8089
-grafana_db_name: grafana
-grafana_db_user: grafana
-grafana_db_password: secrete
-grafana_admin_password: SuperSecrete
-
 # InfluxDB vars
 influxdb_admin_port: 8083
 influxdb_port: 8086

elk_metrics_6x/env.d/elk.yml

@@ -23,12 +23,12 @@ physical_skel:
   elastic-logstash_containers:
     belongs_to:
       - all_containers
-  kibana_containers:
-    belongs_to:
-      - all_containers
   elastic-logstash_hosts:
     belongs_to:
       - hosts
+  kibana_containers:
+    belongs_to:
+      - all_containers
   kibana_hosts:
     belongs_to:
       - hosts

elk_metrics_6x/readme.rst

@@ -27,7 +27,7 @@ like Kibana will use when connecting to elasticsearch. If this option is
 omitted, the first node in the elasticsearch cluster will be used.
 
 Optional | configure haproxy endpoints
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Edit the `/etc/openstack_deploy/user_variables.yml` file and add fiel following
 lines

elk_metrics_6x/vars/variables.yml

@@ -13,5 +13,6 @@ kibana_password: admin
 kibana_nginx_port: 81
 kibana_server_name: "{{ ansible_hostname }}"
 
+
 # logstash vars
 logstash_beat_input_port: 5044

grafana/conf.d/grafana.yml

@@ -0,0 +1,3 @@
+grafana_hosts:
+  logging01:
+    ip: 172.22.8.27

grafana/env.d/grafana.yml

@@ -0,0 +1,20 @@
+---
+component_skel:
+  grafana:
+    belongs_to:
+      - grafana_all
+
+container_skel:
+  grafana_container:
+    belongs_to:
+      - grafana_containers
+    contains:
+      - grafana
+
+physical_skel:
+  grafana_containers:
+    belongs_to:
+      - all_containers
+  grafana_hosts:
+    belongs_to:
+      - hosts

grafana/installGrafana.yml

@@ -0,0 +1,81 @@
+---
+# Copyright 2016, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Deploy Grafana
+  hosts: grafana
+  become: true
+  vars_files:
+    - vars/variables.yml
+
+  pre_tasks:
+    - name: Galera database block
+      block:
+        - name: Check for db password
+          fail:
+            msg: >-
+              The database root login user is undefined
+          when:
+            - galera_root_user is undefined
+
+        - name: Check for db password
+          fail:
+            msg: >-
+              The database root password is undefined
+          when:
+            - galera_root_password is undefined
+
+        - name: Create DB for service
+          mysql_db:
+            login_user: "{{ galera_root_user }}"
+            login_password: "{{ galera_root_password }}"
+            login_host: "{{ galera_address | default('127.0.0.1') }}"
+            name: "{{ grafana_db_name }}"
+            state: "present"
+          delegate_to: "{{ groups['galera_all'][0] }}"
+
+        - name: Grant access to the DB for the service
+          mysql_user:
+            login_user: "{{ galera_root_user }}"
+            login_password: "{{ galera_root_password }}"
+            login_host: "{{ galera_address | default('127.0.0.1') }}"
+            name: "{{ grafana_db_user }}"
+            password: "{{ grafana_db_password }}"
+            host: "{{ item }}"
+            state: "present"
+            priv: "{{ grafana_db_name }}.*:ALL"
+          with_items:
+            - "localhost"
+            - "%"
+          delegate_to: "{{ groups['galera_all'][0] }}"
+
+        - name: Set grafana database fact
+          set_fact:
+            grafana_database:
+              type: mysql
+              host: "{{ galera_address }}:3306"
+              name: "{{ grafana_db_name }}"
+              user: "{{ grafana_db_user }}"
+              password: "{{ grafana_db_password }}"
+      when:
+        - (groups['galera_all'] | default([])) | length > 0
+
+  pre_tasks:
+    - name: Ensure https repos function
+      apt:
+        pkg: "apt-transport-https"
+        state: "latest"
+
+  roles:
+    - role: grafana

grafana/inventory.example.yml

@@ -0,0 +1,29 @@
+---
+
+################################## ALL HOSTS ##################################
+all:
+  hosts:
+    # Local host
+    localhost:
+      ansible_connection: local
+
+################################## REQUIRED ###################################
+    logging01:
+      ansible_host: 172.16.27.100
+      ansible_user: root
+
+  vars: {}
+
+
+################################### GROUPS ####################################
+
+# The hosts group is used to target physical host machines. Enter all physical
+# host machines here.
+hosts:
+  hosts:
+    logging01:
+
+# This is the location where grafana(s) will live
+grafana:
+  hosts:
+    logging01:

grafana/readme.rst

@@ -0,0 +1,73 @@
+Install Grafana
+###############
+:tags: openstack, ansible
+
+About this repository
+---------------------
+
+This set of playbooks will deploy Grafana. If this is being deployed as part of
+an OpenStack all of the inventory needs will be provided for.
+
+**These playbooks require Ansible 2.4+.**
+
+Optional | configure haproxy endpoints
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Edit the `/etc/openstack_deploy/user_variables.yml` file and add fiel following
+lines
+
+.. code-block:: yaml
+
+    haproxy_extra_services:
+      - service:
+          haproxy_service_name: grafana
+          haproxy_ssl: False
+          haproxy_backend_nodes: "{{ groups['grafana'] | default([]) }}"
+          haproxy_port: 3000  # This is set using the "grafana_port" variable
+          haproxy_balance_type: tcp
+
+Deployment Process
+------------------
+
+Clone the grafana-osa repo
+
+.. code-block:: bash
+
+    cd /opt
+    git clone https://github.com/openstack/openstack-ansible-ops
+
+Clone the grafana role
+
+.. code-block:: bash
+
+    cd /opt
+    git clone https://github.com/cloudalchemy/ansible-grafana /etc/ansible/roles/grafana
+
+Copy the env.d file into place
+
+.. code-block:: bash
+
+    cd /opt/openstack-ansible-ops/grafana
+    cp env.d/grafana.yml /etc/openstack_deploy/env.d/
+
+Copy the conf.d file into place
+
+.. code-block:: bash
+
+    cp conf.d/grafana.yml /etc/openstack_deploy/conf.d/
+
+    Create the containers
+
+Create the containers
+
+.. code-block:: bash
+
+   cd /opt/openstack-ansible-playbooks
+   openstack-ansible lxc-containers-create.yml -e 'container_group=grafana'
+
+install grafana
+
+.. code-block:: bash
+
+    cd /opt/openstack-ansible-ops/grafana
+    openstack-ansible installGrafana.yml

grafana/vars/variables.yml

@@ -0,0 +1,12 @@
+# Grafana vars
+grafana_db_name: grafana
+grafana_db_user: grafana
+grafana_db_password: SuperSecrete
+grafana_admin_password: admin
+
+grafana_plugins:
+  - grafana-piechart-panel
+
+grafana_api_keys:
+  - name: "admin"
+    role: "Admin"