From b146181174480ada835929fdc3bea59bdaa5e3a7 Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Tue, 23 Jun 2020 11:41:27 +1000 Subject: [PATCH] Grafana container deployment This uses the Grafana container created with Iddfafe852166fe95b3e433420e2e2a4a6380fc64 to run the grafana.opendev.org service. We retain the old model of an Apache reverse-proxy; it's well tested and understood, it's much easier than trying to map all the SSL termination/renewal/etc. into the Grafana container and we don't have to convince ourselves the container is safe to be directly web-facing. Otherwise this is a fairly straight forward deployment of the container. As before, it uses the graph configuration kept in project-config which is loaded in with grafyaml, which is included in the container. Once nice advantage is that it makes it quite easy to develop graphs locally, using the container which can talk to the public graphite instance. The documentation has been updated with a reference on how to do this. Change-Id: I0cc76d29b6911aecfebc71e5fdfe7cf4fcd071a4 --- doc/source/grafana.rst | 68 +++++++++++---- inventory/service/groups.yaml | 5 +- .../service/host_vars/grafana01.opendev.org | 5 ++ playbooks/roles/grafana/README.rst | 1 + playbooks/roles/grafana/handlers/main.yaml | 4 + playbooks/roles/grafana/tasks/main.yaml | 87 +++++++++++++++++++ .../grafana/templates/docker-compose.yaml.j2 | 18 ++++ .../roles/grafana/templates/grafana.vhost.j2 | 41 +++++++++ .../handlers/main.yaml | 3 + playbooks/service-grafana.yaml | 6 ++ playbooks/zuul/run-base.yaml | 1 + .../host_vars/grafana01.opendev.org.yaml.j2 | 3 + testinfra/test_grafana.py | 27 ++++++ zuul.d/docker-images/grafana.yaml | 2 +- zuul.d/infra-prod.yaml | 17 ++++ zuul.d/project.yaml | 16 ++++ zuul.d/system-config-run.yaml | 30 +++++++ 17 files changed, 314 insertions(+), 20 deletions(-) create mode 100644 inventory/service/host_vars/grafana01.opendev.org create mode 100644 playbooks/roles/grafana/README.rst create mode 100644 playbooks/roles/grafana/handlers/main.yaml create mode 100644 playbooks/roles/grafana/tasks/main.yaml create mode 100644 playbooks/roles/grafana/templates/docker-compose.yaml.j2 create mode 100644 playbooks/roles/grafana/templates/grafana.vhost.j2 create mode 100644 playbooks/service-grafana.yaml create mode 100644 playbooks/zuul/templates/host_vars/grafana01.opendev.org.yaml.j2 create mode 100644 testinfra/test_grafana.py diff --git a/doc/source/grafana.rst b/doc/source/grafana.rst index f5c2c52c56..15044f06d8 100644 --- a/doc/source/grafana.rst +++ b/doc/source/grafana.rst @@ -13,34 +13,66 @@ At a Glance =========== :Hosts: - * http://grafana.openstack.org -:Puppet: - * https://github.com/bfraser/puppet-grafana - * :git_file:`modules/openstack_project/manifests/grafana.pp` + * https://grafana.opendev.org :Projects: - * http://grafana.org + * https://grafana.org :Bugs: * https://storyboard.openstack.org/#!/project/748 Overview ======== -Apache is configured as a reverse proxy and there is a MySQL database -backend. +Apache is configured as a reverse proxy to Grafana running in a +container, listening on port 3000. -Sysadmin -======== +Local Development +================= -After bringing up a Grafana node with puppet, log in and configure Grafana by -hand: +To develop dashboards, you can run the OpenDev Grafana container. +Firstly, get setup: -#. Log in as the admin user. +.. code-block:: shell-session -#. Under 'Data Sources', add a new entry with following: + $ cd + $ mkdir secrets + $ echo "password" > secrets/admin_password + $ echo "admin" > secrets/admin_user + $ echo "key" > secrets/secret_key - - name: OpenStack - - type: Graphite - - default: checked - - url: http://graphite.opendev.org - - access: direct + $ git clone https://opendev.org/openstack/project-config + +Then run the container with the following options: + +.. code-block:: shell-session + + $ cd + $ sudo podman run \ + -p 3000:3000 \ + -v ./secrets:/etc/grafana/secrets \ + -v ./project-config:/opt/project-config \ + -e GF_AUTH_ANONYMOUS_ENABLED=true \ + -e GF_USERS_ALLOW_SIGN_UP=false \ + -e GF_SECURITY_ADMIN_PASSWORD__FILE=/etc/grafana/secrets/admin_password \ + -e GF_SECURITY_ADMIN_USER__FILE=/etc/grafana/secrets/admin_user \ + -e GF_SECURITY_SECRET_KEY__FILE=/etc/grafana/secrets/secret_key \ + docker.io/opendevorg/grafana + +At this point, Grafana will be running and listening on port 3000. +You can log into as ``admin`` with ``password`` (or using your secrets +above). + +This is unconfigured and does not yet talk to the OpenDev Graphite +instance. The dashboard definitions are kept in +``project-config/grafana``. To load them ``exec`` the +``update-grafana`` script in the container (i.e. ``podman exec + update-grafana``). That will run ``grafyaml`` and +load in the updated dashboards (note it relies on things being mapped +as above). To work on dashboards, update the ``yaml`` files in +``project-config`` and re-run ``update-grafana``, then reload them in +the Grafana UI. + +Alternatively, you can use the Grafana editor to make the dashboards, +and then under "Dashboard Settings" (gear icon) select "JSON Model" +and commit that (it seems you have to cut-and-paste, there isn't +currently a way to export the JSON as such). diff --git a/inventory/service/groups.yaml b/inventory/service/groups.yaml index 59e5d58233..9aa74a05cb 100644 --- a/inventory/service/groups.yaml +++ b/inventory/service/groups.yaml @@ -60,7 +60,9 @@ groups: gitea-lb: - gitea-lb[0-9]*.opendev.org grafana: - - grafana[0-9]*.open*.org + - grafana[0-9]*.openstack.org + grafana_opendev: + - grafana[0-9]*.opendev.org graphite: - graphite*.open*.org health: @@ -73,6 +75,7 @@ groups: - etherpad[0-9]*.opendev.org - gitea[0-9]*.opendev.org - graphite01.opendev.org + - grafana[0-9]*.opendev.org - insecure-ci-registry[0-9]*.opendev.org - meetpad[0-9]*.opendev.org - mirror[0-9]*.opendev.org diff --git a/inventory/service/host_vars/grafana01.opendev.org b/inventory/service/host_vars/grafana01.opendev.org new file mode 100644 index 0000000000..9c0d00239d --- /dev/null +++ b/inventory/service/host_vars/grafana01.opendev.org @@ -0,0 +1,5 @@ +letsencrypt_certs: + grafana01-opendev-org-main: + - grafana01.opendev.org + - grafana.opendev.org + - grafana.openstack.org diff --git a/playbooks/roles/grafana/README.rst b/playbooks/roles/grafana/README.rst new file mode 100644 index 0000000000..da7a35deaf --- /dev/null +++ b/playbooks/roles/grafana/README.rst @@ -0,0 +1 @@ +Run Grafana diff --git a/playbooks/roles/grafana/handlers/main.yaml b/playbooks/roles/grafana/handlers/main.yaml new file mode 100644 index 0000000000..768a2ab017 --- /dev/null +++ b/playbooks/roles/grafana/handlers/main.yaml @@ -0,0 +1,4 @@ +- name: grafana Reload apache2 + service: + name: apache2 + state: reloaded diff --git a/playbooks/roles/grafana/tasks/main.yaml b/playbooks/roles/grafana/tasks/main.yaml new file mode 100644 index 0000000000..a811141c22 --- /dev/null +++ b/playbooks/roles/grafana/tasks/main.yaml @@ -0,0 +1,87 @@ +- name: Ensure docker-compose directory exists + file: + state: directory + path: /etc/grafana-docker + +- name: Write settings file + template: + src: docker-compose.yaml.j2 + dest: /etc/grafana-docker/docker-compose.yaml + +- name: Ensure config directory exists + file: + state: directory + path: /etc/grafana + +- name: Ensure secrets config directory exists + file: + state: directory + path: /etc/grafana/secrets + +- name: Make admin_password + copy: + content: '{{ grafana_admin_password }}' + dest: /etc/grafana/secrets/admin_password + +- name: Make admin_user + copy: + content: '{{ grafana_admin_user }}' + dest: /etc/grafana/secrets/admin_user + +- name: Make secret_key + copy: + content: '{{ grafana_secret_key }}' + dest: /etc/grafana/secrets/secret_key + +- name: Install apache2 + apt: + name: + - apache2 + - apache2-utils + state: present + +- name: Apache modules + apache2_module: + state: present + name: "{{ item }}" + loop: + - rewrite + - proxy + - proxy_http + - ssl + - headers + - proxy_wstunnel + +- name: Copy apache config + template: + src: grafana.vhost.j2 + dest: /etc/apache2/sites-enabled/000-default.conf + owner: root + group: root + mode: 0644 + notify: grafana Reload apache2 + +- name: Sync project-config + include_role: + name: sync-project-config + +- name: Run docker-compose pull + shell: + cmd: docker-compose pull + chdir: /etc/grafana-docker/ + +- name: Run docker-compose up + shell: + cmd: docker-compose up -d + chdir: /etc/grafana-docker/ + +- name: Run docker prune to cleanup unneeded images + shell: + cmd: docker image prune -f + +- name: Import dashboards to container + shell: + cmd: | + docker-compose exec -T grafana /usr/local/bin/update-grafana + chdir: /etc/grafana-docker/ + diff --git a/playbooks/roles/grafana/templates/docker-compose.yaml.j2 b/playbooks/roles/grafana/templates/docker-compose.yaml.j2 new file mode 100644 index 0000000000..36dff6400d --- /dev/null +++ b/playbooks/roles/grafana/templates/docker-compose.yaml.j2 @@ -0,0 +1,18 @@ +# Version 2 is the latest that is supported by docker-compose in +# Ubuntu Xenial. +version: '2' + +services: + grafana: + restart: always + image: docker.io/opendevorg/grafana + network_mode: host + environment: + GF_SECURITY_ADMIN_PASSWORD__FILE: '/etc/grafana/secrets/admin_password' + GF_SECURITY_ADMIN_USER__FILE: '/etc/grafana/secrets/admin_user' + GF_SECURITY_SECRET_KEY__FILE: '/etc/grafana/secrets/secret_key' + GF_AUTH_ANONYMOUS_ENABLED: 'true' + GF_USERS_ALLOW_SIGN_UP: 'false' + volumes: + - /opt/project-config:/opt/project-config + - /etc/grafana/secrets:/etc/grafana/secrets diff --git a/playbooks/roles/grafana/templates/grafana.vhost.j2 b/playbooks/roles/grafana/templates/grafana.vhost.j2 new file mode 100644 index 0000000000..fb57227b8f --- /dev/null +++ b/playbooks/roles/grafana/templates/grafana.vhost.j2 @@ -0,0 +1,41 @@ + + ServerName {{ inventory_hostname }} + ServerAdmin webmaster@openstack.org + + ErrorLog ${APACHE_LOG_DIR}/grafana-error.log + + LogLevel warn + + CustomLog ${APACHE_LOG_DIR}/grafana-access.log combined + + Redirect / https://{{ inventory_hostname }}/ + + + + + ServerName {{ inventory_hostname }} + ServerAdmin webmaster@openstack.org + + AllowEncodedSlashes On + + ErrorLog ${APACHE_LOG_DIR}/grafana-ssl-error.log + + LogLevel warn + + CustomLog ${APACHE_LOG_DIR}/grafana-ssl-access.log combined + + SSLEngine on + SSLProtocol All -SSLv2 -SSLv3 + # Note: this list should ensure ciphers that provide forward secrecy + SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:!AES256:!aNULL:!eNULL:!MD5:!DSS:!PSK:!SRP + SSLHonorCipherOrder on + + SSLCertificateFile /etc/letsencrypt-certs/{{ inventory_hostname }}/{{ inventory_hostname }}.cer + SSLCertificateKeyFile /etc/letsencrypt-certs/{{ inventory_hostname }}/{{ inventory_hostname }}.key + SSLCertificateChainFile /etc/letsencrypt-certs/{{ inventory_hostname }}/ca.cer + + ProxyPass / http://localhost:3000/ retry=0 + ProxyPassReverse / http://localhost:3000/ + + + diff --git a/playbooks/roles/letsencrypt-create-certs/handlers/main.yaml b/playbooks/roles/letsencrypt-create-certs/handlers/main.yaml index 289e72d58a..813bf47198 100644 --- a/playbooks/roles/letsencrypt-create-certs/handlers/main.yaml +++ b/playbooks/roles/letsencrypt-create-certs/handlers/main.yaml @@ -117,6 +117,9 @@ - name: letsencrypt updated static01-zuul-ci-org include_tasks: roles/letsencrypt-create-certs/handlers/restart_apache.yaml +- name: letsencrypt updated grafana01-opendev-org-main + include_tasks: roles/letsencrypt-create-certs/handlers/restart_apache.yaml + # nodepool - name: letsencrypt updated nb01-opendev-org-main diff --git a/playbooks/service-grafana.yaml b/playbooks/service-grafana.yaml new file mode 100644 index 0000000000..8fd0607e68 --- /dev/null +++ b/playbooks/service-grafana.yaml @@ -0,0 +1,6 @@ +- hosts: "grafana_opendev:!disabled" + name: "Base: configure grafana" + roles: + - iptables + - install-docker + - grafana diff --git a/playbooks/zuul/run-base.yaml b/playbooks/zuul/run-base.yaml index caa54f7df4..a7da3d5773 100644 --- a/playbooks/zuul/run-base.yaml +++ b/playbooks/zuul/run-base.yaml @@ -74,6 +74,7 @@ - host_vars/letsencrypt02.opendev.org.yaml - host_vars/lists.openstack.org.yaml - host_vars/gitea99.opendev.org.yaml + - host_vars/grafana01.opendev.org.yaml - host_vars/mirror01.openafs.provider.opendev.org.yaml - host_vars/mirror02.openafs.provider.opendev.org.yaml - host_vars/mirror-update01.opendev.org.yaml diff --git a/playbooks/zuul/templates/host_vars/grafana01.opendev.org.yaml.j2 b/playbooks/zuul/templates/host_vars/grafana01.opendev.org.yaml.j2 new file mode 100644 index 0000000000..eae0101c51 --- /dev/null +++ b/playbooks/zuul/templates/host_vars/grafana01.opendev.org.yaml.j2 @@ -0,0 +1,3 @@ +grafana_admin_password: adminpassword +grafana_admin_user: admin +grafana_secret_key: grafanaSecretKey diff --git a/testinfra/test_grafana.py b/testinfra/test_grafana.py new file mode 100644 index 0000000000..bf26f9b3c1 --- /dev/null +++ b/testinfra/test_grafana.py @@ -0,0 +1,27 @@ +# Copyright 2020 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +testinfra_hosts = ['grafana01.opendev.org'] + + +def test_grafana_container_listening(host): + grafana = host.socket("tcp://127.0.0.1:3000") + assert grafana.is_listening + +def tets_grafana_proxy(host): + cmd = host.run('curl --insecure ' + '--resolve grafana.opendev.org:443:127.0.0.1 ' + 'https://grafana.opendev.org') + assert 'Grafana' in cmd.stdout diff --git a/zuul.d/docker-images/grafana.yaml b/zuul.d/docker-images/grafana.yaml index d756ac3425..9586a86073 100644 --- a/zuul.d/docker-images/grafana.yaml +++ b/zuul.d/docker-images/grafana.yaml @@ -9,7 +9,7 @@ - context: docker/grafana repository: opendevorg/grafana files: &grafana_files - - docker/grafana + - docker/grafana/ - job: name: system-config-upload-image-grafana diff --git a/zuul.d/infra-prod.yaml b/zuul.d/infra-prod.yaml index 1b398adfb4..37538e6b23 100644 --- a/zuul.d/infra-prod.yaml +++ b/zuul.d/infra-prod.yaml @@ -509,6 +509,23 @@ - playbooks/roles/accessbot - docker/accessbot/ +- job: + name: infra-prod-service-grafana + parent: infra-prod-service-base + description: Run service-grafana.yaml playbook. + vars: + playbook_name: service-grafana.yaml + files: + - inventory/ + - playbooks/service-grafana.yaml + - inventory/service/host_vars/grafana01.org.yaml + - inventory/service/group_vars/grafana + - playbooks/roles/install-docker/ + - playbooks/roles/pip3/ + - playbooks/roles/grafana + - playbooks/roles/logrotate + - playbooks/roles/iptables/ + # Run AFS changes separately so we can make sure to only do one at a time # (turns out quorum is nice to have) - job: diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index 0b966f31c9..f2e376a347 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -43,6 +43,11 @@ soft: true - name: system-config-build-image-haproxy-statsd soft: true + - system-config-run-grafana: + dependencies: + - name: opendev-buildset-registry + - name: system-config-build-image-grafana + soft: true - system-config-run-review: dependencies: - name: opendev-buildset-registry @@ -122,6 +127,11 @@ soft: true - name: system-config-upload-image-haproxy-statsd soft: true + - system-config-run-grafana: + dependencies: + - name: opendev-buildset-registry + - name: system-config-upload-image-grafana + soft: true - system-config-run-review: dependencies: - name: opendev-buildset-registry @@ -210,6 +220,12 @@ soft: true - name: system-config-promote-image-etherpad soft: true + - infra-prod-service-grafana: + dependencies: + - name: infra-prod-letsencrypt + soft: true + - name: system-config-promote-image-grafana + soft: true - infra-prod-service-meetpad - infra-prod-service-mirror-update - infra-prod-service-mirror diff --git a/zuul.d/system-config-run.yaml b/zuul.d/system-config-run.yaml index b5acf18984..af85b61dc5 100644 --- a/zuul.d/system-config-run.yaml +++ b/zuul.d/system-config-run.yaml @@ -542,6 +542,36 @@ # to run this job as well. - docker/haproxy-statsd/ +- job: + name: system-config-run-grafana + parent: system-config-run-containers + description: | + Run the playbook for the etherpad servers. + timeout: 3600 + requires: grafana-container-image + required-projects: + - opendev/system-config + - openstack/project-config + nodeset: + nodes: + - name: bridge.openstack.org + label: ubuntu-bionic + - name: grafana01.opendev.org + label: ubuntu-focal + vars: + run_playbooks: + - playbooks/letsencrypt.yaml + - playbooks/service-grafana.yaml + files: + - playbooks/bridge.yaml + - playbooks/letsencrypt.yaml + - playbooks/service-grafana.yaml + - playbooks/roles/grafana/ + - playbooks/roles/install-docker/ + - playbooks/roles/pip3/ + - docker/grafana/ + - testinfra/test_grafana.py + - job: name: system-config-run-meetpad parent: system-config-run-containers