Graphite container deployment

This deploys graphite from the upstream container.

We override the statsd configuration to have it listen on ipv6.
Similarly we override the ngnix config to listen on ipv6, enable ssl,
forward port 80 to 443, block the /admin page (we don't use it).

For production we will just want to put some cinder storage in
/opt/graphite/storage on the production host and figure out how to
migrate the old stats.  The is also a bit of cleanup that will follow,
because we half-converted grafana01.opendev.org -- so everything can't
be in the same group till that is gone.

Testing has been added to push some stats and ensure they are seen.

Change-Id: Ie843b3d90a72564ef90805f820c8abc61a71017d
This commit is contained in:
Ian Wienand 2020-06-26 10:23:16 +10:00
parent b146181174
commit 185797a0e5
17 changed files with 299 additions and 4 deletions

View File

@ -0,0 +1,14 @@
iptables_extra_allowed_hosts:
- hostname: bridge.openstack.org
port: 8125
protocol: udp
- hostname: opendev.org
port: 8125
protocol: udp
iptables_extra_allowed_groups:
- {'protocol': 'udp', 'port': '8125', 'group': 'firehose'}
- {'protocol': 'udp', 'port': '8125', 'group': 'mirror-update'}
- {'protocol': 'udp', 'port': '8125', 'group': 'logstash'}
- {'protocol': 'udp', 'port': '8125', 'group': 'nodepool'}
- {'protocol': 'udp', 'port': '8125', 'group': 'zuul'}

View File

@ -65,6 +65,10 @@ groups:
- grafana[0-9]*.opendev.org
graphite:
- graphite*.open*.org
# NOTE(ianw) : to be cleaned up once the half-puppet
# graphite01.opendev.org is gone.
graphite_opendev:
- graphite02.opendev.org
health:
- health[0-9]*.openstack.org
jvb:
@ -74,7 +78,7 @@ groups:
letsencrypt:
- etherpad[0-9]*.opendev.org
- gitea[0-9]*.opendev.org
- graphite01.opendev.org
- graphite[0-9]*.opendev.org
- grafana[0-9]*.opendev.org
- insecure-ci-registry[0-9]*.opendev.org
- meetpad[0-9]*.opendev.org
@ -130,7 +134,10 @@ groups:
- ethercalc[0-9]*.open*.org
- firehose[0-9]*.open*.org
- grafana[0-9]*.open*.org
- graphite*.open*.org
# TODO(ianw) : this is a weird one we half-converted and moved
# into opendev.org in the early days of opendev. remove when
# graphite02 up.
- graphite01.opendev.org
- health[0-9]*.openstack.org
- kdc[0-9]*.open*.org
- lists*.katacontainers.io
@ -167,7 +174,7 @@ groups:
- ethercalc[0-9]*.open*.org
- firehose[0-9]*.open*.org
- grafana[0-9]*.open*.org
- graphite[0-9]*.open*.org
- graphite01.opendev.org
- health[0-9]*.openstack.org
- kdc[0-9]*.open*.org
- lists*.katacontainers.io

View File

@ -0,0 +1,4 @@
letsencrypt_certs:
graphite02-main:
- graphite02.opendev.org
- graphite.opendev.org

View File

@ -0,0 +1 @@
Run Graphite

View File

@ -0,0 +1,4 @@
- name: graphite Reload apache2
service:
name: apache2
state: reloaded

View File

@ -0,0 +1,50 @@
- name: Ensure docker-compose directory exists
file:
state: directory
path: /etc/graphite-docker
- name: Write settings file
template:
src: docker-compose.yaml.j2
dest: /etc/graphite-docker/docker-compose.yaml
- name: Write nginx override config
template:
src: graphite-statsd.conf.j2
dest: /etc/graphite-docker/graphite-statsd.conf
- name: Write statsd override config
template:
src: statsd.js.j2
dest: /etc/graphite-docker/statsd.js
- name: Ensure storage directory exists
file:
state: directory
path: /opt/graphite/storage
- name: Ensure log directory exists
file:
state: directory
path: /var/log/graphite
- name: Run docker-compose pull
shell:
cmd: docker-compose pull
chdir: /etc/graphite-docker/
- name: Run docker-compose up
shell:
cmd: docker-compose up -d
chdir: /etc/graphite-docker/
- name: Run docker prune to cleanup unneeded images
shell:
cmd: docker image prune -f
# This is handy to have on the host for checking stat ingestion
- name: Install netcat
package:
name:
- netcat
state: present

View File

@ -0,0 +1,15 @@
# Version 2 is the latest that is supported by docker-compose in
# Ubuntu Xenial.
version: '2'
services:
graphite:
restart: always
image: docker.io/graphiteapp/graphite-statsd
network_mode: host
volumes:
- /etc/graphite-docker/graphite-statsd.conf:/etc/nginx/sites-enabled/graphite-statsd.conf
- /etc/graphite-docker/statsd.js:/opt/statsd/config/udp.js
- /etc/letsencrypt-certs:/etc/letsencrypt-certs
- /opt/graphite/storage:/opt/graphite/storage
- /var/log/graphite:/var/log/

View File

@ -0,0 +1,57 @@
server {
listen 80 default_server;
listen [::]:80 default_server;
server_name _;
return 301 https://$host$request_uri;
}
server {
listen 443 ssl;
listen [::]:443 ssl;
server_name {{ inventory_hostname }};
ssl_certificate /etc/letsencrypt-certs/{{ inventory_hostname }}/{{ inventory_hostname }}.cer;
ssl_certificate_key /etc/letsencrypt-certs/{{ inventory_hostname }}/{{ inventory_hostname }}.key;
root /opt/graphite/static;
index index.html;
location /nginx_status {
stub_status on;
access_log off;
allow 127.0.0.1;
deny all;
}
# No remote login
location /admin {
allow 127.0.0.1;
deny all;
}
location /media {
# django admin static files
alias /usr/local/lib/python3.6/dist-packages/django/contrib/admin/media/;
}
location /admin/auth/admin {
alias /usr/local/lib/python3.6/dist-packages/django/contrib/admin/static/admin;
}
location /admin/auth/user/admin {
alias /usr/local/lib/python3.6/dist-packages/django/contrib/admin/static/admin;
}
location / {
proxy_pass http://127.0.0.1:8080;
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
add_header 'Access-Control-Allow-Origin' '*';
add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
add_header 'Access-Control-Allow-Headers' 'Authorization, Content-Type';
add_header 'Access-Control-Allow-Credentials' 'true';
}
}

View File

@ -0,0 +1,9 @@
{
"graphiteHost": "127.0.0.1",
"graphitePort": 2003,
"port": 8125,
"flushInterval": 10000,
"servers": [
{ server: "./servers/udp", address: "::", port: 8125, address_ipv6: true }
]
}

View File

@ -18,8 +18,11 @@ results:
- puppet
- puppet4
graphite.opendev.org:
graphite01.opendev.org:
- puppet
- puppet4
- graphite
- letsencrypt
- puppet
- webservers

View File

@ -12,6 +12,9 @@
# https://docs.ansible.com/ansible/latest/porting_guides/porting_guide_2.8.html#imports-as-handlers
- name: letsencrypt updated graphite01-main
include_tasks: roles/letsencrypt-create-certs/handlers/restart_graphite.yaml
- name: letsencrypt updated graphite02-main
include_tasks: roles/letsencrypt-create-certs/handlers/restart_apache.yaml
- name: letsencrypt updated tarballs-main

View File

@ -0,0 +1,12 @@
- name: Check for running nginx
command: pgrep -f nginx
ignore_errors: yes
register: nginx_pids
- name: Restart graphite container
when: nginx_pids.rc == 0
block:
- name: Restart nginx
shell:
cmd: docker-compose restart graphite
chdir: /etc/graphite-docker

View File

@ -0,0 +1,6 @@
- hosts: "graphite_opendev:!disabled"
name: "Base: configure graphite"
roles:
- iptables
- install-docker
- graphite

View File

@ -0,0 +1,58 @@
# Copyright 2020 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import json
import logging
import ssl
import urllib.request
testinfra_hosts = ['graphite02.opendev.org']
def test_graphite_container_web_listening(host):
graphite_http = host.socket("tcp://127.0.0.1:80")
assert graphite_http.is_listening
graphite_https = host.socket("tcp://127.0.0.1:443")
assert graphite_https.is_listening
def test_graphite(host):
cmd = host.run('curl --insecure '
'--resolve graphite.opendev.org:443:127.0.0.1 '
'https://graphite.opendev.org')
assert '<title>Graphite Browser</title>' in cmd.stdout
def test_graphite_data(host):
# seed some data; send it over ipv6
cmd = ('timeout 20 bash -c '
'\'while true; do echo -n "example:$((RANDOM % 100))|c" '
'| nc -6 -w 1 -u localhost 8125; done\'')
host.run(cmd)
url='render?from=-10mins&until=now&target=stats.example&format=json'
# Assert we see some non-null values for this stat
# multi-node-hosts-file has setup graphite02.opendev.org to
# resolve from hosts.
found_value = False
with urllib.request.urlopen('https://graphite02.opendev.org/%s' % (url),
context=ssl._create_unverified_context()) \
as req:
data = json.loads(req.read().decode())
logging.debug('got: %s' % data)
datapoints = (data[0]['datapoints'])
for p in datapoints:
if p[0] != None:
found_value = True
assert found_value

View File

@ -526,6 +526,22 @@
- playbooks/roles/logrotate
- playbooks/roles/iptables/
- job:
name: infra-prod-service-graphite
parent: infra-prod-service-base
description: Run service-graphite.yaml playbook.
vars:
playbook_name: service-graphite.yaml
files:
- inventory/
- playbooks/service-graphite.yaml
- inventory/service/host_vars/graphite02.opendev.org.yaml
- inventory/service/group_vars/graphite
- playbooks/roles/install-docker/
- playbooks/roles/pip3/
- playbooks/roles/graphite/
- playbooks/roles/iptables/
# Run AFS changes separately so we can make sure to only do one at a time
# (turns out quorum is nice to have)
- job:

View File

@ -48,6 +48,7 @@
- name: opendev-buildset-registry
- name: system-config-build-image-grafana
soft: true
- system-config-run-graphite
- system-config-run-review:
dependencies:
- name: opendev-buildset-registry
@ -132,6 +133,7 @@
- name: opendev-buildset-registry
- name: system-config-upload-image-grafana
soft: true
- system-config-run-graphite
- system-config-run-review:
dependencies:
- name: opendev-buildset-registry
@ -226,6 +228,7 @@
soft: true
- name: system-config-promote-image-grafana
soft: true
- infra-prod-service-graphite
- infra-prod-service-meetpad
- infra-prod-service-mirror-update
- infra-prod-service-mirror

View File

@ -572,6 +572,39 @@
- docker/grafana/
- testinfra/test_grafana.py
- job:
name: system-config-run-graphite
parent: system-config-run
description: |
Run the playbook for the graphite servers.
timeout: 3600
required-projects:
- opendev/system-config
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
# NOTE(ianw): 01 is a half-puppet opendev.org
# server
- name: graphite02.opendev.org
label: ubuntu-focal
vars:
run_playbooks:
- playbooks/letsencrypt.yaml
- playbooks/service-graphite.yaml
host-vars:
graphite02.opendev.org:
host_copy_output:
'/var/log/graphite': logs
files:
- playbooks/bridge.yaml
- playbooks/letsencrypt.yaml
- playbooks/service-graphite.yaml
- playbooks/roles/graphite
- playbooks/roles/install-docker/
- playbooks/roles/pip3/
- testinfra/test_graphite.py
- job:
name: system-config-run-meetpad
parent: system-config-run-containers