From 96318fed5aafff3d2a9315838cb7e7c0e76eb8f8 Mon Sep 17 00:00:00 2001 From: Jeffrey Zhang Date: Thu, 22 Sep 2016 00:17:47 +0800 Subject: [PATCH] Integrate gnocchi with ceph Gnocchi previously lacked high availability. We consider a lack of HA in our a vast majority of operator oriented services to be a defective design choice. this change integrates gnocchi with ceph to resolve the the lack of HA. Closes-Bug: #1626623 Change-Id: I71c5137842cb48bc4af0e50a2952df5631d0d6df --- ansible/group_vars/all.yml | 8 ++++ ansible/roles/gnocchi/defaults/main.yml | 13 ++++++ ansible/roles/gnocchi/tasks/ceph.yml | 45 +++++++++++++++++++ ansible/roles/gnocchi/tasks/deploy.yml | 10 +++++ ansible/roles/gnocchi/tasks/external_ceph.yml | 30 +++++++++++++ .../gnocchi/templates/gnocchi-api.json.j2 | 14 +++++- .../gnocchi/templates/gnocchi-metricd.json.j2 | 14 +++++- .../gnocchi/templates/gnocchi-statsd.json.j2 | 14 +++++- .../roles/gnocchi/templates/gnocchi.conf.j2 | 8 ++++ docker/gnocchi/gnocchi-base/Dockerfile.j2 | 14 +++--- etc/kolla/globals.yml | 8 ++++ ...te-gnocchi-with-ceph-a6d5f81f4d8b0391.yaml | 3 ++ 12 files changed, 173 insertions(+), 8 deletions(-) create mode 100644 ansible/roles/gnocchi/tasks/ceph.yml create mode 100644 ansible/roles/gnocchi/tasks/external_ceph.yml create mode 100644 releasenotes/notes/integrate-gnocchi-with-ceph-a6d5f81f4d8b0391.yaml diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index c708e8cdbd..f0f63a43c2 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -348,6 +348,13 @@ glance_backend_ceph: "{{ enable_ceph }}" ceilometer_database_type: "mongodb" +################# +# Gnocchi options +################# +# Vaid options are [file, ceph] +gnocchi_backend_storage: "{{ 'ceph' if enable_ceph|bool else 'file' }}" + + ################################# # Cinder options ################################# @@ -391,6 +398,7 @@ ceph_pool_type: "replicated" ceph_cinder_pool_name: "volumes" ceph_cinder_backup_pool_name: "backups" ceph_glance_pool_name: "images" +ceph_gnocchi_pool_name: "gnocchi" ceph_nova_pool_name: "vms" ceph_erasure_profile: "k=4 m=2 ruleset-failure-domain=host" diff --git a/ansible/roles/gnocchi/defaults/main.yml b/ansible/roles/gnocchi/defaults/main.yml index 3dacdee6ba..cd80d1695d 100644 --- a/ansible/roles/gnocchi/defaults/main.yml +++ b/ansible/roles/gnocchi/defaults/main.yml @@ -2,6 +2,19 @@ project_name: "gnocchi" +#################### +# Ceph +#################### +ceph_gnocchi_pool_type: "{{ ceph_pool_type }}" +ceph_gnocchi_cache_mode: "{{ ceph_cache_mode }}" + +# Due to Ansible issues on include, you cannot override these variables. Please +# override the variables they reference instead. +gnocchi_pool_name: "{{ ceph_gnocchi_pool_name }}" +gnocchi_pool_type: "{{ ceph_gnocchi_pool_type }}" +gnocchi_cache_mode: "{{ ceph_gnocchi_cache_mode }}" + + #################### # Database #################### diff --git a/ansible/roles/gnocchi/tasks/ceph.yml b/ansible/roles/gnocchi/tasks/ceph.yml new file mode 100644 index 0000000000..e5c6d9a169 --- /dev/null +++ b/ansible/roles/gnocchi/tasks/ceph.yml @@ -0,0 +1,45 @@ +--- +- name: Ensuring config directory exists + file: + path: "{{ node_config_directory }}/{{ item }}" + state: "directory" + when: inventory_hostname in groups[item] + with_items: + - "gnocchi-api" + - "gnocchi-metricd" + - "gnocchi-statsd" + +- name: Copying over config(s) + template: + src: roles/ceph/templates/ceph.conf.j2 + dest: "{{ node_config_directory }}/{{ item }}/ceph.conf" + when: inventory_hostname in groups[item] + with_items: + - "gnocchi-api" + - "gnocchi-metricd" + - "gnocchi-statsd" + +- include: ../../ceph_pools.yml + vars: + pool_name: "{{ gnocchi_pool_name }}" + pool_type: "{{ gnocchi_pool_type }}" + cache_mode: "{{ gnocchi_cache_mode }}" + +# TODO(SamYaple): Improve changed_when tests +- name: Pulling cephx keyring + command: docker exec ceph_mon ceph auth get-or-create client.gnocchi mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool={{ ceph_gnocchi_pool_name }}, allow rwx pool={{ ceph_gnocchi_pool_name }}-cache' + register: cephx_key + delegate_to: "{{ groups['ceph-mon'][0] }}" + changed_when: False + run_once: True + +- name: Pushing cephx keyring + copy: + content: "{{ cephx_key.stdout }}\n\r" + dest: "{{ node_config_directory }}/{{ item }}/ceph.client.gnocchi.keyring" + mode: "0600" + when: inventory_hostname in groups[item] + with_items: + - "gnocchi-api" + - "gnocchi-metricd" + - "gnocchi-statsd" diff --git a/ansible/roles/gnocchi/tasks/deploy.yml b/ansible/roles/gnocchi/tasks/deploy.yml index df3d43d96d..0fb27eca91 100644 --- a/ansible/roles/gnocchi/tasks/deploy.yml +++ b/ansible/roles/gnocchi/tasks/deploy.yml @@ -1,4 +1,14 @@ --- +- include: ceph.yml + when: + - enable_ceph|bool + - gnocchi_backend_storage == 'ceph' + +- include: external_ceph.yml + when: + - enable_ceph|bool == False + - gnocchi_backend_storage == 'ceph' + - include: register.yml when: inventory_hostname in groups['gnocchi-api'] diff --git a/ansible/roles/gnocchi/tasks/external_ceph.yml b/ansible/roles/gnocchi/tasks/external_ceph.yml new file mode 100644 index 0000000000..623ed80079 --- /dev/null +++ b/ansible/roles/gnocchi/tasks/external_ceph.yml @@ -0,0 +1,30 @@ +--- +- name: Ensuring config directory exists + file: + path: "{{ node_config_directory }}/{{ item }}" + state: "directory" + when: inventory_hostname in groups[item] + with_items: + - "gnocchi-api" + - "gnocchi-metricd" + - "gnocchi-statsd" + +- name: Copy over ceph.conf file + copy: + src: "{{ node_custom_config }}/{{ item }}/ceph.conf" + dest: "{{ node_config_directory }}/{{ item }}/ceph.conf" + when: inventory_hostname in groups[item] + with_items: + - "gnocchi-api" + - "gnocchi-metricd" + - "gnocchi-statsd" + +- name: Copy over ceph gnocchi keyring + copy: + src: "{{ node_custom_config }}/{{ item }}/ceph.client.gnocchi.keyring" + dest: "{{ node_config_directory }}/{{ item }}/ceph.client.gnocchi.keryring" + when: inventory_hostname in groups[item] + with_items: + - "gnocchi-api" + - "gnocchi-metricd" + - "gnocchi-statsd" diff --git a/ansible/roles/gnocchi/templates/gnocchi-api.json.j2 b/ansible/roles/gnocchi/templates/gnocchi-api.json.j2 index 2e5749f51c..373cb2154b 100644 --- a/ansible/roles/gnocchi/templates/gnocchi-api.json.j2 +++ b/ansible/roles/gnocchi/templates/gnocchi-api.json.j2 @@ -20,7 +20,19 @@ "dest": "/etc/{{ gnocchi_dir }}/wsgi-gnocchi.conf", "owner": "gnocchi", "perm": "0644" - } + }{% if gnocchi_backend_storage == 'ceph' %}, + { + "source": "{{ container_config_directory }}/ceph.conf", + "dest": "/etc/ceph/ceph.conf", + "owner": "gnocchi", + "perm": "0600" + }, + { + "source": "{{ container_config_directory }}/ceph.client.gnocchi.keyring", + "dest": "/etc/ceph/ceph.client.gnocchi.keyring", + "owner": "gnocchi", + "perm": "0600" + }{% endif %} ] } diff --git a/ansible/roles/gnocchi/templates/gnocchi-metricd.json.j2 b/ansible/roles/gnocchi/templates/gnocchi-metricd.json.j2 index b4bfc3f29b..1d41f8e790 100644 --- a/ansible/roles/gnocchi/templates/gnocchi-metricd.json.j2 +++ b/ansible/roles/gnocchi/templates/gnocchi-metricd.json.j2 @@ -6,6 +6,18 @@ "dest": "/etc/gnocchi/gnocchi.conf", "owner": "gnocchi", "perm": "0600" - } + }{% if gnocchi_backend_storage == 'ceph' %}, + { + "source": "{{ container_config_directory }}/ceph.conf", + "dest": "/etc/ceph/ceph.conf", + "owner": "gnocchi", + "perm": "0600" + }, + { + "source": "{{ container_config_directory }}/ceph.client.gnocchi.keyring", + "dest": "/etc/ceph/ceph.client.gnocchi.keyring", + "owner": "gnocchi", + "perm": "0600" + }{% endif %} ] } diff --git a/ansible/roles/gnocchi/templates/gnocchi-statsd.json.j2 b/ansible/roles/gnocchi/templates/gnocchi-statsd.json.j2 index c2c07f9d1c..141e05f7f9 100644 --- a/ansible/roles/gnocchi/templates/gnocchi-statsd.json.j2 +++ b/ansible/roles/gnocchi/templates/gnocchi-statsd.json.j2 @@ -6,6 +6,18 @@ "dest": "/etc/gnocchi/gnocchi.conf", "owner": "gnocchi", "perm": "0600" - } + }{% if gnocchi_backend_storage == 'ceph' %}, + { + "source": "{{ container_config_directory }}/ceph.conf", + "dest": "/etc/ceph/ceph.conf", + "owner": "gnocchi", + "perm": "0600" + }, + { + "source": "{{ container_config_directory }}/ceph.client.gnocchi.keyring", + "dest": "/etc/ceph/ceph.client.gnocchi.keyring", + "owner": "gnocchi", + "perm": "0600" + }{% endif %} ] } diff --git a/ansible/roles/gnocchi/templates/gnocchi.conf.j2 b/ansible/roles/gnocchi/templates/gnocchi.conf.j2 index 6493343b0b..30d8e33c1c 100644 --- a/ansible/roles/gnocchi/templates/gnocchi.conf.j2 +++ b/ansible/roles/gnocchi/templates/gnocchi.conf.j2 @@ -44,5 +44,13 @@ memcache_secret_key = {{ memcache_secret_key }} memcached_servers = {% for host in groups['memcached'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ memcached_port }}{% if not loop.last %},{% endif %}{% endfor %} [storage] +{% if gnocchi_backend_storage == 'file' %} driver = file file_basepath = /var/lib/gnocchi +{% elif gnocchi_backend_storage == 'ceph' %} +driver = ceph +ceph_pool = {{ gnocchi_pool_name }} +ceph_username = gnocchi +ceph_keyring = /etc/ceph/ceph.client.gnocchi.keyring +ceph_conffile = /etc/ceph/ceph.conf +{% endif %} diff --git a/docker/gnocchi/gnocchi-base/Dockerfile.j2 b/docker/gnocchi/gnocchi-base/Dockerfile.j2 index 213a3a610d..1986ca7033 100644 --- a/docker/gnocchi/gnocchi-base/Dockerfile.j2 +++ b/docker/gnocchi/gnocchi-base/Dockerfile.j2 @@ -13,7 +13,8 @@ MAINTAINER {{ maintainer }} 'python-gnocchiclient', 'httpd', 'mod_wsgi', - 'python-ldappool' + 'python-ldappool', + 'python-rados' ] %} {{ macros.install_packages(gnocchi_base_packages | customizable("packages")) }} @@ -24,7 +25,8 @@ RUN sed -i -r 's,^(Listen 80),#\1,' /etc/httpd/conf/httpd.conf 'gnocchi-common', 'apache2', 'libapache2-mod-wsgi', - 'python-ldappool' + 'python-ldappool', + 'python-rados' ] %} {{ macros.install_packages(gnocchi_base_packages | customizable("packages")) }} @@ -37,7 +39,8 @@ RUN truncate -s 0 /etc/apache2/ports.conf {% set gnocchi_base_packages = [ 'httpd', 'mod_wsgi', - 'python-ldappool' + 'python-ldappool', + 'python-rados' ] %} {{ macros.install_packages(gnocchi_base_packages | customizable("packages")) }} RUN mkdir -p /var/www/cgi-bin/gnocchi \ @@ -47,7 +50,8 @@ RUN mkdir -p /var/www/cgi-bin/gnocchi \ {% set gnocchi_base_packages = [ 'apache2', 'libapache2-mod-wsgi', - 'python-ldappool' + 'python-ldappool', + 'python-rados' ] %} {{ macros.install_packages(gnocchi_base_packages | customizable("packages")) }} @@ -58,7 +62,7 @@ ADD gnocchi-base-archive /gnocchi-base-source RUN ln -s gnocchi-base-source/* gnocchi \ && useradd --user-group gnocchi \ - && /var/lib/kolla/venv/bin/pip --no-cache-dir install --upgrade -c requirements/upper-constraints.txt gnocchiclient /gnocchi[keystone,mysql,file] \ + && /var/lib/kolla/venv/bin/pip --no-cache-dir install --upgrade -c requirements/upper-constraints.txt gnocchiclient /gnocchi[keystone,mysql,file,ceph] \ && mkdir -p /etc/gnocchi /var/log/kolla/gnocchi /home/gnocchi \ && cp -r /gnocchi/etc/gnocchi/* /etc/gnocchi/ \ && chown -R gnocchi: /etc/gnocchi /var/log/kolla/gnocchi diff --git a/etc/kolla/globals.yml b/etc/kolla/globals.yml index bfb3d7d507..be588e7e0d 100644 --- a/etc/kolla/globals.yml +++ b/etc/kolla/globals.yml @@ -192,6 +192,14 @@ kolla_internal_vip_address: "10.10.10.254" # Valid options are [ mongodb, mysql ] #ceilometer_database_type: "mongodb" + +####################### +# Gnocchi options +####################### +# Valid options are [ file, ceph ] +#gnocchi_backend_storage: "{{ 'ceph' if enable_ceph|bool else 'file' }}" + + ####################### # Cinder options ####################### diff --git a/releasenotes/notes/integrate-gnocchi-with-ceph-a6d5f81f4d8b0391.yaml b/releasenotes/notes/integrate-gnocchi-with-ceph-a6d5f81f4d8b0391.yaml new file mode 100644 index 0000000000..da5c95e49d --- /dev/null +++ b/releasenotes/notes/integrate-gnocchi-with-ceph-a6d5f81f4d8b0391.yaml @@ -0,0 +1,3 @@ +--- +fixes: + - Integrates gnocchi with ceph to resolve the the lack of HA.