Implement ceilometer ansible role

Change-Id: I13c50a78b2dc7c98b720a4b2e7161104213bd295
Implements: bp ansible-ceilometer

ansible/group_vars/all.yml

@@ -93,6 +93,8 @@ tunnel_interface: "{{ network_interface }}"
 neutron_plugin_agent: "openvswitch"
 
 # The default ports used by each service.
+ceilometer_api_port: "8777"
+
 iscsi_port: "3260"
 
 mariadb_port: "{{ database_port }}"
@@ -191,6 +193,7 @@ enable_nova: "yes"
 enable_rabbitmq: "yes"
 
 # Additional optional OpenStack services are specified here
+enable_ceilometer: "no"
 enable_central_logging: "no"
 enable_ceph: "no"
 enable_ceph_rgw: "no"

ansible/inventory/all-in-one

@@ -84,6 +84,9 @@ control
 [manila:children]
 control
 
+[ceilometer:children]
+control
+
 # Additional control implemented here. These groups allow you to control which
 # services run on which hosts at a per-service level.
 #
@@ -225,3 +228,19 @@ mistral
 
 [mistral-engine:children]
 mistral
+
+# Ceilometer
+[ceilometer-api:children]
+ceilometer
+
+[ceilometer-central:children]
+ceilometer
+
+[ceilometer-notification:children]
+ceilometer
+
+[ceilometer-collector:children]
+ceilometer
+
+[ceilometer-compute:children]
+compute

ansible/inventory/multinode

@@ -96,6 +96,9 @@ control
 [manila:children]
 control
 
+[ceilometer:children]
+control
+
 # Additional control implemented here. These groups allow you to control which
 # services run on which hosts at a per-service level.
 #
@@ -237,3 +240,19 @@ mistral
 
 [mistral-engine:children]
 mistral
+
+# Ceilometer
+[ceilometer-api:children]
+ceilometer
+
+[ceilometer-central:children]
+ceilometer
+
+[ceilometer-notification:children]
+ceilometer
+
+[ceilometer-collector:children]
+ceilometer
+
+[ceilometer-compute:children]
+compute

ansible/roles/ceilometer/defaults/main.yml

@@ -0,0 +1,50 @@
+---
+project_name: "ceilometer"
+
+
+####################
+# Database
+####################
+ceilometer_database_name: "ceilometer"
+ceilometer_database_user: "ceilometer"
+# TODO(HuiKang): Update this so that connection to mongodb could be through
+#                haproxy; haproxy conf needs configurations for mongodb.
+ceilometer_database_address: "{{ hostvars[groups['mongodb'][0]]['ansible_' + hostvars[groups['mongodb'][0]]['api_interface']]['ipv4']['address'] }}"
+
+
+####################
+# Docker
+####################
+ceilometer_notification_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-ceilometer-notification"
+ceilometer_notification_tag: "{{ openstack_release }}"
+ceilometer_notification_image_full: "{{ ceilometer_notification_image }}:{{ ceilometer_notification_tag }}"
+
+ceilometer_api_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-ceilometer-api"
+ceilometer_api_tag: "{{ openstack_release }}"
+ceilometer_api_image_full: "{{ ceilometer_api_image }}:{{ ceilometer_api_tag }}"
+
+ceilometer_central_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-ceilometer-central"
+ceilometer_central_tag: "{{ openstack_release }}"
+ceilometer_central_image_full: "{{ ceilometer_central_image }}:{{ ceilometer_central_tag }}"
+
+ceilometer_collector_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-ceilometer-collector"
+ceilometer_collector_tag: "{{ openstack_release }}"
+ceilometer_collector_image_full: "{{ ceilometer_collector_image }}:{{ ceilometer_collector_tag }}"
+
+ceilometer_compute_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-ceilometer-compute"
+ceilometer_compute_tag: "{{ openstack_release }}"
+ceilometer_compute_image_full: "{{ ceilometer_compute_image }}:{{ ceilometer_compute_tag }}"
+
+
+####################
+# OpenStack
+####################
+ceilometer_admin_endpoint: "{{ admin_protocol }}://{{ kolla_internal_fqdn }}:{{ ceilometer_api_port }}"
+ceilometer_internal_endpoint: "{{ internal_protocol }}://{{ kolla_internal_fqdn }}:{{ ceilometer_api_port }}"
+ceilometer_public_endpoint: "{{ public_protocol }}://{{ kolla_external_fqdn }}:{{ ceilometer_api_port }}"
+
+ceilometer_logging_debug: "{{ openstack_logging_debug }}"
+
+ceilometer_keystone_user: "ceilometer"
+
+openstack_ceilometer_auth: "{'auth_url':'{{ openstack_auth.auth_url }}','username':'{{ openstack_auth.username }}','password':'{{ openstack_auth.password }}','project_name':'{{ openstack_auth.project_name }}','domain_name':'default'}"

ansible/roles/ceilometer/meta/main.yml

@@ -0,0 +1,3 @@
+---
+dependencies:
+  - { role: common, project_yaml: 'ceilometer.yml'  }

ansible/roles/ceilometer/tasks/bootstrap.yml

@@ -0,0 +1,11 @@
+---
+- name: Creating Ceilometer database
+  command: docker exec -t mongodb mongo --host {{ ceilometer_database_address }} --eval 'db = db.getSiblingDB("{{ ceilometer_database_name }}"); db.addUser({user':' "{{ ceilometer_database_user }}", pwd':' "{{ ceilometer_database_password}}", roles':' [ "readWrite", "dbAdmin" ]})'
+  register: database
+  run_once: true
+  failed_when:
+    - "'already' not in database.stdout"
+    - database.stdout.split()[4] != 'connecting'
+  delegate_to: "{{ groups['mongodb'][0] }}"
+
+- include: bootstrap_service.yml

ansible/roles/ceilometer/tasks/bootstrap_service.yml

@@ -0,0 +1,19 @@
+- name: Running Ceilometer bootstrap container
+  kolla_docker:
+    action: "start_container"
+    common_options: "{{ docker_common_options }}"
+    detach: False
+    environment:
+      KOLLA_BOOTSTRAP:
+      KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
+    image: "{{ ceilometer_api_image_full }}"
+    labels:
+      BOOTSTRAP:
+    name: "bootstrap_ceilometer"
+    restart_policy: "never"
+    volumes:
+      - "{{ node_config_directory }}/ceilometer-api/:{{ container_config_directory }}/:ro"
+      - "ceilometer:/var/lib/ceilometer/"
+      - "kolla_logs:/var/log/kolla/"
+  run_once: True
+  delegate_to: "{{ groups['ceilometer-api'][0] }}"

ansible/roles/ceilometer/tasks/config.yml

@@ -0,0 +1,41 @@
+- name: Ensuring config directories exist
+  file:
+    path: "{{ node_config_directory }}/{{ item }}"
+    state: "directory"
+    recurse: yes
+  with_items:
+    - "ceilometer-notification"
+    - "ceilometer-collector"
+    - "ceilometer-api"
+    - "ceilometer-central"
+    - "ceilometer-compute"
+
+- name: Copying over config.json files for services
+  template:
+    src: "{{ item }}.json.j2"
+    dest: "{{ node_config_directory }}/{{ item }}/config.json"
+  with_items:
+    - "ceilometer-notification"
+    - "ceilometer-collector"
+    - "ceilometer-api"
+    - "ceilometer-central"
+    - "ceilometer-compute"
+
+- name: Copying over ceilometer.conf
+  merge_configs:
+    vars:
+      service_name: "{{ item }}"
+    sources:
+      - "{{ role_path }}/templates/ceilometer.conf.j2"
+      - "/etc/kolla/config/global.conf"
+      - "/etc/kolla/config/database.conf"
+      - "/etc/kolla/config/messaging.conf"
+      - "/etc/kolla/config/glance.conf"
+      - "/etc/kolla/config/glance/{{ item }}.conf"
+    dest: "{{ node_config_directory }}/{{ item }}/ceilometer.conf"
+  with_items:
+    - "ceilometer-notification"
+    - "ceilometer-collector"
+    - "ceilometer-api"
+    - "ceilometer-central"
+    - "ceilometer-compute"

ansible/roles/ceilometer/tasks/deploy.yml

@@ -0,0 +1,14 @@
+---
+- include: register.yml
+  when: inventory_hostname in groups['ceilometer']
+
+- include: config.yml
+  when: inventory_hostname in groups['ceilometer'] or
+        inventory_hostname in groups['compute']
+
+- include: bootstrap.yml
+  when: inventory_hostname in groups['ceilometer']
+
+- include: start.yml
+  when: inventory_hostname in groups['ceilometer'] or
+        inventory_hostname in groups['compute']

ansible/roles/ceilometer/tasks/main.yml

@@ -0,0 +1,2 @@
+---
+- include: "{{ action }}.yml"

ansible/roles/ceilometer/tasks/register.yml

@@ -0,0 +1,40 @@
+---
+- name: Creating the Ceilometer service and endpoint
+  command: docker exec -t kolla_toolbox /usr/bin/ansible localhost
+    -m kolla_keystone_service
+    -a "service_name=ceilometer
+        service_type=metering
+        description='Openstack Telemetry'
+        endpoint_region={{ openstack_region_name }}
+        url='{{ item.url }}'
+        interface='{{ item.interface }}'
+        region_name={{ openstack_region_name }}
+        auth={{ '{{ openstack_ceilometer_auth }}' }}"
+    -e "{'openstack_ceilometer_auth':{{ openstack_ceilometer_auth }}}"
+  register: ceilometer_endpoint
+  changed_when: "{{ ceilometer_endpoint.stdout.find('localhost | SUCCESS => ') != -1 and (ceilometer_endpoint.stdout.split('localhost | SUCCESS => ')[1]|from_json).changed }}"
+  until: ceilometer_endpoint.stdout.split()[2] == 'SUCCESS'
+  retries: 10
+  delay: 5
+  run_once: True
+  with_items:
+    - {'interface': 'admin', 'url': '{{ ceilometer_admin_endpoint }}'}
+    - {'interface': 'internal', 'url': '{{ ceilometer_internal_endpoint }}'}
+    - {'interface': 'public', 'url': '{{ ceilometer_public_endpoint }}'}
+
+- name: Creating the Ceilometer project, user, and role
+  command: docker exec -t kolla_toolbox /usr/bin/ansible localhost
+    -m kolla_keystone_user
+    -a "project=service
+        user=ceilometer
+        password={{ ceilometer_keystone_password }}
+        role=admin
+        region_name={{ openstack_region_name }}
+        auth={{ '{{ openstack_ceilometer_auth }}' }}"
+    -e "{'openstack_ceilometer_auth':{{ openstack_ceilometer_auth }}}"
+  register: ceilometer_user
+  changed_when: "{{ ceilometer_user.stdout.find('localhost | SUCCESS => ') != -1 and (ceilometer_user.stdout.split('localhost | SUCCESS => ')[1]|from_json).changed }}"
+  until: ceilometer_user.stdout.split()[2] == 'SUCCESS'
+  retries: 10
+  delay: 5
+  run_once: True

ansible/roles/ceilometer/tasks/start.yml

@@ -0,0 +1,61 @@
+- name: Starting ceilometer-notification container
+  kolla_docker:
+    action: "start_container"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ ceilometer_notification_image_full }}"
+    name: "ceilometer_notification"
+    volumes:
+      - "{{ node_config_directory }}/ceilometer-notification/:{{ container_config_directory }}/:ro"
+      - "kolla_logs:/var/log/kolla/"
+  when: inventory_hostname in groups['ceilometer-notification']
+
+- name: Starting ceilometer-api container
+  kolla_docker:
+    action: "start_container"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ ceilometer_api_image_full }}"
+    name: "ceilometer_api"
+    volumes:
+      - "{{ node_config_directory }}/ceilometer-api/:{{ container_config_directory }}/:ro"
+      - "ceilometer:/var/lib/ceilometer/"
+      - "kolla_logs:/var/log/kolla/"
+  when: inventory_hostname in groups['ceilometer-api']
+
+- name: Starting ceilometer-central container
+  kolla_docker:
+    action: "start_container"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ ceilometer_central_image_full }}"
+    name: "ceilometer_central"
+    volumes:
+      - "{{ node_config_directory }}/ceilometer-central/:{{ container_config_directory }}/:ro"
+      - "ceilometer:/var/lib/ceilometer/"
+      - "kolla_logs:/var/log/kolla/"
+  when: inventory_hostname in groups['ceilometer-central']
+
+- name: Starting ceilometer-collector container
+  kolla_docker:
+    action: "start_container"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ ceilometer_collector_image_full }}"
+    name: "ceilometer_collector"
+    volumes:
+      - "{{ node_config_directory }}/ceilometer-collector/:{{ container_config_directory }}/:ro"
+      - "ceilometer:/var/lib/ceilometer/"
+      - "kolla_logs:/var/log/kolla/"
+  when: inventory_hostname in groups['ceilometer-collector']
+
+- name: Starting ceilometer-compute container
+  kolla_docker:
+    action: "start_container"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ ceilometer_compute_image_full }}"
+    name: "ceilometer_compute"
+    privileged: True
+    volumes:
+      - "{{ node_config_directory }}/ceilometer-compute/:{{ container_config_directory }}/:ro"
+      - "ceilometer:/var/lib/ceilometer/"
+      - "kolla_logs:/var/log/kolla/"
+      - "/run/:/run/"
+      - "nova_libvirt:/var/lib/libvirt"
+  when: inventory_hostname in groups['ceilometer-compute']

ansible/roles/ceilometer/templates/ceilometer-api.json.j2

@@ -0,0 +1,11 @@
+{
+    "command": "ceilometer-api",
+    "config_files": [
+        {
+            "source": "{{ container_config_directory }}/ceilometer.conf",
+            "dest": "/etc/ceilometer/ceilometer.conf",
+            "owner": "ceilometer",
+            "perm": "0600"
+        }
+    ]
+}

ansible/roles/ceilometer/templates/ceilometer-central.json.j2

@@ -0,0 +1,11 @@
+{
+    "command": "ceilometer-polling --polling-namespaces central",
+    "config_files": [
+        {
+            "source": "{{ container_config_directory }}/ceilometer.conf",
+            "dest": "/etc/ceilometer/ceilometer.conf",
+            "owner": "ceilometer",
+            "perm": "0600"
+        }
+    ]
+}

ansible/roles/ceilometer/templates/ceilometer-collector.json.j2

@@ -0,0 +1,11 @@
+{
+    "command": "ceilometer-collector",
+    "config_files": [
+        {
+            "source": "{{ container_config_directory }}/ceilometer.conf",
+            "dest": "/etc/ceilometer/ceilometer.conf",
+            "owner": "ceilometer",
+            "perm": "0600"
+        }
+    ]
+}

ansible/roles/ceilometer/templates/ceilometer-compute.json.j2

@@ -0,0 +1,11 @@
+{
+    "command": "ceilometer-polling --polling-namespaces compute",
+    "config_files": [
+        {
+            "source": "{{ container_config_directory }}/ceilometer.conf",
+            "dest": "/etc/ceilometer/ceilometer.conf",
+            "owner": "ceilometer",
+            "perm": "0600"
+        }
+    ]
+}

ansible/roles/ceilometer/templates/ceilometer-notification.json.j2

@@ -0,0 +1,11 @@
+{
+    "command": "ceilometer-agent-notification",
+    "config_files": [
+        {
+            "source": "{{ container_config_directory }}/ceilometer.conf",
+            "dest": "/etc/ceilometer/ceilometer.conf",
+            "owner": "ceilometer",
+            "perm": "0600"
+        }
+    ]
+}

ansible/roles/ceilometer/templates/ceilometer.conf.j2

@@ -0,0 +1,40 @@
+[DEFAULT]
+debug = {{ ceilometer_logging_debug }}
+
+log_dir = /var/log/kolla/ceilometer
+rpc_backend = rabbit
+
+[collector]
+
+[database]
+connection = mongodb://{{ ceilometer_database_user }}:{{ ceilometer_database_password }}@{{ ceilometer_database_address }}:{{ mongodb_port }}/{{ ceilometer_database_name }}
+
+[keystone_authtoken]
+memcached_servers = {{ kolla_internal_fqdn }}:{{ memcached_port }}
+auth_uri = {{ internal_protocol }}://{{ kolla_internal_fqdn }}:{{ keystone_public_port }}
+project_domain_name = Default
+project_name = service
+user_domain_name = Default
+username = {{ ceilometer_keystone_user }}
+password = {{ ceilometer_keystone_password }}
+auth_url = {{ admin_protocol }}://{{ kolla_internal_fqdn }}:{{ keystone_admin_port }}
+auth_type = password
+
+[notification]
+store_events = true
+
+[oslo_messaging_rabbit]
+rabbit_userid = {{ rabbitmq_user }}
+rabbit_password = {{ rabbitmq_password }}
+rabbit_ha_queues = true
+rabbit_hosts = {% for host in groups['rabbitmq'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ rabbitmq_port }}{% if not loop.last %},{% endif %}{% endfor %}
+
+[service_credentials]
+auth_url = {{ internal_protocol  }}://{{ kolla_internal_fqdn }}:{{ keystone_public_port }}/v3
+region_name = {{ openstack_region_name }}
+password = {{ ceilometer_keystone_password }}
+username = {{ ceilometer_keystone_user }}
+project_name = service
+project_domain_id = default
+user_domain_id = default
+auth_type = password

ansible/roles/glance/templates/glance-api.conf.j2

@@ -51,4 +51,13 @@ filesystem_store_datadir = /var/lib/glance/images/
 {% endif %}
 
 [oslo_messaging_notifications]
+{% if enable_ceilometer | bool %}
+driver = messagingv2
+
+[oslo_messaging_rabbit]
+rabbit_host = {% for host in groups['rabbitmq'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ rabbitmq_port }}{% if not loop.last %},{% endif %}{% endfor %}
+rabbit_userid = {{ rabbitmq_user }}
+rabbit_password = {{ rabbitmq_password }}
+{% else %}
 driver = noop
+{% endif %}

ansible/roles/glance/templates/glance-registry.conf.j2

@@ -7,7 +7,6 @@ log_file = /var/log/kolla/glance/registry.log
 bind_host = {{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}
 bind_port = {{ glance_registry_port }}
 
-
 [database]
 connection = mysql+pymysql://{{ glance_database_user }}:{{ glance_database_password }}@{{ glance_database_address }}/{{ glance_database_name }}
 max_retries = -1
@@ -31,4 +30,13 @@ memcached_servers = {% for host in groups['memcached'] %}{{ hostvars[host]['ansi
 flavor = keystone
 
 [oslo_messaging_notifications]
+{% if enable_ceilometer | bool %}
+driver = messagingv2
+
+[oslo_messaging_rabbit]
+rabbit_host = {% for host in groups['rabbitmq'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ rabbitmq_port }}{% if not loop.last %},{% endif %}{% endfor %}
+rabbit_userid = {{ rabbitmq_user }}
+rabbit_password = {{ rabbitmq_password }}
+{% else %}
 driver = noop
+{% endif %}

ansible/roles/nova/templates/nova.conf.j2

@@ -54,6 +54,13 @@ compute_driver = libvirt.LibvirtDriver
 # Though my_ip is not used directly, lots of other variables use $my_ip
 my_ip = {{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}
 
+{% if enable_ceilometer | bool %}
+instance_usage_audit = True
+instance_usage_audit_period = hour
+notify_on_state_change = vm_and_task_state
+notification_driver = messagingv2
+{% endif %}
+
 {% if nova_console == 'novnc' %}
 [vnc]
 novncproxy_host = {{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}
@@ -165,4 +172,9 @@ hw_disk_discard = unmap
 compute = auto
 
 [oslo_messaging_notifications]
+{% if enable_ceilometer | bool %}
+driver = messagingv2
+topics = notifications
+{% else %}
 driver = noop
+{% endif %}

ansible/site.yml

@@ -222,3 +222,11 @@
     - { role: manila,
         tags: manila,
         when: enable_manila | bool }
+
+- hosts:
+    - ceilometer
+    - compute
+  roles:
+    - { role: ceilometer,
+        tags: ceilometer,
+        when: enable_ceilometer | bool }

docker/ceilometer/ceilometer-api/extend_start.sh

@@ -3,7 +3,7 @@
 # Bootstrap and exit if KOLLA_BOOTSTRAP variable is set. This catches all cases
 # of the KOLLA_BOOTSTRAP variable being set, including empty.
 if [[ "${!KOLLA_BOOTSTRAP[@]}" ]]; then
-    ceilometer-manage db_sync
+    ceilometer-dbsync
     sudo chown -R ceilometer: /var/lib/ceilometer/
     exit 0
 fi

docker/ceilometer/ceilometer-compute/Dockerfile.j2

@@ -38,4 +38,6 @@ RUN /var/lib/kolla/venv/bin/pip --no-cache-dir install --upgrade -c requirements
 
 {{ include_footer }}
 
-USER ceilometer
+# NOTE(huikang): non-root user does not work with libvirt.sock.
+#                Need configuration in nova-libvirt container to allow
+#                USER ceilometer to access libvirt.sock

etc/kolla/globals.yml

@@ -106,6 +106,7 @@ neutron_external_interface: "eth1"
 #nova_console: "novnc"
 
 # OpenStack services can be enabled or disabled with these options
+#enable_ceilometer: "no"
 #enable_central_logging: "no"
 #enable_ceph: "no"
 #enable_ceph_rgw: "no"

etc/kolla/passwords.yml

@@ -58,8 +58,13 @@ magnum_keystone_password:
 mistral_database_password:
 mistral_keystone_password:
 
+ceilometer_database_password:
+ceilometer_keystone_password:
+
 horizon_secret_key:
 
+telemetry_secret_key:
+
 manila_database_password:
 manila_keystone_password: