Add Watcher ansible roles and templates

Previous work on Watcher added the Docker images, this
change adds the ansible configuration.

There is support for HA, via haproxy to balance across the
Watcher API hosts.

There is also a hook into nova.conf to conditionally add
Nova compute Host metrics via Ceilometer if Watcher is enabled.

This defaults to enabled false.

Change-Id: I8763528bb6ff12943b810212c71396d2d7cf6836
Partial-bug: #1598929
Partially-implements: bp watcher
Signed-off-by: Dave Walker (Daviey) <email@daviey.com>

ansible/group_vars/all.yml

@@ -154,6 +154,8 @@ elasticsearch_port: "9200"
 
 manila_api_port: "8786"
 
+watcher_api_port: "9322"
+
 public_protocol: "{{ 'https' if kolla_enable_tls_external | bool else 'http' }}"
 internal_protocol: "http"
 admin_protocol: "http"
@@ -211,6 +213,7 @@ enable_neutron_lbaas: "no"
 enable_neutron_qos: "no"
 enable_swift: "no"
 enable_tempest: "no"
+enable_watcher: "no"
 
 ironic_keystone_user: "ironic"
 neutron_keystone_user: "neutron"

ansible/roles/common/tasks/config.yml

@@ -70,3 +70,4 @@
     - "nova"
     - "rabbitmq"
     - "swift"
+    - "watcher"

ansible/roles/common/templates/cron-logrotate-watcher.conf.j2

@@ -0,0 +1,3 @@
+"/var/log/kolla/watcher/*.log"
+{
+}

ansible/roles/haproxy/templates/haproxy.cfg.j2

@@ -332,6 +332,22 @@ listen magnum_api_external
 {% endif %}
 {% endif %}
 
+{% if enable_watcher | bool and enable_ceilometer | bool %}
+listen watcher_api
+  bind {{ kolla_internal_vip_address }}:{{ watcher_api_port }}
+{% for host in groups['watcher-api'] %}
+  server {{ hostvars[host]['ansible_hostname'] }} {{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ watcher_api_port }} check inter 2000 rise 2 fall 5
+{% endfor %}
+{% if haproxy_enable_external_vip | bool %}
+
+listen watcher_api_external
+  bind {{ kolla_external_vip_address }}:{{ watcher_api_port }} {{ tls_bind_info }}
+{% for host in groups['watcher-api'] %}
+  server {{ hostvars[host]['ansible_hostname'] }} {{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ watcher_api_port }} check inter 2000 rise 2 fall 5
+{% endfor %}
+{% endif %}
+{% endif %}
+
 {% if enable_ceph | bool and enable_ceph_rgw | bool %}
 listen radosgw
   bind {{ kolla_internal_vip_address }}:{{ rgw_port }}

ansible/roles/nova/templates/nova.conf.j2

@@ -55,6 +55,9 @@ my_ip = {{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['add
 instance_usage_audit = True
 instance_usage_audit_period = hour
 notify_on_state_change = vm_and_task_state
+{% if enable_watcher | bool %}
+compute_monitors=nova.compute.monitors.cpu.virt_driver
+{% endif %}
 {% endif %}
 
 {% if nova_console == 'novnc' %}

ansible/roles/watcher/defaults/main.yml

@@ -0,0 +1,38 @@
+---
+project_name: "watcher"
+
+####################
+# Database
+####################
+watcher_database_name: "watcher"
+watcher_database_user: "watcher"
+watcher_database_address: "{{ kolla_internal_fqdn }}:{{ database_port }}"
+
+
+####################
+# Docker
+####################
+watcher_engine_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-watcher-engine"
+watcher_engine_tag: "{{ openstack_release }}"
+watcher_engine_image_full: "{{ watcher_engine_image }}:{{ watcher_engine_tag }}"
+
+watcher_api_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-watcher-api"
+watcher_api_tag: "{{ openstack_release }}"
+watcher_api_image_full: "{{ watcher_api_image }}:{{ watcher_api_tag }}"
+
+watcher_applier_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-watcher-applier"
+watcher_applier_tag: "{{ openstack_release }}"
+watcher_applier_image_full: "{{ watcher_applier_image }}:{{ watcher_applier_tag }}"
+
+####################
+# OpenStack
+####################
+watcher_admin_endpoint: "{{ admin_protocol }}://{{ kolla_internal_fqdn }}:{{ watcher_api_port }}"
+watcher_internal_endpoint: "{{ internal_protocol }}://{{ kolla_internal_fqdn }}:{{ watcher_api_port }}"
+watcher_public_endpoint: "{{ public_protocol }}://{{ kolla_external_fqdn }}:{{ watcher_api_port }}"
+
+watcher_logging_debug: "{{ openstack_logging_debug }}"
+
+watcher_keystone_user: "watcher"
+
+openstack_watcher_auth: "{'auth_url':'{{ openstack_auth.auth_url }}','username':'{{ openstack_auth.username }}','password':'{{ openstack_auth.password }}','project_name':'{{ openstack_auth.project_name }}'}"

ansible/roles/watcher/meta/main.yml

@@ -0,0 +1,3 @@
+---
+dependencies:
+  - { role: common }

ansible/roles/watcher/tasks/bootstrap.yml

@@ -0,0 +1,41 @@
+---
+- name: Creating Watcher database
+  command: docker exec -t kolla_toolbox /usr/bin/ansible localhost
+    -m mysql_db
+    -a "login_host='{{ database_address }}'
+        login_port='{{ database_port }}'
+        login_user='{{ database_user }}'
+        login_password='{{ database_password }}'
+        name='{{ watcher_database_name }}'"
+  register: database
+  changed_when: "{{ database.stdout.find('localhost | SUCCESS => ') != -1 and
+                    (database.stdout.split('localhost | SUCCESS => ')[1]|from_json).changed }}"
+  failed_when: database.stdout.split()[2] != 'SUCCESS'
+  run_once: True
+  delegate_to: "{{ groups['watcher-api'][0] }}"
+
+- name: Reading json from variable
+  set_fact:
+    database_created: "{{ (database.stdout.split('localhost | SUCCESS => ')[1]|from_json).changed }}"
+
+- name: Creating Watcher database user and setting permissions
+  command: docker exec -t kolla_toolbox /usr/bin/ansible localhost
+    -m mysql_user
+    -a "login_host='{{ database_address }}'
+        login_port='{{ database_port }}'
+        login_user='{{ database_user }}'
+        login_password='{{ database_password }}'
+        name='{{ watcher_database_name }}'
+        password='{{ watcher_database_password }}'
+        host='%'
+        priv='{{ watcher_database_name }}.*:ALL'
+        append_privs='yes'"
+  register: database_user_create
+  changed_when: "{{ database_user_create.stdout.find('localhost | SUCCESS => ') != -1 and
+                    (database_user_create.stdout.split('localhost | SUCCESS => ')[1]|from_json).changed }}"
+  failed_when: database_user_create.stdout.split()[2] != 'SUCCESS'
+  run_once: True
+  delegate_to: "{{ groups['watcher-api'][0] }}"
+
+- include: bootstrap_service.yml
+  when: database_created

ansible/roles/watcher/tasks/bootstrap_service.yml

@@ -0,0 +1,20 @@
+---
+- name: Running Watcher bootstrap container
+  kolla_docker:
+    action: "start_container"
+    common_options: "{{ docker_common_options }}"
+    detach: False
+    environment:
+      KOLLA_BOOTSTRAP:
+      KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
+    image: "{{ watcher_api_image_full }}"
+    labels:
+      BOOTSTRAP:
+    name: "bootstrap_watcher"
+    restart_policy: "never"
+    volumes:
+      - "{{ node_config_directory }}/watcher-api/:{{ container_config_directory }}/:ro"
+      - "/etc/localtime:/etc/localtime:ro"
+      - "kolla_logs:/var/log/kolla/"
+  run_once: True
+  delegate_to: "{{ groups['watcher-api'][0] }}"

ansible/roles/watcher/tasks/config.yml

@@ -0,0 +1,37 @@
+---
+- name: Ensuring config directories exist
+  file:
+    path: "{{ node_config_directory }}/{{ item }}"
+    state: "directory"
+    recurse: yes
+  with_items:
+    - "watcher-api"
+    - "watcher-engine"
+    - "watcher-applier"
+
+- name: Copying over config.json files for services
+  template:
+    src: "{{ item }}.json.j2"
+    dest: "{{ node_config_directory }}/{{ item }}/config.json"
+  with_items:
+    - "watcher-api"
+    - "watcher-engine"
+    - "watcher-applier"
+
+- name: Copying over watcher.conf
+  merge_configs:
+    vars:
+      service_name: "{{ item }}"
+    sources:
+      - "{{ role_path }}/templates/watcher.conf.j2"
+      - "{{ node_config_directory }}/config/global.conf"
+      - "{{ node_config_directory }}/config/database.conf"
+      - "{{ node_config_directory }}/config/messaging.conf"
+      - "{{ node_config_directory }}/config/watcher.conf"
+      - "{{ node_config_directory }}/config/watcher/{{ item }}.conf"
+      - "{{ node_config_directory }}/config/watcher/{{ inventory_hostname }}/watcher.conf"
+    dest: "{{ node_config_directory }}/{{ item }}/watcher.conf"
+  with_items:
+    - "watcher-api"
+    - "watcher-engine"
+    - "watcher-applier"

ansible/roles/watcher/tasks/deploy.yml

@@ -0,0 +1,16 @@
+---
+- include: register.yml
+  when: inventory_hostname in groups['watcher-api']
+
+- include: config.yml
+  when: inventory_hostname in groups['watcher-api'] or
+        inventory_hostname in groups['watcher-engine'] or
+        inventory_hostname in groups['watcher-applier']
+
+- include: bootstrap.yml
+  when: inventory_hostname in groups['watcher-api']
+
+- include: start.yml
+  when: inventory_hostname in groups['watcher-api'] or
+        inventory_hostname in groups['watcher-engine'] or
+        inventory_hostname in groups['watcher-applier']

ansible/roles/watcher/tasks/do_reconfigure.yml

@@ -0,0 +1,71 @@
+---
+- name: Ensuring the containers up
+  kolla_docker:
+    name: "{{ item.name }}"
+    action: "get_container_state"
+  register: container_state
+  failed_when: container_state.Running == false
+  when: inventory_hostname in groups[item.group]
+  with_items:
+    - { name: watcher_api, group: watcher-api }
+    - { name: watcher_engine, group: watcher-engine }
+    - { name: watcher_applier, group: watcher-applier }
+
+- include: config.yml
+
+- name: Check the configs
+  command: docker exec {{ item.name }} /usr/local/bin/kolla_set_configs --check
+  changed_when: false
+  failed_when: false
+  register: check_results
+  when: inventory_hostname in groups[item.group]
+  with_items:
+    - { name: watcher_api, group: watcher-api }
+    - { name: watcher_engine, group: watcher-engine }
+    - { name: watcher_applier, group: watcher-applier }
+
+- name: Containers config strategy
+  kolla_docker:
+    name: "{{ item.name }}"
+    action: "get_container_env"
+  register: container_envs
+  when: inventory_hostname in groups[item.group]
+  with_items:
+    - { name: watcher_api, group: watcher-api }
+    - { name: watcher_engine, group: watcher-engine }
+    - { name: watcher_applier, group: watcher-applier }
+
+- name: Remove the containers
+  kolla_docker:
+    name: "{{ item[0]['name'] }}"
+    action: "remove_container"
+  register: remove_containers
+  when:
+    - inventory_hostname in groups[item[0]['group']]
+    - config_strategy == "COPY_ONCE" or item[1]['KOLLA_CONFIG_STRATEGY'] == 'COPY_ONCE'
+    - item[2]['rc'] == 1
+  with_together:
+    - [{ name: watcher_api, group: watcher-api },
+       { name: watcher_engine, group: watcher-engine },
+       { name: watcher_applier, group: watcher-applier }]
+    - container_envs.results
+    - check_results.results
+
+- include: start.yml
+  when: remove_containers.changed
+
+- name: Restart containers
+  kolla_docker:
+    name: "{{ item[0]['name'] }}"
+    action: "restart_container"
+  when:
+    - config_strategy == 'COPY_ALWAYS'
+    - item[1]['KOLLA_CONFIG_STRATEGY'] != 'COPY_ONCE'
+    - item[2]['rc'] == 1
+    - inventory_hostname in groups[item[0]['group']]
+  with_together:
+    - [{ name: watcher_api, group: watcher-api },
+       { name: watcher_engine, group: watcher-engine },
+       { name: watcher_applier, group: watcher-applier }]
+    - container_envs.results
+    - check_results.results

ansible/roles/watcher/tasks/main.yml

@@ -0,0 +1,2 @@
+---
+- include: "{{ action }}.yml"

ansible/roles/watcher/tasks/pull.yml

@@ -0,0 +1,21 @@
+---
+- name: Pulling watcher-api image
+  kolla_docker:
+    action: "pull_image"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ watcher_api_image_full }}"
+  when: inventory_hostname in groups['watcher-api']
+
+- name: Pulling watcher-engine image
+  kolla_docker:
+    action: "pull_image"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ watcher_engine_image_full }}"
+  when: inventory_hostname in groups['watcher-engine']
+
+- name: Pulling watcher-applier image
+  kolla_docker:
+    action: "pull_image"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ watcher_applier_image_full }}"
+  when: inventory_hostname in groups['watcher-applier']

ansible/roles/watcher/tasks/reconfigure.yml

@@ -0,0 +1,6 @@
+---
+- include: do_reconfigure.yml
+  serial: "30%"
+  when: inventory_hostname in groups['watcher-api']
+        or inventory_hostname in groups['watcher-engine']
+        or inventory_hostname in groups['watcher-applier']

ansible/roles/watcher/tasks/register.yml

@@ -0,0 +1,40 @@
+---
+- name: Creating the Watcher service and endpoint
+  command: docker exec -t kolla_toolbox /usr/bin/ansible localhost
+    -m kolla_keystone_service
+    -a "service_name=watcher
+        service_type=infra-optim
+        description='Infrastructure Optimization service'
+        endpoint_region={{ openstack_region_name }}
+        url='{{ item.url }}'
+        interface='{{ item.interface }}'
+        region_name={{ openstack_region_name }}
+        auth={{ '{{ openstack_watcher_auth }}' }}"
+    -e "{'openstack_watcher_auth':{{ openstack_watcher_auth }}}"
+  register: watcher_endpoint
+  changed_when: "{{ watcher_endpoint.stdout.find('localhost | SUCCESS => ') != -1 and (watcher_endpoint.stdout.split('localhost | SUCCESS => ')[1]|from_json).changed }}"
+  until: watcher_endpoint.stdout.split()[2] == 'SUCCESS'
+  retries: 10
+  delay: 5
+  run_once: True
+  with_items:
+    - {'interface': 'admin', 'url': '{{ watcher_admin_endpoint }}'}
+    - {'interface': 'internal', 'url': '{{ watcher_internal_endpoint }}'}
+    - {'interface': 'public', 'url': '{{ watcher_public_endpoint }}'}
+
+- name: Creating the Watcher project, user, and role
+  command: docker exec -t kolla_toolbox /usr/bin/ansible localhost
+    -m kolla_keystone_user
+    -a "project=service
+        user=watcher
+        password={{ watcher_keystone_password }}
+        role=admin
+        region_name={{ openstack_region_name }}
+        auth={{ '{{ openstack_watcher_auth }}' }}"
+    -e "{'openstack_watcher_auth':{{ openstack_watcher_auth }}}"
+  register: watcher_user
+  changed_when: "{{ watcher_user.stdout.find('localhost | SUCCESS => ') != -1 and (watcher_user.stdout.split('localhost | SUCCESS => ')[1]|from_json).changed }}"
+  until: watcher_user.stdout.split()[2] == 'SUCCESS'
+  retries: 10
+  delay: 5
+  run_once: True

ansible/roles/watcher/tasks/start.yml

@@ -0,0 +1,36 @@
+---
+- name: Starting watcher-applier container
+  kolla_docker:
+    action: "start_container"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ watcher_applier_image_full }}"
+    name: "watcher_applier"
+    volumes:
+      - "{{ node_config_directory }}/watcher-applier/:{{ container_config_directory }}/:ro"
+      - "/etc/localtime:/etc/localtime:ro"
+      - "kolla_logs:/var/log/kolla/"
+  when: inventory_hostname in groups['watcher-applier']
+
+- name: Starting watcher-engine container
+  kolla_docker:
+    action: "start_container"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ watcher_engine_image_full }}"
+    name: "watcher_engine"
+    volumes:
+      - "{{ node_config_directory }}/watcher-engine/:{{ container_config_directory }}/:ro"
+      - "/etc/localtime:/etc/localtime:ro"
+      - "kolla_logs:/var/log/kolla/"
+  when: inventory_hostname in groups['watcher-engine']
+
+- name: Starting watcher-api container
+  kolla_docker:
+    action: "start_container"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ watcher_api_image_full }}"
+    name: "watcher_api"
+    volumes:
+      - "{{ node_config_directory }}/watcher-api/:{{ container_config_directory }}/:ro"
+      - "/etc/localtime:/etc/localtime:ro"
+      - "kolla_logs:/var/log/kolla/"
+  when: inventory_hostname in groups['watcher-api']

ansible/roles/watcher/tasks/upgrade.yml

@@ -0,0 +1,7 @@
+---
+- include: config.yml
+
+- include: bootstrap_service.yml
+
+- include: start.yml
+  serial: "30%"

ansible/roles/watcher/templates/watcher-api.json.j2

@@ -0,0 +1,11 @@
+{
+    "command": "watcher-api --config-file /etc/watcher/watcher.conf",
+    "config_files": [
+        {
+            "source": "{{ container_config_directory }}/watcher.conf",
+            "dest": "/etc/watcher/watcher.conf",
+            "owner": "watcher",
+            "perm": "0644"
+        }
+    ]
+}

ansible/roles/watcher/templates/watcher-applier.json.j2

@@ -0,0 +1,11 @@
+{
+    "command": "watcher-applier --config-file /etc/watcher/watcher.conf",
+    "config_files": [
+        {
+            "source": "{{ container_config_directory }}/watcher.conf",
+            "dest": "/etc/watcher/watcher.conf",
+            "owner": "watcher",
+            "perm": "0644"
+        }
+    ]
+}

ansible/roles/watcher/templates/watcher-engine.json.j2

@@ -0,0 +1,11 @@
+{
+    "command": "watcher-decision-engine --config-file /etc/watcher/watcher.conf",
+    "config_files": [
+        {
+            "source": "{{ container_config_directory }}/watcher.conf",
+            "dest": "/etc/watcher/watcher.conf",
+            "owner": "watcher",
+            "perm": "0644"
+        }
+    ]
+}

ansible/roles/watcher/templates/watcher.conf.j2

@@ -0,0 +1,47 @@
+[DEFAULT]
+debug = {{ watcher_logging_debug }}
+
+log_dir = /var/log/kolla/watcher
+
+{% if service_name == 'watcher-api' %}
+[api]
+host = {{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}
+port = {{ watcher_api_port }}
+{% endif %}
+
+[database]
+connection = mysql+pymysql://{{ watcher_database_user }}:{{ watcher_database_password }}@{{ watcher_database_address}}/{{ watcher_database_name }}
+max_retries = -1
+
+[keystone_authtoken]
+auth_uri = {{ internal_protocol }}://{{ kolla_internal_fqdn }}:{{ keystone_public_port }}
+auth_url = {{ admin_protocol }}://{{ kolla_internal_fqdn }}:{{ keystone_admin_port }}
+auth_type = password
+project_domain_id = default
+user_domain_id = default
+project_name = service
+username = {{ watcher_keystone_user }}
+password = {{ watcher_keystone_password }}
+
+memcache_security_strategy = ENCRYPT
+memcache_secret_key = {{ memcache_secret_key }}
+memcached_servers = {% for host in groups['memcached'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ memcached_port }}{% if not loop.last %},{% endif %}{% endfor %}
+
+[watcher_clients_auth]
+auth_uri = {{ internal_protocol }}://{{ kolla_internal_fqdn }}:{{ keystone_public_port }}
+auth_url = {{ admin_protocol }}://{{ kolla_internal_fqdn }}:{{ keystone_admin_port }}
+auth_type = password
+project_domain_id = default
+user_domain_id = default
+project_name = service
+username = {{ watcher_keystone_user }}
+password = {{ watcher_keystone_password }}
+
+[oslo_concurrency]
+lock_path = /var/lib/watcher/tmp
+
+[oslo_messaging_rabbit]
+rabbit_userid = {{ rabbitmq_user }}
+rabbit_password = {{ rabbitmq_password }}
+rabbit_ha_queues = true
+rabbit_hosts = {% for host in groups['rabbitmq'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ rabbitmq_port }}{% if not loop.last %},{% endif %}{% endfor %}

ansible/site.yml

@@ -245,3 +245,14 @@
     - { role: tempest,
         tags: tempest,
         when: enable_tempest | bool }
+
+- hosts:
+    - watcher-api
+    - watcher-engine
+    - watcher-applier
+    - rabbitmq
+    - memcached
+  roles:
+    - { role: watcher,
+        tags: watcher,
+        when: enable_watcher | bool }