Support searchlight ansible role

Searchlight is an Openstack search and index service, this patch
implements the ansible role for searchlight.

Implement blueprint: support-searchlight-deployment

Change-Id: Ibf42d5b259a6740d6596041f896e1009657b7388

README.rst

@@ -73,6 +73,7 @@ Kolla provides images to deploy the following OpenStack projects:
 - `Nova <http://docs.openstack.org/developer/nova/>`__
 - `Rally <http://docs.openstack.org/developer/rally/>`__
 - `Sahara <http://docs.openstack.org/developer/sahara/>`__
+- `Searchlight <http://docs.openstack.org/developer/searchlight/>`__
 - `Senlin <http://docs.openstack.org/developer/senlin/>`__
 - `Swift <http://docs.openstack.org/developer/swift/>`__
 - `Tempest <http://docs.openstack.org/developer/tempest/>`__

ansible/group_vars/all.yml

@@ -200,6 +200,8 @@ etcd_peer_port: "2380"
 
 kuryr_port: "23750"
 
+searchlight_api_port: "9393"
+
 public_protocol: "{{ 'https' if kolla_enable_tls_external | bool else 'http' }}"
 internal_protocol: "http"
 admin_protocol: "http"
@@ -276,6 +278,7 @@ enable_neutron_qos: "no"
 enable_neutron_agent_ha: "no"
 enable_rally: "no"
 enable_sahara: "no"
+enable_searchlight: "no"
 enable_senlin: "no"
 enable_swift: "no"
 enable_telegraf: "no"

ansible/inventory/all-in-one

@@ -132,6 +132,9 @@ control
 [rally:children]
 control
 
+[searchlight:children]
+control
+
 # Additional control implemented here. These groups allow you to control which
 # services run on which hosts at a per-service level.
 #
@@ -382,3 +385,10 @@ senlin
 
 [senlin-engine:children]
 senlin
+
+# Searchlight
+[searchlight-api:children]
+searchlight
+
+[searchlight-listener:children]
+searchlight

ansible/inventory/multinode

@@ -150,6 +150,9 @@ control
 [rally:children]
 control
 
+[searchlight:children]
+control
+
 # Additional control implemented here. These groups allow you to control which
 # services run on which hosts at a per-service level.
 #
@@ -400,3 +403,10 @@ senlin
 
 [senlin-engine:children]
 senlin
+
+# Searchlight
+[searchlight-api:children]
+searchlight
+
+[searchlight-listener:children]
+searchlight

ansible/roles/common/tasks/config.yml

@@ -91,6 +91,7 @@
     - { name: "nova", enabled: "{{ enable_nova }}" }
     - { name: "rabbitmq", enabled: "{{ enable_rabbitmq }}" }
     - { name: "sahara", enabled: "{{ enable_sahara }}" }
+    - { name: "searchlight", enabled: "{{ enable_searchlight }}" }
     - { name: "senlin", enabled: "{{ enable_senlin }}" }
     - { name: "swift", enabled: "{{ enable_swift }}" }
     - { name: "watcher", enabled: "{{ enable_watcher }}" }

ansible/roles/common/templates/cron-logrotate-searchlight.conf.j2

@@ -0,0 +1,3 @@
+"/var/log/kolla/searchlight/*.log"
+{
+}

ansible/roles/common/templates/cron.json.j2

@@ -21,6 +21,7 @@
     ( 'nova', enable_nova ),
     ( 'rabbitmq', enable_rabbitmq ),
     ( 'sahara', enable_sahara ),
+    ( 'searchlight', enable_searchlight ),
     ( 'senlin', enable_senlin ),
     ( 'swift', enable_swift )
 ] %}

ansible/roles/common/templates/heka-openstack.toml.j2

@@ -6,6 +6,6 @@ filename = "lua_decoders/os_openstack_log.lua"
 type = "LogstreamerInput"
 decoder = "openstack_log_decoder"
 log_directory = "/var/log/kolla"
-file_match = '(?P<Service>cloudkitty|nova|glance|keystone|neutron|ceph|cinder|heat|murano|magnum|mistral|manila|senlin|sahara)/(?P<Program>.*)\.log\.?(?P<Seq>\d*)$'
+file_match = '(?P<Service>cloudkitty|nova|glance|keystone|neutron|ceph|cinder|heat|murano|magnum|mistral|manila|searchlight|senlin|sahara)/(?P<Program>.*)\.log\.?(?P<Seq>\d*)$'
 priority = ["^Seq"]
 differentiator = ["Service", "_", "Program"]

ansible/roles/haproxy/templates/haproxy.cfg.j2

@@ -417,6 +417,21 @@ listen sahara_api_external
 {% endif %}
 {% endif %}
 
+{% if enable_searchlight | bool %}
+listen searchlight_api
+  bind {{ kolla_internal_vip_address }}:{{ searchlight_api_port }}
+{% for host in groups['searchlight-api'] %}
+  server {{ hostvars[host]['ansible_hostname'] }} {{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ searchlight_api_port }} check inter 2000 rise 2 fall 5
+{% endfor %}
+{% if haproxy_enable_external_vip | bool %}
+
+listen searchlight_api_external
+  bind {{ kolla_external_vip_address }}:{{ searchlight_api_port }} {{ tls_bind_info }}
+{% for host in groups['searchlight-api'] %}
+  server {{ hostvars[host]['ansible_hostname'] }} {{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ searchlight_api_port }} check inter 2000 rise 2 fall 5
+{% endfor %}
+{% endif %}
+{% endif %}
 
 {% if enable_ceph | bool and enable_ceph_rgw | bool %}
 listen radosgw

ansible/roles/prechecks/tasks/port_checks.yml

@@ -657,6 +657,22 @@
     state: stopped
   when: inventory_hostname in groups['haproxy']
 
+- name: Checking free port for Searchlight API
+  wait_for:
+    host: "{{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}"
+    port: "{{ searchlight_api_port }}"
+    connect_timeout: 1
+    state: stopped
+  when: inventory_hostname in groups['searchlight-api']
+
+- name: Checking free port for Searchlight API HAProxy
+  wait_for:
+    host: "{{ kolla_internal_vip_address }}"
+    port: "{{ searchlight_api_port }}"
+    connect_timeout: 1
+    state: stopped
+  when: inventory_hostname in groups['haproxy']
+
 - name: Checking the network_interface is present
   fail: "msg='Please check the network_interface property - interface {{ network_interface }} not found'"
   when: network_interface not in ansible_interfaces

ansible/roles/searchlight/defaults/main.yml

@@ -0,0 +1,32 @@
+---
+project_name: "searchlight"
+
+####################
+# Elasticsearch
+####################
+searchlight_elasticsearch_url: "{{ kolla_internal_fqdn }}:{{ elasticsearch_port }}"
+
+####################
+# Docker
+####################
+searchlight_listener_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-searchlight-listener"
+searchlight_listener_tag: "{{ openstack_release }}"
+searchlight_listener_image_full: "{{ searchlight_listener_image }}:{{ searchlight_listener_tag }}"
+
+searchlight_api_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-searchlight-api"
+searchlight_api_tag: "{{ openstack_release }}"
+searchlight_api_image_full: "{{ searchlight_api_image }}:{{ searchlight_api_tag }}"
+
+
+####################
+# OpenStack
+####################
+searchlight_admin_endpoint: "{{ admin_protocol }}://{{ kolla_internal_fqdn }}:{{ searchlight_api_port }}"
+searchlight_internal_endpoint: "{{ internal_protocol }}://{{ kolla_internal_fqdn }}:{{ searchlight_api_port }}"
+searchlight_public_endpoint: "{{ public_protocol }}://{{ kolla_external_fqdn }}:{{ searchlight_api_port }}"
+
+searchlight_logging_debug: "{{ openstack_logging_debug }}"
+
+searchlight_keystone_user: "searchlight"
+
+openstack_searchlight_auth: "{'auth_url':'{{ openstack_auth.auth_url }}','username':'{{ openstack_auth.username }}','password':'{{ openstack_auth.password }}','project_name':'{{ openstack_auth.project_name }}'}"

ansible/roles/searchlight/meta/main.yml

@@ -0,0 +1,3 @@
+---
+dependencies:
+  - { role: common }

ansible/roles/searchlight/tasks/bootstrap.yml

@@ -0,0 +1,2 @@
+---
+- include: bootstrap_service.yml

ansible/roles/searchlight/tasks/bootstrap_service.yml

@@ -0,0 +1,20 @@
+---
+- name: Running Searchlight bootstrap container
+  kolla_docker:
+    action: "start_container"
+    common_options: "{{ docker_common_options }}"
+    detach: False
+    environment:
+      KOLLA_BOOTSTRAP:
+      KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
+    image: "{{ searchlight_api_image_full }}"
+    labels:
+      BOOTSTRAP:
+    name: "bootstrap_searchlight"
+    restart_policy: "never"
+    volumes:
+      - "{{ node_config_directory }}/searchlight-api/:{{ container_config_directory }}/:ro"
+      - "/etc/localtime:/etc/localtime:ro"
+      - "kolla_logs:/var/log/kolla/"
+  run_once: True
+  delegate_to: "{{ groups['searchlight-api'][0] }}"

ansible/roles/searchlight/tasks/config.yml

@@ -0,0 +1,32 @@
+---
+- name: Ensuring config directories exist
+  file:
+    path: "{{ node_config_directory }}/{{ item }}"
+    state: "directory"
+    recurse: yes
+  with_items:
+    - "searchlight-api"
+    - "searchlight-listener"
+
+- name: Copying over config.json files for services
+  template:
+    src: "{{ item }}.json.j2"
+    dest: "{{ node_config_directory }}/{{ item }}/config.json"
+  with_items:
+    - "searchlight-api"
+    - "searchlight-listener"
+
+- name: Copying over searchlight.conf
+  merge_configs:
+    vars:
+      service_name: "{{ item }}"
+    sources:
+      - "{{ role_path }}/templates/searchlight.conf.j2"
+      - "{{ node_custom_config }}/global.conf"
+      - "{{ node_custom_config }}/messaging.conf"
+      - "{{ node_custom_config }}/searchlight.conf"
+      - "{{ node_custom_config }}/searchlight/{{ inventory_hostname }}/searchlight.conf"
+    dest: "{{ node_config_directory }}/{{ item }}/searchlight.conf"
+  with_items:
+    - "searchlight-api"
+    - "searchlight-listener"

ansible/roles/searchlight/tasks/deploy.yml

@@ -0,0 +1,12 @@
+---
+- include: register.yml
+  when: inventory_hostname in groups['searchlight']
+
+- include: config.yml
+  when: inventory_hostname in groups['searchlight']
+
+- include: bootstrap.yml
+  when: inventory_hostname in groups['searchlight']
+
+- include: start.yml
+  when: inventory_hostname in groups['searchlight']

ansible/roles/searchlight/tasks/main.yml

@@ -0,0 +1,2 @@
+---
+- include: "{{ action }}.yml"

ansible/roles/searchlight/tasks/pull.yml

@@ -0,0 +1,14 @@
+---
+- name: Pulling searchlight-api image
+  kolla_docker:
+    action: "pull_image"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ searchlight_api_image_full }}"
+  when: inventory_hostname in groups['searchlight-api']
+
+- name: Pulling searchlight-listener image
+  kolla_docker:
+    action: "pull_image"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ searchlight_listener_image_full }}"
+  when: inventory_hostname in groups['searchlight-listener']

ansible/roles/searchlight/tasks/reconfigure.yml

@@ -0,0 +1,66 @@
+---
+- name: Ensuring the containers up
+  kolla_docker:
+    name: "{{ item.name }}"
+    action: "get_container_state"
+  register: container_state
+  failed_when: container_state.Running == false
+  when: inventory_hostname in groups[item.group]
+  with_items:
+    - { name: searchlight_api, group: searchlight-api }
+    - { name: searchlight_listener, group: searchlight-listener }
+
+- include: config.yml
+
+- name: Check the configs
+  command: docker exec {{ item.name }} /usr/local/bin/kolla_set_configs --check
+  changed_when: false
+  failed_when: false
+  register: check_results
+  when: inventory_hostname in groups[item.group]
+  with_items:
+    - { name: searchlight_api, group: searchlight-api }
+    - { name: searchlight_listener, group: searchlight-listener }
+
+- name: Containers config strategy
+  kolla_docker:
+    name: "{{ item.name }}"
+    action: "get_container_env"
+  register: container_envs
+  when: inventory_hostname in groups[item.group]
+  with_items:
+    - { name: searchlight_api, group: searchlight-api }
+    - { name: searchlight_listener, group: searchlight-listener }
+
+- name: Remove the containers
+  kolla_docker:
+    name: "{{ item[0]['name'] }}"
+    action: "remove_container"
+  register: remove_containers
+  when:
+    - config_strategy == "COPY_ONCE" or item[1]['KOLLA_CONFIG_STRATEGY'] == 'COPY_ONCE'
+    - item[2]['rc'] == 1
+    - inventory_hostname in groups[item[0]['group']]
+  with_together:
+    - [{ name: searchlight_api, group: searchlight-api },
+       { name: searchlight_listener, group: searchlight-listener }]
+    - "{{ container_envs.results }}"
+    - "{{ check_results.results }}"
+
+- include: start.yml
+  when: remove_containers.changed
+
+- name: Restart containers
+  kolla_docker:
+    name: "{{ item[0]['name'] }}"
+    action: "restart_container"
+  when:
+    - config_strategy == 'COPY_ALWAYS'
+    - item[1]['KOLLA_CONFIG_STRATEGY'] != 'COPY_ONCE'
+    - item[2]['rc'] == 1
+    - inventory_hostname in groups[item[0]['group']]
+  with_together:
+    - [{ name: searchlight_api, group: searchlight-api },
+       { name: searchlight_listener, group: searchlight-listener }]
+    - "{{ container_envs.results }}"
+    - "{{ check_results.results }}"

ansible/roles/searchlight/tasks/register.yml

@@ -0,0 +1,40 @@
+---
+- name: Creating the Searchlight service and endpoint
+  command: docker exec -t kolla_toolbox /usr/bin/ansible localhost
+    -m kolla_keystone_service
+    -a "service_name=searchlight
+        service_type=search
+        description='Openstack Index Service'
+        endpoint_region={{ openstack_region_name }}
+        url='{{ item.url }}'
+        interface='{{ item.interface }}'
+        region_name={{ openstack_region_name }}
+        auth={{ '{{ openstack_searchlight_auth }}' }}"
+    -e "{'openstack_searchlight_auth':{{ openstack_searchlight_auth }}}"
+  register: searchlight_endpoint
+  changed_when: "{{ searchlight_endpoint.stdout.find('localhost | SUCCESS => ') != -1 and (searchlight_endpoint.stdout.split('localhost | SUCCESS => ')[1]|from_json).changed }}"
+  until: searchlight_endpoint.stdout.split()[2] == 'SUCCESS'
+  retries: 10
+  delay: 5
+  run_once: True
+  with_items:
+    - {'interface': 'admin', 'url': '{{ searchlight_admin_endpoint }}'}
+    - {'interface': 'internal', 'url': '{{ searchlight_internal_endpoint }}'}
+    - {'interface': 'public', 'url': '{{ searchlight_public_endpoint }}'}
+
+- name: Creating the Searchlight project, user, and role
+  command: docker exec -t kolla_toolbox /usr/bin/ansible localhost
+    -m kolla_keystone_user
+    -a "project=service
+        user=searchlight
+        password={{ searchlight_keystone_password }}
+        role=admin
+        region_name={{ openstack_region_name }}
+        auth={{ '{{ openstack_searchlight_auth }}' }}"
+    -e "{'openstack_searchlight_auth':{{ openstack_searchlight_auth }}}"
+  register: searchlight_user
+  changed_when: "{{ searchlight_user.stdout.find('localhost | SUCCESS => ') != -1 and (searchlight_user.stdout.split('localhost | SUCCESS => ')[1]|from_json).changed }}"
+  until: searchlight_user.stdout.split()[2] == 'SUCCESS'
+  retries: 10
+  delay: 5
+  run_once: True

ansible/roles/searchlight/tasks/start.yml

@@ -0,0 +1,24 @@
+---
+- name: Starting searchlight-api container
+  kolla_docker:
+    action: "start_container"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ searchlight_api_image_full }}"
+    name: "searchlight_api"
+    volumes:
+      - "{{ node_config_directory }}/searchlight-api/:{{ container_config_directory }}/:ro"
+      - "/etc/localtime:/etc/localtime:ro"
+      - "kolla_logs:/var/log/kolla/"
+  when: inventory_hostname in groups['searchlight-api']
+
+- name: Starting searchlight-listener container
+  kolla_docker:
+    action: "start_container"
+    common_options: "{{ docker_common_options }}"
+    image: "{{ searchlight_listener_image_full }}"
+    name: "searchlight_listener"
+    volumes:
+      - "{{ node_config_directory }}/searchlight-listener/:{{ container_config_directory }}/:ro"
+      - "/etc/localtime:/etc/localtime:ro"
+      - "kolla_logs:/var/log/kolla/"
+  when: inventory_hostname in groups['searchlight-listener']

ansible/roles/searchlight/tasks/upgrade.yml

@@ -0,0 +1,6 @@
+---
+- include: config.yml
+
+- include: bootstrap_service.yml
+
+- include: start.yml

ansible/roles/searchlight/templates/searchlight-api.json.j2

@@ -0,0 +1,18 @@
+{
+    "command": "searchlight-api",
+    "config_files": [
+        {
+            "source": "{{ container_config_directory }}/searchlight.conf",
+            "dest": "/etc/searchlight/searchlight.conf",
+            "owner": "searchlight",
+            "perm": "0600"
+        }
+    ],
+    "permissions": [
+        {
+            "path": "/var/log/kolla/searchlight",
+            "owner": "searchlight:searchlight",
+            "recurse": true
+        }
+    ]
+}

ansible/roles/searchlight/templates/searchlight-listener.json.j2

@@ -0,0 +1,18 @@
+{
+    "command": "searchlight-listener",
+    "config_files": [
+        {
+            "source": "{{ container_config_directory }}/searchlight.conf",
+            "dest": "/etc/searchlight/searchlight.conf",
+            "owner": "searchlight",
+            "perm": "0600"
+        }
+    ],
+    "permissions": [
+        {
+            "path": "/var/log/kolla/searchlight",
+            "owner": "searchlight:searchlight",
+            "recurse": true
+        }
+    ]
+}

ansible/roles/searchlight/templates/searchlight.conf.j2

@@ -0,0 +1,90 @@
+[DEFAULT]
+debug = {{ searchlight_logging_debug }}
+
+policy_file = /etc/searchlight/policy.json
+log_dir = /var/log/kolla/searchlight
+transport_url = rabbit://{% for host in groups['rabbitmq'] %}{{ rabbitmq_user }}:{{ rabbitmq_password }}@{% if orchestration_engine == 'KUBERNETES' %}rabbitmq{% else %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}{% endif %}:{{ rabbitmq_port }}{% if not loop.last %},{% endif %}{% endfor %}
+
+
+[api]
+port = {{ searchlight_api_port }}
+bind_host = {{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}
+public_endpoint = {{ public_protocol }}://{{ kolla_external_fqdn }}:{{ searchlight_api_port }}
+
+[elasticsearch]
+hosts = {{ searchlight_elasticsearch_url }}
+
+[listener]
+workers = {{ openstack_service_workers }}
+notifications_pool = searchlight-listener
+
+[paste_deploy]
+flavor = keystone
+
+[keystone_authtoken]
+auth_uri = {{ internal_protocol }}://{{ kolla_internal_fqdn }}:{{ keystone_public_port }}
+auth_url = {{ admin_protocol }}://{{ kolla_internal_fqdn }}:{{ keystone_admin_port }}
+project_domain_name = default
+project_name = service
+user_domain_name = default
+username = {{ searchlight_keystone_user }}
+password = {{ searchlight_keystone_password }}
+auth_type = password
+
+memcache_security_strategy = ENCRYPT
+memcache_secret_key = {{ memcache_secret_key }}
+memcached_servers = {% for host in groups['memcached'] %}{% if orchestration_engine == 'KUBERNETES' %}memcached{% else %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}{% endif %}:{{ memcached_port }}{% if not loop.last %},{% endif %}{% endfor %}
+
+[service_credentials]
+auth_uri = {{ internal_protocol }}://{{ kolla_internal_fqdn }}:{{ keystone_public_port }}
+auth_url = {{ admin_protocol }}://{{ kolla_internal_fqdn }}:{{ keystone_admin_port }}
+region_name = {{ openstack_region_name }}
+project_domain_name = default
+project_name = service
+user_domain_name = default
+username = {{ searchlight_keystone_user }}
+password = {{ searchlight_keystone_password }}
+auth_type = password
+auth_plugin = password
+
+memcache_security_strategy = ENCRYPT
+memcache_secret_key = {{ memcache_secret_key }}
+memcached_servers = {% for host in groups['memcached'] %}{% if orchestration_engine == 'KUBERNETES' %}memcached{% else %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}{% endif %}:{{ memcached_port }}{% if not loop.last %},{% endif %}{% endfor %}
+
+[resource_plugin:os_designate_zone]
+enabled = False
+
+[resource_plugin:os_designate_recordset]
+enabled = False
+
+{% if not enable_swift | bool %}
+[resource_plugin:os_swift_account]
+enabled = False
+
+[resource_plugin:os_swift_container]
+enabled = False
+
+[resource_plugin:os_swift_object]
+enabled = False
+{% endif %}
+
+{% if not enable_nova | bool %}
+[resource_plugin:os_server_nova]
+enabled = False
+{% endif %}
+
+{% if not enable_glance | bool %}
+[resource_plugin:os_glance_image]
+enabled = False
+
+[resource_plugin:os_glance_metadef]
+enabled = False
+{% endif %}
+
+{% if not enable_neutron | bool %}
+[resource_plugin:os_neutron_net]
+enabled = False
+
+[resource_plugin:os_neutron_port]
+enabled = False
+{% endif %}

ansible/site.yml

@@ -378,3 +378,11 @@
     - { role: senlin,
         tags: senlin,
         when: enable_senlin | bool }
+
+- hosts:
+    - searchlight-api
+    - searchlight-listener
+  roles:
+    - { role: searchlight,
+        tags: searchlight,
+        when: enable_searchlight | bool }

etc/kolla/globals.yml

@@ -146,6 +146,7 @@ kolla_internal_vip_address: "10.10.10.254"
 #enable_neutron_vpnaas: "no"
 #enable_rally: "no"
 #enable_sahara: "no"
+#enable_searchlight: "no"
 #enable_senlin: "no"
 #enable_swift: "no"
 #enable_telegraf: "no"

etc/kolla/passwords.yml

@@ -100,6 +100,8 @@ telemetry_secret_key:
 manila_database_password:
 manila_keystone_password:
 
+searchlight_keystone_password:
+
 memcache_secret_key:
 
 nova_ssh_key: