Adding a playbook for deploying Sahara

This change adds a playbook for deploying sahara as part of the
sahara role integration.

This change also adds variables needed for the installation of
the sahara-dashboard and its support by the ceilometer service,
which will be added on patches in their respective roles.

Change-Id: I782d74e09d1796744ece75d12aa9c65c9453be19

ansible-role-requirements.yml

@@ -102,6 +102,10 @@
   scm: git
   src: https://git.openstack.org/openstack/openstack-ansible-os_rally
   version: master
+- name: os_sahara
+  scm: git
+  src: https://git.openstack.org/openstack/openstack-ansible-os_sahara
+  version: master
 - name: os_swift
   scm: git
   src: https://git.openstack.org/openstack/openstack-ansible-os_swift

etc/openstack_deploy/conf.d/sahara.yml.aio

@@ -0,0 +1,16 @@
+---
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+sahara-infra_hosts:
+  aio1:
+    ip: 172.29.236.100

etc/openstack_deploy/user_secrets.yml

@@ -107,6 +107,11 @@ nova_v21_service_password:
 nova_s3_service_password:
 nova_rabbitmq_password:
 
+## Sahara Options
+sahara_container_mysql_password:
+sahara_rabbitmq_password:
+sahara_service_password:
+
 ## Swift Options:
 swift_service_password:
 swift_container_mysql_password:

playbooks/defaults/repo_packages/openstack_services.yml

@@ -101,6 +101,12 @@ neutron_lbaas_dashboard_git_install_branch: aba14a4ccde177141b6314b9fafafb0868b8
 neutron_lbaas_dashboard_git_dest: "/opt/neutron_lbaas_dashboard_{{ neutron_lbaas_dashboard_git_install_branch | replace('/', '_') }}"
 neutron_lbaas_dashboard_git_project_group: horizon_all
 
+## Horizon Sahara dashboard plugin
+sahara_dashboard_git_repo: https://git.openstack.org/openstack/sahara-dashboard
+sahara_dashboard_git_install_branch: 1a19b8851c7d7d2daa2e22db4894ecd47eb9498d # HEAD of "master" as of 15.08.2016
+sahara_dashboard_git_dest: "/opt/sahara_dashboard_{{ sahara_dashboard_git_install_branch | replace('/', '_') }}"
+sahara_dashboard_git_project_group: horizon_all
+
 
 ## Keystone service
 keystone_git_repo: https://git.openstack.org/openstack/keystone
@@ -157,6 +163,13 @@ nova_lxd_git_dest: "/opt/nova_lxd_{{ nova_lxd_git_install_branch | replace('/',
 nova_lxd_git_project_group: nova_compute
 
 
+## Sahara service
+sahara_git_repo: https://git.openstack.org/openstack/sahara
+sahara_git_install_branch: 8f8f817d779b4eee22a2ecda3dc8d7344facd7c6 # HEAD of "master" as of 15.08.2016
+sahara_git_dest: "/opt/sahara_{{ sahara_git_install_branch | replace('/', '_') }}"
+sahara_git_project_group: sahara_all
+
+
 ## Swift service
 swift_git_repo: https://git.openstack.org/openstack/swift
 swift_git_install_branch: 9c859ccfeeb0a3e11ba01d770d704a5c9ccda612 # HEAD of "master" as of 15.08.2016

playbooks/inventory/env.d/sahara.yml

@@ -0,0 +1,38 @@
+---
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+component_skel:
+  sahara_api:
+    belongs_to:
+      - sahara_all
+  sahara_engine:
+    belongs_to:
+      - sahara_all
+
+container_skel:
+  sahara_container:
+    belongs_to:
+      - sahara-infra_containers
+    contains:
+      - sahara_api
+      - sahara_engine
+    properties:
+      service_name: sahara
+
+physical_skel:
+  sahara-infra_containers:
+    belongs_to:
+      - all_containers
+  sahara-infra_hosts:
+    belongs_to:
+      - hosts

playbooks/inventory/group_vars/all.yml

@@ -348,6 +348,26 @@ nova_rabbitmq_telemetry_host_group: "{{ nova_rabbitmq_host_group }}"
 # If there are any Ceilometer hosts in the environment, then enable its usage
 nova_ceilometer_enabled: "{{ (groups['ceilometer_all'] is defined) and (groups['ceilometer_all'] | length > 0) }}"
 
+## Sahara
+#RPC
+sahara_rabbitmq_userid: sahara
+sahara_rabbitmq_vhost: /sahara
+sahara_rabbitmq_port: "{{ rabbitmq_port }}"
+sahara_rabbitmq_servers: "{{ rabbitmq_servers }}"
+sahara_rabbitmq_use_ssl: "{{ rabbitmq_use_ssl }}"
+sahara_rabbitmq_host_group: "{{ rabbitmq_host_group }}"
+
+# Telemetry notifications
+sahara_rabbitmq_telemetry_userid: "{{ sahara_rabbitmq_userid }}"
+sahara_rabbitmq_telemetry_password: "{{ sahara_rabbitmq_password }}"
+sahara_rabbitmq_telemetry_vhost: "{{ sahara_rabbitmq_vhost }}"
+sahara_rabbitmq_telemetry_port: "{{ sahara_rabbitmq_port }}"
+sahara_rabbitmq_telemetry_servers: "{{ sahara_rabbitmq_servers }}"
+sahara_rabbitmq_telemetry_use_ssl: "{{ sahara_rabbitmq_use_ssl }}"
+sahara_rabbitmq_telemetry_host_group: "{{ sahara_rabbitmq_host_group }}"
+
+# If there are any Ceilometer and Sahara hosts in the environment, then enable its usage
+sahara_ceilometer_enabled: "{{ (groups['ceilometer_all'] is defined) and (groups['sahara_all'] is defined) and (groups['ceilometer_all'] | length > 0) and (groups['sahara_all'] | length > 0) }}"
 
 ## Swift
 swift_system_user_name: swift

playbooks/inventory/group_vars/horizon_all.yml

@@ -29,3 +29,6 @@ horizon_rabbitmq_vhost: /horizon
 
 # Ensure that the package state matches the global setting
 horizon_package_state: "{{ package_state }}"
+
+# If there are any Sahara hosts in the environment, then enable sahara-dashboard
+horizon_enable_sahara_ui: "{{ (groups['sahara_all'] is defined) and (groups['sahara_all'] | length > 0) }}"

playbooks/inventory/group_vars/sahara_all.yml

@@ -0,0 +1,17 @@
+---
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+sahara_service_region: "{{ service_region }}"
+sahara_service_in_ldap: "{{ service_ldap_backend_enabled }}"
+
+sahara_galera_address: "{{ galera_address }}"

playbooks/inventory/group_vars/utility_all.yml

@@ -51,6 +51,7 @@ utility_pip_packages:
   - python-neutronclient
   - python-novaclient
   - python-ironicclient
+  - python-saharaclient
   - python-swiftclient
   - python-memcached
   - pycrypto

playbooks/os-sahara-install.yml

@@ -0,0 +1,78 @@
+---
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Install sahara server
+  hosts: sahara_all
+  gather_facts: "{{ gather_facts | default(True) }}"
+  user: root
+  pre_tasks:
+    - include: common-tasks/rabbitmq-servers-sort.yml
+      vars:
+        sort_group_name: "sahara_all"
+    - include: common-tasks/os-lxc-container-setup.yml
+    - include: common-tasks/rabbitmq-vhost-user.yml
+      vars:
+        user: "{{ sahara_rabbitmq_userid }}"
+        password: "{{ sahara_rabbitmq_password }}"
+        vhost: "{{ sahara_rabbitmq_vhost }}"
+        _rabbitmq_host_group: "{{ sahara_rabbitmq_host_group }}"
+      when:
+        - inventory_hostname == groups['sahara_all'][0]
+        - groups[sahara_rabbitmq_host_group] | length > 0
+    - include: common-tasks/rabbitmq-vhost-user.yml
+      vars:
+        user: "{{ sahara_rabbitmq_telemetry_userid }}"
+        password: "{{ sahara_rabbitmq_telemetry_password }}"
+        vhost: "{{ sahara_rabbitmq_telemetry_vhost }}"
+        _rabbitmq_host_group: "{{ sahara_rabbitmq_telemetry_host_group }}"
+      when:
+        - sahara_ceilometer_enabled | bool
+        - inventory_hostname == groups['sahara_all'][0]
+        - groups[sahara_rabbitmq_telemetry_host_group] is defined
+        - groups[sahara_rabbitmq_telemetry_host_group] | length > 0
+        - groups[sahara_rabbitmq_telemetry_host_group] != groups[sahara_rabbitmq_host_group]
+    - include: common-tasks/os-log-dir-setup.yml
+      vars:
+        log_dirs:
+          - src: "/openstack/log/{{ inventory_hostname }}-sahara"
+            dest: "/var/log/sahara"
+    - include: common-tasks/mysql-db-user.yml
+      vars:
+        user_name: "{{ sahara_galera_user }}"
+        password: "{{ sahara_container_mysql_password }}"
+        login_host: "{{ sahara_galera_address }}"
+        db_name: "{{ sahara_galera_database }}"
+      when: inventory_hostname == groups['sahara_all'][0]
+    - include: common-tasks/package-cache-proxy.yml
+  roles:
+    - role: "os_sahara"
+      sahara_venv_tag: "{{ openstack_release }}"
+      sahara_venv_download_url: "{{ openstack_repo_url }}/venvs/{{ openstack_release }}/{{ ansible_distribution | lower }}/sahara-{{ openstack_release }}-{{ ansible_architecture | lower }}.tgz"
+      tags:
+        - "os-sahara"
+    - role: "openstack_openrc"
+      tags:
+        - "openstack-openrc"
+    - role: "rsyslog_client"
+      rsyslog_client_log_rotate_file: sahara_log_rotate
+      rsyslog_client_log_dir: "/var/log/sahara"
+      rsyslog_client_config_name: "99-sahara-rsyslog-client.conf"
+      tags:
+        - "sahara-rsyslog-client"
+        - "rsyslog-client"
+    - role: "system_crontab_coordination"
+      tags:
+        - "system-crontab-coordination"
+  vars:
+    is_metal: "{{ properties.is_metal|default(false) }}"
+    pip_lock_to_internal_repo: "{{ (pip_links | length) >= 1 }}"

playbooks/setup-openstack.yml

@@ -26,3 +26,4 @@
 - include: os-gnocchi-install.yml
 - include: os-ironic-install.yml
 - include: os-magnum-install.yml
+- include: os-sahara-install.yml

playbooks/vars/configs/haproxy_config.yml

@@ -190,6 +190,15 @@ haproxy_default_services:
       haproxy_balance_alg: source
       haproxy_backend_options:
         - "httpchk HEAD /"
+  - service:
+      haproxy_service_name: sahara_api
+      haproxy_backend_nodes: "{{ groups['sahara_api'] | default([]) }}"
+      haproxy_ssl: "{{ haproxy_ssl }}"
+      haproxy_balance_alg: source
+      haproxy_port: 8386
+      haproxy_balance_type: http
+      haproxy_backend_options:
+        - "httpchk /healthcheck"
   - service:
       haproxy_service_name: swift_proxy
       haproxy_backend_nodes: "{{ groups['swift_proxy'] | default([]) }}"

releasenotes/notes/sahara-data-processing-service-8e63ebed6baf08bc.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - Experimental support has been added to allow the deployment of the
+    the Sahara data-processing service. To deploy sahara hosts should be
+    present in the host group ``sahara-infra_hosts``.

tests/test_inventory.py

@@ -212,6 +212,12 @@ class TestAnsibleInventoryFormatConstraints(unittest.TestCase):
         'rsyslog',
         'rsyslog_all',
         'rsyslog_container',
+        'sahara_all',
+        'sahara_api',
+        'sahara_container',
+        'sahara_engine',
+        'sahara-infra_containers',
+        'sahara-infra_hosts',
         'shared-infra_all',
         'shared-infra_containers',
         'shared-infra_hosts',