Add ansible support for Cinder

Change-Id: I0694e5ad95aa7f924ebd3a29dff1f85e6307b7ae
Partially-implements: blueprint ansible-cinder
Partially-Implements: blueprint remove-abstraction-ansible
Co-Authored-By: Ryan Hallisey <rhallise@redhat.com>

ansible/inventory/all-in-one

@@ -9,6 +9,8 @@ localhost       ansible_connection=local
 [compute]
 localhost       ansible_connection=local
 
+[storage]
+localhost       ansible_connection=local
 
 # You can explicitly specify which hosts run each project by updating the
 # groups in the sections below. Common services are grouped together.
@@ -33,6 +35,9 @@ support
 [neutron:children]
 network
 
+[cinder:children]
+support
+
 
 # Additional control implemented here. These groups allow you to control which
 # services run on which hosts at a per-service level.
@@ -70,3 +75,16 @@ neutron
 
 [neutron-agents:children]
 neutron
+
+#Cinder
+[cinder-api:children]
+cinder
+
+[cinder-backup:children]
+storage
+
+[cinder-scheduler:children]
+cinder
+
+[cinder-volume:children]
+storage

ansible/inventory/multinode

@@ -15,6 +15,8 @@ network01
 [compute]
 compute01
 
+[storage]
+storage01
 
 # You can explicitly specify which hosts run each project by updating the
 # groups in the sections below. Common services are grouped together.
@@ -39,6 +41,9 @@ support
 [neutron:children]
 network
 
+[cinder:children]
+support
+
 
 # Additional control implemented here. These groups allow you to control which
 # services run on which hosts at a per-service level.
@@ -76,3 +81,16 @@ neutron
 
 [neutron-agents:children]
 neutron
+
+#Cinder
+[cinder-api:children]
+cinder
+
+[cinder-backup:children]
+storage
+
+[cinder-scheduler:children]
+cinder
+
+[cinder-volume:children]
+storage

ansible/roles/cinder/defaults/main.yml

@@ -0,0 +1,44 @@
+---
+project_name: "cinder"
+
+####################
+# Database
+####################
+cinder_database_name: "cinder"
+cinder_database_user: "cinder"
+cinder_database_address: "{{ kolla_internal_address }}"
+
+
+####################
+# Docker
+####################
+cinder_volume_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-cinder-volume"
+cinder_volume_tag: "{{ openstack_release }}"
+cinder_volume_image_full: "{{ cinder_volume_image }}:{{ cinder_volume_tag }}"
+
+cinder_scheduler_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-cinder-scheduler"
+cinder_scheduler_tag: "{{ openstack_release }}"
+cinder_scheduler_image_full: "{{ cinder_scheduler_image }}:{{ cinder_scheduler_tag }}"
+
+cinder_backup_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-cinder-backup"
+cinder_backup_tag: "{{ openstack_release }}"
+cinder_backup_image_full: "{{ cinder_backup_image }}:{{ cinder_backup_tag }}"
+
+cinder_api_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-cinder-api"
+cinder_api_tag: "{{ openstack_release }}"
+cinder_api_image_full: "{{ cinder_api_image }}:{{ cinder_api_tag }}"
+
+
+####################
+# Openstack
+####################
+cinder_public_address: "{{ kolla_external_address }}"
+cinder_admin_address: "{{ kolla_internal_address }}"
+cinder_internal_address: "{{ kolla_internal_address }}"
+
+cinder_logging_verbose: "{{ openstack_logging_verbose }}"
+cinder_logging_debug: "{{ openstack_logging_debug }}"
+
+cinder_keystone_user: "cinder"
+
+openstack_cinder_auth: "{'auth_url':'{{ openstack_auth_v2.auth_url }}','username':'{{ openstack_auth_v2.username }}','password':'{{ openstack_auth_v2.password }}','project_name':'{{ openstack_auth_v2.project_name }}'}"

ansible/roles/cinder/tasks/bootstrap.yml

@@ -0,0 +1,80 @@
+---
+- name: Creating Cinder database
+  command: docker exec -t kolla_ansible /usr/bin/ansible localhost
+    -m mysql_db
+    -a "login_host='{{ database_address }}'
+        login_user='{{ database_user }}'
+        login_password='{{ database_password }}'
+        name='{{ cinder_database_name }}'"
+  register: database
+  changed_when: "{{ database.stdout.find('localhost | SUCCESS => ') != -1 and (database.stdout.split('localhost | SUCCESS => ')[1]|from_json).changed }}"
+  failed_when: database.stdout.split()[2] != 'SUCCESS'
+  run_once: True
+
+- name: Creating Cinder database user and setting permissions
+  command: docker exec -t kolla_ansible /usr/bin/ansible localhost
+    -m mysql_user
+    -a "login_host='{{ database_address }}'
+        login_user='{{ database_user }}'
+        login_password='{{ database_password }}'
+        name='{{ cinder_database_name }}'
+        password='{{ cinder_database_password }}'
+        host='%'
+        priv='{{ cinder_database_name }}.*:ALL'
+        append_privs='yes'"
+  register: database_user
+  changed_when: "{{ database.stdout.find('localhost | SUCCESS => ') != -1 and (database_user.stdout.split('localhost | SUCCESS => ')[1]|from_json).changed }}"
+  failed_when: database_user.stdout.split()[2] != 'SUCCESS'
+  run_once: True
+
+- name: Starting Cinder data container
+  docker:
+    docker_api_version: "{{ docker_api_version }}"
+    net: host
+    pull: "{{ docker_pull_policy }}"
+    restart_policy: "{{ docker_restart_policy }}"
+    restart_policy_retry: "{{ docker_restart_policy_retry }}"
+    state: reloaded
+    registry: "{{ docker_registry }}"
+    username: "{{ docker_registry_username }}"
+    password: "{{ docker_registry_password }}"
+    insecure_registry: "{{ docker_insecure_registry }}"
+    name: cinder_data
+    image: "{{ cinder_api_image_full }}"
+    command: "/bin/sleep infinity"
+    volumes: "/var/lib/cinder/"
+
+- name: Starting Cinder bootstrap container
+  docker:
+    detach: False
+    docker_api_version: "{{ docker_api_version }}"
+    net: host
+    pull: "{{ docker_pull_policy }}"
+    restart_policy: "no"
+    state: reloaded
+    registry: "{{ docker_registry }}"
+    username: "{{ docker_registry_username }}"
+    password: "{{ docker_registry_password }}"
+    insecure_registry: "{{ docker_insecure_registry }}"
+    name: bootstrap_cinder
+    image: "{{ cinder_api_image_full }}"
+    volumes: "{{ node_config_directory }}/cinder/:/opt/kolla/cinder/:ro"
+    volumes_from: cinder_data
+    env:
+      KOLLA_BOOSTRAP:
+      KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
+  run_once: True
+  when: database.stdout.find('localhost | SUCCESS => ') != -1 and (database.stdout.split('localhost | SUCCESS => ')[1]|from_json).changed
+
+# https://github.com/ansible/ansible-modules-core/pull/1031
+
+- name: Waiting for bootstrap container to exit
+  command: docker wait bootstrap_cinder
+  when: database.stdout.find('localhost | SUCCESS => ') != -1 and (database.stdout.split('localhost | SUCCESS => ')[1]|from_json).changed
+
+- name: Cleaning up Cinder boostrap container
+  docker:
+    name: bootstrap_cinder
+    image: "{{ cinder_api_image_full }}"
+    state: absent
+  when: database.stdout.find('localhost | SUCCESS => ') != -1 and (database.stdout.split('localhost | SUCCESS => ')[1]|from_json).changed

ansible/roles/cinder/tasks/config.yml

@@ -0,0 +1,80 @@
+---
+- include: ../../config.yml
+  vars:
+    service_name: "cinder-api"
+    config_source:
+      - "roles/{{ project_name }}/templates/cinder.conf.j2"
+      - "/etc/kolla/config/global.conf"
+      - "/etc/kolla/config/database.conf"
+      - "/etc/kolla/config/messaging.conf"
+      - "/etc/kolla/config/{{ project_name }}.conf"
+      - "/etc/kolla/config/{{ project_name }}/{{ service_name }}.conf"
+    config_template_dest:
+      - "{{ node_templates_directory }}/{{ service_name }}/{{ project_name }}.conf_minimal"
+      - "{{ node_templates_directory }}/{{ service_name }}/{{ project_name }}.conf_global"
+      - "{{ node_templates_directory }}/{{ service_name }}/{{ project_name }}.conf_database"
+      - "{{ node_templates_directory }}/{{ service_name }}/{{ project_name }}.conf_messaging"
+      - "{{ node_templates_directory }}/{{ service_name }}/{{ project_name }}.conf_augment"
+      - "{{ node_templates_directory }}/{{ service_name }}/{{ service_name }}.conf_augment"
+    config_dest: "{{ node_config_directory }}/{{ service_name }}/cinder.conf"
+  when: inventory_hostname in groups['cinder-api']
+
+- include: ../../config.yml
+  vars:
+    service_name: "cinder-backup"
+    config_source:
+      - "roles/{{ project_name }}/templates/cinder.conf.j2"
+      - "/etc/kolla/config/global.conf"
+      - "/etc/kolla/config/database.conf"
+      - "/etc/kolla/config/messaging.conf"
+      - "/etc/kolla/config/{{ project_name }}.conf"
+      - "/etc/kolla/config/{{ project_name }}/{{ service_name }}.conf"
+    config_template_dest:
+      - "{{ node_templates_directory }}/{{ service_name }}/{{ project_name }}.conf_minimal"
+      - "{{ node_templates_directory }}/{{ service_name }}/{{ project_name }}.conf_global"
+      - "{{ node_templates_directory }}/{{ service_name }}/{{ project_name }}.conf_database"
+      - "{{ node_templates_directory }}/{{ service_name }}/{{ project_name }}.conf_messaging"
+      - "{{ node_templates_directory }}/{{ service_name }}/{{ project_name }}.conf_augment"
+      - "{{ node_templates_directory }}/{{ service_name }}/{{ service_name }}.conf_augment"
+    config_dest: "{{ node_config_directory }}/{{ service_name }}/cinder.conf"
+  when: inventory_hostname in groups['cinder-backup']
+
+- include: ../../config.yml
+  vars:
+    service_name: "cinder-scheduler"
+    config_source:
+      - "roles/{{ project_name }}/templates/cinder.conf.j2"
+      - "/etc/kolla/config/global.conf"
+      - "/etc/kolla/config/database.conf"
+      - "/etc/kolla/config/messaging.conf"
+      - "/etc/kolla/config/{{ project_name }}.conf"
+      - "/etc/kolla/config/{{ project_name }}/{{ service_name }}.conf"
+    config_template_dest:
+      - "{{ node_templates_directory }}/{{ service_name }}/{{ project_name }}.conf_minimal"
+      - "{{ node_templates_directory }}/{{ service_name }}/{{ project_name }}.conf_global"
+      - "{{ node_templates_directory }}/{{ service_name }}/{{ project_name }}.conf_database"
+      - "{{ node_templates_directory }}/{{ service_name }}/{{ project_name }}.conf_messaging"
+      - "{{ node_templates_directory }}/{{ service_name }}/{{ project_name }}.conf_augment"
+      - "{{ node_templates_directory }}/{{ service_name }}/{{ service_name }}.conf_augment"
+    config_dest: "{{ node_config_directory }}/{{ service_name }}/cinder.conf"
+  when: inventory_hostname in groups['cinder-scheduler']
+
+- include: ../../config.yml
+  vars:
+    service_name: "cinder-volume"
+    config_source:
+      - "roles/{{ project_name }}/templates/cinder.conf.j2"
+      - "/etc/kolla/config/global.conf"
+      - "/etc/kolla/config/database.conf"
+      - "/etc/kolla/config/messaging.conf"
+      - "/etc/kolla/config/{{ project_name }}.conf"
+      - "/etc/kolla/config/{{ project_name }}/{{ service_name }}.conf"
+    config_template_dest:
+      - "{{ node_templates_directory }}/{{ service_name }}/{{ project_name }}.conf_minimal"
+      - "{{ node_templates_directory }}/{{ service_name }}/{{ project_name }}.conf_global"
+      - "{{ node_templates_directory }}/{{ service_name }}/{{ project_name }}.conf_database"
+      - "{{ node_templates_directory }}/{{ service_name }}/{{ project_name }}.conf_messaging"
+      - "{{ node_templates_directory }}/{{ service_name }}/{{ project_name }}.conf_augment"
+      - "{{ node_templates_directory }}/{{ service_name }}/{{ service_name }}.conf_augment"
+    config_dest: "{{ node_config_directory }}/{{ service_name }}/cinder.conf"
+  when: inventory_hostname in groups['cinder-volume']

ansible/roles/cinder/tasks/main.yml

@@ -0,0 +1,8 @@
+---
+- include: register.yml
+
+- include: config.yml
+
+- include: bootstrap.yml
+
+- include: start.yml

ansible/roles/cinder/tasks/register.yml

@@ -0,0 +1,37 @@
+---
+- name: Creating the Cinder service and endpoint
+  command: docker exec -t kolla_ansible /usr/bin/ansible localhost
+    -m kolla_keystone_service
+    -a "service_name=cinder
+        service_type='Block Storage'
+        description='Openstack Block Storage'
+        endpoint_region={{ openstack_region_name }}
+        admin_url='http://{{ kolla_internal_address }}:{{ cinder_api_port }}'
+        internal_url='http://{{ kolla_internal_address }}:{{ cinder_api_port }}'
+        public_url='http://{{ kolla_external_address }}:{{ cinder_api_port }}'
+        region_name={{ openstack_region_name }}
+        auth={{ '{{ openstack_cinder_auth }}' }}"
+    -e  "{'openstack_cinder_auth':{{ openstack_cinder_auth }}}"
+  register: cinder_endpoint
+  changed_when: "{{ cinder_endpoint.stdout.find('localhost | SUCCESS => ') != -1 and (cinder_endpoint.stdout.split('localhost | SUCCESS => ')[1]|from_json).changed }}"
+  until: cinder_endpoint.stdout.split()[2] == 'SUCCESS'
+  retries: 10
+  delay: 5
+  run_once: True
+
+- name: Creating the Cinder project, user, and role
+  command: docker exec -t kolla_ansible /usr/bin/ansible localhost
+    -m kolla_keystone_user
+    -a "project=service
+        user=cinder
+        password={{ cinder_keystone_password }}
+        role=admin
+        region_name={{ openstack_region_name }}
+        auth={{ '{{ openstack_cinder_auth }}' }}"
+    -e "{'openstack_cinder_auth':{{ openstack_cinder_auth }}}"
+  register: cinder_user
+  changed_when: "{{ cinder_user.stdout.find('localhost | SUCCESS => ') != -1 and (cinder_user.stdout.split('localhost | SUCCESS => ')[1]|from_json).changed }}"
+  until: cinder_user.stdout.split()[2] == 'SUCCESS'
+  retries: 10
+  delay: 5
+  run_once: True

ansible/roles/cinder/tasks/start.yml

@@ -0,0 +1,75 @@
+- name: Starting Cinder-api container
+  docker:
+    docker_api_version: "{{ docker_api_version }}"
+    net: host
+    pull: "{{ docker_pull_policy }}"
+    restart_policy: "{{ docker_restart_policy }}"
+    restart_policy_retry: "{{ docker_restart_policy_retry }}"
+    state: reloaded
+    registry: "{{ docker_registry }}"
+    username: "{{ docker_registry_username }}"
+    password: "{{ docker_registry_password }}"
+    insecure_registry: "{{ docker_insecure_registry }}"
+    name: cinder_api
+    image: "{{ cinder_api_image_full }}"
+    volumes: "{{ node_config_directory }}/cinder-api/:/opt/kolla/cinder/:ro"
+    env:
+      KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
+  when: inventory_hostname in groups['cinder-api']
+
+- name: Starting Cinder-backup container
+  docker:
+    docker_api_version: "{{ docker_api_version }}"
+    net: host
+    pull: "{{ docker_pull_policy }}"
+    restart_policy: "{{ docker_restart_policy }}"
+    restart_policy_retry: "{{ docker_restart_policy_retry }}"
+    state: reloaded
+    registry: "{{ docker_registry }}"
+    username: "{{ docker_registry_username }}"
+    password: "{{ docker_registry_password }}"
+    insecure_registry: "{{ docker_insecure_registry }}"
+    name: cinder_backup
+    image: "{{ cinder_backup_image_full }}"
+    volumes: "{{ node_config_directory }}/cinder-backup/:/opt/kolla/cinder-backup/:ro"
+    env:
+      KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
+  when: inventory_hostname in groups['cinder-backup']
+
+- name: Starting Cinder-scheduler container
+  docker:
+    docker_api_version: "{{ docker_api_version }}"
+    net: host
+    pull: "{{ docker_pull_policy }}"
+    restart_policy: "{{ docker_restart_policy }}"
+    restart_policy_retry: "{{ docker_restart_policy_retry }}"
+    state: reloaded
+    registry: "{{ docker_registry }}"
+    username: "{{ docker_registry_username }}"
+    password: "{{ docker_registry_password }}"
+    insecure_registry: "{{ docker_insecure_registry }}"
+    name: cinder_scheduler
+    image: "{{ cinder_scheduler_image_full }}"
+    volumes: "{{ node_config_directory }}/cinder-scheduler/:/opt/kolla/cinder-scheduler/:ro"
+    env:
+      KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
+  when: inventory_hostname in groups['cinder-scheduler']
+
+- name: Starting Cinder-volume container
+  docker:
+    docker_api_version: "{{ docker_api_version }}"
+    net: host
+    pull: "{{ docker_pull_policy }}"
+    restart_policy: "{{ docker_restart_policy }}"
+    restart_policy_retry: "{{ docker_restart_policy_retry }}"
+    state: reloaded
+    registry: "{{ docker_registry }}"
+    username: "{{ docker_registry_username }}"
+    password: "{{ docker_registry_password }}"
+    insecure_registry: "{{ docker_insecure_registry }}"
+    name: cinder_volume
+    image: "{{ cinder_volume_image_full }}"
+    volumes: "{{ node_config_directory }}/cinder-volume/:/opt/kolla/cinder-volume/:ro"
+    env:
+      KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
+  when: inventory_hostname in groups['cinder-volume']

ansible/roles/cinder/templates/cinder.conf.j2

@@ -0,0 +1,55 @@
+[DEFAULT]
+verbose = true
+debug = true
+enable_v1_api=false
+volume_name_template = %s
+glance_api_servers = http://{{ kolla_internal_address }}:{{ glance_api_port }}
+os_region_name = {{ openstack_region_name }}
+{% if cinder_volume_driver == "lvm" %}
+default_volume_type = lvmdriver-1
+enabled_backends = lvmdriver-1
+{% elif cinder_volume_driver == "ceph" %}
+default_volume_type = rbd-1
+enabled_backends = rbd-1
+{% endif %}
+osapi_volume_listen = {{ hostvars[inventory_hostname]['ansible_' + api_interface]['ipv4']['address'] }}
+osapi_volume_listen_port = {{ cinder_api_port }}
+api_paste_config = /etc/cinder/api-paste.ini
+nova_catalog_info = compute:nova:internalURL
+
+[database]
+connection = mysql://{{ cinder_database_user }}:{{ cinder_database_password }}@{{ cinder_database_address }}/{{ cinder_database_name }}
+
+[keystone_authtoken]
+auth_uri = http://{{ kolla_internal_address }}:{{ keystone_public_port }}
+auth_url = http://{{ kolla_internal_address }}:{{ keystone_admin_port }}
+auth_plugin = password
+project_domain_id = default
+user_domain_id = default
+project_name = service
+username = {{ cinder_keystone_user }}
+password = {{ cinder_keystone_password }}
+
+[oslo_concurrency]
+lock_path = /var/lib/cinder/tmp
+
+[oslo_messaging_rabbit]
+rabbit_userid = {{ rabbitmq_user }}
+rabbit_password = {{ rabbitmq_password }}
+rabbit_host = {{ kolla_internal_address }}
+rabbit_ha_queues = true
+
+{% if cinder_volume_driver == "lvm" %}
+[lvmdriver-1]
+lvm_type = default
+volume_group = stack-volumes-lvmdriver-1
+volume_driver = cinder.volume.drivers.lvm.LVMVolumeDriver
+volume_backend_name = lvmdriver-1
+{% elif cinder_volume_driver == "ceph" %}
+[rbd-1]
+volume_driver = cinder.volume.drivers.rbd.RBDDriver
+rbd_pool = volumes
+rbd_user = cinder
+rbd_secret_uuid = {{ metadata_secret }}
+{% endif %}
+

ansible/roles/haproxy/templates/haproxy.cfg.j2

@@ -78,3 +78,9 @@ listen neutron_server
 {% for host in groups['neutron-server'] %}
   server {{ hostvars[host]['ansible_hostname'] }} {{ hostvars[host]['ansible_' + api_interface]['ipv4']['address'] }}:{{ neutron_server_port }} check inter 2000 rise 2 fall 5
 {% endfor %}
+
+listen cinder_api
+  bind {{ kolla_internal_address }}:{{ cinder_api_port }}
+{% for host in groups['cinder-api'] %}
+  server {{ hostvars[host]['ansible_hostname'] }} {{ hostvars[host]['ansible_' + api_interface]['ipv4']['address'] }}:{{ cinder_api_port }} check inter 2000 rise 2 fall 5
+{% endfor %}

ansible/site.yml

@@ -26,3 +26,7 @@
 - hosts: [neutron-server, neutron-agents, compute]
   roles:
     - { role: neutron, tags: neutron, when: enable_neutron | bool }
+
+- hosts: [cinder-api, cinder-backup, cinder-scheduler, cinder-volume]
+  roles:
+    - { role: cinder, tags: cinder, when: enable_cinder | bool }

etc/kolla/defaults.yml

@@ -33,6 +33,7 @@ nova_api_port: "8774"
 
 neutron_server_port: "9696"
 
+cinder_api_port: "8776"
 
 # These roles are required for Kolla to be operation, however a savvy deployer
 # could disable some of these required roles and run thier own services.

etc/kolla/globals.yml

@@ -93,3 +93,10 @@ openstack_auth_v2:
 # RabbitMQ options
 ####################
 rabbitmq_user: "openstack"
+
+
+#################################
+## Cinder - Block Storage options
+#################################
+enable_cinder: "no"
+cinder_volume_driver: "lvm"

etc/kolla/passwords.yml

@@ -33,6 +33,9 @@ neutron_keystone_password: "password"
 
 metadata_secret: "password"
 
+cinder_database_password: "password"
+cinder_keystone_password: "password"
+
 ####################
 # RabbitMQ options
 ####################