diff --git a/group_vars/all/all.yml b/group_vars/all/all.yml index fe7603cb37..f6f857bfde 100644 --- a/group_vars/all/all.yml +++ b/group_vars/all/all.yml @@ -113,3 +113,7 @@ ansible_ssh_extra_args: > -o ForwardX11=no -o ForwardAgent=yes -T + +# Toggle whether the service is deployed in a container or not +is_metal: "{{ properties.is_metal | default(false) }}" + diff --git a/group_vars/cinder_all.yml b/group_vars/cinder_all.yml index 8b9fab7763..8b92b275f0 100644 --- a/group_vars/cinder_all.yml +++ b/group_vars/cinder_all.yml @@ -13,8 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +# The MySQL details for the cinder service +cinder_galera_user: cinder +cinder_galera_database: cinder +cinder_galera_address: "{{ galera_address }}" + +# The address used to listen for communications cinder_management_address: "{{ ansible_host }}" +# The address used for communications with the glance service cinder_glance_host: "{{ internal_lb_vip_address }}" cinder_glance_service_port: "{{ glance_service_port }}" @@ -31,6 +38,9 @@ cinder_glance_api_servers: "{{ glance_api_servers }}" # Ensure that the package state matches the global setting cinder_package_state: "{{ package_state }}" +# The system user for all cinder services +cinder_system_user_name: cinder + # venv fetch configuration cinder_venv_tag: "{{ venv_tag }}" cinder_bin: "/openstack/venvs/cinder-{{ cinder_venv_tag }}/bin" diff --git a/playbooks/common-playbooks/cinder.yml b/playbooks/common-playbooks/cinder.yml new file mode 100644 index 0000000000..8adb16b381 --- /dev/null +++ b/playbooks/common-playbooks/cinder.yml @@ -0,0 +1,130 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Install cinder services + hosts: "{{ cinder_hosts }}" + serial: "{{ cinder_serial }}" + gather_facts: "{{ gather_facts | default(True) }}" + user: root + environment: "{{ deployment_environment_variables | default({}) }}" + tags: + - cinder + pre_tasks: + + # In order to ensure that any container, software or + # config file changes which causes a container/service + # restart do not cause an unexpected outage, we drain + # the load balancer back end for this container. + - include: ../common-tasks/haproxy-endpoint-manage.yml + vars: + haproxy_backend: cinder_api-back + haproxy_state: disabled + when: + - "'cinder_api' in group_names" + - "groups['cinder_api'] | length > 1" + + - name: Determine storage bridge IP address + include: ../common-tasks/dynamic-address-fact.yml + vars: + network_address: "storage_address" + + - name: Configure container (cinder-volume) + include: ../common-tasks/os-lxc-container-setup.yml + static: no + vars: + aa_profile: "unconfined" + extra_container_config: + - "lxc.autodev=0" + - "lxc.cgroup.devices.allow=a *:* rmw" + - "lxc.mount.entry=udev dev devtmpfs defaults 0 0" + extra_container_config_no_restart: + - "lxc.start.order=79" + when: + - "'cinder_volume' in group_names" + - "cinder_backend_lvm_inuse | bool" + + - name: Configure container (other services) + include: ../common-tasks/os-lxc-container-setup.yml + static: no + when: + - "'cinder_volume' not in group_names" + + - name: Configure log directories (on metal) + include: ../common-tasks/os-log-dir-setup.yml + vars: + log_dirs: + - src: "/openstack/log/{{ inventory_hostname }}-cinder" + dest: "/var/log/cinder" + + - name: Configure package proxy cache + include: ../common-tasks/package-cache-proxy.yml + + - name: Add volume group block device to cinder + shell: | + {% if item.value.volume_group is defined %} + if [ "$(pvdisplay | grep -B1 {{ item.value.volume_group }} | awk '/PV/ {print $3}')" ];then + for device in `pvdisplay | grep -B1 {{ item.value.volume_group }} | awk '/PV/ {print $3}'` + do lxc-device -n {{ container_name }} add $device + done + fi + {% else %} + echo "{{ item.key }} volume_group not defined" + {% endif %} + with_dict: "{{ cinder_backends | default({}) }}" + when: + - physical_host != container_name + - cinder_backend_lvm_inuse | bool + delegate_to: "{{ physical_host }}" + + - name: udevadm trigger + command: udevadm trigger + delegate_to: "{{ physical_host }}" + when: cinder_backend_lvm_inuse | bool + + roles: + - role: "os_cinder" + cinder_storage_address: "{{ storage_address }}" + + - role: "ceph_client" + openstack_service_system_user: "{{ cinder_system_user_name }}" + openstack_service_venv_bin: "{{ cinder_bin }}" + when: + - "'cinder_volume' in group_names" + - "cinder_backend_rbd_inuse | default(false) | bool" + tags: + - ceph + + - role: "rsyslog_client" + rsyslog_client_log_rotate_file: cinder_log_rotate + rsyslog_client_log_dir: "/var/log/cinder" + rsyslog_client_config_name: "99-cinder-rsyslog-client.conf" + tags: + - rsyslog + + - role: "system_crontab_coordination" + tags: + - crontab + + post_tasks: + # Now that container changes are done, we can set + # the load balancer back end for this container + # to available again. + - include: ../common-tasks/haproxy-endpoint-manage.yml + vars: + haproxy_backend: cinder_api-back + haproxy_state: enabled + when: + - "'cinder_api' in group_names" + - "groups['cinder_api'] | length > 1" diff --git a/playbooks/common-tasks/restart-service.yml b/playbooks/common-tasks/restart-service.yml new file mode 100644 index 0000000000..007e3cd326 --- /dev/null +++ b/playbooks/common-tasks/restart-service.yml @@ -0,0 +1,45 @@ +--- +# Copyright 2017, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is a generic task set which can be used to execute +# a service action on target hosts for any services. This +# is useful for executing a SIGHUP (reload) to load up any +# configuration changes or to restart services as required. +# +# Inputs: +# - service_name: Any service found matching this prefix will be acted on. +# - service_action: The action to execute [stop, start, restart, reload]. + +- name: Gather service list + shell: "systemctl list-unit-files --state=enabled --type=service | awk '/{{ service_name }}.* enabled$/ {print $1}'" + args: + executable: "/bin/bash" + register: _enabled_services + changed_when: false + +- name: Execute service action + service: + name: "{{ service_file }}" + state: "{{ service_action }}" + with_items: "{{ _enabled_services.stdout_lines }}" + loop_control: + loop_var: service_file + +- name: Disable the service restart requirement + ini_file: + dest: "/etc/ansible/facts.d/openstack_ansible.fact" + section: "{{ service_fact | default(service_name) }}" + option: need_service_restart + value: False diff --git a/playbooks/os-cinder-install.yml b/playbooks/os-cinder-install.yml index 5d3dc07041..6cf3da8759 100644 --- a/playbooks/os-cinder-install.yml +++ b/playbooks/os-cinder-install.yml @@ -13,115 +13,220 @@ # See the License for the specific language governing permissions and # limitations under the License. -- name: Install cinder server +- name: Prepare MQ/DB services hosts: cinder_all gather_facts: "{{ gather_facts | default(True) }}" - max_fail_percentage: 20 user: root - pre_tasks: - - include: common-tasks/dynamic-address-fact.yml - vars: - network_address: "storage_address" - - include: common-tasks/os-lxc-container-setup.yml - static: no - vars: - aa_profile: "unconfined" - extra_container_config: - - "lxc.autodev=0" - - "lxc.cgroup.devices.allow=a *:* rmw" - - "lxc.mount.entry=udev dev devtmpfs defaults 0 0" - extra_container_config_no_restart: - - "lxc.start.order=79" - when: - - inventory_hostname in groups['cinder_volume'] - - cinder_backend_lvm_inuse | bool - - include: common-tasks/os-lxc-container-setup.yml - static: no - when: - - inventory_hostname not in groups['cinder_volume'] - - include: common-tasks/rabbitmq-vhost-user.yml - static: no + environment: "{{ deployment_environment_variables | default({}) }}" + tags: + - cinder + tasks: + + - name: Configure rabbitmq vhost/user + include: common-tasks/rabbitmq-vhost-user.yml vars: user: "{{ cinder_rabbitmq_userid }}" password: "{{ cinder_rabbitmq_password }}" vhost: "{{ cinder_rabbitmq_vhost }}" _rabbitmq_host_group: "{{ cinder_rabbitmq_host_group }}" when: - - inventory_hostname == groups['cinder_all'][0] - - groups[cinder_rabbitmq_host_group] | length > 0 - - include: common-tasks/rabbitmq-vhost-user.yml - static: no + - "groups[cinder_rabbitmq_host_group] | length > 0" + run_once: yes + + - name: Configure rabbitmq vhost/user (telemetry) + include: common-tasks/rabbitmq-vhost-user.yml vars: user: "{{ cinder_rabbitmq_telemetry_userid }}" password: "{{ cinder_rabbitmq_telemetry_password }}" vhost: "{{ cinder_rabbitmq_telemetry_vhost }}" _rabbitmq_host_group: "{{ cinder_rabbitmq_telemetry_host_group }}" when: - - cinder_ceilometer_enabled | bool - - inventory_hostname == groups['cinder_all'][0] - - groups[cinder_rabbitmq_telemetry_host_group] is defined - - groups[cinder_rabbitmq_telemetry_host_group] | length > 0 - - groups[cinder_rabbitmq_telemetry_host_group] != groups[cinder_rabbitmq_host_group] - - include: common-tasks/os-log-dir-setup.yml - vars: - log_dirs: - - src: "/openstack/log/{{ inventory_hostname }}-cinder" - dest: "/var/log/cinder" - - include: common-tasks/mysql-db-user.yml - static: no + - "cinder_ceilometer_enabled | bool" + - "groups[cinder_rabbitmq_telemetry_host_group] is defined" + - "groups[cinder_rabbitmq_telemetry_host_group] | length > 0" + - "groups[cinder_rabbitmq_telemetry_host_group] != groups[cinder_rabbitmq_host_group]" + run_once: yes + + - name: Configure MySQL user + include: common-tasks/mysql-db-user.yml vars: user_name: "{{ cinder_galera_user }}" password: "{{ cinder_container_mysql_password }}" login_host: "{{ cinder_galera_address }}" db_name: "{{ cinder_galera_database }}" - when: inventory_hostname == groups['cinder_all'][0] - - include: common-tasks/package-cache-proxy.yml + run_once: yes - - name: Add volume group block device to cinder - shell: | - {% if item.value.volume_group is defined %} - if [ "$(pvdisplay | grep -B1 {{ item.value.volume_group }} | awk '/PV/ {print $3}')" ];then - for device in `pvdisplay | grep -B1 {{ item.value.volume_group }} | awk '/PV/ {print $3}'` - do lxc-device -n {{ container_name }} add $device - done - fi - {% else %} - echo "{{ item.key }} volume_group not defined" - {% endif %} - with_dict: "{{ cinder_backends | default({}) }}" - when: - - physical_host != container_name - - cinder_backend_lvm_inuse | bool - delegate_to: "{{ physical_host }}" - - name: udevadm trigger - command: udevadm trigger - delegate_to: "{{ physical_host }}" - when: cinder_backend_lvm_inuse | bool - roles: - - role: "os_cinder" - cinder_storage_address: "{{ storage_address }}" - - role: "ceph_client" - openstack_service_system_user: "{{ cinder_system_user_name }}" - openstack_service_venv_bin: "{{ cinder_bin }}" - when: - - inventory_hostname in groups['cinder_volume'] - - cinder_backend_rbd_inuse | default(false) | bool - tags: - - ceph - - role: "rsyslog_client" - rsyslog_client_log_rotate_file: cinder_log_rotate - rsyslog_client_log_dir: "/var/log/cinder" - rsyslog_client_config_name: "99-cinder-rsyslog-client.conf" - tags: - - rsyslog - - role: "system_crontab_coordination" - tags: - - crontab + + +- name: Install cinder API services + include: common-playbooks/cinder.yml vars: - is_metal: "{{ properties.is_metal|default(false) }}" - cinder_galera_user: cinder - cinder_galera_database: cinder - cinder_galera_address: "{{ galera_address }}" + cinder_hosts: "cinder_api" + cinder_serial: "{{ cinder_api_serial | default(['1', '100%']) }}" + + + +- name: Install cinder scheduler services + include: common-playbooks/cinder.yml + vars: + cinder_hosts: "cinder_scheduler:!cinder_api" + cinder_serial: "{{ cinder_scheduler_serial | default(['1', '100%']) }}" + + + +- name: Install cinder volume services + include: common-playbooks/cinder.yml + vars: + cinder_hosts: "cinder_volume:!cinder_scheduler:!cinder_api" + cinder_serial: "{{ cinder_backend_serial | default('1') }}" + + + +- name: Install cinder backup services + include: common-playbooks/cinder.yml + vars: + cinder_hosts: "cinder_backup:!cinder_volume:!cinder_scheduler:!cinder_api" + cinder_serial: "{{ cinder_backend_serial | default('1') }}" + + + +# These facts are set against the deployment host to ensure that +# they are fast to access. This is done in preference to setting +# them against each target as the hostvars extraction will take +# a long time if executed against a large inventory. +- name: Refresh local facts after all software changes are made + hosts: cinder_all + max_fail_percentage: 20 + user: root environment: "{{ deployment_environment_variables | default({}) }}" tags: - cinder + tasks: + - name: refresh local facts + setup: + filter: ansible_local + gather_subset: "!all" + + # This variable contains the values of the local fact set for the cinder + # venv tag for all hosts in the 'cinder_all' host group. + - name: Gather software version list + set_fact: + cinder_all_software_versions: "{{ (groups['cinder_all'] | map('extract', hostvars, ['ansible_local', 'openstack_ansible', 'cinder', 'venv_tag'])) | list }}" + delegate_to: localhost + run_once: yes + + # This variable outputs a boolean value which is True when + # cinder_all_software_versions contains a list of defined + # values. If they are not defined, it means that not all + # hosts have their software deployed yet. + - name: Set software deployed fact + set_fact: + cinder_all_software_deployed: "{{ (cinder_all_software_versions | select('defined')) | list == cinder_all_software_versions }}" + delegate_to: localhost + run_once: yes + + # This variable outputs a boolean when all the values in + # cinder_all_software_versions are the same and the software + # has been deployed to all hosts in the group. + - name: Set software updated fact + set_fact: + cinder_all_software_updated: "{{ ((cinder_all_software_versions | unique) | length == 1) and (cinder_all_software_deployed | bool) }}" + delegate_to: localhost + run_once: yes + + + +- name: Restart cinder agents to ensure new RPC object version is used + hosts: cinder_backup,cinder_volume,cinder_scheduler + gather_facts: no + serial: "{{ cinder_backend_serial | default('1') }}" + max_fail_percentage: 20 + user: root + environment: "{{ deployment_environment_variables | default({}) }}" + tags: + - cinder + tasks: + - name: Execute cinder service reload + include: common-tasks/restart-service.yml + vars: + service_name: "{{ item }}" + service_action: "reloaded" + service_fact: "cinder" + with_items: + - cinder-backup + - cinder-volume + - cinder-scheduler + when: + - "cinder_all_software_updated | bool" + - "ansible_local['openstack_ansible']['cinder']['need_service_restart'] | bool" + + + +- name: Restart cinder API to ensure new RPC object version is used + hosts: cinder_api + gather_facts: no + serial: "{{ cinder_api_serial | default(['1','100%']) }}" + max_fail_percentage: 20 + user: root + environment: "{{ deployment_environment_variables | default({}) }}" + tags: + - cinder + tasks: + # In order to ensure that the service restart does not + # cause an unexpected outage, we drain the load balancer + # back end for this container. + - include: common-tasks/haproxy-endpoint-manage.yml + vars: + haproxy_state: disabled + when: + - "cinder_all_software_updated | bool" + - "ansible_local['openstack_ansible']['cinder']['need_service_restart'] | bool" + - "groups['cinder_api'] | length > 1" + + - name: Execute cinder service restart + include: common-tasks/restart-service.yml + vars: + service_name: "cinder-api" + service_action: "restarted" + service_fact: "cinder" + when: + - "cinder_all_software_updated | bool" + - "ansible_local['openstack_ansible']['cinder']['need_service_restart'] | bool" + + # Now that service restart is done, we can set + # the load balancer back end for this container + # to available again. + - include: common-tasks/haproxy-endpoint-manage.yml + vars: + haproxy_state: enabled + when: "groups['cinder_api'] | length > 1" + + + +- name: Perform online database migrations + hosts: cinder_api[0] + gather_facts: no + user: root + environment: "{{ deployment_environment_variables | default({}) }}" + tags: + - cinder + tasks: + - name: Perform online data migrations + command: "{{ cinder_bin }}/cinder-manage db online-data-migrations" + become: yes + become_user: "{{ cinder_system_user_name }}" + when: + - "cinder_all_software_updated | bool" + - "ansible_local['openstack_ansible']['cinder']['need_online_data_migrations'] | bool" + changed_when: false + register: data_migrations + + - name: Disable the online migrations requirement + ini_file: + dest: "/etc/ansible/facts.d/openstack_ansible.fact" + section: cinder + option: need_online_data_migrations + value: False + when: + - data_migrations | succeeded + diff --git a/releasenotes/notes/cinder-rolling-upgrades-f069187fc9139276.yaml b/releasenotes/notes/cinder-rolling-upgrades-f069187fc9139276.yaml new file mode 100644 index 0000000000..d00d7f54a6 --- /dev/null +++ b/releasenotes/notes/cinder-rolling-upgrades-f069187fc9139276.yaml @@ -0,0 +1,10 @@ +--- +features: + - | + The ``os-cinder-install.yml`` playbook will now execute a rolling + upgrade of cinder including database migrations (both schema and + online) as per the procedure described in the + `cinder documentation `_. + When haproxy is used as the load balancer, the backend being + changed will be drained before changes are made, then added back + to the pool once the changes are complete.