From 0152e51d7ee0de38377ea81cba6c5ec18f9a861b Mon Sep 17 00:00:00 2001 From: Duong Ha-Quang Date: Mon, 25 Jun 2018 17:03:31 +0700 Subject: [PATCH] Apply Ironic rolling upgrade logic This patchset apply Ironic rolling upgrade logic [1][2] [1] https://docs.openstack.org/ironic/latest/contributor/rolling-upgrades.html [2] https://docs.openstack.org/ironic/latest/admin/upgrade-guide.html#rolling-upgrades Depends-On: https://review.openstack.org/#/c/575594/ Co-author: Ha Manh Dong Change-Id: Id68244951dc66d5c3423ef44324bd72058f4ba67 Implements: blueprint apply-service-upgrade-procedure --- ansible/group_vars/all.yml | 3 ++ ansible/roles/ironic/defaults/main.yml | 2 +- ansible/roles/ironic/handlers/main.yml | 48 +++++++++---------- .../roles/ironic/tasks/bootstrap_service.yml | 10 ++-- ansible/roles/ironic/tasks/legacy_upgrade.yml | 7 +++ .../roles/ironic/tasks/rolling_upgrade.yml | 44 +++++++++++++++++ ansible/roles/ironic/tasks/upgrade.yml | 14 ++---- ansible/roles/ironic/templates/ironic.conf.j2 | 4 ++ etc/kolla/globals.yml | 7 +++ ...onic-rolling-upgrade-c45536fe4814212e.yaml | 5 ++ 10 files changed, 106 insertions(+), 38 deletions(-) create mode 100644 ansible/roles/ironic/tasks/legacy_upgrade.yml create mode 100644 ansible/roles/ironic/tasks/rolling_upgrade.yml create mode 100644 releasenotes/notes/implement-ironic-rolling-upgrade-c45536fe4814212e.yaml diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index f2efd03656..8431796505 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -352,6 +352,9 @@ openstack_logging_debug: "False" openstack_region_name: "RegionOne" +# Variable defined the pin_release_version, apply for rolling upgrade process +openstack_previous_release_name: "queens" + # A list of policy file formats that are supported by Oslo.policy supported_policy_format_list: - policy.yaml diff --git a/ansible/roles/ironic/defaults/main.yml b/ansible/roles/ironic/defaults/main.yml index 64a76e13c4..c21b23897f 100644 --- a/ansible/roles/ironic/defaults/main.yml +++ b/ansible/roles/ironic/defaults/main.yml @@ -143,7 +143,7 @@ ironic_dnsmasq_boot_file: "{% if enable_ironic_ipxe | bool %}undionly.kpxe{% els ironic_cleaning_network: ironic_console_serial_speed: "115200n8" ironic_ipxe_url: http://{{ api_interface_address }}:{{ ironic_ipxe_port }} - +ironic_enable_rolling_upgrade: "yes" #################### ## Kolla diff --git a/ansible/roles/ironic/handlers/main.yml b/ansible/roles/ironic/handlers/main.yml index 0fb1aa1538..15ded2fafb 100644 --- a/ansible/roles/ironic/handlers/main.yml +++ b/ansible/roles/ironic/handlers/main.yml @@ -1,28 +1,4 @@ --- -- name: Restart ironic-api container - vars: - service_name: "ironic-api" - service: "{{ ironic_services[service_name] }}" - config_json: "{{ ironic_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}" - ironic_conf: "{{ ironic_confs.results|selectattr('item.key', 'equalto', service_name)|first }}" - policy_json: "{{ ironic_policy_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}" - ironic_api_container: "{{ check_ironic_containers.results|selectattr('item.key', 'equalto', service_name)|first }}" - become: true - kolla_docker: - action: "recreate_or_restart_container" - common_options: "{{ docker_common_options }}" - name: "{{ service.container_name }}" - image: "{{ service.image }}" - volumes: "{{ service.volumes|reject('equalto', '')|list }}" - when: - - kolla_action != "config" - - inventory_hostname in groups[service.group] - - service.enabled | bool - - config_json.changed | bool - or ironic_conf.changed | bool - or policy_json.changed | bool - or ironic_api_container.changed | bool - - name: Restart ironic-conductor container vars: service_name: "ironic-conductor" @@ -48,6 +24,30 @@ or policy_json.changed | bool or ironic_conductor_container.changed | bool +- name: Restart ironic-api container + vars: + service_name: "ironic-api" + service: "{{ ironic_services[service_name] }}" + config_json: "{{ ironic_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}" + ironic_conf: "{{ ironic_confs.results|selectattr('item.key', 'equalto', service_name)|first }}" + policy_json: "{{ ironic_policy_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}" + ironic_api_container: "{{ check_ironic_containers.results|selectattr('item.key', 'equalto', service_name)|first }}" + become: true + kolla_docker: + action: "recreate_or_restart_container" + common_options: "{{ docker_common_options }}" + name: "{{ service.container_name }}" + image: "{{ service.image }}" + volumes: "{{ service.volumes|reject('equalto', '')|list }}" + when: + - kolla_action != "config" + - inventory_hostname in groups[service.group] + - service.enabled | bool + - config_json.changed | bool + or ironic_conf.changed | bool + or policy_json.changed | bool + or ironic_api_container.changed | bool + - name: Restart ironic-inspector container vars: service_name: "ironic-inspector" diff --git a/ansible/roles/ironic/tasks/bootstrap_service.yml b/ansible/roles/ironic/tasks/bootstrap_service.yml index fa1a6df65f..cbb27021ca 100644 --- a/ansible/roles/ironic/tasks/bootstrap_service.yml +++ b/ansible/roles/ironic/tasks/bootstrap_service.yml @@ -2,14 +2,18 @@ - name: Running Ironic bootstrap container vars: ironic_api: "{{ ironic_services['ironic-api'] }}" + bootstrap_environment: + KOLLA_BOOTSTRAP: + KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}" + upgrade_environment: + KOLLA_UPGRADE: + KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}" become: true kolla_docker: action: "start_container" common_options: "{{ docker_common_options }}" detach: False - environment: - KOLLA_BOOTSTRAP: - KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}" + environment: "{{ upgrade_environment if ironic_enable_rolling_upgrade|bool else bootstrap_environment }}" image: "{{ ironic_api.image }}" labels: BOOTSTRAP: diff --git a/ansible/roles/ironic/tasks/legacy_upgrade.yml b/ansible/roles/ironic/tasks/legacy_upgrade.yml new file mode 100644 index 0000000000..c38db1adf4 --- /dev/null +++ b/ansible/roles/ironic/tasks/legacy_upgrade.yml @@ -0,0 +1,7 @@ +--- +- include: config.yml + +- include: bootstrap_service.yml + +- name: Flush handlers + meta: flush_handlers diff --git a/ansible/roles/ironic/tasks/rolling_upgrade.yml b/ansible/roles/ironic/tasks/rolling_upgrade.yml new file mode 100644 index 0000000000..f77d6060d8 --- /dev/null +++ b/ansible/roles/ironic/tasks/rolling_upgrade.yml @@ -0,0 +1,44 @@ +--- +- include: pull.yml + +# Pin release version +- include: config.yml + vars: + pin_release_version: "{{ openstack_previous_release_name }}" + +- include: bootstrap_service.yml + +# TODO(donghm): Flush_handlers to restart ironic services +# should be run in serial nodes to decrease downtime. Update when +# the module ansible strategy for rolling upgrade is finished. + +# Restart ironic services with pinned release version +- name: Flush handlers + meta: flush_handlers + +# Unpin version +- include: config.yml + +# Restart ironic services with unpinned release version +- name: Flush handlers + meta: flush_handlers + +- name: Running Ironic online data migration + vars: + ironic_api: "{{ ironic_services['ironic-api'] }}" + kolla_docker: + action: "start_container" + common_options: "{{ docker_common_options }}" + detach: False + environment: + KOLLA_OSM: + KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}" + image: "{{ ironic_api.image }}" + labels: + BOOTSTRAP: + name: "bootstrap_ironic" + restart_policy: "never" + volumes: "{{ ironic_api.volumes }}" + run_once: True + delegate_to: "{{ groups[ironic_api.group][0] }}" + when: inventory_hostname in groups[ironic_api.group] diff --git a/ansible/roles/ironic/tasks/upgrade.yml b/ansible/roles/ironic/tasks/upgrade.yml index 62e5323fdd..3533e93670 100644 --- a/ansible/roles/ironic/tasks/upgrade.yml +++ b/ansible/roles/ironic/tasks/upgrade.yml @@ -1,12 +1,6 @@ --- -- include: register.yml - when: enable_keystone | bool and - (inventory_hostname in groups['ironic-api'] or - inventory_hostname in groups['ironic-inspector']) +- include: rolling_upgrade.yml + when: ironic_enable_rolling_upgrade | bool -- include: config.yml - -- include: bootstrap_service.yml - -- name: Flush handlers - meta: flush_handlers +- include: legacy_upgrade.yml + when: not ironic_enable_rolling_upgrade | bool diff --git a/ansible/roles/ironic/templates/ironic.conf.j2 b/ansible/roles/ironic/templates/ironic.conf.j2 index b8f71897c8..4033a098cc 100644 --- a/ansible/roles/ironic/templates/ironic.conf.j2 +++ b/ansible/roles/ironic/templates/ironic.conf.j2 @@ -15,6 +15,10 @@ log_dir = /var/log/kolla/ironic transport_url = {{ rpc_transport_url }} +{% if pin_release_version is defined %} +pin_release_version = {{ pin_release_version }} +{% endif %} + [oslo_messaging_notifications] transport_url = {{ notify_transport_url }} diff --git a/etc/kolla/globals.yml b/etc/kolla/globals.yml index 5757ceb090..868e78a0f7 100644 --- a/etc/kolla/globals.yml +++ b/etc/kolla/globals.yml @@ -398,6 +398,13 @@ ironic_dnsmasq_dhcp_range: # PXE bootloader file for Ironic Inspector, relative to /tftpboot. #ironic_dnsmasq_boot_file: "pxelinux.0" +# Configure ironic upgrade option, due to currently kolla support +# two upgrade ways for ironic: legacy_upgrade and rolling_upgrade +# The variable "ironic_enable_rolling_upgrade: yes" is meaning legacy_upgrade +# were enabled and opposite +# Rolling upgrade were enable by default +#ironic_enable_rolling_upgrade: "yes" + ###################################### # Manila - Shared File Systems Options ###################################### diff --git a/releasenotes/notes/implement-ironic-rolling-upgrade-c45536fe4814212e.yaml b/releasenotes/notes/implement-ironic-rolling-upgrade-c45536fe4814212e.yaml new file mode 100644 index 0000000000..8be7841dd7 --- /dev/null +++ b/releasenotes/notes/implement-ironic-rolling-upgrade-c45536fe4814212e.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Implement Ironic rolling upgrade logic, enabled by default at + ironic_enable_rolling_upgrade: "yes" in etc/kolla/globals.yml file.