diff --git a/ansible/roles/cinder/templates/cinder.conf.j2 b/ansible/roles/cinder/templates/cinder.conf.j2 index f4b72fa43b..e89e2008e0 100644 --- a/ansible/roles/cinder/templates/cinder.conf.j2 +++ b/ansible/roles/cinder/templates/cinder.conf.j2 @@ -259,5 +259,7 @@ backend_url = {{ redis_connection_string }} # NOTE(yoctozepto): we must use etcd3gw (aka etcd3+http) due to issues with alternative (etcd3) and eventlet (as used by cinder) # see https://bugs.launchpad.net/kolla-ansible/+bug/1854932 # and https://review.opendev.org/466098 for details -backend_url = etcd3+{{ internal_protocol }}://{{ kolla_internal_fqdn }}:{{ etcd_client_port }}{% if openstack_cacert %}?ca_cert={{ openstack_cacert }}{% endif %} +# NOTE(jan.gutter): etcd v3.4 removed the default `v3alpha` api_version. Until +# tooz defaults to a newer version, we should explicitly specify `v3` +backend_url = etcd3+{{ internal_protocol }}://{{ kolla_internal_fqdn }}:{{ etcd_client_port }}?api_version=v3{% if openstack_cacert %}?ca_cert={{ openstack_cacert }}{% endif %} {% endif %} diff --git a/ansible/roles/etcd/defaults/main.yml b/ansible/roles/etcd/defaults/main.yml index c19bd2f328..02b3d80d9f 100644 --- a/ansible/roles/etcd/defaults/main.yml +++ b/ansible/roles/etcd/defaults/main.yml @@ -32,7 +32,8 @@ etcd_base_environment: ETCD_ADVERTISE_CLIENT_URLS: "{{ etcd_client_internal_endpoint }}" ETCD_LISTEN_CLIENT_URLS: "{{ etcd_client_internal_endpoint }}" ETCD_LISTEN_PEER_URLS: "{{ etcd_peer_internal_endpoint }}" - ETCD_OUT_FILE: "/var/log/kolla/etcd/etcd.log" + ETCD_LOGGER: "zap" + ETCD_LOG_OUTPUTS: "stderr,/var/log/kolla/etcd/etcd.log" KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}" ETCD_CERT_FILE: "{% if etcd_enable_tls | bool %}/etc/etcd/certs/etcd-cert.pem{% endif %}" ETCD_KEY_FILE: "{% if etcd_enable_tls | bool %}/etc/etcd/certs/etcd-key.pem{% endif %}" diff --git a/ansible/roles/etcd/handlers/main.yml b/ansible/roles/etcd/handlers/main.yml index 5813fcfaf5..408b97b6e5 100644 --- a/ansible/roles/etcd/handlers/main.yml +++ b/ansible/roles/etcd/handlers/main.yml @@ -25,12 +25,25 @@ listen: - Bootstrap etcd services +# When upgrading an etcd cluster we have to do it one by one +- name: Upgrade etcd non-leaders + include_tasks: 'restart_services.yml' + when: + - inventory_hostname not in (groups.etcd_is_leader_True | default([])) + - kolla_action == "upgrade" + listen: + - Restart etcd container + loop: "{{ groups.etcd }}" + +# When there is no upgrade we can restart 25% of the services without +# losing quorum. - name: Rolling restart of etcd non-leaders include_tasks: 'restart_services.yml' when: - inventory_hostname not in (groups.etcd_is_leader_True | default([])) - groups.etcd.index(inventory_hostname) % 4 == item - kolla_action != "config" + - kolla_action != "upgrade" listen: - Restart etcd container - Bootstrap etcd services diff --git a/ansible/roles/etcd/tasks/restart_services.yml b/ansible/roles/etcd/tasks/restart_services.yml index b3c4de5264..622e1b467e 100644 --- a/ansible/roles/etcd/tasks/restart_services.yml +++ b/ansible/roles/etcd/tasks/restart_services.yml @@ -23,3 +23,19 @@ until: check_etcd_client_port is success retries: 10 delay: 6 + +- name: Wait for etcd endpoints to be healthy + become: true + vars: + service_name: "etcd" + service: "{{ etcd_services[service_name] }}" + command: >- + {{ kolla_container_engine }} exec {{ service.container_name }} + etcdctl endpoint health + changed_when: false + register: result + until: + - result is success + - ((result.stdout | from_json | first)['health'] | default(False) | bool) + retries: 10 + delay: 6 diff --git a/ansible/roles/ironic/templates/ironic-inspector.conf.j2 b/ansible/roles/ironic/templates/ironic-inspector.conf.j2 index d0fe3a3bf0..8781919efe 100644 --- a/ansible/roles/ironic/templates/ironic-inspector.conf.j2 +++ b/ansible/roles/ironic/templates/ironic-inspector.conf.j2 @@ -86,5 +86,7 @@ backend_url = {{ redis_connection_string }} # NOTE(yoctozepto): we must use etcd3gw (aka etcd3+http) due to issues with alternative (etcd3) and eventlet (as used by cinder) # see https://bugs.launchpad.net/kolla-ansible/+bug/1854932 # and https://review.opendev.org/466098 for details -backend_url = etcd3+{{ internal_protocol }}://{{ kolla_internal_fqdn }}:{{ etcd_client_port }}{% if openstack_cacert %}?ca_cert={{ openstack_cacert }}{% endif %} +# NOTE(jan.gutter): etcd v3.4 removed the default `v3alpha` api_version. Until +# tooz defaults to a newer version, we should explicitly specify `v3` +backend_url = etcd3+{{ internal_protocol }}://{{ kolla_internal_fqdn }}:{{ etcd_client_port }}?api_version=v3{% if openstack_cacert %}?ca_cert={{ openstack_cacert }}{% endif %} {% endif %} diff --git a/ansible/roles/masakari/templates/masakari.conf.j2 b/ansible/roles/masakari/templates/masakari.conf.j2 index 081688aed1..7f4b8f3fba 100644 --- a/ansible/roles/masakari/templates/masakari.conf.j2 +++ b/ansible/roles/masakari/templates/masakari.conf.j2 @@ -76,6 +76,8 @@ connection = mysql+pymysql://{{ masakari_database_user }}:{{ masakari_database_p {% if masakari_coordination_backend == 'redis' %} backend_url = {{ redis_connection_string }} {% elif masakari_coordination_backend == 'etcd' %} -backend_url = etcd3+{{ internal_protocol }}://{{ kolla_internal_fqdn }}:{{ etcd_client_port }}{% if openstack_cacert %}?ca_cert={{ openstack_cacert }}{% endif %} +# NOTE(jan.gutter): etcd v3.4 removed the default `v3alpha` api_version. Until +# tooz defaults to a newer version, we should explicitly specify `v3` +backend_url = etcd3+{{ internal_protocol }}://{{ kolla_internal_fqdn }}:{{ etcd_client_port }}?api_version=v3{% if openstack_cacert %}?ca_cert={{ openstack_cacert }}{% endif %} {% endif %} {% endif %} diff --git a/releasenotes/notes/etcd-3.4-8ced41674f25674f.yaml b/releasenotes/notes/etcd-3.4-8ced41674f25674f.yaml new file mode 100644 index 0000000000..b3ded9c1d4 --- /dev/null +++ b/releasenotes/notes/etcd-3.4-8ced41674f25674f.yaml @@ -0,0 +1,13 @@ +--- +upgrade: + - | + `etcd` has been upgraded to version `3.4` in this release. Operators are + highly encouraged to read the `upgrade notes `__ + for impacts on `etcd` clients. Upgrades are only supported from + `etcd v3.3`: Skip version upgrades are not supported. Please ensure that + adequate backups are taken before running the upgrade to guard against + dataloss. + - | + `etcd` version `3.4` drops support for the `v3alpha` endpoint. Internal + kolla-ansible endpoints have been updated, but operators are strongly + encouraged to audit any customizations or external users of `etcd`.