etcd: update to v3.4

* Updates etcd to v3.4
* Updated the config to use v3.4's logging mechanism
* Deprecated etcd CA parameters aren't used, so we are not affected
  by their removal.
* Note that we are not currently guarding against skip-version updates for
  etcd.

Notable non-voting jobs exercising some of this:
* kolla-ansible-ubuntu-upgrade-cephadm (cinder->tooz->etcd3gw->etcd)
* kolla-ansible-ubuntu-zun (see
  https://review.opendev.org/c/openstack/openstack-ansible/+/883194 )

Depends-On: https://review.opendev.org/c/openstack/kolla/+/890464
Change-Id: I086e7bbc7db64421445731a533265e7056fbdb43

ansible/roles/cinder/templates/cinder.conf.j2

@@ -259,5 +259,7 @@ backend_url = {{ redis_connection_string }}
 # NOTE(yoctozepto): we must use etcd3gw (aka etcd3+http) due to issues with alternative (etcd3) and eventlet (as used by cinder)
 # see https://bugs.launchpad.net/kolla-ansible/+bug/1854932
 # and https://review.opendev.org/466098 for details
-backend_url = etcd3+{{ internal_protocol }}://{{ kolla_internal_fqdn }}:{{ etcd_client_port }}{% if openstack_cacert %}?ca_cert={{ openstack_cacert }}{% endif %}
+# NOTE(jan.gutter): etcd v3.4 removed the default `v3alpha` api_version. Until
+# tooz defaults to a newer version, we should explicitly specify `v3`
+backend_url = etcd3+{{ internal_protocol }}://{{ kolla_internal_fqdn }}:{{ etcd_client_port }}?api_version=v3{% if openstack_cacert %}?ca_cert={{ openstack_cacert }}{% endif %}
 {% endif %}

ansible/roles/etcd/defaults/main.yml

@@ -32,7 +32,8 @@ etcd_base_environment:
   ETCD_ADVERTISE_CLIENT_URLS: "{{ etcd_client_internal_endpoint }}"
   ETCD_LISTEN_CLIENT_URLS: "{{ etcd_client_internal_endpoint }}"
   ETCD_LISTEN_PEER_URLS: "{{ etcd_peer_internal_endpoint }}"
-  ETCD_OUT_FILE: "/var/log/kolla/etcd/etcd.log"
+  ETCD_LOGGER: "zap"
+  ETCD_LOG_OUTPUTS: "stderr,/var/log/kolla/etcd/etcd.log"
   KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
   ETCD_CERT_FILE: "{% if etcd_enable_tls | bool %}/etc/etcd/certs/etcd-cert.pem{% endif %}"
   ETCD_KEY_FILE: "{% if etcd_enable_tls | bool %}/etc/etcd/certs/etcd-key.pem{% endif %}"

ansible/roles/etcd/handlers/main.yml

@@ -25,12 +25,25 @@
   listen:
     - Bootstrap etcd services
 
+# When upgrading an etcd cluster we have to do it one by one
+- name: Upgrade etcd non-leaders
+  include_tasks: 'restart_services.yml'
+  when:
+    - inventory_hostname not in (groups.etcd_is_leader_True | default([]))
+    - kolla_action == "upgrade"
+  listen:
+    - Restart etcd container
+  loop: "{{ groups.etcd }}"
+
+# When there is no upgrade we can restart 25% of the services without
+# losing quorum.
 - name: Rolling restart of etcd non-leaders
   include_tasks: 'restart_services.yml'
   when:
     - inventory_hostname not in (groups.etcd_is_leader_True | default([]))
     - groups.etcd.index(inventory_hostname) % 4 == item
     - kolla_action != "config"
+    - kolla_action != "upgrade"
   listen:
     - Restart etcd container
     - Bootstrap etcd services

ansible/roles/etcd/tasks/restart_services.yml

@@ -23,3 +23,19 @@
   until: check_etcd_client_port is success
   retries: 10
   delay: 6
+
+- name: Wait for etcd endpoints to be healthy
+  become: true
+  vars:
+    service_name: "etcd"
+    service: "{{ etcd_services[service_name] }}"
+  command: >-
+    {{ kolla_container_engine }} exec {{ service.container_name }}
+    etcdctl endpoint health
+  changed_when: false
+  register: result
+  until:
+    - result is success
+    - ((result.stdout | from_json | first)['health'] | default(False) | bool)
+  retries: 10
+  delay: 6

ansible/roles/ironic/templates/ironic-inspector.conf.j2

@@ -86,5 +86,7 @@ backend_url = {{ redis_connection_string }}
 # NOTE(yoctozepto): we must use etcd3gw (aka etcd3+http) due to issues with alternative (etcd3) and eventlet (as used by cinder)
 # see https://bugs.launchpad.net/kolla-ansible/+bug/1854932
 # and https://review.opendev.org/466098 for details
-backend_url = etcd3+{{ internal_protocol }}://{{ kolla_internal_fqdn }}:{{ etcd_client_port }}{% if openstack_cacert %}?ca_cert={{ openstack_cacert }}{% endif %}
+# NOTE(jan.gutter): etcd v3.4 removed the default `v3alpha` api_version. Until
+# tooz defaults to a newer version, we should explicitly specify `v3`
+backend_url = etcd3+{{ internal_protocol }}://{{ kolla_internal_fqdn }}:{{ etcd_client_port }}?api_version=v3{% if openstack_cacert %}?ca_cert={{ openstack_cacert }}{% endif %}
 {% endif %}

ansible/roles/masakari/templates/masakari.conf.j2

@@ -76,6 +76,8 @@ connection = mysql+pymysql://{{ masakari_database_user }}:{{ masakari_database_p
 {% if masakari_coordination_backend == 'redis' %}
 backend_url = {{ redis_connection_string }}
 {% elif masakari_coordination_backend == 'etcd' %}
-backend_url = etcd3+{{ internal_protocol }}://{{ kolla_internal_fqdn }}:{{ etcd_client_port }}{% if openstack_cacert %}?ca_cert={{ openstack_cacert }}{% endif %}
+# NOTE(jan.gutter): etcd v3.4 removed the default `v3alpha` api_version. Until
+# tooz defaults to a newer version, we should explicitly specify `v3`
+backend_url = etcd3+{{ internal_protocol }}://{{ kolla_internal_fqdn }}:{{ etcd_client_port }}?api_version=v3{% if openstack_cacert %}?ca_cert={{ openstack_cacert }}{% endif %}
 {% endif %}
 {% endif %}

releasenotes/notes/etcd-3.4-8ced41674f25674f.yaml

@@ -0,0 +1,13 @@
+---
+upgrade:
+  - |
+    `etcd` has been upgraded to version `3.4` in this release. Operators are
+    highly encouraged to read the `upgrade notes <https://etcd.io/docs/v3.4/upgrades/upgrade_3_4/>`__
+    for impacts on `etcd` clients. Upgrades are only supported from
+    `etcd v3.3`: Skip version upgrades are not supported. Please ensure that
+    adequate backups are taken before running the upgrade to guard against
+    dataloss.
+  - |
+    `etcd` version `3.4` drops support for the `v3alpha` endpoint. Internal
+    kolla-ansible endpoints have been updated, but operators are strongly
+    encouraged to audit any customizations or external users of `etcd`.