Implement zero downtime upgrades

This patch implements upgrading keystone with zero downtime as the default installation process. Handlers have been modified to ensure that the first keystone node is stopped, facilitates the database migrations, and that it is started and available before restarting any other keystone nodes. Migrations also now only occur when there is a change within the installed keystone venv. This process is documented at http://docs.openstack.org/developer/keystone/upgrading.html#upgrading-without-downtime A new test scenario has been added for testing basic upgradability between releases. Implements: blueprint upgrade-testing Change-Id: I0d3cfcb80b64d005d60f4c8445f991855f844796
2016-10-04 13:26:05 -07:00 · 2016-10-04 13:26:05 -07:00 · 04737f5dbd
parent 68fd798f31
commit 04737f5dbd
22 changed files with 263 additions and 54 deletions
--- a/handlers/main.yml
+++ b/handlers/main.yml
@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

- name: Restart service
+- name: Restart service on first node
  service:
    name: "{{ keystone_system_service_name }}"
    state: restarted
@ -22,9 +22,13 @@
  until: _restart|success
  retries: 5
  delay: 2
-  when: (keystone_apache_enabled | bool) or (keystone_mod_wsgi_enabled | bool)
+  when:
+    - inventory_hostname == groups['keystone_all'][0]
+    - (keystone_apache_enabled | bool) or (keystone_mod_wsgi_enabled | bool)
+  notify:
+    - Wait for keystone service port

- name: Restart Nginx
+- name: Restart Nginx on first node
  service:
    name: nginx
    state: restarted
@ -32,9 +36,11 @@
  until: keystone_restart | success
  retries: 5
  delay: 2
-  when: not keystone_apache_enabled | bool
+  when:
+    - inventory_hostname == groups['keystone_all'][0]
+    - not keystone_apache_enabled | bool

- name: Restart Keystone APIs
+- name: Restart Keystone APIs on first node
  service:
    name: "{{ item }}"
    state: "restarted"
@ -43,7 +49,58 @@
  retries: 5
  delay: 2
  with_items: "{{ keystone_wsgi_program_names }}"
-  when: not keystone_mod_wsgi_enabled | bool
+  when:
+    - inventory_hostname == groups['keystone_all'][0]
+    - not keystone_mod_wsgi_enabled | bool
+  notify:
+    - Wait for keystone service port
+
+- name: Wait for keystone service port
+  wait_for:
+    port: "{{ keystone_service_port }}"
+    timeout: 25
+    delay: 10
+  register: keystone_wait_check
+  until: keystone_wait_check | success
+  retries: 5
+
+- name: Restart service on other nodes
+  service:
+    name: "{{ keystone_system_service_name }}"
+    state: restarted
+    pattern: "{{ keystone_system_service_name }}"
+  register: _restart
+  until: _restart|success
+  retries: 5
+  delay: 2
+  when:
+    - inventory_hostname != groups['keystone_all'][0]
+    - (keystone_apache_enabled | bool) or (keystone_mod_wsgi_enabled | bool)
+
+- name: Restart Nginx on other nodes
+  service:
+    name: nginx
+    state: restarted
+  register: keystone_restart
+  until: keystone_restart | success
+  retries: 5
+  delay: 2
+  when:
+    - inventory_hostname != groups['keystone_all'][0]
+    - not keystone_apache_enabled | bool
+
+- name: Restart Keystone APIs on other nodes
+  service:
+    name: "{{ item }}"
+    state: "restarted"
+  register: keystone_restart
+  until: keystone_restart | success
+  retries: 5
+  delay: 2
+  with_items: "{{ keystone_wsgi_program_names }}"
+  when:
+    - inventory_hostname != groups['keystone_all'][0]
+    - not keystone_mod_wsgi_enabled | bool

 - name: Restart Shibd
  service:
@ -54,3 +111,8 @@
  until: shibd_restart|success
  retries: 5
  delay: 2
+
+- name: Perform a Keystone DB sync contract
+  command: "{{ keystone_bin }}/keystone-manage db_sync --contract"
+  become: yes
+  become_user: "{{ keystone_system_user_name }}"
--- a/releasenotes/notes/os-keystone-zero-downtime-upgrade-5f19ab84183490b9.yaml
+++ b/releasenotes/notes/os-keystone-zero-downtime-upgrade-5f19ab84183490b9.yaml
@ -0,0 +1,5 @@
+---
+features:
+  - The os_keystone role now performs a rolling upgrade
+    without downtime during installation. This process
+    is documented `here <http://docs.openstack.org/developer/keystone/upgrading.html#upgrading-without-downtime>`_.
--- a/tasks/keystone_apache.yml
+++ b/tasks/keystone_apache.yml
@ -43,7 +43,8 @@
  when:
    - ansible_pkg_mgr == 'apt'
  notify:
-    - Restart service
+    - Restart service on first node
+    - Restart service on other nodes

 ## NOTE(andymccr):
 ## We need to enable a module for httpd on RedHat/CentOS using LoadModule inside conf files
@ -55,7 +56,8 @@
  when:
   - ansible_pkg_mgr == 'yum'
  notify:
-    - Restart service
+    - Restart service on first node
+    - Restart service on other nodes

 - name: Drop apache2 config files
  template:
@ -65,7 +67,8 @@
    group: "root"
  with_items: "{{ keystone_apache_configs }}"
  notify:
-    - Restart service
+    - Restart service on first node
+    - Restart service on other nodes

 - name: Disable default apache site
  file:
@ -73,7 +76,8 @@
    state: "absent"
  with_items: "{{ keystone_apache_default_sites }}"
  notify:
-    - Restart service
+    - Restart service on first node
+    - Restart service on other nodes

 - name: Enabled keystone vhost
  file:
@ -84,14 +88,16 @@
    - keystone_apache_site_available is defined
    - keystone_apache_site_enabled is defined
  notify:
-    - Restart service
+    - Restart service on first node
+    - Restart service on other nodes

 - name: Ensure Apache ServerName
  lineinfile:
    dest: "{{ keystone_apache_conf }}"
    line: "ServerName {{ ansible_hostname }}"
  notify:
-    - Restart service
+    - Restart service on first node
+    - Restart service on other nodes

 - name: Ensure Apache ServerTokens
  lineinfile:
@ -99,7 +105,8 @@
    regexp: '^ServerTokens'
    line: "ServerTokens {{ keystone_apache_servertokens }}"
  notify:
-    - Restart service
+    - Restart service on first node
+    - Restart service on other nodes

 - name: Ensure Apache ServerSignature
  lineinfile:
@ -107,7 +114,8 @@
    regexp: '^ServerSignature'
    line: "ServerSignature {{ keystone_apache_serversignature }}"
  notify:
-    - Restart service
+    - Restart service on first node
+    - Restart service on other nodes

 ## NOTE(mgariepy):
 ## We need to enable httpd on CentOS if not it won't start when the container is restarted.
@ -116,4 +124,5 @@
    name: "{{ keystone_system_service_name }}"
    enabled: "yes"
  notify:
-    - Restart service
+    - Restart service on first node
+    - Restart service on other nodes
--- a/tasks/keystone_db_setup.yml
+++ b/tasks/keystone_db_setup.yml
@ -13,8 +13,30 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

- name: Perform a Keystone DB sync
-  command: "{{ keystone_bin }}/keystone-manage db_sync"
-  changed_when: false
+- name: Ensure keystone service stopped on first node
+  service:
+    name: "{{ item }}"
+    state: stopped
+  register: keystone_stop
+  failed_when:
+    - keystone_stop.msg is defined
+    - "'no service or tool' not in keystone_stop.msg"
+    - "'systemd could not find' not in keystone_stop.msg"
+    - "'Could not find the requested service' not in keystone_stop.msg"
+  with_items:
+    - "{{ keystone_wsgi_program_names }}"
+    - "{{ keystone_system_service_name }}"
+
+- name: Perform a Keystone DB sync expand
+  command: "{{ keystone_bin }}/keystone-manage db_sync --expand"
+  changed_when: true
  become: yes
  become_user: "{{ keystone_system_user_name }}"
+
+- name: Perform a Keystone DB sync migrate
+  command: "{{ keystone_bin }}/keystone-manage db_sync --migrate"
+  changed_when: true
+  become: yes
+  become_user: "{{ keystone_system_user_name }}"
+  notify:
+    - Perform a Keystone DB sync contract
--- a/tasks/keystone_federation_sp_setup.yml
+++ b/tasks/keystone_federation_sp_setup.yml
@ -33,7 +33,8 @@
  changed_when: false
  when: inventory_hostname == groups['keystone_all'][0]
  notify:
-    - Restart service
+    - Restart service on first node
+    - Restart service on other nodes
    - Restart Shibd

 - name: Store Shibboleth SP key-pair
@ -70,7 +71,8 @@
  delay: 2
  when: inventory_hostname != groups['keystone_all'][0]
  notify:
-    - Restart service
+    - Restart service on first node
+    - Restart service on other nodes
    - Restart Shibd

 - name: Set appropriate file ownership on the Shibboleth SP key-pair
@ -83,5 +85,6 @@
    - "/etc/shibboleth/sp-key.pem"
  when: inventory_hostname != groups['keystone_all'][0]
  notify:
-    - Restart service
+    - Restart service on first node
+    - Restart service on other nodes
    - Restart Shibd
--- a/tasks/keystone_idp_metadata.yml
+++ b/tasks/keystone_idp_metadata.yml
@ -20,5 +20,7 @@
  become_user: "{{ keystone_system_user_name }}"
  when: keystone_idp != {}
  notify:
-    - Restart Keystone APIs
-    - Restart service
+    - Restart Keystone APIs on first node
+    - Restart Keystone APIs on other nodes
+    - Restart service on first node
+    - Restart service on other nodes
--- a/tasks/keystone_idp_self_signed_create.yml
+++ b/tasks/keystone_idp_self_signed_create.yml
@ -33,7 +33,8 @@
  when: >
    inventory_hostname == groups['keystone_all'][0]
  notify:
-    - Restart service
+    - Restart service on first node
+    - Restart service on other nodes

 - name: Set appropriate file ownership on the IdP self-signed cert
  file:
--- a/tasks/keystone_idp_self_signed_distribute.yml
+++ b/tasks/keystone_idp_self_signed_distribute.yml
@ -30,7 +30,8 @@
  retries: 5
  delay: 2
  notify:
-    - Restart service
+    - Restart service on first node
+    - Restart service on other nodes

 - name: Set appropriate file ownership on the IdP self-signed cert
  file:
--- a/tasks/keystone_init_systemd.yml
+++ b/tasks/keystone_init_systemd.yml
@ -45,4 +45,7 @@
  command: "systemctl daemon-reload"
  when: systemd_init | changed
  notify:
-    - Restart Keystone APIs
+    - Restart Keystone APIs on first node
+    - Restart Keystone APIs on other nodes
+    - Restart service on first node
+    - Restart service on other nodes
--- a/tasks/keystone_init_upstart.yml
+++ b/tasks/keystone_init_upstart.yml
@ -21,11 +21,18 @@
    owner: "root"
    group: "root"
  register: upstart_init
-  notify: Restart Keystone APIs
+  notify:
+    - Restart Keystone APIs on first node
+    - Restart Keystone APIs on other nodes
+    - Restart service on first node
+    - Restart service on other nodes

 - name: Reload init scripts
  command: initctl reload-configuration
  changed_when: false
  when: upstart_init | changed
  notify:
-    - Restart Keystone APIs
+    - Restart Keystone APIs on first node
+    - Restart Keystone APIs on other nodes
+    - Restart service on first node
+    - Restart service on other nodes
--- a/tasks/keystone_install.yml
+++ b/tasks/keystone_install.yml
@ -84,8 +84,10 @@
    - not keystone_developer_mode | bool
    - keystone_get_venv | changed or keystone_venv_dir | changed
  notify:
-    - Restart Keystone APIs
-    - Restart service
+    - Restart Keystone APIs on first node
+    - Restart Keystone APIs on other nodes
+    - Restart service on first node
+    - Restart service on other nodes

 - name: Install pip packages
  pip:
@ -103,8 +105,10 @@
  delay: 2
  when: keystone_developer_mode | bool
  notify:
-    - Restart Keystone APIs
-    - Restart service
+    - Restart Keystone APIs on first node
+    - Restart Keystone APIs on other nodes
+    - Restart service on first node
+    - Restart service on other nodes

 - name: Update virtualenv path
  command: >
--- a/tasks/keystone_ldap_setup.yml
+++ b/tasks/keystone_ldap_setup.yml
@ -35,8 +35,10 @@
    mode: "0644"
  with_dict: "{{ keystone_ldap }}"
  notify:
-    - Restart Keystone APIs
-    - Restart service
+    - Restart Keystone APIs on first node
+    - Restart Keystone APIs on other nodes
+    - Restart service on first node
+    - Restart service on other nodes

 # Bug 1547542 - Older versions of the keystone role would deploy a blank
 # keystone.Default.conf and this will cause errors when adding LDAP-backed
@ -47,5 +49,7 @@
    state: absent
  when: keystone_ldap.Default is not defined
  notify:
-    - Restart Keystone APIs
-    - Restart service
+    - Restart Keystone APIs on first node
+    - Restart Keystone APIs on other nodes
+    - Restart service on first node
+    - Restart service on other nodes
--- a/tasks/keystone_nginx.yml
+++ b/tasks/keystone_nginx.yml
@ -24,14 +24,18 @@
  file:
    path: /etc/nginx/sites-enabled/default
    state: absent
-  notify: Restart Nginx
+  notify:
+    - Restart Nginx on first node
+    - Restart Nginx on other nodes

 - name: Configure custom nginx log format
  lineinfile:
    insertbefore: access_log
    dest: "/etc/nginx/nginx.conf"
    line: "log_format custom '{{ keystone_nginx_access_log_format_combined }} {{ keystone_nginx_access_log_format_extras }}';"
-  notify: Restart Nginx
+  notify:
+    - Restart Nginx on first node
+    - Restart Nginx on other nodes

 # Configure app
 - name: Configure virtual hosts
@ -39,7 +43,9 @@
    src: keystone_nginx.conf.j2
    dest: "/etc/nginx/{{ keystone_nginx_conf_path }}/{{ item }}.conf"
  with_items: "{{ keystone_wsgi_program_names }}"
-  notify: Restart Nginx
+  notify:
+    - Restart Nginx on first node
+    - Restart Nginx on other nodes

 - name: Link to enable virtual hosts
  file:
@ -48,4 +54,6 @@
    state: link
  with_items: "{{ keystone_wsgi_program_names }}"
  when: ansible_os_family == "Debian"
-  notify: Restart Nginx
+  notify:
+    - Restart Nginx on first node
+    - Restart Nginx on other nodes
--- a/tasks/keystone_post_install.yml
+++ b/tasks/keystone_post_install.yml
@ -36,8 +36,10 @@
      config_overrides: "{{ keystone_policy_overrides }}"
      config_type: "json"
  notify:
-    - Restart Keystone APIs
-    - Restart service
+    - Restart Keystone APIs on first node
+    - Restart Keystone APIs on other nodes
+    - Restart service on first node
+    - Restart service on other nodes

 - name: Copy Keystone Federation SP SSO callback template
  copy:
@ -49,8 +51,10 @@
  when:
    - keystone_idp != {}
  notify:
-    - Restart Keystone APIs
-    - Restart service
+    - Restart Keystone APIs on first node
+    - Restart Keystone APIs on other nodes
+    - Restart service on first node
+    - Restart service on other nodes

 - name: Clean up Keystone Federation SP SSO callback template
  file:
@ -59,5 +63,7 @@
  when:
    - keystone_idp == {}
  notify:
-    - Restart Keystone APIs
-    - Restart service
+    - Restart Keystone APIs on first node
+    - Restart Keystone APIs on other nodes
+    - Restart service on first node
+    - Restart service on other nodes
--- a/tasks/keystone_ssl_key_create.yml
+++ b/tasks/keystone_ssl_key_create.yml
@ -29,7 +29,8 @@
    -extensions v3_ca
    creates={{ keystone_ssl_cert }}
  notify:
-    - Restart service
+    - Restart service on first node
+    - Restart service on other nodes

 - name: Ensure keystone user owns the self-signed key and certificate
  file:
@ -41,4 +42,5 @@
    - "{{ keystone_ssl_key }}"
    - "{{ keystone_ssl_cert }}"
  notify:
-    - Restart service
+    - Restart service on first node
+    - Restart service on other nodes
--- a/tasks/keystone_ssl_user_provided.yml
+++ b/tasks/keystone_ssl_user_provided.yml
@ -22,7 +22,8 @@
    mode: "0644"
  when: keystone_user_ssl_cert is defined
  notify:
-    - Restart service
+    - Restart service on first node
+    - Restart service on other nodes

 - name: Drop user provided ssl key
  copy:
@ -33,7 +34,8 @@
    mode: "0640"
  when: keystone_user_ssl_key is defined
  notify:
-    - Restart service
+    - Restart service on first node
+    - Restart service on other nodes

 - name: Drop user provided ssl CA cert
  copy:
@ -44,4 +46,5 @@
    mode: "0644"
  when: keystone_user_ssl_ca_cert is defined
  notify:
-    - Restart service
+    - Restart service on first node
+    - Restart service on other nodes
--- a/tasks/keystone_uwsgi.yml
+++ b/tasks/keystone_uwsgi.yml
@ -27,7 +27,9 @@
    config_overrides: "{{ keystone_uwsgi_ini_overrides }}"
    config_type: ini
  with_items: "{{ keystone_wsgi_program_names }}"
-  notify: Restart Keystone APIs
+  notify:
+    - Restart Keystone APIs on first node
+    - Restart Keystone APIs on other nodes

 - include: keystone_init_common.yml
  vars:
@ -36,7 +38,9 @@
    system_user: "{{ keystone_system_user_name }}"
    system_group: "{{ keystone_system_group_name }}"
    service_home: "{{ keystone_system_user_home }}"
-  notify: Restart Keystone APIs
+  notify:
+    - Restart Keystone APIs on first node
+    - Restart Keystone APIs on other nodes

 - include: keystone_init_common.yml
  vars:
@ -45,7 +49,9 @@
    system_user: "{{ keystone_system_user_name }}"
    system_group: "{{ keystone_system_group_name }}"
    service_home: "{{ keystone_system_user_home }}"
-  notify: Restart Keystone APIs
+  notify:
+    - Restart Keystone APIs on first node
+    - Restart Keystone APIs on other nodes

 - name: Ensure uwsgi service started
  service:
--- a/tasks/main.yml
+++ b/tasks/main.yml
@ -85,6 +85,7 @@
  when:
    - keystone_database_enabled | bool
    - inventory_hostname == groups['keystone_all'][0]
+    - keystone_get_venv | changed or keystone_venv_dir | changed or install_packages | changed
  tags:
    - keystone-install

--- a/tests/ansible-role-requirements.yml
+++ b/tests/ansible-role-requirements.yml
@ -38,6 +38,10 @@
  src: https://git.openstack.org/openstack/openstack-ansible-openstack_openrc
  scm: git
  version: master
+- name: os_previous_keystone
+  src: https://git.openstack.org/openstack/openstack-ansible-os_keystone
+  scm: git
+  version: stable/newton
 - name: os_tempest
  src: https://git.openstack.org/openstack/openstack-ansible-os_tempest
  scm: git
--- a/tests/test-install-previous-keystone.yml
+++ b/tests/test-install-previous-keystone.yml
@ -0,0 +1,35 @@
+---
+# Copyright 2015, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Playbook for deploying previous keystone
+  hosts: keystone_all
+  user: root
+  gather_facts: true
+  pre_tasks:
+    - name: Set keystone_messaging fact
+      set_fact:
+        keystone_messaging_enabled: "{{ groups['rabbitmq_all'] is defined }}"
+    - include: common/ensure-rabbitmq.yml
+      vhost_name: "{{ keystone_rabbitmq_vhost }}"
+      user_name: "{{ keystone_rabbitmq_userid }}"
+      user_password: "{{ keystone_rabbitmq_password }}"
+      when: "{{ groups['rabbitmq_all'] is defined }}"
+    - include: common/create-grant-db.yml
+      db_name: "{{ keystone_galera_database }}"
+      db_password: "{{ keystone_container_mysql_password }}"
+  roles:
+    - role: "os_previous_keystone"
+  vars_files:
+    - common/previous/test-vars.yml
--- a/tests/test.yml
+++ b/tests/test.yml
@ -25,6 +25,12 @@
 # Install RabbitMQ/MariaDB
 - include: common/test-install-infra.yml

+# Install previous Keystone
+- include: test-install-previous-keystone.yml
+  when:
+    - keystone_upgrade is defined
+    - keystone_upgrade | bool
+
 # Install Keystone
 - include: common/test-install-keystone.yml

--- a/tox.ini
+++ b/tox.ini
@ -110,6 +110,21 @@ commands =
    bash -c "{toxinidir}/tests/common/test-ansible-functional.sh"


+[testenv:upgrade]
+deps =
+    {[testenv:ansible]deps}
+setenv =
+    {[testenv]setenv}
+    ANSIBLE_PARAMETERS=-vvv -e keystone_upgrade=True
+commands =
+    {[testenv:tests_clone]commands}
+    bash -c "if [ ! -d "{toxinidir}/tests/common/previous" ]; then \
+               git clone https://git.openstack.org/openstack/openstack-ansible-tests -b stable/newton \
+                 {toxinidir}/tests/common/previous; \
+             fi"
+    bash -c "{toxinidir}/tests/common/test-ansible-functional.sh"
+
+
 [testenv:uwsgi_apache]
 deps =
    {[testenv:ansible]deps}