Serialize Keystone and manage haproxy endpoints

Serialize keystone to run the playbook on 1 container first, then
run on the remainder of the containers in the environment.

This also adds a common-task include to disable the haproxy endpoints
for the containers where the role is running, then re-enable them
in the post tasks.

The haproxy endpoint management task will wait for up to 2 minutes
by default for the endpoint to report a healthy/UP or drained/MAINT
state.

Change-Id: I0df71255154cfb28b62bb14736ba0c8fc1f6ae66

playbooks/common-tasks/haproxy-endpoint-manage.yml

@@ -0,0 +1,26 @@
+---
+# Copyright 2017, Logan Vig <logan2211@gmail.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Set haproxy service state
+  haproxy:
+    socket: /var/run/haproxy.stat
+    host: "{{ inventory_hostname }}"
+    state: "{{ haproxy_state | default('enabled') }}"
+    shutdown_sessions: "{{ haproxy_shutdown_sessions | default(False) | bool }}"
+    wait: "{{ haproxy_wait | default(True) | bool }}"
+    wait_interval: "{{ haproxy_wait_interval | default(5) }}"
+    wait_retries: "{{ haproxy_wait_retries | default(24) }}"
+  delegate_to: "{{ item }}"
+  with_items: "{{ groups['haproxy'] }}"

playbooks/os-keystone-install.yml

@@ -15,6 +15,7 @@
 
 - name: Installation and setup of Keystone
   hosts: keystone_all
+  serial: "{{ keystone_serial }}"
   gather_facts: "{{ gather_facts | default(True) }}"
   max_fail_percentage: 20
   user: root
@@ -76,6 +77,9 @@
       with_items:
         - { key: "net.ipv4.ip_local_reserved_ports", value: "{{ keystone_admin_port }}"}
       when: is_metal | bool
+    - include: common-tasks/haproxy-endpoint-manage.yml
+      haproxy_state: disabled
+      when: "{{ groups['keystone_all'] | length > 1 }}"
   roles:
     - role: "os_keystone"
     - role: "openstack_openrc"
@@ -90,8 +94,15 @@
     - role: "system_crontab_coordination"
       tags:
         - crontab
+  post_tasks:
+    - include: common-tasks/haproxy-endpoint-manage.yml
+      haproxy_state: enabled
+      when: "{{ groups['keystone_all'] | length > 1 }}"
   vars:
     is_metal: "{{ properties.is_metal|default(false) }}"
+    keystone_serial:
+      - 1
+      - 100%
     keystone_admin_port: 35357
     keystone_galera_user: keystone
     keystone_galera_database: keystone