Merge "Implementation of keepalived for haproxy"

ansible-role-requirements.yml

@@ -0,0 +1,9 @@
+# Use this file to fill in your third party roles that you'd like to have added to the list of available roles.
+# Example:
+#  - github_api: https://api.github.com/repos/os-cloud/opc_role-galera_client
+#    name: galera_client
+#    src: https://github.com/os-cloud/opc_role-galera_client
+#    version: master
+- src: evrardjp.keepalived
+  name: keepalived
+  version: '1.3'

doc/source/install-guide/configure-haproxy.rst

@@ -23,8 +23,61 @@ balancer prior to deploying OSA.
          123458-infra03:
            ip: 172.29.236.53
 
+Making HAProxy highly-available
+###############################
+
+HAProxy will be deployed in a highly-available manner, by installing
+keepalived if multiple hosts are found in the inventory.
+
+To skip the deployment of keepalived along HAProxy when installing
+HAProxy on multiple hosts, edit the
+``/etc/openstack_deploy/user_variables.yml`` by setting:
+
+.. code-block:: yaml
+
+   haproxy_use_keepalived: False
+
+Otherwise, edit at least the following variables in
+``user_variables.yml`` to make keepalived work:
+
+.. code-block:: yaml
+
+   haproxy_keepalived_external_vip_cidr: 192.168.0.4/25
+   haproxy_keepalived_internal_vip_cidr: 172.29.236.54/16
+   haproxy_keepalived_external_interface: br-flat
+   haproxy_keepalived_internal_interface: br-mgmt
+
+``haproxy_keepalived_internal_interface`` represents the interface
+on the deployed node where the keepalived master will bind the
+internal vip. By default the ``br-mgmt`` will be used.
+
+``haproxy_keepalived_external_interface`` represents the interface
+on the deployed node where the keepalived master will bind the
+external vip. By default the ``br-mgmt`` will be used.
+
+``haproxy_keepalived_external_vip_cidr`` represents the external
+vip (and its netmask) that will be used on keepalived master host.
+
+``haproxy_keepalived_internal_vip_cidr`` represents the internal
+vip (and its netmask) that will be used on keepalived master host.
+
+Additional variables can be set to adapt keepalived in the deployed
+environment. Please refer to the ``user_variables.yml``
+for more descriptions.
+
+All the variables mentionned here before are used in the variable
+files ``vars/configs/keepalived_haproxy_master.yml`` and
+``vars/configs/keepalived_haproxy_backup.yml`` to feed the
+keepalived role. More information can be found on the keepalived
+role documentation. You can use your own file by setting their path
+in your ``/etc/openstack_deploy/user_variables.yml``:
+
+.. code-block:: yaml
+
+   haproxy_keepalived_vars_file:
+
 Securing HAProxy communication with SSL certificates
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+####################################################
 
 The openstack-ansible project provides the ability to secure HAProxy
 communications with self-signed or user-provided SSL certificates.

doc/source/install-guide/install-foundation-run.rst

@@ -24,7 +24,26 @@ Running the foundation playbook
        ...
        deployment_host                :  ok=18   changed=11   unreachable=0    failed=0
 
-#. If using HAProxy, run the playbook to deploy it:
+#. If using HAProxy:
+
+   .. note::
+
+     If you plan to run haproxy on multiple hosts, you'll need keepalived
+     to make haproxy highly-available. The keepalived role should have
+     been downloaded during the bootstrap-ansible stage. If not, you should
+     rerun the following command before running the haproxy playbook:
+
+     .. code-block:: shell
+
+        $ ../scripts/bootstrap-ansible.sh
+
+     or
+
+     .. code-block:: shell
+
+        $ ansible-galaxy install -r ../ansible-role-requirements.yml
+
+  Run the playbook to deploy haproxy:
 
   .. code-block:: bash
 

etc/openstack_deploy/conf.d/haproxy.yml.example

@@ -0,0 +1,6 @@
+# The nodes where haproxy will run
+haproxy_hosts:
+  infra1:
+    ip: 172.20.236.110
+  infra2:
+    ip: 172.20.236.111

etc/openstack_deploy/env.d/haproxy.yml

@@ -0,0 +1,39 @@
+---
+# Copyright 2015, Jean-Philippe Evrard <jean-philippe@evrard.me>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+component_skel:
+  haproxy:
+    belongs_to:
+      # This is a meta group of a given component type.
+      - haproxy_all
+
+container_skel:
+  haproxy_container:
+    belongs_to:
+      - haproxy_containers
+    contains:
+      - haproxy
+    properties:
+      service_name: haproxy
+      #container_release: trusty
+      is_metal: true
+
+physical_skel:
+  haproxy_containers:
+    belongs_to:
+      - all_containers
+  haproxy_hosts:
+    belongs_to:
+      - hosts

etc/openstack_deploy/user_secrets.yml

@@ -95,3 +95,4 @@ swift_hash_path_prefix:
 
 ## haproxy stats password
 haproxy_stats_password:
+haproxy_keepalived_authentication_password:

etc/openstack_deploy/user_variables.yml

@@ -167,3 +167,25 @@ ssl_cipher_suite: "ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AE
 # If an increased delay for the ssh connection check is desired,
 # uncomment this variable and set it appropriately.
 #ssh_delay: 5
+
+## HAProxy
+# Uncomment this to disable keepalived installation (cf. documentation)
+#haproxy_use_keepalived: False
+#
+# HAProxy Keepalived configuration (cf. documentation)
+haproxy_keepalived_external_vip_cidr: "{{external_lb_vip_address}}/32"
+haproxy_keepalived_internal_vip_cidr: "{{internal_lb_vip_address}}/32"
+#haproxy_keepalived_external_interface:
+#haproxy_keepalived_internal_interface:
+# Defines the default VRRP id used for keepalived with haproxy.
+# Overwrite it to your value to make sure you don't overlap
+# with existing VRRPs id on your network. Default is 10 for the external and 11 for the
+# internal VRRPs
+#haproxy_keepalived_external_virtual_router_id:
+#haproxy_keepalived_internal_virtual_router_id:
+# Defines the VRRP master/backup priority. Defaults respectively to 100 and 20
+#haproxy_keepalived_priority_master:
+#haproxy_keepalived_priority_backup:
+# All the previous variables are used in a var file, fed to the keepalived role.
+# To use another file to feed the role, override the following var:
+#haproxy_keepalived_vars_file: 'vars/configs/keepalived_haproxy.yml'

playbooks/haproxy-install.yml

@@ -13,6 +13,25 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+- hosts: haproxy_hosts
+  vars_files:
+    - "{{ haproxy_keepalived_vars_file | default('vars/configs/keepalived_haproxy.yml')}}"
+  roles:
+    - role: "keepalived"
+      keepalived_sync_groups: "{{ keepalived_master_sync_groups }}"
+      keepalived_scripts: "{{ keepalived_master_scripts }}"
+      keepalived_instances: "{{ keepalived_master_instances }}"
+      when: >
+        haproxy_use_keepalived|bool and
+        inventory_hostname in groups['haproxy_hosts'][0]
+    - role: "keepalived"
+      keepalived_sync_groups: "{{ keepalived_backup_sync_groups }}"
+      keepalived_scripts: "{{ keepalived_backup_scripts }}"
+      keepalived_instances: "{{ keepalived_backup_instances }}"
+      when: >
+        haproxy_use_keepalived|bool and
+        inventory_hostname in groups['haproxy_hosts'][1:]
+
 - name: Install haproxy
   hosts: haproxy_hosts
   max_fail_percentage: 20

playbooks/inventory/group_vars/hosts.yml

@@ -233,3 +233,4 @@ swift_service_region: "{{ service_region }}"
 
 ## HAProxy
 haproxy_bind_on_non_local: "{% if groups.haproxy_hosts[1] is defined and internal_lb_vip_address != external_lb_vip_address %}True{% else %}False{% endif %}"
+haproxy_use_keepalived: "{% if groups.haproxy_hosts|length > 1 %}True{% else %}False{% endif %}"

playbooks/vars/configs/keepalived_haproxy.yml

@@ -0,0 +1,90 @@
+---
+# Copyright 2015, Jean-Philippe Evrard <jean-philippe@evrard.me>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+keepalived_global_sync_groups:
+  haproxy:
+    instances:
+      - external
+      - internal
+    notify_script: /etc/keepalived/haproxy_notify.sh
+    ##if a src_*_script is defined, it will be uploaded from src_*_script on the deploy host to the *_script location. Make sure *_script is a location in that case.
+    #src_notify_script: /opt/os-ansible-deployment/playbooks/vars/configs/keepalived_haproxy_notifications.sh
+    src_notify_script: vars/configs/keepalived_haproxy_notifications.sh
+
+# Master and backup sync groups should normally be the same.
+keepalived_master_sync_groups: "{{ keepalived_global_sync_groups }}"
+keepalived_backup_sync_groups: "{{ keepalived_global_sync_groups }}"
+
+keepalived_global_scripts:
+  haproxy_check_script:
+    check_script: "killall -0 haproxy"
+  pingable_check_script:
+    check_script: "ping -c 1 193.0.14.129 1>&2"
+    interval: 10
+    fall: 2
+    rise: 4
+
+# Master and backup scripts should be the same.
+# The two variables (master/backup) are kept if the deployer wants different checks for backup and master.
+keepalived_master_scripts: "{{ keepalived_global_scripts }}"
+keepalived_backup_scripts: "{{ keepalived_global_scripts }}"
+
+keepalived_master_instances:
+  external:
+    interface: "{{ haproxy_keepalived_external_interface | default(management_bridge) }}"
+    state: MASTER
+    virtual_router_id: "{{ haproxy_keepalived_external_virtual_router_id | default ('10') }}"
+    priority: "{{ haproxy_keepalived_priority_master | default('100') }}"
+    authentication_password: "{{ haproxy_keepalived_authentication_password }}"
+    vips:
+      - "{{ haproxy_keepalived_external_vip_cidr }} dev {{ haproxy_keepalived_external_interface | default(management_bridge) }}"
+    track_scripts:
+      - haproxy_check_script
+      - pingable_check_script
+  internal:
+    interface: "{{ haproxy_keepalived_internal_interface | default(management_bridge) }}"
+    state: MASTER
+    virtual_router_id: "{{ haproxy_keepalived_internal_virtual_router_id | default ('11') }}"
+    priority: "{{ haproxy_keepalived_priority_master | default('100') }}"
+    authentication_password: "{{ haproxy_keepalived_authentication_password }}"
+    track_scripts:
+      - haproxy_check_script
+      - pingable_check_script
+    vips:
+      - "{{ haproxy_keepalived_internal_vip_cidr }} dev {{ haproxy_keepalived_internal_interface | default(management_bridge) }}"
+
+keepalived_backup_instances:
+  external:
+    interface: "{{ haproxy_keepalived_external_interface | default(management_bridge) }}"
+    state: BACKUP
+    virtual_router_id: "{{ haproxy_keepalived_external_virtual_router_id | default ('10') }}"
+    priority: "{{ haproxy_keepalived_priority_backup | default('20') }}"
+    authentication_password: "{{ haproxy_keepalived_authentication_password }}"
+    vips:
+      - "{{ haproxy_keepalived_external_vip_cidr }} dev {{ haproxy_keepalived_external_interface | default(management_bridge) }}"
+    track_scripts:
+      - haproxy_check_script
+      - pingable_check_script
+  internal:
+    interface: "{{ haproxy_keepalived_internal_interface | default(management_bridge) }}"
+    state: BACKUP
+    virtual_router_id: "{{ haproxy_keepalived_internal_virtual_router_id | default ('11') }}"
+    priority: "{{ haproxy_keepalived_priority_backup | default('20') }}"
+    authentication_password: "{{ haproxy_keepalived_authentication_password }}"
+    track_scripts:
+      - haproxy_check_script
+      - pingable_check_script
+    vips:
+      - "{{ haproxy_keepalived_internal_vip_cidr }} dev {{ haproxy_keepalived_internal_interface | default(management_bridge) }}"

playbooks/vars/configs/keepalived_haproxy_notifications.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+# Copyright 2015, Jean-Philippe Evrard <jean-philippe@evrard.me>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+TYPE=$1
+NAME=$2
+NOW=`date "+%Y-%m-%d %H:%M:%S"`
+NEWSTATE=$3
+OLDSTATE=$(cat /var/run/keepalived.state)
+
+echo "$NEWSTATE" > /var/run/keepalived.state
+
+case $NEWSTATE in
+        "FAULT") echo "$NOW Trying to restart haproxy to get out"\
+                  "of faulty state" >> /var/log/keepalived-notifications.log
+                 /etc/init.d/haproxy stop
+                 /etc/init.d/haproxy start
+                 exit 0
+                 ;;
+        *) echo "$NOW Unknown state" >> /var/log/keepalived-notifications.log
+           exit 1
+           ;;
+esac

scripts/gate-check-lint.sh

@@ -24,6 +24,8 @@ info_block "Checking for required libraries." 2> /dev/null || source $(dirname $
 ## Main ----------------------------------------------------------------------
 info_block "Running Basic Ansible Lint Check"
 
+# next, bootstrap Ansible
+source $(dirname ${0})/bootstrap-ansible.sh
 
 # Install the development requirements.
 if [ -f "dev-requirements.txt" ]; then